feat: resolve ingest conflicts and complete storage, Docker, and API fixes

Dockerfile

@@ -1,5 +1,8 @@
 FROM python:3.13-slim@sha256:6771159cd4fa5d9bba1258caf0b82e6b73458c694d178ad97c5e925c2d0e1a91 AS base
 
+ARG LOCALRAG_BUILD_SHA=unknown
+ENV LOCALRAG_BUILD_SHA=${LOCALRAG_BUILD_SHA}
+
 WORKDIR /app
 
 # tesseract-ocr: required at runtime for scanned/image-only PDF pages (see docs/ocr.md).

Taskfile.yml

@@ -127,7 +127,12 @@ tasks:
   docker-up:
     desc: Start the Compose stack in the background without removing volumes
     cmds:
-      - '{{.COMPOSE}} --project-name "{{.PROJECT}}" -f "{{.COMPOSE_FILE}}" {{if .COMPOSE_OVERRIDE}}-f "{{.COMPOSE_OVERRIDE}}" {{end}}up --build -d'
+      - 'LOCALRAG_BUILD_SHA="$$(git rev-parse HEAD)" {{.COMPOSE}} --project-name "{{.PROJECT}}" -f "{{.COMPOSE_FILE}}" {{if .COMPOSE_OVERRIDE}}-f "{{.COMPOSE_OVERRIDE}}" {{end}}up --build -d'
+
+  docker-check:
+    desc: Compare the running API image identity with the current Git revision
+    cmds:
+      - '{{.PYTHON}} -c ''import json, os, subprocess, sys, urllib.request; expected=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(); request=urllib.request.Request("{{.API_URL}}/build-info", headers={"X-API-Key": os.environ["API_KEY"]}); actual=json.load(urllib.request.urlopen(request))["build_sha"]; print(f"running={actual} source={expected}"); sys.exit(0 if actual == expected else 1)'''
 
   docker-down:
     desc: Stop the Compose stack; named volumes are preserved

docker-compose.yml

@@ -31,7 +31,10 @@ services:
   # Runs from the same built image as localrag-api, so no separate script to
   # maintain. Exiting 0 is expected here — it's not meant to stay running.
   localrag-setup:
-    build: .
+    build:
+      context: .
+      args:
+        LOCALRAG_BUILD_SHA: ${LOCALRAG_BUILD_SHA:-unknown}
     image: localrag-api:local
     command: /app/.venv/bin/localrag setup
     environment:
@@ -45,7 +48,10 @@ services:
     restart: "no"
 
   localrag-api:
-    build: .
+    build:
+      context: .
+      args:
+        LOCALRAG_BUILD_SHA: ${LOCALRAG_BUILD_SHA:-unknown}
     image: localrag-api:local
     restart: unless-stopped
     ports:
@@ -81,7 +87,10 @@ services:
       start_period: 30s
 
   localrag-mcp:
-    build: .
+    build:
+      context: .
+      args:
+        LOCALRAG_BUILD_SHA: ${LOCALRAG_BUILD_SHA:-unknown}
     image: localrag-api:local
     restart: unless-stopped
     ports:

docs/adr/040-request-scoped-collection-selection.md

@@ -0,0 +1,34 @@
+# ADR 040: Request-Scoped Collection Selection
+
+## Status
+
+Accepted
+
+## Context
+
+The HTTP query API previously always used the collection configured at process
+startup. That made `GET /collections` expose namespaces that HTTP clients could
+not query, while the CLI could select them with a settings override.
+
+## Decision
+
+`POST /query`, `/query/contexts`, and `/query/stream` accept an optional
+`collection` field. An omitted field retains the configured
+`chroma_collection_name`. A supplied name selects an existing Chroma collection
+for that request by creating a request-scoped vector store, BM25 snapshot, and
+retriever/engine view. Process-wide settings and the cached default engine are
+not mutated.
+
+The endpoint remains behind `API_KEY`. Collection names are not treated as
+tenant authorization: deployments that use collections for tenant separation
+must enforce per-key authorization at a gateway or provide separate persist
+paths. This is an explicit single-deployment namespace selection feature, not a
+multi-tenant isolation boundary.
+
+## Consequences
+
+- JSON, contexts, and SSE query paths can address any existing collection.
+- A request-scoped hybrid query rebuilds the BM25 snapshot for the selected collection.
+- Unknown collections return a query `404` rather than exposing a backend error.
+- Retriever plugins that do not expose the built-in collection seam cannot use
+  request-scoped selection.

docs/adr/README.md

@@ -61,6 +61,7 @@ default.
 | [037](037-grouped-configuration-model.md) | Grouped configuration model behind flat public names | Accepted | Configuration |
 | [038](038-application-and-mcp-boundaries.md) | Transport-agnostic application boundary and MCP adapter | Amended by [039](039-fastmcp-sdk-adoption.md) | Architecture / MCP |
 | [039](039-fastmcp-sdk-adoption.md) | Adopt the FastMCP SDK for the MCP adapter | Accepted | Architecture / MCP |
+| [040](040-request-scoped-collection-selection.md) | Request-scoped HTTP collection selection | Accepted | API / Retrieval |
 
 ## Research spikes
 

docs/agent-friction.md

@@ -0,0 +1,37 @@
+# Agent Friction Notes
+
+This is a short record of repository traps discovered while resolving and
+verifying changes. Keep entries concrete and delete them only when the
+underlying workflow or contract changes.
+
+## Merge Conflicts
+
+PR #169 conflicted with the progress-output work because both branches changed
+the same CLI ingest section, ingest entry point, and ingestion-service tests.
+The correct resolution was additive: keep progress callbacks and wrap the whole
+write path in the cross-process ingest lock. Rebase the PR branch onto current
+`main` before reconstructing tests; resolving only the documentation conflict
+leaves the test file structurally broken.
+
+## Docker Drift
+
+An already-running Compose stack serves the old image even when the source tree
+has changed. Rebuild before integration tests. The image now carries
+`LOCALRAG_BUILD_SHA`; `task docker-up` stamps the current revision and
+`task docker-check` compares it through the authenticated `/build-info`
+endpoint.
+
+## Chroma Lifecycle
+
+Deleting a collection removes Chroma metadata but can leave its persisted HNSW
+directory. Resolve the vector segment ID before deletion, remove only that
+directory after successful metadata deletion, and invalidate cached retrieval
+objects. Otherwise the next query in the same long-running API can use the
+deleted collection UUID and return a misleading 503.
+
+## Integration Test Assumptions
+
+The Compose setup pulls `gemma3:4b`; integration tests must not request an
+unprovisioned model such as `qwen2.5:0.5b`. Run the full integration suite only
+after rebuilding the image and starting the stack, using
+`LOCALRAG_TEST_API_KEY` for protected endpoints.

docs/agent-navigation.md

@@ -36,7 +36,7 @@ override unless `COMPOSE_OVERRIDE` is supplied.
 | Transport-agnostic use cases | `localrag/application/`, `localrag/application/service.py`, `localrag/application/container.py` |
 | MCP tools and transports | `localrag/mcp/server.py`, `localrag/mcp/app.py`, `docs/mcp.md` |
 | API request/response OpenAPI models | `localrag/api/schemas.py` |
-| Application use cases (health, ingest rules, query JSON + SSE, collections including rebuild) | `localrag/application/service.py` |
+| Application use cases (health, ingest rules, query JSON + SSE, request-scoped collections, collections including rebuild) | `localrag/application/service.py`, `localrag/rag/engine.py`, `localrag/rag/retriever.py` |
 | API schema/error adapter | `localrag/api/service.py`, `localrag/api/exceptions.py` |
 | Application persistence boundary (Chroma collections) | `localrag/application/repository.py` |
 | API app factory (lifespan, middleware, error handlers) | `localrag/api/main.py` |
@@ -48,7 +48,7 @@ override unless `COMPOSE_OVERRIDE` is supplied.
 | Log format, levels, request ID | `localrag/logging_config.py`, `localrag/api/middleware.py`, `LOG_LEVEL` in `localrag/settings.py` |
 | Optional tracing / observability | `localrag/observability/tracing.py`, `OTEL_*` in `localrag/settings.py`, [observability.md](observability.md), [ADR 030](adr/030-optional-otel-observability-boundary.md) |
 | API key auth | `localrag/api/dependencies.py` (`require_api_key`), `API_KEY` in `localrag/settings.py` |
-| Prometheus metrics endpoint | `localrag/api/routers/metrics.py` |
+| Prometheus metrics and running build identity | `localrag/api/routers/metrics.py`, `Taskfile.yml`, `Dockerfile`, [observability.md](observability.md) |
 | LLM provider abstraction | `localrag/llm/providers/`, `localrag/llm/factory.py` |
 | Cost estimation | `localrag/llm/costs.py` |
 | Agent tool-use (search_documents / answer_directly) | `localrag/agent/service.py`, `localrag/api/routers/agent.py` |
@@ -63,6 +63,7 @@ override unless `COMPOSE_OVERRIDE` is supplied.
 | Ingestion embedding cache | `localrag/embedding/cache.py`, embedding cache settings, [ADR 024](adr/024-embedding-cache-contract.md), `benchmarks/embedding_cache_benchmark.py` |
 | Ingest orchestration | `localrag/ingestion/service.py` |
 | Chroma collection / persist path | `localrag/storage/vector_store.py`, settings |
+| Concurrent-ingest exclusion (one writer per persist path) | `localrag/storage/persist_lock.py`, `localrag/ingestion/service.py`, [ADR 035](adr/035-atomic-ingestion-replacement.md), [cli.md](cli.md) |
 | Retrieval mode / hybrid ranking / freshness decay / HyDE experiment | `localrag/rag/retriever.py`, `localrag/rag/hyde.py`, `localrag/rag/bm25_index.py`, `localrag/settings.py`, [ADR 025](adr/025-hyde-retrieval-experiment.md) |
 | Retriever plugin contract / discovery | `localrag/plugins/retriever.py`, [plugin-author-guide.md](plugin-author-guide.md), [ADR 032](adr/032-retriever-plugin-contract.md) |
 | Bounded adaptive retrieval policy / trace | `localrag/rag/adaptive.py`, `localrag/rag/engine.py`, adaptive settings, [ADR 023](adr/023-bounded-adaptive-retrieval.md) |
@@ -102,6 +103,8 @@ uv run ruff check .
 
 Pre-commit and contribution workflow: [`CONTRIBUTING.md`](../CONTRIBUTING.md).
 For portable task commands and their variable contract, use [`Taskfile.yml`](../Taskfile.yml).
+Known merge, Docker, Chroma, and integration-test friction is recorded in
+[`agent-friction.md`](agent-friction.md).
 
 ## External dependencies
 

docs/architecture.md

@@ -62,10 +62,11 @@ flowchart LR
 ```
 
   - **Ingest:** files → `loader` / `ingestion/parsers/*` → text → the shared `Chunk` contract (`localrag/ingestion/contract.py`) implemented by fixed, structural, or recursive strategies → the factory-created `EmbeddingProvider` → `VectorStore` (Chroma, persistent path from settings). Contract IDs are deterministic from source, strategy, index, and text; offsets are explicitly absent because current strategies normalize or repack text. Empty input emits no chunks and oversized atomic input is retained with an `oversized` marker. The same provider instance embeds retrieval queries. Collection metadata records provider/model/dimension and rejects incompatible operations; changing the embedding space requires an explicit rebuild. The application ingest use cases in `localrag/application/service.py` own path decode, existence checks, `INGEST_ROOTS`, upload limits, and background jobs; the HTTP adapter maps their DTOs and errors to OpenAPI responses. CLI and MCP calls use the same application use cases directly. See [ADR 021](adr/021-chunking-strategy-contract.md) and [ADR 038](adr/038-application-and-mcp-boundaries.md).
+  - **One writer per persist path:** Chroma's embedded client holds HNSW segments in per-process memory with no cross-process invalidation, so concurrent writers from separate processes silently lose writes ([ADR 035](adr/035-atomic-ingestion-replacement.md) already puts them out of contract). `IngestionService.ingest_paths` and `rebuild_collection` therefore take `localrag/storage/persist_lock.py::ingest_lock` — an advisory `flock` on `<CHROMA_PERSIST_PATH>/.ingest.lock` — for the duration of the write; a competing process gets `ConcurrentIngestError` immediately (CLI: stderr message and exit `1`; HTTP: `409 Conflict`). The lock deliberately sits at the ingest use-case boundary rather than inside `VectorStore`, because `api/dependencies.py` and `application/runtime.py` cache a `VectorStore` for the whole process lifetime and a store-scoped lock would let a running API block every CLI ingest forever. Read and query paths are never locked, and an unopenable persist directory or a filesystem without `flock` degrades to a logged warning rather than a hard failure. `VectorStore.delete_collection` removes the deleted collection's recorded persisted HNSW segment directory after Chroma metadata deletion, without sweeping unrelated directories.
 - **Rebuild:** `POST /collections/rebuild` and `localrag collections rebuild` list distinct `source` values in the active collection, drop vectors for missing files, and re-chunk/re-embed remaining paths (optional `embed_model` override). Implemented in `IngestionService.rebuild_collection`.
-  - **Query (JSON):** `POST /query` returns a complete `QueryResponse` (answer, sources, latency_ms, model) from the application `query_json` use case, adapted by `localrag/api/service.py`. Retrieval supports vector-only and hybrid (vector + BM25 with reciprocal-rank fusion), optional bounded query expansion, then applies optional freshness decay based on chunk `ingested_at`. When `ADAPTIVE_ENABLED=true`, `AdaptiveRetrievalPolicy` performs bounded evidence evaluation/escalation/refinement and adds an observable trace; thresholds are corpus-tuned heuristics, not calibrated confidence. JSON and SSE use the same engine policy path. `RAGEngine` generates the answer via its injected `provider` (a `BaseLLMProvider` built by `llm/factory.py::build_provider`, resilience-wrapped), so `LLM_BACKEND` genuinely governs which backend answers `/query` — it is no longer hard-wired to Ollama. Requires `X-API-Key` when `API_KEY` is set.
+  - **Query (JSON):** `POST /query` returns a complete `QueryResponse` (answer, sources, latency_ms, model) from the application `query_json` use case, adapted by `localrag/api/service.py`. Retrieval supports vector-only and hybrid (vector + BM25 with reciprocal-rank fusion), optional bounded query expansion, then applies optional freshness decay based on chunk `ingested_at`. When `ADAPTIVE_ENABLED=true`, `AdaptiveRetrievalPolicy` performs bounded evidence evaluation/escalation/refinement and adds an observable trace; thresholds are corpus-tuned heuristics, not calibrated confidence. JSON and SSE use the same engine policy path. `RAGEngine` generates the answer via its injected `provider` (a `BaseLLMProvider` built by `llm/factory.py::build_provider`, resilience-wrapped), so `LLM_BACKEND` genuinely governs which backend answers `/query` — it is no longer hard-wired to Ollama. An optional request `collection` selects another collection for that request only; omitted values use `chroma_collection_name` ([ADR 040](adr/040-request-scoped-collection-selection.md)). Requires `X-API-Key` when `API_KEY` is set.
 - **Query (SSE stream):** `POST /query/stream` streams tokens as Server-Sent Events. Retrieval runs synchronously first (`get_query_contexts`) so errors map to HTTP before SSE starts, then tokens are mapped via `iter_query_sse_events`. Token streaming likewise goes through `RAGEngine.provider.stream_from_prompt(...)`, so `LLM_BACKEND` governs the streaming path too.
-- **Metrics:** `GET /metrics` exposes Prometheus metrics via `prometheus_client` (router at `localrag/api/routers/metrics.py`). No auth required.
+- **Metrics:** `GET /metrics` exposes Prometheus metrics via `prometheus_client` (router at `localrag/api/routers/metrics.py`). Metrics and `GET /build-info` require `X-API-Key` when `API_KEY` is set; build info reports `LOCALRAG_BUILD_SHA` for `task docker-check`.
 
 ## Package map
 
@@ -89,6 +90,7 @@ flowchart LR
 | Embedding | `localrag/embedding/`, `localrag/ingestion/embedder.py` | Provider protocol/factory, Ollama **`POST /api/embed`**, optional sentence-transformers backend, collection identity checks, and the disabled-by-default provider-aware ingestion vector cache |
 | Embedding cache | `localrag/embedding/cache.py` | Versioned hashed vector-only disk entries, atomic writes, process/thread locking, checksum validation, bounded LRU cleanup, and fail-open cache I/O |
 | Storage | `localrag/storage/vector_store.py` | Chroma client wrapper |
+| Ingest ownership lock | `localrag/storage/persist_lock.py` | `ingest_lock` — advisory `flock` on `<persist_path>/.ingest.lock` giving one process exclusive write ownership of a Chroma persist path; fails fast with `ConcurrentIngestError` |
 | RAG | `localrag/rag/retriever.py`, `bm25_index.py`, `engine.py`, `prompt.py` | Hybrid retrieval (vector + BM25), freshness decay reranking, prompt build, LLM call |
 | Retriever plugins | `localrag/plugins/retriever.py`, `docs/plugin-author-guide.md` | Versioned `localrag.retrievers` entry points; deterministic selection and lifecycle ownership |
 | Context compression | `localrag/rag/compressor.py` | Disabled-by-default deterministic extractive compression after parent expansion; preserves retrieval provenance and hard token/character budgets |

docs/cli.md

@@ -58,6 +58,36 @@ Files that no parser can handle — and binary content in a file with a textual
 extension — are reported and skipped rather than being read as text. See
 [document-formats.md](document-formats.md).
 
+`ingest` accepts a file or a directory and writes into the configured Chroma
+persist path:
+
+```bash
+uv run localrag ingest ./docs
+uv run localrag ingest ./docs/guide.md
+```
+
+**One writer per persist path.** Chroma's embedded client keeps its HNSW
+segments in per-process memory with no cross-process invalidation, so two
+processes writing the same `CHROMA_PERSIST_PATH` corrupt each other's view —
+the boundary [ADR 035](adr/035-atomic-ingestion-replacement.md) already declares
+out of contract. Every ingest therefore takes an exclusive `flock` on
+`<CHROMA_PERSIST_PATH>/.ingest.lock` for the duration of the run, and the same
+lock covers collection rebuilds.
+
+A second concurrent ingest **fails immediately** rather than queueing — an
+ingest can run for minutes, so a caller is better told to retry than left
+hanging:
+
+```
+status=error reason=concurrent_ingest detail=another ingest is already running against ./data/chroma; wait for it to finish or use a different collection
+```
+
+The message goes to stderr and the command exits `1`. The equivalent HTTP
+ingest endpoints return `409 Conflict` with the same detail. Query and other
+read paths are never locked. The lock is advisory: if the persist directory
+cannot be opened, or the filesystem does not support `flock` (some network
+mounts) the ingest proceeds unguarded and logs a warning.
+
 ## Inspect
 
 `inspect` is read-only and never creates a missing collection, calls an LLM, or

docs/deployment.md

@@ -16,6 +16,11 @@ Set secrets in the environment or `.env` before starting:
 API_KEY='change-me' GRAFANA_ADMIN_PASSWORD='change-me-too' docker compose up -d
 ```
 
+`task docker-up` passes the current Git revision into the image as
+`LOCALRAG_BUILD_SHA`. Run `API_KEY=... task docker-check` after source changes;
+it calls the protected `/build-info` endpoint and exits non-zero when the
+running image is stale. Rebuild the stack before trusting end-to-end results.
+
 Ollama runs without a default GPU reservation so the stack also works on CPU
 CI runners. Configure GPU scheduling separately for hosts that support it.
 

docs/observability.md

@@ -35,15 +35,20 @@ telemetry.
 
 ## Prometheus Metrics
 
-The `/metrics` endpoint exposes bounded counters and histograms for query
+The authenticated `/metrics` endpoint exposes bounded counters and histograms for query
 duration, retrieved chunks, generated tokens, query/provider failures, cache
 hits and misses, ingestion failures, background job terminal status, upload
 cleanup, audit-log rotation, and HTTP failures by status class. Labels are limited to transport/outcome
  operation names and provider identity; model names, question text, source
  paths, and request IDs are never labels. JSON, adaptive, and SSE queries record
  the same duration, retrieval, token, audit, and failure signals. Fallback use,
  upload quota rejection, oversized audit records, and audit write/cleanup
- failures have dedicated bounded counters.
+failures have dedicated bounded counters.
+
+`GET /build-info` is protected by the same API key and returns the image's
+`LOCALRAG_BUILD_SHA`. `task docker-up` stamps this value from the current Git
+revision, and `task docker-check` compares it with the running API so a stale
+Compose image is visible before diagnosing application behavior.
 
 Useful alerts include `rate(localrag_query_failures_total[5m]) > 0`, a sustained
 `localrag_ingest_jobs_pending` near the configured cap, and any increase in