feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

tests/integration/test_chat_api.py

@@ -94,7 +94,7 @@ def seeded_kb(db) -> Iterator[FakeRagLLM]:
     db.commit()
     llm = FakeRagLLM()
     summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
-    assert summary.added == 3
+    assert summary.added == 8  # A9 formats; .hidden/ skipped
     yield llm
     db.execute(text("TRUNCATE chunks, documents, query_log"))
     db.commit()
@@ -163,12 +163,19 @@ def test_chat_writes_query_log_row(client, db, seeded_kb: FakeRagLLM) -> None:
     assert row.question == QUESTION
     assert row.deflected is False
     total_chunks = db.scalar(select(func.count()).select_from(Chunk))
-    assert row.chunk_hits == min(get_settings().top_k_chunks, total_chunks)
+    # chunk_hits is the fused candidate set (cosine top-N ∪ FTS top-N).
+    assert 1 <= row.chunk_hits <= total_chunks
     assert row.top_score > 0.0  # genuine token-overlap cosine, best hit
-    assert row.top_score >= get_settings().relevance_threshold  # why the gate answered
     assert row.top_score <= 1.0
     assert "docs/homelab/kubernetes.md" in row.sources
     assert row.latency_ms >= 0
+    # Why the gate answered (A8 revised): cosine over the threshold OR a
+    # lexical hit. The mock-calibrated threshold (0.30, see tests/conftest.py)
+    # makes the cosine branch true here; the FTS branch is covered too —
+    # "kubernetes" / "cluster" match the doc's tsvector.
+    thr = get_settings().relevance_threshold
+    assert row.top_score >= thr or (row.fts_hits or 0) > 0
+    assert (row.fts_hits or 0) >= 1  # the lexical branch really fired
 
 
 def test_off_topic_question_deflects_honestly(client, db, seeded_kb: FakeRagLLM) -> None:
@@ -202,14 +209,46 @@ def test_off_topic_question_deflects_honestly(client, db, seeded_kb: FakeRagLLM)
     assert "Talos Linux" not in system["content"]  # full doc content never sent
     assert "<documents>" not in system["content"]
 
-    # Durable record: deflected=true + the weak top_score.
+    # Durable record: deflected=true + the weak top_score. Deflection is
+    # only reached when the cosine is under the threshold AND no chunk
+    # FTS-matches the question — so fts_hits must be zero here.
     row = db.scalars(select(QueryLog)).one()
     assert row.question == OFF_TOPIC
     assert row.deflected is True
     assert 0.0 < row.top_score < get_settings().relevance_threshold
+    assert row.fts_hits == 0
     assert row.chunk_hits >= 1
 
 
+def test_keyword_question_grounded_by_lexical_hit_despite_weak_cosine(
+    client, db, seeded_kb: FakeRagLLM
+) -> None:
+    """Phase 09: a name-your-tool question the vector model barely ranks
+    ("kafkabridge" only appears in static-dns.json) must still be grounded
+    via the FTS branch — LOW only fires at weak cosine AND zero hits."""
+    fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
+    try:
+        _, _, frames = _stream_chat(client, "How does kafkabridge work?")
+    finally:
+        fastapi_app.dependency_overrides.clear()
+
+    done = frames[-1]
+    assert done["type"] == "done"
+    assert done["deflected"] is False  # weak cosine, but a lexical hit
+    assert done["suggestions"] == []
+    sources = done["sources"]
+    assert sources and sources[0]["path"] == "homelab/networking/static-dns.json"
+
+    (system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
+    assert "<relevance>HIGH</relevance>" in system["content"]  # grounded prompt
+
+    row = db.scalars(select(QueryLog)).one()
+    assert row.deflected is False
+    assert row.top_score < get_settings().relevance_threshold  # weak vector score
+    assert (row.fts_hits or 0) >= 1  # …and it is the FTS hit that grounds it
+    assert "docs/homelab/networking/static-dns.json" in row.sources
+
+
 def test_chat_empty_kb_streams_empty_sources(client, db) -> None:
     db.execute(text("TRUNCATE chunks, documents, query_log"))
     db.commit()

tests/integration/test_importer_e2e.py

@@ -22,6 +22,11 @@ EXPECTED_DOCS = {
     ("docs", "homelab/kubernetes.md"),
     ("docs", "homelab/backups.md"),
     ("docs", "deployments/new-service.md"),
+    ("docs", "homelab/container_gitlab/gitlab.md"),
+    ("docs", "homelab/container_gitlab/gitlab-compose.yaml"),
+    ("docs", "homelab/networking/static-dns.json"),
+    ("docs", "homelab/scripts/uptime_probe.py"),
+    ("docs", "homelab/ssh/ssh_aliases.txt"),
 }
 
 
@@ -31,14 +36,27 @@ def test_import_fixtures_end_to_end(client, db) -> None:
     llm = FakeEmbedder()
 
     summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
-    assert (summary.files, summary.added, summary.unchanged) == (3, 3, 0)
-    assert summary.chunks >= 3
+    # Eight A9-format files; .hidden/junk.md is out of scope (A9 revised).
+    assert (summary.files, summary.added, summary.unchanged) == (8, 8, 0)
+    assert summary.chunks >= 8
+    assert summary.formats == {"md": 4, "yaml": 1, "json": 1, "py": 1, "txt": 1}
+    # PLAN §9 per-format summary line: highest count first, then alpha.
+    assert summary.format_counts() == "md:4,json:1,py:1,txt:1,yaml:1"
 
     docs = db.scalars(select(Document)).all()
     assert {(d.source, d.path) for d in docs} == EXPECTED_DOCS
     titles = {d.path: d.title for d in docs}
     assert titles["homelab/kubernetes.md"] == "Kubernetes Homelab Cluster"
     assert titles["deployments/new-service.md"] == "Deploying a New Service"
+    assert titles["homelab/container_gitlab/gitlab.md"] == "Gitlab"
+    # Non-markdown titles come from the file stem (a leading ``#`` or docstring
+    # line is a comment there, not a heading).
+    assert titles["homelab/container_gitlab/gitlab-compose.yaml"] == "gitlab-compose"
+    assert titles["homelab/scripts/uptime_probe.py"] == "uptime_probe"
+    assert titles["homelab/networking/static-dns.json"] == "static-dns"
+    assert titles["homelab/ssh/ssh_aliases.txt"] == "ssh_aliases"
+    # Hidden junk was never imported.
+    assert not any(".hidden" in d.path for d in docs)
     # Full content is stored — that is what the RAG context will be.
     k8s = next(d for d in docs if d.path == "homelab/kubernetes.md")
     assert "Talos Linux" in k8s.content and k8s.content_hash
@@ -52,13 +70,13 @@ def test_import_fixtures_end_to_end(client, db) -> None:
     r = client.get("/api/docs")
     assert r.status_code == 200
     body = r.json()
-    assert len(body["documents"]) == 3
+    assert len(body["documents"]) == 8
     assert all(d["chunks"] >= 1 for d in body["documents"])
 
     # Idempotent re-run: nothing re-embedded.
     calls_before = len(llm.calls)
     s2 = asyncio.run(import_sources([FIXTURES], llm, session=db))
-    assert s2.unchanged == 3 and s2.added == 0
+    assert s2.unchanged == 8 and s2.added == 0
     assert len(llm.calls) == calls_before  # unchanged → no embedding requests
 
     db.execute(text("TRUNCATE chunks, documents, query_log"))

tests/integration/test_migration_0002.py

@@ -0,0 +1,72 @@
+"""Integration: migration 0002 (hybrid retrieval) schema contract.
+
+Asserts the state the migration must leave on the live schema:
+``chunks.tsv`` as a stored generated tsvector, its GIN index, and the
+nullable ``query_log.fts_hits`` column (pre-0002 rows stay NULL, so it
+must accept NULL and an int). Requires ``podman compose up -d db``.
+"""
+from __future__ import annotations
+
+from sqlalchemy import text
+
+
+def test_migration_0002_schema_contract(db) -> None:
+    tsv_col = db.execute(
+        text(
+            "SELECT count(*) FROM information_schema.columns"
+            " WHERE table_name = 'chunks' AND column_name = 'tsv'"
+        )
+    ).scalar()
+    assert tsv_col == 1, "chunks.tsv (stored tsvector) is missing"
+
+    gin = db.execute(
+        text(
+            "SELECT count(*) FROM pg_indexes"
+            " WHERE tablename = 'chunks' AND indexdef ILIKE '%USING gin%'"
+            " AND indexdef ILIKE '%tsv%'"
+        )
+    ).scalar()
+    assert gin == 1, "GIN index on chunks.tsv is missing"
+
+    fts = db.execute(
+        text(
+            "SELECT is_nullable = 'YES' FROM information_schema.columns"
+            " WHERE table_name = 'query_log' AND column_name = 'fts_hits'"
+        )
+    ).scalar()
+    assert fts is True, "query_log.fts_hits must exist and be nullable (pre-0002 rows)"
+
+
+def test_tsv_is_generated_and_lexically_queryable(db) -> None:
+    """The tsvector is generated from ``content`` (not maintained by app
+    code) and answers a tsquery — the retrieval path's lexical branch."""
+    doc_id = db.execute(text("SELECT gen_random_uuid()")).scalar()
+    try:
+        db.execute(
+            text(
+                "INSERT INTO documents (id, source, path, full_path, title, content,"
+                " content_hash, indexed_at) VALUES"
+                " (:id, 'mig_test', 't.md', '/t.md', 'T', 'kafkabridge routes here',"
+                " repeat('0', 64), now())"
+            ),
+            {"id": doc_id},
+        )
+        db.execute(
+            text(
+                "INSERT INTO chunks (id, document_id, position, content) VALUES"
+                " (gen_random_uuid(), :id, 0, 'kafkabridge routes here')"
+            ),
+            {"id": doc_id},
+        )
+        db.commit()
+        hit = db.execute(
+            text(
+                "SELECT count(*) FROM chunks"
+                " WHERE tsv @@ to_tsquery('english', 'kafkabridge')"
+            )
+        ).scalar()
+        assert hit == 1
+    finally:
+        db.execute(text("DELETE FROM chunks WHERE document_id = :id"), {"id": doc_id})
+        db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id})
+        db.commit()