feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

tests/conftest.py

@@ -1,14 +1,22 @@
 """Shared fixtures for unit + integration tests."""
 from __future__ import annotations
 
+import os
 from collections.abc import Iterator
 
 import pytest
 from fastapi.testclient import TestClient
 from sqlalchemy.orm import Session
 
-from app.db import SessionLocal, db_available
-from app.main import app as fastapi_app
+# In-process integration/E2E tests drive the app with *mock* embeddings
+# (bag-of-words, cosine ~0.1–0.8), not the live aipi model — so the honesty
+# gate is calibrated to the mock's distribution, mirroring tests/e2e/
+# conftest.py. Must be set before ``app.main`` (below) caches settings.
+# The production default stays 0.62 (app/config.py, A8 revised).
+os.environ.setdefault("BOR_RELEVANCE_THRESHOLD", "0.30")
+
+from app.db import SessionLocal, db_available  # noqa: E402
+from app.main import app as fastapi_app  # noqa: E402
 
 
 @pytest.fixture()

tests/e2e/conftest.py

@@ -82,6 +82,12 @@ def app_server(mock_llm: int) -> Iterator[str]:
         if USE_REAL_LLM
         else f"http://127.0.0.1:{mock_llm}/v1"
     )
+    # The E2E mock's token-overlap embeddings have their own score
+    # distribution (phase 09) — the app under test gets the mock-calibrated
+    # threshold so every story suite keeps its deterministic gate behavior.
+    # The production default stays 0.62 (re-tuned against the real
+    # `embed` model's 0.41–0.84 cosine range, PLAN A8).
+    env["BOR_RELEVANCE_THRESHOLD"] = "0.30"
     env.setdefault("BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese")
     proc = subprocess.Popen(
         [sys.executable, "-m", "uvicorn", "app.main:app",

tests/e2e/test_chat_rag.py

@@ -76,7 +76,7 @@ def test_on_topic_question_streams_grounded_answer(
     page: Page, app_url: str, mock_llm: int, db_ready: None
 ) -> None:
     summary = _reset_db(mock_llm, seed=True)
-    assert summary is not None and summary.added == 3
+    assert summary is not None and summary.added == 8  # A9 formats
     page.set_default_timeout(30_000)
     page.goto(app_url)
 

tests/e2e/test_dark_tech_theme.py

@@ -111,7 +111,7 @@ def _seed_kb(mock_port: int) -> ImportSummary:
         db.execute(text("TRUNCATE chunks, documents, query_log"))
         db.commit()
     summary = _run_in_thread(_import_fixtures(mock_port))
-    assert summary is not None and summary.added == 3
+    assert summary is not None and summary.added == 8  # A9 formats
     return summary
 
 

tests/e2e/test_honest_deflection.py

@@ -84,7 +84,7 @@ def test_off_topic_question_deflects_honestly(
     page: Page, app_url: str, mock_llm: int, db_ready: None
 ) -> None:
     summary = _reset_db(mock_llm, seed=True)
-    assert summary is not None and summary.added == 3
+    assert summary is not None and summary.added == 8  # A9 formats
     page.set_default_timeout(30_000)
     page.goto(app_url)
     expect(page.locator("#kb-banner")).to_be_hidden()

tests/e2e/test_import_documents.py

@@ -31,6 +31,11 @@ EXPECTED_ROWS = (
     "homelab/kubernetes.md",
     "homelab/backups.md",
     "deployments/new-service.md",
+    "homelab/container_gitlab/gitlab.md",
+    "homelab/container_gitlab/gitlab-compose.yaml",
+    "homelab/networking/static-dns.json",
+    "homelab/scripts/uptime_probe.py",
+    "homelab/ssh/ssh_aliases.txt",
 )
 
 
@@ -76,16 +81,21 @@ def test_sources_page_lists_indexed_docs(
     page: Page, app_url: str, mock_llm: int, db_ready: None
 ) -> None:
     summary = _reset_db(mock_llm, seed=True)
-    assert summary is not None and summary.added == 3
+    # Eight A9-format files are imported; .hidden/junk.md is out of scope
+    # (A9 revised — hidden path components are never walked).
+    assert summary is not None and summary.added == 8
+    assert summary.formats == {"md": 4, "yaml": 1, "json": 1, "py": 1, "txt": 1}
 
     page.goto(f"{app_url}/sources.html")
-    expect(page.locator("#stat-docs")).to_have_text("3")
+    expect(page.locator("#stat-docs")).to_have_text("8")
     expect(page.locator("#stat-chunks")).to_have_text(str(summary.chunks))
     expect(page.locator("#stat-last")).not_to_have_text("–")
     expect(page.locator("#sources-empty")).to_be_hidden()
 
     for row_path in EXPECTED_ROWS:
         expect(page.locator("#docs-tbody tr", has_text=row_path)).to_have_count(1)
+    # The hidden junk was never indexed (A9 scope).
+    expect(page.locator("#docs-tbody tr", has_text=".hidden")).to_have_count(0)
     # The path column carries the full path for hover (ellipsis is visual only).
     expect(page.locator("#docs-tbody tr", has_text="homelab/kubernetes.md")
            .get_by_role("cell").nth(1)).to_have_attribute("title", "homelab/kubernetes.md")

tests/e2e/test_loading_feedback.py

@@ -176,7 +176,7 @@ def test_typing_indicator_during_slow_think(
     """AC1/AC5: the 3s mock warm-up must show the typing indicator for
     >=2s before any text appears, then it is gone once the answer lands."""
     summary = _reset_db(mock_llm, seed=True)
-    assert summary is not None and summary.added == 3
+    assert summary is not None and summary.added == 8  # A9 formats
     page.set_default_timeout(30_000)
     page.goto(app_url)
 

tests/e2e/test_retrieval_quality.py

@@ -0,0 +1,197 @@
+"""Phase 09 E2E (Playwright): retrieval quality — hybrid search end to end.
+
+Story: ``.agent/user_stories/retrieval-quality.md``
+Run in isolation (DB must be up: ``podman compose up -d db``):
+
+    uv run pytest tests/e2e/test_retrieval_quality.py -v --no-cov
+
+Seeding reuses the real importer against ``tests/fixtures/docs/`` with the
+deterministic mock embeddings (same pattern as the earlier story suites).
+The four tests map the story's acceptance criteria:
+
+1. multi-format fixture import — hidden doc excluded, ``/api/docs`` counts
+2. "How did I install gitlab?" — grounded (not deflected), gitlab chip,
+   ``query_log`` row with the gitlab doc in ``sources``
+3. keyword-only question ("kafkabridge") beats the vector ranking — the
+   FTS-OR gate grounds it end to end despite weak cosine
+4. "sourdough" — deflected bubble + ≥2 "Maybe try" chips
+"""
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from threading import Thread
+from typing import Any
+
+import httpx
+from playwright.sync_api import Page, expect
+from sqlalchemy import select, text
+
+from app.config import Settings, get_settings
+from app.db import SessionLocal
+from app.models import QueryLog
+from app.rag.importer import ImportSummary, import_sources
+from app.rag.llm import LLMClient
+
+REPO = Path(__file__).resolve().parents[2]
+FIXTURES = REPO / "tests" / "fixtures" / "docs"
+GITLAB_QUESTION = "How did I install gitlab?"
+KEYWORD_QUESTION = "How does kafkabridge work?"
+OFF_TOPIC = "sourdough starter"
+MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
+
+
+async def _import_fixtures(mock_port: int) -> ImportSummary:
+    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
+    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
+    return await import_sources([FIXTURES], LLMClient(settings))
+
+
+def _run_in_thread(coro: Any) -> Any:
+    """Run a coroutine on a worker thread (Playwright owns the test loop)."""
+    box: dict[str, Any] = {}
+
+    def runner() -> None:
+        try:
+            box["value"] = asyncio.run(coro)
+        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
+            box["error"] = e
+
+    t = Thread(target=runner)
+    t.start()
+    t.join()
+    if "error" in box:
+        raise box["error"]
+    return box["value"]
+
+
+def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
+    """Truncate the KB (and query log), then optionally re-import fixtures."""
+    with SessionLocal() as db:
+        db.execute(text("TRUNCATE chunks, documents, query_log"))
+        db.commit()
+    if not seed:
+        return None
+    return _run_in_thread(_import_fixtures(mock_port))
+
+
+def _ask(page: Page, message: str) -> None:
+    page.fill("#message-input", message)
+    page.click("#send-btn")
+
+
+def test_multi_format_import_hidden_doc_excluded(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """A9 (revised): all seven formats import; hidden (dot) paths never do."""
+    summary = _reset_db(mock_llm, seed=True)
+    assert summary is not None
+    # Eight A9-format fixture files; .hidden/junk.md must never be walked.
+    assert summary.added == 8
+    assert summary.formats == {"md": 4, "yaml": 1, "json": 1, "py": 1, "txt": 1}
+
+    r = httpx.get(f"{app_url}/api/docs", timeout=10)
+    assert r.status_code == 200
+    docs = r.json()["documents"]
+    assert len(docs) == 8
+    assert all(".hidden" not in d["path"] for d in docs)
+    assert {d["path"] for d in docs} >= {
+        "homelab/container_gitlab/gitlab.md",
+        "homelab/container_gitlab/gitlab-compose.yaml",
+        "homelab/networking/static-dns.json",
+        "homelab/scripts/uptime_probe.py",
+        "homelab/ssh/ssh_aliases.txt",
+    }
+
+    # The Sources page reflects the same set.
+    page.goto(f"{app_url}/sources.html")
+    expect(page.locator("#stat-docs")).to_have_text("8")
+    expect(page.locator("#docs-tbody tr", has_text=".hidden")).to_have_count(0)
+
+
+def test_gitlab_question_is_grounded_with_gitlab_chip(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """The ranking problem that motivated this phase: a tool-name question
+    must land on the tool's own document — not a generic template."""
+    _reset_db(mock_llm, seed=True)
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    _ask(page, GITLAB_QUESTION)
+    expect(page.locator(".msg.user .bubble")).to_contain_text(GITLAB_QUESTION)
+
+    bubble = page.locator(".msg.brain .bubble").first
+    bubble.wait_for(state="visible", timeout=30_000)
+    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000)
+    # Grounded: no deflected bubble at all.
+    expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)
+
+    # The gitlab document is cited (a chip carrying its path).
+    chip = page.locator(".msg.brain .source-chip", has_text="container_gitlab/gitlab.md")
+    expect(chip).to_have_count(1, timeout=30_000)
+
+    # Durable record: not deflected, and the gitlab doc is in sources.
+    with SessionLocal() as db:
+        row = db.scalars(select(QueryLog)).one()
+    assert row.question == GITLAB_QUESTION
+    assert row.deflected is False
+    assert "container_gitlab/gitlab.md" in row.sources
+
+
+def test_keyword_only_question_beats_vector_ranking(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """The FTS-OR gate end to end: "kafkabridge" appears in exactly one
+    fixture doc (static-dns.json) and the question's cosine overlap is
+    weak — the lexical branch is what grounds the answer."""
+    _reset_db(mock_llm, seed=True)
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    _ask(page, KEYWORD_QUESTION)
+    bubble = page.locator(".msg.brain .bubble").first
+    bubble.wait_for(state="visible", timeout=30_000)
+    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000)
+    expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)
+
+    # The FTS-matched doc is the TOP source chip (it beats the vector rank).
+    first_chip = page.locator(".msg.brain .source-chip").first
+    first_chip.wait_for(state="visible", timeout=30_000)
+    expect(first_chip).to_contain_text("static-dns.json")
+
+    with SessionLocal() as db:
+        row = db.scalars(select(QueryLog)).one()
+    # Weak vector score…
+    assert row.top_score < get_settings().relevance_threshold
+    # …but a lexical hit grounded it (the FTS-OR branch).
+    assert (row.fts_hits or 0) >= 1
+    assert row.deflected is False
+    assert "homelab/networking/static-dns.json" in row.sources
+
+
+def test_off_topic_still_deflects_with_chips(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """A8 (revised): deflection requires weak cosine AND zero FTS hits.
+    "sourdough" matches nothing in the KB lexically → honest deflection."""
+    _reset_db(mock_llm, seed=True)
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    _ask(page, OFF_TOPIC)
+    bubble = page.locator(".msg.brain.is-deflected .bubble").first
+    bubble.wait_for(state="visible", timeout=30_000)
+    expect(page.locator(".msg.brain.is-deflected")).to_have_count(1)
+
+    chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip")
+    expect(chips.first).to_be_visible(timeout=30_000)
+    assert chips.count() >= 2, "deflection must offer 2-3 alternative chips"
+    assert all(c.strip() for c in chips.all_inner_texts())
+
+    with SessionLocal() as db:
+        row = db.scalars(select(QueryLog)).one()
+    assert row.question == OFF_TOPIC
+    assert row.deflected is True
+    assert 0.0 < row.top_score < get_settings().relevance_threshold
+    assert row.fts_hits == 0  # deflection is only reached with zero hits

tests/e2e/test_suggestion_chips.py

@@ -63,7 +63,7 @@ def _seed_kb(mock_port: int) -> ImportSummary:
         db.execute(text("TRUNCATE chunks, documents, query_log"))
         db.commit()
     summary = _run_in_thread(_import_fixtures(mock_port))
-    assert summary is not None and summary.added == 3
+    assert summary is not None and summary.added == 8  # A9 formats
     return summary
 
 

tests/fixtures/docs/.hidden/junk.md

@@ -0,0 +1,5 @@
+# Vendored Junk
+
+This file lives under a dot-prefixed directory and must **never** be
+imported into the knowledge base (A9 hidden-dir skip). It exists so the
+retrieval-quality E2E can prove the filter works.

tests/fixtures/docs/homelab/container_gitlab/gitlab-compose.yaml

@@ -0,0 +1,24 @@
+# gitlab stack — single container + gitlab-data volume
+services:
+  gitlab:
+    image: gitlab/gitlab-ce:17.2.1-ce.0
+    container_name: gitlab
+    restart: unless-stopped
+    hostname: "gitlab.reeseapps.com"
+    environment:
+      GITLAB_OMNIBUS_CONFIG: |
+        external_url 'https://gitlab.reeseapps.com'
+        gitlab_rails['gitlab_shell_ssh_port'] = 2222
+    ports:
+      - "8929:80"
+      - "2222:22"
+    volumes:
+      - gitlab-data:/var/opt/gitlab
+    shm_size: "256m"
+    deploy:
+      resources:
+        limits:
+          memory: 4G
+
+volumes:
+  gitlab-data:

tests/fixtures/docs/homelab/container_gitlab/gitlab.md

@@ -0,0 +1,27 @@
+# Gitlab
+
+Gitlab CE runs as a single Docker container on the `gitlab` host
+(`10.0.1.14`), managed by Ansible (`deployments/gitlab/`).
+
+## Install
+
+1. Install Docker and the compose plugin on the host.
+2. Create the `gitlab-data` volume: `docker volume create gitlab-data`.
+3. Run the stack from `gitlab-compose.yaml`:
+   `docker compose -f gitlab-compose.yaml up -d`
+4. Wait ~2 minutes for the initial gitlab migration to finish.
+
+## Access
+
+- Web UI: https://gitlab.reeseapps.com (Traefik routes it to port 8929).
+- Root password: `gitlab-root-password` file in the repo (rotated yearly).
+- Backup: nightly `gitlab-backup create` at 03:30, copy to BorgBase.
+
+## Operations
+
+- Upgrade gitlab: bump the image tag in the compose file,
+  `docker compose up -d gitlab`, watch the logs for the version banner.
+- Logs: `docker logs -f gitlab` or the gitlab admin area → Admin
+  area → Logs.
+- If the container is OOM-killed, raise the memory limit in the compose
+  file (it needs 4GB free).

tests/fixtures/docs/homelab/networking/static-dns.json

@@ -0,0 +1,14 @@
+{
+  "comment": "Static DNS overrides for the homelab Pi-hole (applied by the ddns updater).",
+  "hosts": {
+    "kafkabridge": "10.0.3.7",
+    "k3s-control": "10.0.1.10",
+    "gitea": "10.0.2.21",
+    "ntfy": "10.0.2.30"
+  },
+  "domains": [
+    "reeseapps.com",
+    "homelab.lan"
+  ],
+  "expiry_days": 365
+}

tests/fixtures/docs/homelab/scripts/uptime_probe.py

@@ -0,0 +1,60 @@
+"""Uptime probe — the homelab healthcheck runner.
+
+Polls every service listed in ``CHECKS`` every 5 minutes and posts a
+failure to the ntfy topic ``homelab-alerts``.
+"""
+from __future__ import annotations
+
+import subprocess
+
+#: (name, health URL) for every long-running service.
+CHECKS: list[tuple[str, str]] = [
+    ("k3s", "https://10.0.1.10:6443/healthz"),
+    ("gitea", "https://gitea.reeseapps.com/api/healthz"),
+    ("ntfy", "https://ntfy.reeseapps.com/health"),
+    ("gitlab", "https://gitlab.reeseapps.com/-/health_check"),
+]
+
+
+def probe(name: str, url: str) -> bool:
+    """One HTTP check; returns True when the service answered 200."""
+    result = subprocess.run(
+        ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "10", url],
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip() == "200"
+
+
+def notify_failure(name: str) -> None:
+    """Push an alert to ntfy (best effort — alerting must not crash the probe)."""
+    subprocess.run(
+        [
+            "curl",
+            "-s",
+            "-X",
+            "POST",
+            "https://ntfy.reeseapps.com/homelab-alerts",
+            "-H",
+            "Title: homelab check failed",
+            "-d",
+            f"{name} is down",
+        ],
+        capture_output=True,
+    )
+
+
+def run_round() -> int:
+    """Probe everything once; returns the number of failing services."""
+    failed = 0
+    for name, url in CHECKS:
+        if not probe(name, url):
+            failed += 1
+            notify_failure(name)
+    return failed
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(run_round())

tests/fixtures/docs/homelab/ssh/ssh_aliases.txt

@@ -0,0 +1,14 @@
+SSH notes for the homelab jump host.
+
+All admin hosts are reachable through the jump box at 10.0.1.2
+(`ssh reese@jump`). The `~/.ssh/config` aliases:
+
+k3s — the kubernetes control plane node (10.0.1.10, user talos)
+gitlab — the gitlab container host (10.0.1.14)
+nuc — the low-power media box (10.0.1.20)
+
+Keys: ed25519 per host, no passwords. The old RSA key was retired in
+2025 and its line removed from authorized_keys on every host.
+
+Forwarding X11 stays off everywhere; use `ssh -L` port forwards for the
+occasional GUI tool instead.

tests/integration/test_chat_api.py

@@ -94,7 +94,7 @@ def seeded_kb(db) -> Iterator[FakeRagLLM]:
     db.commit()
     llm = FakeRagLLM()
     summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
-    assert summary.added == 3
+    assert summary.added == 8  # A9 formats; .hidden/ skipped
     yield llm
     db.execute(text("TRUNCATE chunks, documents, query_log"))
     db.commit()
@@ -163,12 +163,19 @@ def test_chat_writes_query_log_row(client, db, seeded_kb: FakeRagLLM) -> None:
     assert row.question == QUESTION
     assert row.deflected is False
     total_chunks = db.scalar(select(func.count()).select_from(Chunk))
-    assert row.chunk_hits == min(get_settings().top_k_chunks, total_chunks)
+    # chunk_hits is the fused candidate set (cosine top-N ∪ FTS top-N).
+    assert 1 <= row.chunk_hits <= total_chunks
     assert row.top_score > 0.0  # genuine token-overlap cosine, best hit
-    assert row.top_score >= get_settings().relevance_threshold  # why the gate answered
     assert row.top_score <= 1.0
     assert "docs/homelab/kubernetes.md" in row.sources
     assert row.latency_ms >= 0
+    # Why the gate answered (A8 revised): cosine over the threshold OR a
+    # lexical hit. The mock-calibrated threshold (0.30, see tests/conftest.py)
+    # makes the cosine branch true here; the FTS branch is covered too —
+    # "kubernetes" / "cluster" match the doc's tsvector.
+    thr = get_settings().relevance_threshold
+    assert row.top_score >= thr or (row.fts_hits or 0) > 0
+    assert (row.fts_hits or 0) >= 1  # the lexical branch really fired
 
 
 def test_off_topic_question_deflects_honestly(client, db, seeded_kb: FakeRagLLM) -> None:
@@ -202,14 +209,46 @@ def test_off_topic_question_deflects_honestly(client, db, seeded_kb: FakeRagLLM)
     assert "Talos Linux" not in system["content"]  # full doc content never sent
     assert "<documents>" not in system["content"]
 
-    # Durable record: deflected=true + the weak top_score.
+    # Durable record: deflected=true + the weak top_score. Deflection is
+    # only reached when the cosine is under the threshold AND no chunk
+    # FTS-matches the question — so fts_hits must be zero here.
     row = db.scalars(select(QueryLog)).one()
     assert row.question == OFF_TOPIC
     assert row.deflected is True
     assert 0.0 < row.top_score < get_settings().relevance_threshold
+    assert row.fts_hits == 0
     assert row.chunk_hits >= 1
 
 
+def test_keyword_question_grounded_by_lexical_hit_despite_weak_cosine(
+    client, db, seeded_kb: FakeRagLLM
+) -> None:
+    """Phase 09: a name-your-tool question the vector model barely ranks
+    ("kafkabridge" only appears in static-dns.json) must still be grounded
+    via the FTS branch — LOW only fires at weak cosine AND zero hits."""
+    fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
+    try:
+        _, _, frames = _stream_chat(client, "How does kafkabridge work?")
+    finally:
+        fastapi_app.dependency_overrides.clear()
+
+    done = frames[-1]
+    assert done["type"] == "done"
+    assert done["deflected"] is False  # weak cosine, but a lexical hit
+    assert done["suggestions"] == []
+    sources = done["sources"]
+    assert sources and sources[0]["path"] == "homelab/networking/static-dns.json"
+
+    (system, _user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
+    assert "<relevance>HIGH</relevance>" in system["content"]  # grounded prompt
+
+    row = db.scalars(select(QueryLog)).one()
+    assert row.deflected is False
+    assert row.top_score < get_settings().relevance_threshold  # weak vector score
+    assert (row.fts_hits or 0) >= 1  # …and it is the FTS hit that grounds it
+    assert "docs/homelab/networking/static-dns.json" in row.sources
+
+
 def test_chat_empty_kb_streams_empty_sources(client, db) -> None:
     db.execute(text("TRUNCATE chunks, documents, query_log"))
     db.commit()

tests/integration/test_importer_e2e.py

@@ -22,6 +22,11 @@ EXPECTED_DOCS = {
     ("docs", "homelab/kubernetes.md"),
     ("docs", "homelab/backups.md"),
     ("docs", "deployments/new-service.md"),
+    ("docs", "homelab/container_gitlab/gitlab.md"),
+    ("docs", "homelab/container_gitlab/gitlab-compose.yaml"),
+    ("docs", "homelab/networking/static-dns.json"),
+    ("docs", "homelab/scripts/uptime_probe.py"),
+    ("docs", "homelab/ssh/ssh_aliases.txt"),
 }
 
 
@@ -31,14 +36,27 @@ def test_import_fixtures_end_to_end(client, db) -> None:
     llm = FakeEmbedder()
 
     summary = asyncio.run(import_sources([FIXTURES], llm, session=db))
-    assert (summary.files, summary.added, summary.unchanged) == (3, 3, 0)
-    assert summary.chunks >= 3
+    # Eight A9-format files; .hidden/junk.md is out of scope (A9 revised).
+    assert (summary.files, summary.added, summary.unchanged) == (8, 8, 0)
+    assert summary.chunks >= 8
+    assert summary.formats == {"md": 4, "yaml": 1, "json": 1, "py": 1, "txt": 1}
+    # PLAN §9 per-format summary line: highest count first, then alpha.
+    assert summary.format_counts() == "md:4,json:1,py:1,txt:1,yaml:1"
 
     docs = db.scalars(select(Document)).all()
     assert {(d.source, d.path) for d in docs} == EXPECTED_DOCS
     titles = {d.path: d.title for d in docs}
     assert titles["homelab/kubernetes.md"] == "Kubernetes Homelab Cluster"
     assert titles["deployments/new-service.md"] == "Deploying a New Service"
+    assert titles["homelab/container_gitlab/gitlab.md"] == "Gitlab"
+    # Non-markdown titles come from the file stem (a leading ``#`` or docstring
+    # line is a comment there, not a heading).
+    assert titles["homelab/container_gitlab/gitlab-compose.yaml"] == "gitlab-compose"
+    assert titles["homelab/scripts/uptime_probe.py"] == "uptime_probe"
+    assert titles["homelab/networking/static-dns.json"] == "static-dns"
+    assert titles["homelab/ssh/ssh_aliases.txt"] == "ssh_aliases"
+    # Hidden junk was never imported.
+    assert not any(".hidden" in d.path for d in docs)
     # Full content is stored — that is what the RAG context will be.
     k8s = next(d for d in docs if d.path == "homelab/kubernetes.md")
     assert "Talos Linux" in k8s.content and k8s.content_hash
@@ -52,13 +70,13 @@ def test_import_fixtures_end_to_end(client, db) -> None:
     r = client.get("/api/docs")
     assert r.status_code == 200
     body = r.json()
-    assert len(body["documents"]) == 3
+    assert len(body["documents"]) == 8
     assert all(d["chunks"] >= 1 for d in body["documents"])
 
     # Idempotent re-run: nothing re-embedded.
     calls_before = len(llm.calls)
     s2 = asyncio.run(import_sources([FIXTURES], llm, session=db))
-    assert s2.unchanged == 3 and s2.added == 0
+    assert s2.unchanged == 8 and s2.added == 0
     assert len(llm.calls) == calls_before  # unchanged → no embedding requests
 
     db.execute(text("TRUNCATE chunks, documents, query_log"))

tests/integration/test_migration_0002.py

@@ -0,0 +1,72 @@
+"""Integration: migration 0002 (hybrid retrieval) schema contract.
+
+Asserts the state the migration must leave on the live schema:
+``chunks.tsv`` as a stored generated tsvector, its GIN index, and the
+nullable ``query_log.fts_hits`` column (pre-0002 rows stay NULL, so it
+must accept NULL and an int). Requires ``podman compose up -d db``.
+"""
+from __future__ import annotations
+
+from sqlalchemy import text
+
+
+def test_migration_0002_schema_contract(db) -> None:
+    tsv_col = db.execute(
+        text(
+            "SELECT count(*) FROM information_schema.columns"
+            " WHERE table_name = 'chunks' AND column_name = 'tsv'"
+        )
+    ).scalar()
+    assert tsv_col == 1, "chunks.tsv (stored tsvector) is missing"
+
+    gin = db.execute(
+        text(
+            "SELECT count(*) FROM pg_indexes"
+            " WHERE tablename = 'chunks' AND indexdef ILIKE '%USING gin%'"
+            " AND indexdef ILIKE '%tsv%'"
+        )
+    ).scalar()
+    assert gin == 1, "GIN index on chunks.tsv is missing"
+
+    fts = db.execute(
+        text(
+            "SELECT is_nullable = 'YES' FROM information_schema.columns"
+            " WHERE table_name = 'query_log' AND column_name = 'fts_hits'"
+        )
+    ).scalar()
+    assert fts is True, "query_log.fts_hits must exist and be nullable (pre-0002 rows)"
+
+
+def test_tsv_is_generated_and_lexically_queryable(db) -> None:
+    """The tsvector is generated from ``content`` (not maintained by app
+    code) and answers a tsquery — the retrieval path's lexical branch."""
+    doc_id = db.execute(text("SELECT gen_random_uuid()")).scalar()
+    try:
+        db.execute(
+            text(
+                "INSERT INTO documents (id, source, path, full_path, title, content,"
+                " content_hash, indexed_at) VALUES"
+                " (:id, 'mig_test', 't.md', '/t.md', 'T', 'kafkabridge routes here',"
+                " repeat('0', 64), now())"
+            ),
+            {"id": doc_id},
+        )
+        db.execute(
+            text(
+                "INSERT INTO chunks (id, document_id, position, content) VALUES"
+                " (gen_random_uuid(), :id, 0, 'kafkabridge routes here')"
+            ),
+            {"id": doc_id},
+        )
+        db.commit()
+        hit = db.execute(
+            text(
+                "SELECT count(*) FROM chunks"
+                " WHERE tsv @@ to_tsquery('english', 'kafkabridge')"
+            )
+        ).scalar()
+        assert hit == 1
+    finally:
+        db.execute(text("DELETE FROM chunks WHERE document_id = :id"), {"id": doc_id})
+        db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id})
+        db.commit()

tests/unit/test_chat_gate.py

@@ -45,13 +45,19 @@ def _doc(title: str, content: str) -> Document:
     )
 
 
-def _chunk(doc: Document, score: float) -> RetrievedChunk:
+def _chunk(
+    doc: Document, score: float, cosine: float | None = None, fts_hit: bool = False
+) -> RetrievedChunk:
+    """Fake candidate: *score* is the fused rank score; *cosine* (defaults to
+    *score*) is the vector-similarity gate input."""
     return RetrievedChunk(
         chunk_id=uuid.uuid4(),
         position=0,
         content=doc.content[:32],
         score=score,
         document=doc,
+        cosine=score if cosine is None else cosine,
+        fts_hit=fts_hit,
     )
 
 
@@ -89,6 +95,68 @@ def test_gate_is_env_tunable_via_settings() -> None:
     assert chat_api.plan_turn(hits, _settings(threshold=0.25)).deflected is False
 
 
+# ---------- hybrid gate matrix (A8, revised: cosine AND fts) ----------
+
+
+def test_gate_weak_cosine_with_fts_hit_still_answers() -> None:
+    """cosine < threshold but a lexical hit ⇒ HIGH — the FTS-OR branch.
+    This is the name-your-tool case: "kafkabridge" grounds despite weak
+    vector overlap."""
+    doc = _doc("Static DNS", "DNS_DOC_CONTENT")
+    plan = chat_api.plan_turn(
+        [_chunk(doc, 0.02, cosine=0.10, fts_hit=True)], _settings(threshold=0.30)
+    )
+    assert plan.deflected is False
+    assert plan.top_score == pytest.approx(0.10)  # gate input is the cosine
+    assert plan.fts_hits == 1
+    assert "DNS_DOC_CONTENT" in plan.system_prompt
+    assert plan.suggestions == []
+
+
+def test_gate_weak_cosine_zero_fts_deflects() -> None:
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    plan = chat_api.plan_turn([_chunk(doc, 0.02, cosine=0.10)], _settings(threshold=0.30))
+    assert plan.deflected is True
+    assert plan.top_score == pytest.approx(0.10)
+    assert plan.fts_hits == 0
+
+
+def test_gate_strong_cosine_without_fts_answers() -> None:
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    plan = chat_api.plan_turn([_chunk(doc, 0.90, cosine=0.90)], _settings(threshold=0.30))
+    assert plan.deflected is False
+    assert plan.fts_hits == 0
+
+
+def test_gate_fts_hits_counts_all_lexical_candidates() -> None:
+    a = _doc("Alpha", "ALPHA_CONTENT")
+    b = _doc("Beta", "BETA_CONTENT")
+    chunks = [
+        _chunk(a, 0.03, cosine=0.05, fts_hit=True),
+        _chunk(a, 0.02, cosine=0.04, fts_hit=True),  # same doc, second chunk
+        _chunk(b, 0.01, cosine=0.03),
+    ]
+    plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
+    assert plan.deflected is False
+    assert plan.fts_hits == 2  # per chunk, not per doc
+
+
+def test_gate_lexical_only_chunk_does_not_inflate_cosine() -> None:
+    """top_score stays the best *vector* cosine even when a lexical-only
+    chunk (cosine 0.0 by construction) carries the highest fused score."""
+    a = _doc("Alpha", "ALPHA_CONTENT")
+    b = _doc("Beta", "BETA_CONTENT")
+    chunks = [
+        _chunk(a, 0.50, cosine=0.55),  # vector rank 1
+        _chunk(b, 0.90, cosine=0.0, fts_hit=True),  # lexical rank 1 wins the ranking
+    ]
+    plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
+    assert plan.top_score == pytest.approx(0.55)
+    assert plan.deflected is False  # 0.55 >= 0.30 anyway
+    # ranking follows the fused score: Beta's doc is the top source
+    assert plan.docs[0].title == "Beta"
+
+
 def test_gate_zero_chunks_deflects_with_fallback_chips() -> None:
     plan = chat_api.plan_turn([], _settings())
     assert plan.deflected is True
@@ -233,6 +301,13 @@ def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _C
     llm = _CannedLLM()
     monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_db, lambda: session)
     monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_llm, lambda: llm)
+    # These tests assert against a specific gate threshold; keep it stable
+    # regardless of the production default (0.62) or any .env.
+    monkeypatch.setattr(
+        chat_api,
+        "get_settings",
+        lambda: Settings(_env_file=None, relevance_threshold=0.30),  # pyright: ignore[reportCallIssue]
+    )
     yield session, llm
     fastapi_app.dependency_overrides.clear()
 
@@ -254,7 +329,7 @@ def _ask(client: TestClient, message: str) -> list[dict[str, Any]]:
 
 
 def _fake_retriever(chunks: list[RetrievedChunk]) -> Any:
-    def retrieve(_db: Any, _vec: list[float]) -> list[RetrievedChunk]:
+    def retrieve(_db: Any, _question: str, _vec: list[float]) -> list[RetrievedChunk]:
         return chunks
 
     return retrieve

tests/unit/test_chunker.py

@@ -1,11 +1,25 @@
-"""Unit tests: markdown-aware chunker (PLAN §5 policy)."""
+"""Unit tests: format-aware chunker (PLAN §5 policy, A9 formats).
+
+The markdown policy tests are the original contract (md output stays
+unchanged); the per-format tests cover the phase-09 dispatcher
+(yaml/yml, json, py, txt) and the 1200-char hard cap for every format.
+"""
 from __future__ import annotations
 
 from itertools import pairwise
 
 import pytest
 
-from app.rag.chunker import HARD_MAX_CHARS, chunk_markdown, extract_title
+from app.rag.chunker import (
+    HARD_MAX_CHARS,
+    chunk_document,
+    chunk_json,
+    chunk_markdown,
+    chunk_python,
+    chunk_text,
+    chunk_yaml,
+    extract_title,
+)
 
 ANCHOR = "## Big"
 ANCHOR_PREFIX = f"{ANCHOR}\n\n"
@@ -157,3 +171,213 @@ def test_extract_title_prefers_h1() -> None:
     assert extract_title("## not a title\n\nbody") == ""
     assert extract_title("## sub only", fallback="stem") == "stem"
     assert extract_title("", fallback="fallback") == "fallback"
+
+
+# ---------------------------------------------------------------------------
+# Format dispatcher (chunk_document) — A9 multi-format ingestion
+# ---------------------------------------------------------------------------
+
+
+def test_dispatch_by_lowercased_suffix() -> None:
+    md = "# T\n\n## A\n\nbody\n"
+    assert chunk_document(md, "notes/Doc.MD") == chunk_markdown(md)
+    assert chunk_document(md, "notes/doc.MARKDOWN") == chunk_markdown(md)
+    assert chunk_document("p1\n\np2\n", "x.TXT") == chunk_text("p1\n\np2\n")
+    assert chunk_document("a: 1\n", "x.YAML") == chunk_yaml("a: 1\n")
+    assert chunk_document("a: 1\n", "x.Yml") == chunk_yaml("a: 1\n")
+    assert chunk_document('{"a": 1}', "x.Json") == chunk_json('{"a": 1}')
+    assert chunk_document("def f(): pass\n", "x.PY") == chunk_python("def f(): pass\n")
+
+
+def test_dispatch_unknown_suffix_falls_back_to_paragraphs() -> None:
+    assert chunk_document("hello\n\nworld", "data.csv") == ["hello\nworld"]
+
+
+def test_dispatch_ignores_directory_part_of_path() -> None:
+    assert chunk_document("def f(): pass\n", "a/b/c/script.py") == chunk_python("def f(): pass\n")
+
+
+# ---------------------------------------------------------------------------
+# yaml / yml
+# ---------------------------------------------------------------------------
+
+
+def test_yaml_splits_on_top_level_keys_and_keeps_key_anchors() -> None:
+    doc = (
+        "# leading comment\n"
+        "services:\n"
+        "  gitlab:\n"
+        "    image: gitlab/gitlab-ce\n"
+        "  prometheus:\n"
+        "    image: prom/prometheus\n"
+        "volumes:\n"
+        "  gitlab-data:\n"
+    )
+    chunks = chunk_yaml(doc)
+    joined = "\n".join(chunks)
+    for key in ("services:", "volumes:"):
+        assert key in joined
+    # Indented keys are NOT block starts — they stay inside their parent block.
+    assert not any(c.startswith("  gitlab:") for c in chunks)
+    # The leading comment stays with the first block (preamble).
+    assert chunks[0].startswith("# leading comment")
+    assert "gitlab/gitlab-ce" in joined and "prom/prometheus" in joined
+
+
+def test_yaml_document_separators_start_new_blocks() -> None:
+    a = "site_a: " + "a" * 500 + "\n"
+    b = "site_b: " + "b" * 500 + "\n"
+    chunks = chunk_yaml(a + "---\n" + b, target_chars=600, overlap_chars=0)
+    # Each site is long enough to force its own chunk; the separator must not
+    # glue them into one over-budget chunk.
+    assert len(chunks) >= 2
+    assert all(len(c) <= 600 for c in chunks)
+    assert not any("site_a" in c and "site_b" in c for c in chunks)
+
+
+def test_yaml_oversized_key_block_is_split_under_hard_cap() -> None:
+    doc = "big_list:\n" + ("  - " + "x" * 60 + "\n") * 60  # one ~3800-char block
+    chunks = chunk_yaml(doc)
+    assert len(chunks) >= 2
+    assert all(len(c) <= HARD_MAX_CHARS for c in chunks)
+    # Overlap re-prints (≤50 chars per split), so only a little content is
+    # re-stated — the bulk of the block must survive.
+    assert sum(len(c) for c in chunks) >= len(doc) - 300
+
+
+def test_yaml_empty_content() -> None:
+    assert chunk_yaml("") == []
+    assert chunk_yaml("\n\n  \n") == []
+
+
+# ---------------------------------------------------------------------------
+# json
+# ---------------------------------------------------------------------------
+
+
+def test_json_splits_on_top_level_keys_pretty_printed() -> None:
+    doc = '{"hosts": {"kafkabridge": "10.0.3.7"}, "count": 3}'
+    chunks = chunk_json(doc, target_chars=45, overlap_chars=0)  # force 1 chunk/block
+    assert len(chunks) == 2
+    first, second = chunks
+    assert '"hosts"' in first and "kafkabridge" in first
+    assert '"count"' in second
+    # Pretty-printed (indent=2), not the compact input form.
+    assert '"kafkabridge": "10.0.3.7"' in first
+    assert not any('{"hosts"' in c for c in chunks)
+
+
+def test_json_each_key_block_is_self_contained() -> None:
+    doc = '{"a": "x", "b": "y"}'
+    chunks = chunk_json(doc, target_chars=13, overlap_chars=0)  # force 1 chunk/block
+    assert [c for c in chunks if '"a"' in c] and [c for c in chunks if '"b"' in c]
+    assert not any('"a"' in c and '"b"' in c for c in chunks)
+
+
+def test_json_oversized_value_falls_under_hard_cap() -> None:
+    doc = '{"blob": "' + "z" * 4000 + '"}'
+    chunks = chunk_json(doc)
+    assert len(chunks) >= 2
+    assert all(len(c) <= HARD_MAX_CHARS for c in chunks)
+    assert "".join(chunks).count("z") >= 4000
+
+
+def test_json_top_level_list_is_one_pretty_block() -> None:
+    chunks = chunk_json("[1, 2, 3]")
+    assert chunks == ["[\n  1,\n  2,\n  3\n]"]
+
+
+def test_json_unparseable_falls_back_to_paragraph_packing() -> None:
+    doc = "{broken json\n\nsecond paragraph here\n"
+    assert chunk_json(doc) == chunk_text(doc)
+    assert chunk_json("not json at all") == chunk_text("not json at all")
+
+
+# ---------------------------------------------------------------------------
+# python
+# ---------------------------------------------------------------------------
+
+
+def test_python_splits_on_top_level_defs_and_classes() -> None:
+    doc = (
+        '"""Module doc."""\n'
+        "import asyncio\n"
+        "\n"
+        "CONST = 1\n"
+        "\n"
+        "def alpha():\n"
+        "    return 1\n"
+        "\n"
+        "class Beta:\n"
+        "    def run(self):\n"
+        "        return 2\n"
+    )
+    chunks = chunk_python(doc, target_chars=60, overlap_chars=0)  # force 1 chunk/block
+    assert len(chunks) == 3
+    assert chunks[0].startswith('"""Module doc."""')
+    assert "CONST = 1" in chunks[0]  # preamble ends at the first def/class
+    assert chunks[1].startswith("def alpha")
+    assert chunks[2].startswith("class Beta")
+    assert "def run" in chunks[2]  # nested def stays inside the class block
+
+
+def test_python_decorators_stay_with_their_definition() -> None:
+    doc = "@app.get('/x')\ndef handler():\n    return 'x'\n"
+    chunks = chunk_python(doc)
+    assert chunks[0].startswith("@app.get")
+
+
+def test_python_oversized_function_falls_back_to_line_packing() -> None:
+    doc = "def big():\n" + "\n".join(f"    val_{i:03d} = {i}  # padding" for i in range(80))
+    chunks = chunk_python(doc)
+    assert len(chunks) >= 2
+    assert all(len(c) <= HARD_MAX_CHARS for c in chunks)
+    assert "val_000" in chunks[0]
+    assert "val_079" in chunks[-1]
+    assert sum(len(c) for c in chunks) >= len(doc) - 100
+
+
+def test_python_unparseable_source_falls_back_to_paragraphs() -> None:
+    src = "def broken(:\n\nstill text\n"
+    assert chunk_python(src) == chunk_text(src)
+
+
+# ---------------------------------------------------------------------------
+# txt
+# ---------------------------------------------------------------------------
+
+
+def test_txt_paragraph_packing() -> None:
+    doc = "alpha\n\nbeta\n\ngamma\n"
+    chunks = chunk_text(doc)
+    assert chunks == ["alpha\nbeta\ngamma"]  # all three fit the target
+
+
+def test_txt_long_doc_packs_with_overlap() -> None:
+    doc = "\n\n".join(f"para {i} " + "l" * 300 for i in range(6))
+    chunks = chunk_text(doc, target_chars=800, overlap_chars=100)
+    assert len(chunks) >= 2
+    assert all(len(c) <= 800 for c in chunks)
+    assert all(f"para {i}" in "\n".join(chunks) for i in range(6))
+
+
+# ---------------------------------------------------------------------------
+# Hard cap across every format (aipi ~1024-token request cap)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("content", "path"),
+    [
+        ("# T\n\n" + "word " * 1200, "big.md"),
+        ("key: " + "v" * 5000 + "\n", "big.yaml"),
+        ('{"blob": "' + "z" * 5000 + '"}', "big.json"),
+        ("def f():\n" + "    x = 1\n" * 1000, "big.py"),
+        ("line of text\n\n" * 800, "big.txt"),
+    ],
+)
+def test_hard_cap_holds_for_every_format(content: str, path: str) -> None:
+    chunks = chunk_document(content, path)
+    assert chunks, "expected at least one chunk"
+    for c in chunks:
+        assert len(c) <= HARD_MAX_CHARS, f"{path}: {len(c)} chars"

tests/unit/test_config.py

@@ -5,6 +5,7 @@ import json
 from typing import Any
 
 import pytest
+from pydantic import ValidationError
 from pydantic_settings import SettingsError
 
 from app.config import Settings
@@ -16,16 +17,28 @@ def _settings(**kwargs: Any) -> Settings:
     return Settings(**kwargs)  # pyright: ignore[reportCallIssue]  (kwarg exists at runtime)
 
 
-def test_defaults_match_locked_decisions() -> None:
+def test_defaults_match_locked_decisions(monkeypatch: pytest.MonkeyPatch) -> None:
+    # The test process sets BOR_RELEVANCE_THRESHOLD=0.30 for the mock-
+    # calibrated in-process suites (see tests/conftest.py) — the *default*
+    # under test is the production one.
+    monkeypatch.delenv("BOR_RELEVANCE_THRESHOLD", raising=False)
     s = _settings()
     assert s.llm_chat_model == "turbo"
     assert s.llm_embed_model == "embed"
     assert s.embedding_dim == 768
     assert s.llm_base_url.endswith("/v1")
-    assert 0 < s.relevance_threshold < 1
-    assert s.top_k_chunks >= 1
+    # A8 (revised): the honesty gate input is the best cosine, default 0.62.
+    assert s.relevance_threshold == 0.62
+    # A7 (revised): hybrid retrieval — cosine top-N ∪ FTS top-N, RRF-fused.
+    assert s.hybrid_vector_candidates >= 1
+    assert s.hybrid_lexical_candidates >= 1
+    assert s.rrf_k >= 1
     assert s.top_n_docs >= 1
     assert len(s.suggestions) >= 3
+    # A9 (revised): the import scope covers the seven A9 formats.
+    assert s.import_extension_set == {
+        ".md", ".markdown", ".txt", ".yaml", ".yml", ".json", ".py"
+    }
 
 
 def test_env_override(monkeypatch) -> None:
@@ -36,6 +49,26 @@ def test_env_override(monkeypatch) -> None:
     assert s.llm_chat_model == "juggernaut"
 
 
+def test_import_extensions_env_override_is_a_csv_list(monkeypatch) -> None:
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,yml")
+    s = _settings()
+    assert s.import_extension_set == {".md", ".yml"}
+
+
+def test_import_extensions_rejects_unknown_format(monkeypatch) -> None:
+    """A typo in the CSV fails at startup (loudly), not by silently
+    walking zero files."""
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,docx")
+    with pytest.raises(ValidationError, match="docx"):
+        _settings()
+
+
+def test_import_extensions_rejects_empty(monkeypatch) -> None:
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "   ")
+    with pytest.raises(ValidationError):
+        _settings()
+
+
 def test_suggestions_default_is_three_plus_real_questions() -> None:
     s = _settings()
     assert len(s.suggestions) >= 3

tests/unit/test_importer.py

@@ -16,11 +16,15 @@ from app.models import Chunk, Document
 from app.rag.importer import (
     EXCLUDED_DIRS,
     import_sources,
-    iter_markdown_files,
+    iter_importable_files,
 )
 from app.rag.llm import EmbeddingError
 from tests.fakes import FakeEmbedder
 
+#: A9 default extension set as dotted suffixes (what the importer passes to
+#: the walker when no override is configured).
+DEFAULT_EXTS = frozenset({".md", ".markdown", ".txt", ".yaml", ".yml", ".json", ".py"})
+
 
 class _PoisonEmbedder(FakeEmbedder):
     """Fails (like a real endpoint) on any text containing 'poison'."""
@@ -50,7 +54,9 @@ def _cleanup_source(db, source: str) -> None:
     db.commit()
 
 
-def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
+def test_iter_importable_files_excludes_noncontent_dirs_and_hidden(tmp_path: Path) -> None:
+    """Well-known non-content dirs, hidden (dot-) dirs/files, and non-A9
+    extensions are all skipped; the A9 formats pass."""
     root = tmp_path / "proj"
     for d in (
         "notes/sub",
@@ -61,11 +67,20 @@ def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
         ".pytest_cache",
         "dist",
         "build",
+        ".esphome/.espressif",  # vendored hidden cache — the real A9 case
     ):
         (root / d).mkdir(parents=True)
     files = {
+        # content that must be found:
         "README.md": "readme",
         "notes/sub/deep.md": "deep",
+        "compose.yaml": "services: {}",
+        "legacy.YML": "a: b",  # case-insensitive suffix
+        "notes/sub/agent.py": "x = 1",
+        "config.json": "{}",
+        "README.txt": "plain",
+        "notes/sub/deep.markdown": "md2",
+        # must be skipped:
         ".venv/lib/junk.md": "junk",
         "node_modules/x/j.md": "j",
         ".git/c.md": "g",
@@ -73,17 +88,40 @@ def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
         ".pytest_cache/c.md": "pc",
         "dist/d.md": "d",
         "build/b.md": "b",
+        ".esphome/.espressif/secret.md": "vendor",
+        ".secret.md": "hidden file",  # dot-prefixed FILE, not just dir
+        "notes/sub/notes.csv": "a,b",  # not an A9 format
+        "notes/sub/file.md.bak": "x",
     }
     for rel, text in files.items():
         (root / rel).write_text(text)
-    (root / "notes" / "not-md.txt").write_text("skip me")
 
-    found = {p.relative_to(root).as_posix() for p in iter_markdown_files(root)}
-    assert found == {"README.md", "notes/sub/deep.md"}
+    found = {p.relative_to(root).as_posix() for p in iter_importable_files(root, DEFAULT_EXTS)}
+    assert found == {
+        "README.md",
+        "notes/sub/deep.md",
+        "compose.yaml",
+        "legacy.YML",
+        "notes/sub/agent.py",
+        "config.json",
+        "README.txt",
+        "notes/sub/deep.markdown",
+    }
 
 
-def test_iter_markdown_files_missing_dir_yields_nothing(tmp_path: Path) -> None:
-    assert iter_markdown_files(tmp_path / "definitely-missing") == []
+def test_iter_importable_files_missing_dir_yields_nothing(tmp_path: Path) -> None:
+    assert iter_importable_files(tmp_path / "definitely-missing", DEFAULT_EXTS) == []
+
+
+def test_iter_importable_files_respects_custom_extension_filter(tmp_path: Path) -> None:
+    """A narrower filter (e.g. md only) excludes the other A9 formats."""
+    root = tmp_path / "filtered"
+    root.mkdir()
+    (root / "a.md").write_text("a")
+    (root / "b.yaml").write_text("a: b")
+    (root / "c.py").write_text("x = 1")
+    found = {p.name for p in iter_importable_files(root, frozenset([".md"]))}
+    assert found == {"a.md"}
 
 
 def test_excluded_dirs_match_plan_anchor_a9() -> None:
@@ -277,3 +315,65 @@ def test_chunk_positions_and_titles(db, tmp_path: Path) -> None:
         assert positions == list(range(len(doc.chunks))) and len(doc.chunks) >= 2
     finally:
         _cleanup_source(db, root.name)
+
+
+def test_multi_format_import_counts_per_format_and_titles_stem(db, tmp_path: Path) -> None:
+    """A9 formats all import; the summary records per-format counts, and
+    non-markdown titles come from the file stem (a ``#`` line is a comment
+    there, not a heading)."""
+    root = tmp_path / "multi"
+    (root / "svc").mkdir(parents=True)
+    (root / "guide.md").write_text("# Real Heading\n\nbody\n")
+    (root / "svc" / "compose.yaml").write_text("# a comment\nservices:\n  gitlab: {}\n")
+    (root / "svc" / "agent.py").write_text("# docstring-like comment\ndef ping():\n    return 1\n")
+    (root / "inventory.json").write_text('{"hosts": []}\n')
+    (root / "notes.txt").write_text("plain text notes\n")
+    llm = FakeEmbedder()
+    try:
+        summary = asyncio.run(import_sources([root], llm, session=db))
+        assert summary.files == 5
+        assert summary.added == 5
+        assert summary.formats == {"md": 1, "yaml": 1, "py": 1, "json": 1, "txt": 1}
+        # PLAN §9 summary line: counts, highest first, ext:name pairs.
+        assert summary.format_counts() == "json:1,md:1,py:1,txt:1,yaml:1"
+
+        titles = {
+            d.path: d.title
+            for d in db.scalars(select(Document).where(Document.source == root.name)).all()
+        }
+        assert titles["guide.md"] == "Real Heading"  # markdown keeps the H1
+        assert titles["svc/compose.yaml"] == "compose"  # …comment is not a heading
+        assert titles["svc/agent.py"] == "agent"
+        assert titles["inventory.json"] == "inventory"
+        assert titles["notes.txt"] == "notes"
+    finally:
+        _cleanup_source(db, root.name)
+
+
+def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -> None:
+    """Previously-imported junk leaves the index: a file that no longer
+    matches the A9 extension filter is pruned on the next ``prune=True`` run.
+    This is how dot-dir READMEs imported before the scope fix get cleaned up."""
+    root = tmp_path / "cleanup"
+    root.mkdir()
+    (root / "keep.md").write_text("# Keep\n\nkept\n")
+    (root / "junk.md.bak").write_text("old junk that was once imported\n")
+    llm = FakeEmbedder()
+    try:
+        # Seed: import both files as if they were valid at the time.
+        (root / "junk.md").write_text("old junk that was once imported\n")
+        (root / "junk.md.bak").unlink()
+        asyncio.run(import_sources([root], llm, session=db))
+        # Rename the junk out of the A9 formats, then prune.
+        (root / "junk.md").rename(root / "junk.md.bak")
+        summary = asyncio.run(import_sources([root], llm, session=db, prune=True))
+        assert summary.pruned == 1
+        assert summary.unchanged == 1  # keep.md survived
+        assert db.scalar(
+            select(Document).where(Document.source == root.name, Document.path == "junk.md")
+        ) is None
+        assert db.scalar(
+            select(Document).where(Document.source == root.name, Document.path == "keep.md")
+        ) is not None
+    finally:
+        _cleanup_source(db, root.name)

tests/unit/test_retriever.py

@@ -8,6 +8,8 @@ from __future__ import annotations
 
 import uuid
 
+import pytest
+
 from app.models import Document
 from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk, select_documents
 
@@ -94,3 +96,102 @@ def test_under_budget_no_truncation() -> None:
 
 def test_empty_hits_yield_no_documents() -> None:
     assert select_documents([], n=2, max_chars=24_000) == []
+
+
+# ---------------------------------------------------------------------------
+# Hybrid retrieval (A7): RRF fusion + lexical tsquery
+# ---------------------------------------------------------------------------
+
+from app.rag.retriever import fuse, lexical_tsquery  # noqa: E402
+
+
+def _rc(
+    doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0
+) -> RetrievedChunk:
+    return RetrievedChunk(
+        chunk_id=uuid.uuid4(),
+        position=position,
+        content="x" * 20,
+        score=0.0,
+        document=_doc(doc_path, "x" * 20),
+        cosine=cosine,
+        fts_hit=fts_hit,
+    )
+
+
+def test_lexical_tsquery_tokens_lowercased_deduped_in_order() -> None:
+    assert lexical_tsquery("How did I Install GITLAB gitlab?") == "how | did | i | install | gitlab"
+
+
+def test_lexical_tsquery_punctuation_and_umlauts_ignored() -> None:
+    assert lexical_tsquery("c3-r00t? -- what's up!") == "c3 | r00t | what | s | up"
+
+
+def test_lexical_tsquery_pure_symbols_return_none() -> None:
+    assert lexical_tsquery("??? ???") is None
+    assert lexical_tsquery("") is None
+
+
+def test_lexical_tsquery_stopwords_left_to_postgres() -> None:
+    # lexical_tsquery passes raw tokens through; Postgres's to_tsquery
+    # lexing drops the stopwords (verified against real PG in
+    # test_retrieve_empty_kb / integration tests).
+    assert lexical_tsquery("how do i") == "how | do | i"
+
+
+def test_fuse_combines_both_lists_for_double_hits() -> None:
+    v1 = _rc("a.md", cosine=0.9)
+    v2 = _rc("b.md", cosine=0.5)
+    l1 = _rc("a.md", cosine=0.1)  # same chunk id -> matched in place
+    a_id = v1.chunk_id
+    l1.chunk_id = a_id
+    out = fuse([v1, v2], [l1], k=60)
+    by_id = {rc.chunk_id: rc for rc in out}
+    # a: 1/61 (vector rank 1) + 1/61 (lexical rank 1); b: 1/62 only.
+    assert by_id[a_id].score == pytest.approx(2 / 61)
+    assert by_id[a_id].fts_hit is True
+    assert by_id[v2.chunk_id].score == pytest.approx(1 / 62)
+    assert by_id[v2.chunk_id].fts_hit is False
+    assert [rc.chunk_id for rc in out] == [a_id, v2.chunk_id]
+
+
+def test_fuse_lexical_only_chunks_enter_with_zero_cosine() -> None:
+    vector = [_rc("a.md", cosine=0.8)]
+    lexical = [_rc("b.md", cosine=0.0, fts_hit=True)]
+    out = fuse(vector, lexical, k=60)
+    assert len(out) == 2
+    b = next(rc for rc in out if rc.document.path == "b.md")
+    assert b.cosine == 0.0
+    assert b.fts_hit is True
+    # Still ranked by its (only) RRF term.
+    assert b.score == pytest.approx(1 / 61)
+
+
+def test_fuse_orders_by_score_then_cosine_then_path() -> None:
+    # Two chunks share an RRF score (both rank 1 in different lists):
+    # the higher-cosine one must sort first.
+    hi = _rc("z.md", cosine=0.9)
+    lo = _rc("a.md", cosine=0.2)
+    out = fuse([hi], [lo], k=60)
+    assert [rc.document.path for rc in out] == ["z.md", "a.md"]
+    # Equal score AND cosine -> path order.
+    p1 = _rc("b.md", cosine=0.5)
+    p2 = _rc("a.md", cosine=0.5)
+    out = fuse([p1], [p2], k=60)
+    assert [rc.document.path for rc in out] == ["a.md", "b.md"]
+    # Equal score, cosine, path -> position order.
+    s1 = _rc("a.md", cosine=0.5, position=1)
+    s2 = _rc("a.md", cosine=0.5, position=0)
+    out = fuse([s1], [s2], k=60)
+    assert [rc.position for rc in out] == [0, 1]
+
+
+def test_fuse_rejects_nonpositive_k() -> None:
+    with pytest.raises(ValueError):
+        fuse([], [], k=0)
+    with pytest.raises(ValueError):
+        fuse([], [], k=-1)
+
+
+def test_fuse_empty_lists() -> None:
+    assert fuse([], [], k=60) == []