feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

tests/unit/test_chat_gate.py

@@ -45,13 +45,19 @@ def _doc(title: str, content: str) -> Document:
     )
 
 
-def _chunk(doc: Document, score: float) -> RetrievedChunk:
+def _chunk(
+    doc: Document, score: float, cosine: float | None = None, fts_hit: bool = False
+) -> RetrievedChunk:
+    """Fake candidate: *score* is the fused rank score; *cosine* (defaults to
+    *score*) is the vector-similarity gate input."""
     return RetrievedChunk(
         chunk_id=uuid.uuid4(),
         position=0,
         content=doc.content[:32],
         score=score,
         document=doc,
+        cosine=score if cosine is None else cosine,
+        fts_hit=fts_hit,
     )
 
 
@@ -89,6 +95,68 @@ def test_gate_is_env_tunable_via_settings() -> None:
     assert chat_api.plan_turn(hits, _settings(threshold=0.25)).deflected is False
 
 
+# ---------- hybrid gate matrix (A8, revised: cosine AND fts) ----------
+
+
+def test_gate_weak_cosine_with_fts_hit_still_answers() -> None:
+    """cosine < threshold but a lexical hit ⇒ HIGH — the FTS-OR branch.
+    This is the name-your-tool case: "kafkabridge" grounds despite weak
+    vector overlap."""
+    doc = _doc("Static DNS", "DNS_DOC_CONTENT")
+    plan = chat_api.plan_turn(
+        [_chunk(doc, 0.02, cosine=0.10, fts_hit=True)], _settings(threshold=0.30)
+    )
+    assert plan.deflected is False
+    assert plan.top_score == pytest.approx(0.10)  # gate input is the cosine
+    assert plan.fts_hits == 1
+    assert "DNS_DOC_CONTENT" in plan.system_prompt
+    assert plan.suggestions == []
+
+
+def test_gate_weak_cosine_zero_fts_deflects() -> None:
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    plan = chat_api.plan_turn([_chunk(doc, 0.02, cosine=0.10)], _settings(threshold=0.30))
+    assert plan.deflected is True
+    assert plan.top_score == pytest.approx(0.10)
+    assert plan.fts_hits == 0
+
+
+def test_gate_strong_cosine_without_fts_answers() -> None:
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    plan = chat_api.plan_turn([_chunk(doc, 0.90, cosine=0.90)], _settings(threshold=0.30))
+    assert plan.deflected is False
+    assert plan.fts_hits == 0
+
+
+def test_gate_fts_hits_counts_all_lexical_candidates() -> None:
+    a = _doc("Alpha", "ALPHA_CONTENT")
+    b = _doc("Beta", "BETA_CONTENT")
+    chunks = [
+        _chunk(a, 0.03, cosine=0.05, fts_hit=True),
+        _chunk(a, 0.02, cosine=0.04, fts_hit=True),  # same doc, second chunk
+        _chunk(b, 0.01, cosine=0.03),
+    ]
+    plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
+    assert plan.deflected is False
+    assert plan.fts_hits == 2  # per chunk, not per doc
+
+
+def test_gate_lexical_only_chunk_does_not_inflate_cosine() -> None:
+    """top_score stays the best *vector* cosine even when a lexical-only
+    chunk (cosine 0.0 by construction) carries the highest fused score."""
+    a = _doc("Alpha", "ALPHA_CONTENT")
+    b = _doc("Beta", "BETA_CONTENT")
+    chunks = [
+        _chunk(a, 0.50, cosine=0.55),  # vector rank 1
+        _chunk(b, 0.90, cosine=0.0, fts_hit=True),  # lexical rank 1 wins the ranking
+    ]
+    plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
+    assert plan.top_score == pytest.approx(0.55)
+    assert plan.deflected is False  # 0.55 >= 0.30 anyway
+    # ranking follows the fused score: Beta's doc is the top source
+    assert plan.docs[0].title == "Beta"
+
+
 def test_gate_zero_chunks_deflects_with_fallback_chips() -> None:
     plan = chat_api.plan_turn([], _settings())
     assert plan.deflected is True
@@ -233,6 +301,13 @@ def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _C
     llm = _CannedLLM()
     monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_db, lambda: session)
     monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_llm, lambda: llm)
+    # These tests assert against a specific gate threshold; keep it stable
+    # regardless of the production default (0.62) or any .env.
+    monkeypatch.setattr(
+        chat_api,
+        "get_settings",
+        lambda: Settings(_env_file=None, relevance_threshold=0.30),  # pyright: ignore[reportCallIssue]
+    )
     yield session, llm
     fastapi_app.dependency_overrides.clear()
 
@@ -254,7 +329,7 @@ def _ask(client: TestClient, message: str) -> list[dict[str, Any]]:
 
 
 def _fake_retriever(chunks: list[RetrievedChunk]) -> Any:
-    def retrieve(_db: Any, _vec: list[float]) -> list[RetrievedChunk]:
+    def retrieve(_db: Any, _question: str, _vec: list[float]) -> list[RetrievedChunk]:
         return chunks
 
     return retrieve

tests/unit/test_chunker.py

@@ -1,11 +1,25 @@
-"""Unit tests: markdown-aware chunker (PLAN §5 policy)."""
+"""Unit tests: format-aware chunker (PLAN §5 policy, A9 formats).
+
+The markdown policy tests are the original contract (md output stays
+unchanged); the per-format tests cover the phase-09 dispatcher
+(yaml/yml, json, py, txt) and the 1200-char hard cap for every format.
+"""
 from __future__ import annotations
 
 from itertools import pairwise
 
 import pytest
 
-from app.rag.chunker import HARD_MAX_CHARS, chunk_markdown, extract_title
+from app.rag.chunker import (
+    HARD_MAX_CHARS,
+    chunk_document,
+    chunk_json,
+    chunk_markdown,
+    chunk_python,
+    chunk_text,
+    chunk_yaml,
+    extract_title,
+)
 
 ANCHOR = "## Big"
 ANCHOR_PREFIX = f"{ANCHOR}\n\n"
@@ -157,3 +171,213 @@ def test_extract_title_prefers_h1() -> None:
     assert extract_title("## not a title\n\nbody") == ""
     assert extract_title("## sub only", fallback="stem") == "stem"
     assert extract_title("", fallback="fallback") == "fallback"
+
+
+# ---------------------------------------------------------------------------
+# Format dispatcher (chunk_document) — A9 multi-format ingestion
+# ---------------------------------------------------------------------------
+
+
+def test_dispatch_by_lowercased_suffix() -> None:
+    md = "# T\n\n## A\n\nbody\n"
+    assert chunk_document(md, "notes/Doc.MD") == chunk_markdown(md)
+    assert chunk_document(md, "notes/doc.MARKDOWN") == chunk_markdown(md)
+    assert chunk_document("p1\n\np2\n", "x.TXT") == chunk_text("p1\n\np2\n")
+    assert chunk_document("a: 1\n", "x.YAML") == chunk_yaml("a: 1\n")
+    assert chunk_document("a: 1\n", "x.Yml") == chunk_yaml("a: 1\n")
+    assert chunk_document('{"a": 1}', "x.Json") == chunk_json('{"a": 1}')
+    assert chunk_document("def f(): pass\n", "x.PY") == chunk_python("def f(): pass\n")
+
+
+def test_dispatch_unknown_suffix_falls_back_to_paragraphs() -> None:
+    assert chunk_document("hello\n\nworld", "data.csv") == ["hello\nworld"]
+
+
+def test_dispatch_ignores_directory_part_of_path() -> None:
+    assert chunk_document("def f(): pass\n", "a/b/c/script.py") == chunk_python("def f(): pass\n")
+
+
+# ---------------------------------------------------------------------------
+# yaml / yml
+# ---------------------------------------------------------------------------
+
+
+def test_yaml_splits_on_top_level_keys_and_keeps_key_anchors() -> None:
+    doc = (
+        "# leading comment\n"
+        "services:\n"
+        "  gitlab:\n"
+        "    image: gitlab/gitlab-ce\n"
+        "  prometheus:\n"
+        "    image: prom/prometheus\n"
+        "volumes:\n"
+        "  gitlab-data:\n"
+    )
+    chunks = chunk_yaml(doc)
+    joined = "\n".join(chunks)
+    for key in ("services:", "volumes:"):
+        assert key in joined
+    # Indented keys are NOT block starts — they stay inside their parent block.
+    assert not any(c.startswith("  gitlab:") for c in chunks)
+    # The leading comment stays with the first block (preamble).
+    assert chunks[0].startswith("# leading comment")
+    assert "gitlab/gitlab-ce" in joined and "prom/prometheus" in joined
+
+
+def test_yaml_document_separators_start_new_blocks() -> None:
+    a = "site_a: " + "a" * 500 + "\n"
+    b = "site_b: " + "b" * 500 + "\n"
+    chunks = chunk_yaml(a + "---\n" + b, target_chars=600, overlap_chars=0)
+    # Each site is long enough to force its own chunk; the separator must not
+    # glue them into one over-budget chunk.
+    assert len(chunks) >= 2
+    assert all(len(c) <= 600 for c in chunks)
+    assert not any("site_a" in c and "site_b" in c for c in chunks)
+
+
+def test_yaml_oversized_key_block_is_split_under_hard_cap() -> None:
+    doc = "big_list:\n" + ("  - " + "x" * 60 + "\n") * 60  # one ~3800-char block
+    chunks = chunk_yaml(doc)
+    assert len(chunks) >= 2
+    assert all(len(c) <= HARD_MAX_CHARS for c in chunks)
+    # Overlap re-prints (≤50 chars per split), so only a little content is
+    # re-stated — the bulk of the block must survive.
+    assert sum(len(c) for c in chunks) >= len(doc) - 300
+
+
+def test_yaml_empty_content() -> None:
+    assert chunk_yaml("") == []
+    assert chunk_yaml("\n\n  \n") == []
+
+
+# ---------------------------------------------------------------------------
+# json
+# ---------------------------------------------------------------------------
+
+
+def test_json_splits_on_top_level_keys_pretty_printed() -> None:
+    doc = '{"hosts": {"kafkabridge": "10.0.3.7"}, "count": 3}'
+    chunks = chunk_json(doc, target_chars=45, overlap_chars=0)  # force 1 chunk/block
+    assert len(chunks) == 2
+    first, second = chunks
+    assert '"hosts"' in first and "kafkabridge" in first
+    assert '"count"' in second
+    # Pretty-printed (indent=2), not the compact input form.
+    assert '"kafkabridge": "10.0.3.7"' in first
+    assert not any('{"hosts"' in c for c in chunks)
+
+
+def test_json_each_key_block_is_self_contained() -> None:
+    doc = '{"a": "x", "b": "y"}'
+    chunks = chunk_json(doc, target_chars=13, overlap_chars=0)  # force 1 chunk/block
+    assert [c for c in chunks if '"a"' in c] and [c for c in chunks if '"b"' in c]
+    assert not any('"a"' in c and '"b"' in c for c in chunks)
+
+
+def test_json_oversized_value_falls_under_hard_cap() -> None:
+    doc = '{"blob": "' + "z" * 4000 + '"}'
+    chunks = chunk_json(doc)
+    assert len(chunks) >= 2
+    assert all(len(c) <= HARD_MAX_CHARS for c in chunks)
+    assert "".join(chunks).count("z") >= 4000
+
+
+def test_json_top_level_list_is_one_pretty_block() -> None:
+    chunks = chunk_json("[1, 2, 3]")
+    assert chunks == ["[\n  1,\n  2,\n  3\n]"]
+
+
+def test_json_unparseable_falls_back_to_paragraph_packing() -> None:
+    doc = "{broken json\n\nsecond paragraph here\n"
+    assert chunk_json(doc) == chunk_text(doc)
+    assert chunk_json("not json at all") == chunk_text("not json at all")
+
+
+# ---------------------------------------------------------------------------
+# python
+# ---------------------------------------------------------------------------
+
+
+def test_python_splits_on_top_level_defs_and_classes() -> None:
+    doc = (
+        '"""Module doc."""\n'
+        "import asyncio\n"
+        "\n"
+        "CONST = 1\n"
+        "\n"
+        "def alpha():\n"
+        "    return 1\n"
+        "\n"
+        "class Beta:\n"
+        "    def run(self):\n"
+        "        return 2\n"
+    )
+    chunks = chunk_python(doc, target_chars=60, overlap_chars=0)  # force 1 chunk/block
+    assert len(chunks) == 3
+    assert chunks[0].startswith('"""Module doc."""')
+    assert "CONST = 1" in chunks[0]  # preamble ends at the first def/class
+    assert chunks[1].startswith("def alpha")
+    assert chunks[2].startswith("class Beta")
+    assert "def run" in chunks[2]  # nested def stays inside the class block
+
+
+def test_python_decorators_stay_with_their_definition() -> None:
+    doc = "@app.get('/x')\ndef handler():\n    return 'x'\n"
+    chunks = chunk_python(doc)
+    assert chunks[0].startswith("@app.get")
+
+
+def test_python_oversized_function_falls_back_to_line_packing() -> None:
+    doc = "def big():\n" + "\n".join(f"    val_{i:03d} = {i}  # padding" for i in range(80))
+    chunks = chunk_python(doc)
+    assert len(chunks) >= 2
+    assert all(len(c) <= HARD_MAX_CHARS for c in chunks)
+    assert "val_000" in chunks[0]
+    assert "val_079" in chunks[-1]
+    assert sum(len(c) for c in chunks) >= len(doc) - 100
+
+
+def test_python_unparseable_source_falls_back_to_paragraphs() -> None:
+    src = "def broken(:\n\nstill text\n"
+    assert chunk_python(src) == chunk_text(src)
+
+
+# ---------------------------------------------------------------------------
+# txt
+# ---------------------------------------------------------------------------
+
+
+def test_txt_paragraph_packing() -> None:
+    doc = "alpha\n\nbeta\n\ngamma\n"
+    chunks = chunk_text(doc)
+    assert chunks == ["alpha\nbeta\ngamma"]  # all three fit the target
+
+
+def test_txt_long_doc_packs_with_overlap() -> None:
+    doc = "\n\n".join(f"para {i} " + "l" * 300 for i in range(6))
+    chunks = chunk_text(doc, target_chars=800, overlap_chars=100)
+    assert len(chunks) >= 2
+    assert all(len(c) <= 800 for c in chunks)
+    assert all(f"para {i}" in "\n".join(chunks) for i in range(6))
+
+
+# ---------------------------------------------------------------------------
+# Hard cap across every format (aipi ~1024-token request cap)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("content", "path"),
+    [
+        ("# T\n\n" + "word " * 1200, "big.md"),
+        ("key: " + "v" * 5000 + "\n", "big.yaml"),
+        ('{"blob": "' + "z" * 5000 + '"}', "big.json"),
+        ("def f():\n" + "    x = 1\n" * 1000, "big.py"),
+        ("line of text\n\n" * 800, "big.txt"),
+    ],
+)
+def test_hard_cap_holds_for_every_format(content: str, path: str) -> None:
+    chunks = chunk_document(content, path)
+    assert chunks, "expected at least one chunk"
+    for c in chunks:
+        assert len(c) <= HARD_MAX_CHARS, f"{path}: {len(c)} chars"

tests/unit/test_config.py

@@ -5,6 +5,7 @@ import json
 from typing import Any
 
 import pytest
+from pydantic import ValidationError
 from pydantic_settings import SettingsError
 
 from app.config import Settings
@@ -16,16 +17,28 @@ def _settings(**kwargs: Any) -> Settings:
     return Settings(**kwargs)  # pyright: ignore[reportCallIssue]  (kwarg exists at runtime)
 
 
-def test_defaults_match_locked_decisions() -> None:
+def test_defaults_match_locked_decisions(monkeypatch: pytest.MonkeyPatch) -> None:
+    # The test process sets BOR_RELEVANCE_THRESHOLD=0.30 for the mock-
+    # calibrated in-process suites (see tests/conftest.py) — the *default*
+    # under test is the production one.
+    monkeypatch.delenv("BOR_RELEVANCE_THRESHOLD", raising=False)
     s = _settings()
     assert s.llm_chat_model == "turbo"
     assert s.llm_embed_model == "embed"
     assert s.embedding_dim == 768
     assert s.llm_base_url.endswith("/v1")
-    assert 0 < s.relevance_threshold < 1
-    assert s.top_k_chunks >= 1
+    # A8 (revised): the honesty gate input is the best cosine, default 0.62.
+    assert s.relevance_threshold == 0.62
+    # A7 (revised): hybrid retrieval — cosine top-N ∪ FTS top-N, RRF-fused.
+    assert s.hybrid_vector_candidates >= 1
+    assert s.hybrid_lexical_candidates >= 1
+    assert s.rrf_k >= 1
     assert s.top_n_docs >= 1
     assert len(s.suggestions) >= 3
+    # A9 (revised): the import scope covers the seven A9 formats.
+    assert s.import_extension_set == {
+        ".md", ".markdown", ".txt", ".yaml", ".yml", ".json", ".py"
+    }
 
 
 def test_env_override(monkeypatch) -> None:
@@ -36,6 +49,26 @@ def test_env_override(monkeypatch) -> None:
     assert s.llm_chat_model == "juggernaut"
 
 
+def test_import_extensions_env_override_is_a_csv_list(monkeypatch) -> None:
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,yml")
+    s = _settings()
+    assert s.import_extension_set == {".md", ".yml"}
+
+
+def test_import_extensions_rejects_unknown_format(monkeypatch) -> None:
+    """A typo in the CSV fails at startup (loudly), not by silently
+    walking zero files."""
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,docx")
+    with pytest.raises(ValidationError, match="docx"):
+        _settings()
+
+
+def test_import_extensions_rejects_empty(monkeypatch) -> None:
+    monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "   ")
+    with pytest.raises(ValidationError):
+        _settings()
+
+
 def test_suggestions_default_is_three_plus_real_questions() -> None:
     s = _settings()
     assert len(s.suggestions) >= 3

tests/unit/test_importer.py

@@ -16,11 +16,15 @@ from app.models import Chunk, Document
 from app.rag.importer import (
     EXCLUDED_DIRS,
     import_sources,
-    iter_markdown_files,
+    iter_importable_files,
 )
 from app.rag.llm import EmbeddingError
 from tests.fakes import FakeEmbedder
 
+#: A9 default extension set as dotted suffixes (what the importer passes to
+#: the walker when no override is configured).
+DEFAULT_EXTS = frozenset({".md", ".markdown", ".txt", ".yaml", ".yml", ".json", ".py"})
+
 
 class _PoisonEmbedder(FakeEmbedder):
     """Fails (like a real endpoint) on any text containing 'poison'."""
@@ -50,7 +54,9 @@ def _cleanup_source(db, source: str) -> None:
     db.commit()
 
 
-def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
+def test_iter_importable_files_excludes_noncontent_dirs_and_hidden(tmp_path: Path) -> None:
+    """Well-known non-content dirs, hidden (dot-) dirs/files, and non-A9
+    extensions are all skipped; the A9 formats pass."""
     root = tmp_path / "proj"
     for d in (
         "notes/sub",
@@ -61,11 +67,20 @@ def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
         ".pytest_cache",
         "dist",
         "build",
+        ".esphome/.espressif",  # vendored hidden cache — the real A9 case
     ):
         (root / d).mkdir(parents=True)
     files = {
+        # content that must be found:
         "README.md": "readme",
         "notes/sub/deep.md": "deep",
+        "compose.yaml": "services: {}",
+        "legacy.YML": "a: b",  # case-insensitive suffix
+        "notes/sub/agent.py": "x = 1",
+        "config.json": "{}",
+        "README.txt": "plain",
+        "notes/sub/deep.markdown": "md2",
+        # must be skipped:
         ".venv/lib/junk.md": "junk",
         "node_modules/x/j.md": "j",
         ".git/c.md": "g",
@@ -73,17 +88,40 @@ def test_iter_markdown_files_excludes_noncontent_dirs(tmp_path: Path) -> None:
         ".pytest_cache/c.md": "pc",
         "dist/d.md": "d",
         "build/b.md": "b",
+        ".esphome/.espressif/secret.md": "vendor",
+        ".secret.md": "hidden file",  # dot-prefixed FILE, not just dir
+        "notes/sub/notes.csv": "a,b",  # not an A9 format
+        "notes/sub/file.md.bak": "x",
     }
     for rel, text in files.items():
         (root / rel).write_text(text)
-    (root / "notes" / "not-md.txt").write_text("skip me")
 
-    found = {p.relative_to(root).as_posix() for p in iter_markdown_files(root)}
-    assert found == {"README.md", "notes/sub/deep.md"}
+    found = {p.relative_to(root).as_posix() for p in iter_importable_files(root, DEFAULT_EXTS)}
+    assert found == {
+        "README.md",
+        "notes/sub/deep.md",
+        "compose.yaml",
+        "legacy.YML",
+        "notes/sub/agent.py",
+        "config.json",
+        "README.txt",
+        "notes/sub/deep.markdown",
+    }
 
 
-def test_iter_markdown_files_missing_dir_yields_nothing(tmp_path: Path) -> None:
-    assert iter_markdown_files(tmp_path / "definitely-missing") == []
+def test_iter_importable_files_missing_dir_yields_nothing(tmp_path: Path) -> None:
+    assert iter_importable_files(tmp_path / "definitely-missing", DEFAULT_EXTS) == []
+
+
+def test_iter_importable_files_respects_custom_extension_filter(tmp_path: Path) -> None:
+    """A narrower filter (e.g. md only) excludes the other A9 formats."""
+    root = tmp_path / "filtered"
+    root.mkdir()
+    (root / "a.md").write_text("a")
+    (root / "b.yaml").write_text("a: b")
+    (root / "c.py").write_text("x = 1")
+    found = {p.name for p in iter_importable_files(root, frozenset([".md"]))}
+    assert found == {"a.md"}
 
 
 def test_excluded_dirs_match_plan_anchor_a9() -> None:
@@ -277,3 +315,65 @@ def test_chunk_positions_and_titles(db, tmp_path: Path) -> None:
         assert positions == list(range(len(doc.chunks))) and len(doc.chunks) >= 2
     finally:
         _cleanup_source(db, root.name)
+
+
+def test_multi_format_import_counts_per_format_and_titles_stem(db, tmp_path: Path) -> None:
+    """A9 formats all import; the summary records per-format counts, and
+    non-markdown titles come from the file stem (a ``#`` line is a comment
+    there, not a heading)."""
+    root = tmp_path / "multi"
+    (root / "svc").mkdir(parents=True)
+    (root / "guide.md").write_text("# Real Heading\n\nbody\n")
+    (root / "svc" / "compose.yaml").write_text("# a comment\nservices:\n  gitlab: {}\n")
+    (root / "svc" / "agent.py").write_text("# docstring-like comment\ndef ping():\n    return 1\n")
+    (root / "inventory.json").write_text('{"hosts": []}\n')
+    (root / "notes.txt").write_text("plain text notes\n")
+    llm = FakeEmbedder()
+    try:
+        summary = asyncio.run(import_sources([root], llm, session=db))
+        assert summary.files == 5
+        assert summary.added == 5
+        assert summary.formats == {"md": 1, "yaml": 1, "py": 1, "json": 1, "txt": 1}
+        # PLAN §9 summary line: counts, highest first, ext:name pairs.
+        assert summary.format_counts() == "json:1,md:1,py:1,txt:1,yaml:1"
+
+        titles = {
+            d.path: d.title
+            for d in db.scalars(select(Document).where(Document.source == root.name)).all()
+        }
+        assert titles["guide.md"] == "Real Heading"  # markdown keeps the H1
+        assert titles["svc/compose.yaml"] == "compose"  # …comment is not a heading
+        assert titles["svc/agent.py"] == "agent"
+        assert titles["inventory.json"] == "inventory"
+        assert titles["notes.txt"] == "notes"
+    finally:
+        _cleanup_source(db, root.name)
+
+
+def test_prune_removes_files_now_excluded_by_format_filter(db, tmp_path: Path) -> None:
+    """Previously-imported junk leaves the index: a file that no longer
+    matches the A9 extension filter is pruned on the next ``prune=True`` run.
+    This is how dot-dir READMEs imported before the scope fix get cleaned up."""
+    root = tmp_path / "cleanup"
+    root.mkdir()
+    (root / "keep.md").write_text("# Keep\n\nkept\n")
+    (root / "junk.md.bak").write_text("old junk that was once imported\n")
+    llm = FakeEmbedder()
+    try:
+        # Seed: import both files as if they were valid at the time.
+        (root / "junk.md").write_text("old junk that was once imported\n")
+        (root / "junk.md.bak").unlink()
+        asyncio.run(import_sources([root], llm, session=db))
+        # Rename the junk out of the A9 formats, then prune.
+        (root / "junk.md").rename(root / "junk.md.bak")
+        summary = asyncio.run(import_sources([root], llm, session=db, prune=True))
+        assert summary.pruned == 1
+        assert summary.unchanged == 1  # keep.md survived
+        assert db.scalar(
+            select(Document).where(Document.source == root.name, Document.path == "junk.md")
+        ) is None
+        assert db.scalar(
+            select(Document).where(Document.source == root.name, Document.path == "keep.md")
+        ) is not None
+    finally:
+        _cleanup_source(db, root.name)

tests/unit/test_retriever.py

@@ -8,6 +8,8 @@ from __future__ import annotations
 
 import uuid
 
+import pytest
+
 from app.models import Document
 from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk, select_documents
 
@@ -94,3 +96,102 @@ def test_under_budget_no_truncation() -> None:
 
 def test_empty_hits_yield_no_documents() -> None:
     assert select_documents([], n=2, max_chars=24_000) == []
+
+
+# ---------------------------------------------------------------------------
+# Hybrid retrieval (A7): RRF fusion + lexical tsquery
+# ---------------------------------------------------------------------------
+
+from app.rag.retriever import fuse, lexical_tsquery  # noqa: E402
+
+
+def _rc(
+    doc_path: str, cosine: float = 0.0, fts_hit: bool = False, position: int = 0
+) -> RetrievedChunk:
+    return RetrievedChunk(
+        chunk_id=uuid.uuid4(),
+        position=position,
+        content="x" * 20,
+        score=0.0,
+        document=_doc(doc_path, "x" * 20),
+        cosine=cosine,
+        fts_hit=fts_hit,
+    )
+
+
+def test_lexical_tsquery_tokens_lowercased_deduped_in_order() -> None:
+    assert lexical_tsquery("How did I Install GITLAB gitlab?") == "how | did | i | install | gitlab"
+
+
+def test_lexical_tsquery_punctuation_and_umlauts_ignored() -> None:
+    assert lexical_tsquery("c3-r00t? -- what's up!") == "c3 | r00t | what | s | up"
+
+
+def test_lexical_tsquery_pure_symbols_return_none() -> None:
+    assert lexical_tsquery("??? ???") is None
+    assert lexical_tsquery("") is None
+
+
+def test_lexical_tsquery_stopwords_left_to_postgres() -> None:
+    # lexical_tsquery passes raw tokens through; Postgres's to_tsquery
+    # lexing drops the stopwords (verified against real PG in
+    # test_retrieve_empty_kb / integration tests).
+    assert lexical_tsquery("how do i") == "how | do | i"
+
+
+def test_fuse_combines_both_lists_for_double_hits() -> None:
+    v1 = _rc("a.md", cosine=0.9)
+    v2 = _rc("b.md", cosine=0.5)
+    l1 = _rc("a.md", cosine=0.1)  # same chunk id -> matched in place
+    a_id = v1.chunk_id
+    l1.chunk_id = a_id
+    out = fuse([v1, v2], [l1], k=60)
+    by_id = {rc.chunk_id: rc for rc in out}
+    # a: 1/61 (vector rank 1) + 1/61 (lexical rank 1); b: 1/62 only.
+    assert by_id[a_id].score == pytest.approx(2 / 61)
+    assert by_id[a_id].fts_hit is True
+    assert by_id[v2.chunk_id].score == pytest.approx(1 / 62)
+    assert by_id[v2.chunk_id].fts_hit is False
+    assert [rc.chunk_id for rc in out] == [a_id, v2.chunk_id]
+
+
+def test_fuse_lexical_only_chunks_enter_with_zero_cosine() -> None:
+    vector = [_rc("a.md", cosine=0.8)]
+    lexical = [_rc("b.md", cosine=0.0, fts_hit=True)]
+    out = fuse(vector, lexical, k=60)
+    assert len(out) == 2
+    b = next(rc for rc in out if rc.document.path == "b.md")
+    assert b.cosine == 0.0
+    assert b.fts_hit is True
+    # Still ranked by its (only) RRF term.
+    assert b.score == pytest.approx(1 / 61)
+
+
+def test_fuse_orders_by_score_then_cosine_then_path() -> None:
+    # Two chunks share an RRF score (both rank 1 in different lists):
+    # the higher-cosine one must sort first.
+    hi = _rc("z.md", cosine=0.9)
+    lo = _rc("a.md", cosine=0.2)
+    out = fuse([hi], [lo], k=60)
+    assert [rc.document.path for rc in out] == ["z.md", "a.md"]
+    # Equal score AND cosine -> path order.
+    p1 = _rc("b.md", cosine=0.5)
+    p2 = _rc("a.md", cosine=0.5)
+    out = fuse([p1], [p2], k=60)
+    assert [rc.document.path for rc in out] == ["a.md", "b.md"]
+    # Equal score, cosine, path -> position order.
+    s1 = _rc("a.md", cosine=0.5, position=1)
+    s2 = _rc("a.md", cosine=0.5, position=0)
+    out = fuse([s1], [s2], k=60)
+    assert [rc.position for rc in out] == [0, 1]
+
+
+def test_fuse_rejects_nonpositive_k() -> None:
+    with pytest.raises(ValueError):
+        fuse([], [], k=0)
+    with pytest.raises(ValueError):
+        fuse([], [], k=-1)
+
+
+def test_fuse_empty_lists() -> None:
+    assert fuse([], [], k=60) == []