feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

tests/unit/test_chat_gate.py

@@ -45,13 +45,19 @@ def _doc(title: str, content: str) -> Document:
     )
 
 
-def _chunk(doc: Document, score: float) -> RetrievedChunk:
+def _chunk(
+    doc: Document, score: float, cosine: float | None = None, fts_hit: bool = False
+) -> RetrievedChunk:
+    """Fake candidate: *score* is the fused rank score; *cosine* (defaults to
+    *score*) is the vector-similarity gate input."""
     return RetrievedChunk(
         chunk_id=uuid.uuid4(),
         position=0,
         content=doc.content[:32],
         score=score,
         document=doc,
+        cosine=score if cosine is None else cosine,
+        fts_hit=fts_hit,
     )
 
 
@@ -89,6 +95,68 @@ def test_gate_is_env_tunable_via_settings() -> None:
     assert chat_api.plan_turn(hits, _settings(threshold=0.25)).deflected is False
 
 
+# ---------- hybrid gate matrix (A8, revised: cosine AND fts) ----------
+
+
+def test_gate_weak_cosine_with_fts_hit_still_answers() -> None:
+    """cosine < threshold but a lexical hit ⇒ HIGH — the FTS-OR branch.
+    This is the name-your-tool case: "kafkabridge" grounds despite weak
+    vector overlap."""
+    doc = _doc("Static DNS", "DNS_DOC_CONTENT")
+    plan = chat_api.plan_turn(
+        [_chunk(doc, 0.02, cosine=0.10, fts_hit=True)], _settings(threshold=0.30)
+    )
+    assert plan.deflected is False
+    assert plan.top_score == pytest.approx(0.10)  # gate input is the cosine
+    assert plan.fts_hits == 1
+    assert "DNS_DOC_CONTENT" in plan.system_prompt
+    assert plan.suggestions == []
+
+
+def test_gate_weak_cosine_zero_fts_deflects() -> None:
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    plan = chat_api.plan_turn([_chunk(doc, 0.02, cosine=0.10)], _settings(threshold=0.30))
+    assert plan.deflected is True
+    assert plan.top_score == pytest.approx(0.10)
+    assert plan.fts_hits == 0
+
+
+def test_gate_strong_cosine_without_fts_answers() -> None:
+    doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
+    plan = chat_api.plan_turn([_chunk(doc, 0.90, cosine=0.90)], _settings(threshold=0.30))
+    assert plan.deflected is False
+    assert plan.fts_hits == 0
+
+
+def test_gate_fts_hits_counts_all_lexical_candidates() -> None:
+    a = _doc("Alpha", "ALPHA_CONTENT")
+    b = _doc("Beta", "BETA_CONTENT")
+    chunks = [
+        _chunk(a, 0.03, cosine=0.05, fts_hit=True),
+        _chunk(a, 0.02, cosine=0.04, fts_hit=True),  # same doc, second chunk
+        _chunk(b, 0.01, cosine=0.03),
+    ]
+    plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
+    assert plan.deflected is False
+    assert plan.fts_hits == 2  # per chunk, not per doc
+
+
+def test_gate_lexical_only_chunk_does_not_inflate_cosine() -> None:
+    """top_score stays the best *vector* cosine even when a lexical-only
+    chunk (cosine 0.0 by construction) carries the highest fused score."""
+    a = _doc("Alpha", "ALPHA_CONTENT")
+    b = _doc("Beta", "BETA_CONTENT")
+    chunks = [
+        _chunk(a, 0.50, cosine=0.55),  # vector rank 1
+        _chunk(b, 0.90, cosine=0.0, fts_hit=True),  # lexical rank 1 wins the ranking
+    ]
+    plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
+    assert plan.top_score == pytest.approx(0.55)
+    assert plan.deflected is False  # 0.55 >= 0.30 anyway
+    # ranking follows the fused score: Beta's doc is the top source
+    assert plan.docs[0].title == "Beta"
+
+
 def test_gate_zero_chunks_deflects_with_fallback_chips() -> None:
     plan = chat_api.plan_turn([], _settings())
     assert plan.deflected is True
@@ -233,6 +301,13 @@ def gate_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[tuple[_FakeSession, _C
     llm = _CannedLLM()
     monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_db, lambda: session)
     monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_llm, lambda: llm)
+    # These tests assert against a specific gate threshold; keep it stable
+    # regardless of the production default (0.62) or any .env.
+    monkeypatch.setattr(
+        chat_api,
+        "get_settings",
+        lambda: Settings(_env_file=None, relevance_threshold=0.30),  # pyright: ignore[reportCallIssue]
+    )
     yield session, llm
     fastapi_app.dependency_overrides.clear()
 
@@ -254,7 +329,7 @@ def _ask(client: TestClient, message: str) -> list[dict[str, Any]]:
 
 
 def _fake_retriever(chunks: list[RetrievedChunk]) -> Any:
-    def retrieve(_db: Any, _vec: list[float]) -> list[RetrievedChunk]:
+    def retrieve(_db: Any, _question: str, _vec: list[float]) -> list[RetrievedChunk]:
         return chunks
 
     return retrieve