feat(rag): hybrid FTS+vector retrieval and multi-format ingestion — name-your-tool questions find the right document

app/api/chat.py

@@ -1,18 +1,21 @@
 """POST /api/chat — a RAG chat turn streamed over SSE (PLAN §3/§4).
 
-Flow (LOCKED A7/A15): embed the question → pgvector cosine top-K chunks →
-the **honesty gate** (A8: best score < ``BOR_RELEVANCE_THRESHOLD`` ⇒
-deflection) → locked persona prompt (PLAN §6) → ``turbo`` streamed as
-``delta`` events → final ``done`` event (``deflected``, ``sources``,
-``suggestions``) + ``query_log`` row + the per-turn log line (PLAN §9).
+Flow (LOCKED A7/A15): embed the question → hybrid retrieval (cosine
+top-N ∪ Postgres FTS top-N, RRF-fused) → the **honesty gate** → locked
+persona prompt (PLAN §6) → ``turbo`` streamed as ``delta`` events → final
+``done`` event (``deflected``, ``sources``, ``suggestions``) +
+``query_log`` row + the per-turn log line (PLAN §9).
 Mid-stream failures become a structured ``error`` event; a pre-stream DB
 outage is a plain 503 JSON.
 
-Honesty gate: a weak retrieval (score strictly below the threshold — or
-an empty KB) flips the turn to deflection mode: the LOW prompt carries
-weak-hit *titles only* (never document content) plus deterministic
-"Maybe try" chips, and the ``done`` event / ``query_log`` row record
-``deflected=true`` with the weak score.
+Honesty gate (A8, revised 2026-08-21): LOW — deflection — only when the
+best cosine is strictly below ``BOR_RELEVANCE_THRESHOLD`` **and** no
+candidate chunk FTS-matches the question (``fts_hits == 0``). A
+name-your-tool question with weak vector overlap but a lexical hit still
+gets a grounded answer. Deflection mode carries weak-hit *titles only*
+(never document content) plus deterministic "Maybe try" chips, and the
+``done`` event / ``query_log`` row record ``deflected=true``, the weak
+score and the ``fts_hits`` count.
 """
 from __future__ import annotations
 
@@ -62,7 +65,8 @@ def sse_event(payload: dict[str, Any]) -> str:
 class TurnPlan:
     """What one chat turn sends to the LLM and reports on ``done``."""
 
-    top_score: float
+    top_score: float  # best cosine across candidates (query_log.top_score)
+    fts_hits: int  # lexical (OR-tsquery) candidates matched
     deflected: bool
     system_prompt: str
     docs: list[Document]  # cited sources (weak hits when deflected)
@@ -70,25 +74,32 @@ class TurnPlan:
 
 
 def plan_turn(chunks: Sequence[RetrievedChunk], settings: Settings) -> TurnPlan:
-    """Apply the honesty gate (A8) and assemble prompt + context for a turn.
+    """Apply the honesty gate (A8, revised) and assemble prompt + context.
 
-    * ``top_score >= threshold`` → grounded: HIGH prompt with the full
-      top-N documents, no suggestions. A score exactly at the threshold
-      is an answer — the gate is strict (``score < threshold``).
-    * ``top_score < threshold`` (or no hits at all) → deflected: LOW
-      prompt (``DEFLECT_MODE``) with weak-hit titles only — never document
-      content — plus deterministic alternative-question chips derived
-      from those titles.
+    * **HIGH (grounded)** when ``best_cosine >= threshold`` **or**
+      ``fts_hits > 0``: HIGH prompt with the full top-N documents, no
+      suggestions. A cosine exactly at the threshold is an answer — the
+      gate is strict (``< threshold``).
+    * **LOW (deflected)** only when ``best_cosine < threshold`` **and**
+      ``fts_hits == 0`` (or no hits at all): LOW prompt (``DEFLECT_MODE``)
+      with weak-hit titles only — never document content — plus
+      deterministic alternative-question chips derived from those titles.
+
+    ``top_score`` (stored in ``query_log``) is the best cosine, so the
+    gate input is always a pure vector-similarity number; the lexical
+    signal is recorded separately as ``fts_hits``.
     """
-    top_score = chunks[0].score if chunks else 0.0
-    if top_score >= settings.relevance_threshold:
+    best_cosine = max((c.cosine for c in chunks), default=0.0)
+    fts_hits = sum(1 for c in chunks if c.fts_hit)
+    if best_cosine >= settings.relevance_threshold or fts_hits > 0:
         docs = select_documents(
             chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars
         )
-        return TurnPlan(top_score, False, build_high_prompt(docs), docs, [])
+        return TurnPlan(best_cosine, fts_hits, False, build_high_prompt(docs), docs, [])
     titles = weak_hit_titles(chunks)
     return TurnPlan(
-        top_score,
+        best_cosine,
+        fts_hits,
         True,
         build_deflect_prompt(titles),
         select_documents(chunks, n=settings.top_n_docs, max_chars=settings.max_context_chars),
@@ -143,7 +154,7 @@ async def chat(
         #    HIGH (grounded) or LOW (deflected) prompt + context.
         settings = get_settings()
         try:
-            chunks = retrieve(db, question_vec)
+            chunks = retrieve(db, request.message, question_vec)
             plan = plan_turn(chunks, settings)
         except Exception:  # noqa: BLE001 — DB failure mid-turn
             logger.exception(
@@ -188,6 +199,7 @@ async def chat(
                 QueryLog(
                     question=request.message,
                     top_score=plan.top_score,
+                    fts_hits=plan.fts_hits,
                     chunk_hits=len(chunks),
                     deflected=plan.deflected,
                     sources=", ".join(source_paths),
@@ -199,11 +211,12 @@ async def chat(
             logger.exception("chat: failed to write query_log question=%r", request.message)
 
         logger.info(
-            "question=%r embed_ms=%d top_score=%.3f threshold=%.2f deflected=%s "
-            "sources=%r total_ms=%d",
+            "question=%r embed_ms=%d top_score=%.3f fts_hits=%d threshold=%.2f "
+            "deflected=%s sources=%r total_ms=%d",
             request.message,
             embed_ms,
             plan.top_score,
+            plan.fts_hits,
             settings.relevance_threshold,
             plan.deflected,
             source_paths,