phase: 112_honesty_gate_weak_hits

**Phase 112 — final verification pass (all 4 tasks already complete in `complete/`):** - Verified gate fix: `app/api/chat.py::plan_turn` — HIGH iff `best_cosine >= relevance_threshold` OR (`fts_hits > 0` AND `best_cosine >= lexical_support_floor`); `lexical_support_floor` (default 0.35, `BOR_LEXICAL_SUPPORT_FLOOR`, bounds-validated) in `app/config.py` + `.env.example`; A8 revision note (2026-09-14) in `.agents/PLAN.md`. - Verified prompt contract: `app/rag/prompts.py` diff is docstring-only (dated owner-decision-iii entry); `tests/unit/test_prompt_lock.py` byte-pins PERSONA/TOOLS_SECTION/DEFLECT body (sha256+length). - Verified README: L11 + L575 deflection copy refreshed; `grep "haven't done anything" README.md` → no hits; disclosed-answer behavior documented. - Tests: `uv run pytest --cov=app --cov-report=term-missing` → **2378 passed, 99% coverage (>90%)**; includes Mongolia-quadrant unit pins (fts>0 + cosine<floor → LOW). - E2E in isolation: `uv run pytest tests/e2e/test_honest_deflection.py -v --no-cov` → **4 passed** (out-of-KB question: `deflected=true`, `sources==[]`, 2–3 suggestions); regression `test_chat_rag.py` + `test_retrieval_quality.py` → **7 passed**. - Lint/types: `uv run ruff check .` → clean; `uv run pyright` → 0 errors. **Completion criteria:** weak-FTS→LOW unit-pinned ✅ · no false citations + 2–3 alternatives E2E ✅ · prompts byte-identical (test-pinned) + README matches ✅ · suite/coverage/e2e/lint all green ✅ · commit + phase move → left to harness (no `git commit` run, per rules; changes in working tree). **Deviations:** none. Next pending phase: `113_source_chip_quality`.
2026-09-15 00:37:38 -04:00
parent 2683128876
commit 1374faf136
36 changed files with 1240 additions and 67 deletions
@@ -34,6 +34,17 @@ from e2e.auth_helpers import ADMIN_PASSWORD, login
 REPO = Path(__file__).resolve().parents[2]
 FIXTURES = REPO / "tests" / "fixtures" / "docs"
 OFF_TOPIC = "How do I bake sourdough bread?"
+# Phase 112 (A8 revised 2026-09-14, TODO L2a — the "Mongolia case"): a
+# question the LLM knows (Gershwin) but the fixture KB does not cover.
+# Unlike the plain no-hit deflection above, it carries WEAK FTS hits
+# (the "compos" stem matches the compose fixture docs — fts_hits >= 1)
+# while its best mock token-overlap cosine (~0.12) sits BELOW
+# lexical_support_floor (0.15, the mock-calibrated conftest value). The
+# pre-phase gate (fts>0 → HIGH) grounded it and injected irrelevant docs
+# into the prompt; the revised gate (cosine corroboration) must keep it
+# LOW. The mock keys on DEFLECT_MODE, so the test pins the gate, not
+# model compliance.
+OUT_OF_KB = "Who composed Rhapsody in Blue?"
 # The mock's deflection answer (tests/e2e/mock_llm.py) must match this.
 DEFLECT_PHRASE = r"haven't done anything like that"
 MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
@@ -201,3 +212,67 @@ def test_deflected_done_event_and_query_log(app_url: str, mock_llm: int, db_read
    assert row.deflected is True
    assert 0.0 < row.top_score < get_settings().relevance_threshold
    assert row.chunk_hits >= 1
+
+
+def test_out_of_kb_question_deflects_without_citations(
+    app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """Phase 112 acceptance (TODO L2): a known-out-of-KB question whose
+    weak lexical hits NO LONGER promote (the fts>0 / cosine<floor
+    quadrant, pinned end-to-end) deflects with ZERO source citations —
+    done.sources is empty (the UI chips nothing under a deflected
+    answer) and 2-3 concrete alternative questions are offered.
+
+    Raw SSE (like the done-event test above): the done frame is the
+    contract surface; the mock's DEFLECT_MODE phrasing proves the
+    server sent the LOW prompt (the gate, not the model, decides).
+    """
+    _reset_db(mock_llm, seed=True)
+
+    client = httpx.Client(timeout=60.0)
+    r = client.post(f"{app_url}/api/login", json={"password": ADMIN_PASSWORD})
+    assert r.status_code == 204
+
+    frames: list[dict[str, Any]] = []
+    with client.stream(
+        "POST", f"{app_url}/api/chat", json={"message": OUT_OF_KB}, timeout=60.0
+    ) as r:
+        assert r.status_code == 200
+        assert r.headers["content-type"].startswith("text/event-stream")
+        buf = ""
+        for part in r.iter_text():
+            buf += part
+            while "\n\n" in buf:
+                frame, buf = buf.split("\n\n", 1)
+                if frame.strip().startswith("data:"):
+                    frames.append(json.loads(frame.strip().removeprefix("data:").strip()))
+    assert buf.strip() == ""  # stream ends cleanly on a frame boundary
+
+    deltas = [f for f in frames if f.get("type") == "delta"]
+    answer = "".join(d["text"] for d in deltas)
+    # The DEFLECT_MODE phrasing streamed ⇒ the LOW prompt reached the
+    # model (the mock answers it only for the deflection system prompt).
+    assert re.search(DEFLECT_PHRASE, answer, re.IGNORECASE)
+
+    done = frames[-1]
+    assert done["type"] == "done"
+    assert done["deflected"] is True
+    # No false citations (TODO L2): the weak hits never ride the wire as
+    # sources — a deflected answer cites nothing.
+    assert done["sources"] == []
+    # 2-3 concrete alternative questions, all non-empty.
+    assert 2 <= len(done["suggestions"]) <= 3
+    assert all(s.strip() for s in done["suggestions"])
+
+    # Durable record: the quadrant pinned end-to-end — the lexical leg
+    # FIRED (fts_hits > 0, the pre-phase gate's promotion trigger) while
+    # the vector signal never cleared lexical_support_floor, so the
+    # revised gate deflected. The retrieval itself stays recorded
+    # (query_log = observability, not citations).
+    with SessionLocal() as db:
+        row = db.scalars(select(QueryLog)).one()
+    assert row.question == OUT_OF_KB
+    assert row.deflected is True
+    assert (row.fts_hits or 0) >= 1
+    assert row.top_score < get_settings().lexical_support_floor
+    assert row.sources
@@ -13,7 +13,8 @@ The four tests map the story's acceptance criteria:
 2. "How did I install gitlab?" — grounded (not deflected), gitlab chip,
   ``query_log`` row with the gitlab doc in ``sources``
 3. keyword-only question ("kafkabridge") beats the vector ranking — the
-   FTS-OR gate grounds it end to end despite weak cosine
+   corroborated-lexical gate (A8 revised 2026-09-14) grounds it end to
+   end: weak cosine, but an FTS hit AND cosine >= lexical_support_floor
 4. "sourdough" — deflected bubble + ≥2 "Maybe try" chips
 """
 from __future__ import annotations
@@ -37,7 +38,15 @@ from e2e.auth_helpers import login
 REPO = Path(__file__).resolve().parents[2]
 FIXTURES = REPO / "tests" / "fixtures" / "docs"
 GITLAB_QUESTION = "How did I install gitlab?"
-KEYWORD_QUESTION = "How does kafkabridge work?"
+# Phase 112 (A8 revised): the pre-phase question ("How does kafkabridge
+# work?" — mock cosine 0.134) now sits BELOW lexical_support_floor
+# (0.15, mock-calibrated) with fts>0 — the new gate's deflection
+# quadrant, so it can no longer demonstrate the grounded lexical path.
+# "handle DNS" adds static-dns.json's own tokens: cosine ≈0.24 — still
+# weak (below the 0.30 threshold) yet corroborated (>= floor) with the
+# same single-doc FTS hit, and the FTS-matched doc still tops the fused
+# ranking (the test's actual assertion).
+KEYWORD_QUESTION = "How does kafkabridge handle DNS?"
 OFF_TOPIC = "sourdough starter"
 MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"

@@ -162,9 +171,11 @@ def test_gitlab_question_is_grounded_with_gitlab_chip(
 def test_keyword_only_question_beats_vector_ranking(
    page: Page, app_url: str, mock_llm: int, db_ready: None
 ) -> None:
-    """The FTS-OR gate end to end: "kafkabridge" appears in exactly one
-    fixture doc (static-dns.json) and the question's cosine overlap is
-    weak — the lexical branch is what grounds the answer."""
+    """The corroborated-lexical gate end to end (A8 revised 2026-09-14):
+    "kafkabridge" appears in exactly one fixture doc (static-dns.json)
+    and the question's cosine overlap is weak (below the threshold) —
+    the lexical hit plus cosine >= lexical_support_floor is what grounds
+    the answer (a lexical-only hit below the floor would deflect)."""
    _reset_db(mock_llm, seed=True)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated
@@ -182,9 +193,11 @@ def test_keyword_only_question_beats_vector_ranking(

    with SessionLocal() as db:
        row = db.scalars(select(QueryLog)).one()
-    # Weak vector score…
+    # Weak vector score — below the answer threshold…
    assert row.top_score < get_settings().relevance_threshold
-    # …but a lexical hit grounded it (the FTS-OR branch).
+    # …but cleared the lexical support floor, and a lexical hit fired —
+    # the corroborated-lexical path (A8 revised 2026-09-14) grounded it.
+    assert row.top_score >= get_settings().lexical_support_floor
    assert (row.fts_hits or 0) >= 1
    assert row.deflected is False
    assert "homelab/networking/static-dns.json" in row.sources