phase: 112_honesty_gate_weak_hits
**Phase 112 — final verification pass (all 4 tasks already complete in `complete/`):** - Verified gate fix: `app/api/chat.py::plan_turn` — HIGH iff `best_cosine >= relevance_threshold` OR (`fts_hits > 0` AND `best_cosine >= lexical_support_floor`); `lexical_support_floor` (default 0.35, `BOR_LEXICAL_SUPPORT_FLOOR`, bounds-validated) in `app/config.py` + `.env.example`; A8 revision note (2026-09-14) in `.agents/PLAN.md`. - Verified prompt contract: `app/rag/prompts.py` diff is docstring-only (dated owner-decision-iii entry); `tests/unit/test_prompt_lock.py` byte-pins PERSONA/TOOLS_SECTION/DEFLECT body (sha256+length). - Verified README: L11 + L575 deflection copy refreshed; `grep "haven't done anything" README.md` → no hits; disclosed-answer behavior documented. - Tests: `uv run pytest --cov=app --cov-report=term-missing` → **2378 passed, 99% coverage (>90%)**; includes Mongolia-quadrant unit pins (fts>0 + cosine<floor → LOW). - E2E in isolation: `uv run pytest tests/e2e/test_honest_deflection.py -v --no-cov` → **4 passed** (out-of-KB question: `deflected=true`, `sources==[]`, 2–3 suggestions); regression `test_chat_rag.py` + `test_retrieval_quality.py` → **7 passed**. - Lint/types: `uv run ruff check .` → clean; `uv run pyright` → 0 errors. **Completion criteria:** weak-FTS→LOW unit-pinned ✅ · no false citations + 2–3 alternatives E2E ✅ · prompts byte-identical (test-pinned) + README matches ✅ · suite/coverage/e2e/lint all green ✅ · commit + phase move → left to harness (no `git commit` run, per rules; changes in working tree). **Deviations:** none. Next pending phase: `113_source_chip_quality`.
This commit is contained in:
@@ -34,6 +34,17 @@ from e2e.auth_helpers import ADMIN_PASSWORD, login
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "docs"
|
||||
OFF_TOPIC = "How do I bake sourdough bread?"
|
||||
# Phase 112 (A8 revised 2026-09-14, TODO L2a — the "Mongolia case"): a
|
||||
# question the LLM knows (Gershwin) but the fixture KB does not cover.
|
||||
# Unlike the plain no-hit deflection above, it carries WEAK FTS hits
|
||||
# (the "compos" stem matches the compose fixture docs — fts_hits >= 1)
|
||||
# while its best mock token-overlap cosine (~0.12) sits BELOW
|
||||
# lexical_support_floor (0.15, the mock-calibrated conftest value). The
|
||||
# pre-phase gate (fts>0 → HIGH) grounded it and injected irrelevant docs
|
||||
# into the prompt; the revised gate (cosine corroboration) must keep it
|
||||
# LOW. The mock keys on DEFLECT_MODE, so the test pins the gate, not
|
||||
# model compliance.
|
||||
OUT_OF_KB = "Who composed Rhapsody in Blue?"
|
||||
# The mock's deflection answer (tests/e2e/mock_llm.py) must match this.
|
||||
DEFLECT_PHRASE = r"haven't done anything like that"
|
||||
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
|
||||
@@ -201,3 +212,67 @@ def test_deflected_done_event_and_query_log(app_url: str, mock_llm: int, db_read
|
||||
assert row.deflected is True
|
||||
assert 0.0 < row.top_score < get_settings().relevance_threshold
|
||||
assert row.chunk_hits >= 1
|
||||
|
||||
|
||||
def test_out_of_kb_question_deflects_without_citations(
|
||||
app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
"""Phase 112 acceptance (TODO L2): a known-out-of-KB question whose
|
||||
weak lexical hits NO LONGER promote (the fts>0 / cosine<floor
|
||||
quadrant, pinned end-to-end) deflects with ZERO source citations —
|
||||
done.sources is empty (the UI chips nothing under a deflected
|
||||
answer) and 2-3 concrete alternative questions are offered.
|
||||
|
||||
Raw SSE (like the done-event test above): the done frame is the
|
||||
contract surface; the mock's DEFLECT_MODE phrasing proves the
|
||||
server sent the LOW prompt (the gate, not the model, decides).
|
||||
"""
|
||||
_reset_db(mock_llm, seed=True)
|
||||
|
||||
client = httpx.Client(timeout=60.0)
|
||||
r = client.post(f"{app_url}/api/login", json={"password": ADMIN_PASSWORD})
|
||||
assert r.status_code == 204
|
||||
|
||||
frames: list[dict[str, Any]] = []
|
||||
with client.stream(
|
||||
"POST", f"{app_url}/api/chat", json={"message": OUT_OF_KB}, timeout=60.0
|
||||
) as r:
|
||||
assert r.status_code == 200
|
||||
assert r.headers["content-type"].startswith("text/event-stream")
|
||||
buf = ""
|
||||
for part in r.iter_text():
|
||||
buf += part
|
||||
while "\n\n" in buf:
|
||||
frame, buf = buf.split("\n\n", 1)
|
||||
if frame.strip().startswith("data:"):
|
||||
frames.append(json.loads(frame.strip().removeprefix("data:").strip()))
|
||||
assert buf.strip() == "" # stream ends cleanly on a frame boundary
|
||||
|
||||
deltas = [f for f in frames if f.get("type") == "delta"]
|
||||
answer = "".join(d["text"] for d in deltas)
|
||||
# The DEFLECT_MODE phrasing streamed ⇒ the LOW prompt reached the
|
||||
# model (the mock answers it only for the deflection system prompt).
|
||||
assert re.search(DEFLECT_PHRASE, answer, re.IGNORECASE)
|
||||
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done"
|
||||
assert done["deflected"] is True
|
||||
# No false citations (TODO L2): the weak hits never ride the wire as
|
||||
# sources — a deflected answer cites nothing.
|
||||
assert done["sources"] == []
|
||||
# 2-3 concrete alternative questions, all non-empty.
|
||||
assert 2 <= len(done["suggestions"]) <= 3
|
||||
assert all(s.strip() for s in done["suggestions"])
|
||||
|
||||
# Durable record: the quadrant pinned end-to-end — the lexical leg
|
||||
# FIRED (fts_hits > 0, the pre-phase gate's promotion trigger) while
|
||||
# the vector signal never cleared lexical_support_floor, so the
|
||||
# revised gate deflected. The retrieval itself stays recorded
|
||||
# (query_log = observability, not citations).
|
||||
with SessionLocal() as db:
|
||||
row = db.scalars(select(QueryLog)).one()
|
||||
assert row.question == OUT_OF_KB
|
||||
assert row.deflected is True
|
||||
assert (row.fts_hits or 0) >= 1
|
||||
assert row.top_score < get_settings().lexical_support_floor
|
||||
assert row.sources
|
||||
|
||||
Reference in New Issue
Block a user