phase: 113_source_chip_quality
Build and Push Containers / build-and-push-app (push) Successful in 2m2s
Build and Push Containers / build-and-push-db (push) Successful in 15s

All gates green — no defects found; this pass was verification only.

**Phase 113 final verification pass — report**

- Verified (no code changes needed): `select_documents_tiered` cited/related tiering + `select_documents` wrapper, `TurnPlan.related_docs`, `ChatDoneEvent.related` (additive, old payloads parse), `appendRelated` UI row (`.related-doc`, never `.source-chip`), done-frame + restore-path wiring, two settings with validators, `.env.example` entries
- `uv run pytest --cov=app --cov-report=term-missing` → 2422 passed, app/ coverage **99%** (>90% gate)
- `uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov` (isolated) → 2 passed
- Regression E2E `test_retrieval_quality.py` + `test_honest_deflection.py` + `test_chat_rag.py` + `test_sources_midstream_bug.py` → 17 passed
- `uv run ruff check . && uv run pyright` → clean (0 errors); `bash .agents/validate.sh` → "validation OK"

Completion criteria:
1. Single-doc question → exactly one `.source-chip` (E2E): ✅ passed
2. Weak 2nd doc only in de-emphasized related row, never `.source-chip` (unit + E2E): ✅ passed
3. Deflected turn → zero citation chips, weak hits in related row: ✅ passed
4. Full suite green, coverage >90%, isolated E2E green, lint/types clean: ✅ passed
5. `--no-gpg-sign` commit + phase dir move: left to harness per pass rules (task files already in `complete/`)

No deviations. Next pending phase: `114_embed_question_length`.
This commit is contained in:
2026-09-15 03:11:05 -04:00
parent 1374faf136
commit 97d663d16d
31 changed files with 2370 additions and 52 deletions
+135 -2
View File
@@ -179,8 +179,11 @@ def test_gate_lexical_only_chunk_does_not_inflate_cosine() -> None:
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
assert plan.top_score == pytest.approx(0.55)
assert plan.deflected is False # 0.55 >= 0.30 anyway
# ranking follows the fused score: Beta's doc is the top source
assert plan.docs[0].title == "Beta"
# Phase 113 (the usefulness bar): Beta's doc ranks first by fused
# score, but a lexical-only doc (cosine 0.0 by construction) cannot
# clear the bar — it lands in the RELATED tier, never the cited one.
assert plan.docs[0].title == "Alpha"
assert plan.related_docs[0].title == "Beta"
# ---------- lexical support floor (A8 revised 2026-09-14) ----------
@@ -370,6 +373,136 @@ def test_gate_zero_chunks_deflects_with_fallback_chips() -> None:
assert 2 <= len(plan.suggestions) <= MAX_SUGGESTIONS
# ---------- usefulness bar tiering (phase 113, LOCKED A2/A4) ----------
def _bar_settings(
threshold: float = 0.62,
lex_floor: float = 0.35,
source_floor: float = 0.35,
related_cap: int = 2,
top_n: int = 2,
) -> Settings:
"""Explicit code defaults (production calibration) — the env's mock-
calibrated floor (tests/conftest.py) is overridden per test."""
return Settings(
_env_file=None, # pyright: ignore[reportCallIssue]
relevance_threshold=threshold,
lexical_support_floor=lex_floor,
source_usefulness_floor=source_floor,
related_max_docs=related_cap,
top_n_docs=top_n,
)
def test_plan_turn_high_tiers_strong_plus_weak() -> None:
"""Grounded turn: the bar-clearing doc is cited (and in the prompt),
the weak 2nd doc loses its citation slot and lands in related_docs —
the recurring incident's fix at the plan level."""
strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT")
weak = _doc("Backup Strategy", "WEAK_DOC_CONTENT")
chunks = [_chunk(strong, 0.90, cosine=0.80), _chunk(weak, 0.80, cosine=0.20)]
plan = chat_api.plan_turn(chunks, _bar_settings())
assert plan.deflected is False
assert [d.title for d in plan.docs] == ["Kubernetes Homelab Cluster"]
assert [d.title for d in plan.related_docs] == ["Backup Strategy"]
# The HIGH prompt carries the cited doc's content only.
assert "STRONG_DOC_CONTENT" in plan.system_prompt
assert "WEAK_DOC_CONTENT" not in plan.system_prompt
def test_plan_turn_high_single_strong_doc_yields_one_cited() -> None:
"""top_n_docs is a CEILING, not a quota: one strong doc ⇒ one cited doc,
an empty related tier (LOCKED A2)."""
strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT")
plan = chat_api.plan_turn([_chunk(strong, 0.90, cosine=0.80)], _bar_settings())
assert plan.deflected is False
assert [d.title for d in plan.docs] == ["Kubernetes Homelab Cluster"]
assert plan.related_docs == []
def test_plan_turn_low_weak_hits_fall_to_related() -> None:
"""Deflected turn: nothing clears the bar ⇒ the cited tier is empty
and the weak hits fall to related_docs (the done frame's home for
their visibility). The LOW prompt is unchanged (titles only)."""
a = _doc("Alpha", "ALPHA_DOC_NEVER_SENT")
b = _doc("Beta", "BETA_DOC_NEVER_SENT")
chunks = [_chunk(a, 0.30, cosine=0.20), _chunk(b, 0.20, cosine=0.15)]
plan = chat_api.plan_turn(chunks, _bar_settings())
assert plan.deflected is True
assert plan.docs == [] # no citation slot below the bar
assert [d.title for d in plan.related_docs] == ["Alpha", "Beta"] # rank order
assert "ALPHA_DOC_NEVER_SENT" not in plan.system_prompt
assert "Beta" in plan.system_prompt # weak-hit titles still carried
assert plan.suggestions # chips unchanged
def test_plan_turn_related_cap_zero_kills_the_related_tier() -> None:
"""related_max_docs=0 is the kill switch: weak docs are scored but
neither cited nor related (the pre-phase-113 visibility, minus the
false citation — a deflected turn cites nothing)."""
a = _doc("Alpha", "AAA")
b = _doc("Beta", "BBB")
chunks = [_chunk(a, 0.30, cosine=0.20), _chunk(b, 0.20, cosine=0.15)]
plan = chat_api.plan_turn(chunks, _bar_settings(related_cap=0))
assert plan.deflected is True
assert plan.docs == []
assert plan.related_docs == []
def test_plan_turn_floor_zero_keeps_legacy_cited_docs() -> None:
"""source_usefulness_floor=0 disables the bar: plan.docs is the legacy
rank-ordered top-N (any cosine, incl. 0.0 lexical-only) and the
related tier is empty."""
a = _doc("Alpha", "AAA")
b = _doc("Beta", "BBB")
chunks = [
_chunk(a, 0.90, cosine=0.0, fts_hit=True), # lexical-only, rank 1
_chunk(b, 0.80, cosine=0.10),
]
plan = chat_api.plan_turn(
chunks, _bar_settings(source_floor=0.0, lex_floor=0.05)
)
assert plan.deflected is False # 0.10 + the fts hit clears the 0.05 lex floor
assert [d.title for d in plan.docs] == ["Alpha", "Beta"] # legacy order
assert plan.related_docs == []
def test_plan_turn_lexically_grounded_below_source_floor_has_no_cited_docs() -> None:
"""The degenerate operator config (citation bar STRICTER than the
grounding bar): a turn grounded by a corroborated-lexical hit whose
cosine sits between the two floors has an EMPTY cited tier — the HIGH
prompt carries no document content (the tools remain the escape
hatch). The bar is a citation filter, not a gate input."""
doc = _doc("Static DNS", "DNS_DOC_CONTENT")
plan = chat_api.plan_turn(
[_chunk(doc, 0.50, cosine=0.35, fts_hit=True)],
_bar_settings(threshold=0.62, lex_floor=0.30, source_floor=0.50),
)
assert plan.deflected is False # 0.35 >= lex floor 0.30, fts fired
assert plan.docs == [] # 0.35 < source floor 0.50 — no citation slot
assert "DNS_DOC_CONTENT" not in plan.system_prompt
# The doc still SCORED — it rides the related tier (the "nearby docs"
# row), it is not invisible.
assert [d.title for d in plan.related_docs] == ["Static DNS"]
def test_plan_turn_related_tier_capped_in_rank_order() -> None:
"""Grounded turn, four bar-clearing docs, ceiling 2: cited = the top-2
in rank order; related = the next two (the ceiling overflow, any
cosine), capped at related_max_docs."""
docs_in = [
_doc(f"Doc {i}", f"DOC_CONTENT_{i}") for i in range(4)
]
chunks = [
_chunk(d, 0.9 - 0.1 * i, cosine=0.8 - 0.05 * i) for i, d in enumerate(docs_in)
]
plan = chat_api.plan_turn(chunks, _bar_settings(top_n=2, related_cap=2))
assert plan.deflected is False
assert [d.title for d in plan.docs] == ["Doc 0", "Doc 1"]
assert [d.title for d in plan.related_docs] == ["Doc 2", "Doc 3"]
# ---------- summary hits (phase 30: summary → full source document) ----------