phase: 113_source_chip_quality
All gates green — no defects found; this pass was verification only. **Phase 113 final verification pass — report** - Verified (no code changes needed): `select_documents_tiered` cited/related tiering + `select_documents` wrapper, `TurnPlan.related_docs`, `ChatDoneEvent.related` (additive, old payloads parse), `appendRelated` UI row (`.related-doc`, never `.source-chip`), done-frame + restore-path wiring, two settings with validators, `.env.example` entries - `uv run pytest --cov=app --cov-report=term-missing` → 2422 passed, app/ coverage **99%** (>90% gate) - `uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov` (isolated) → 2 passed - Regression E2E `test_retrieval_quality.py` + `test_honest_deflection.py` + `test_chat_rag.py` + `test_sources_midstream_bug.py` → 17 passed - `uv run ruff check . && uv run pyright` → clean (0 errors); `bash .agents/validate.sh` → "validation OK" Completion criteria: 1. Single-doc question → exactly one `.source-chip` (E2E): ✅ passed 2. Weak 2nd doc only in de-emphasized related row, never `.source-chip` (unit + E2E): ✅ passed 3. Deflected turn → zero citation chips, weak hits in related row: ✅ passed 4. Full suite green, coverage >90%, isolated E2E green, lint/types clean: ✅ passed 5. `--no-gpg-sign` commit + phase dir move: left to harness per pass rules (task files already in `complete/`) No deviations. Next pending phase: `114_embed_question_length`.
This commit is contained in:
@@ -179,8 +179,11 @@ def test_gate_lexical_only_chunk_does_not_inflate_cosine() -> None:
|
||||
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
|
||||
assert plan.top_score == pytest.approx(0.55)
|
||||
assert plan.deflected is False # 0.55 >= 0.30 anyway
|
||||
# ranking follows the fused score: Beta's doc is the top source
|
||||
assert plan.docs[0].title == "Beta"
|
||||
# Phase 113 (the usefulness bar): Beta's doc ranks first by fused
|
||||
# score, but a lexical-only doc (cosine 0.0 by construction) cannot
|
||||
# clear the bar — it lands in the RELATED tier, never the cited one.
|
||||
assert plan.docs[0].title == "Alpha"
|
||||
assert plan.related_docs[0].title == "Beta"
|
||||
|
||||
|
||||
# ---------- lexical support floor (A8 revised 2026-09-14) ----------
|
||||
@@ -370,6 +373,136 @@ def test_gate_zero_chunks_deflects_with_fallback_chips() -> None:
|
||||
assert 2 <= len(plan.suggestions) <= MAX_SUGGESTIONS
|
||||
|
||||
|
||||
# ---------- usefulness bar tiering (phase 113, LOCKED A2/A4) ----------
|
||||
|
||||
|
||||
def _bar_settings(
|
||||
threshold: float = 0.62,
|
||||
lex_floor: float = 0.35,
|
||||
source_floor: float = 0.35,
|
||||
related_cap: int = 2,
|
||||
top_n: int = 2,
|
||||
) -> Settings:
|
||||
"""Explicit code defaults (production calibration) — the env's mock-
|
||||
calibrated floor (tests/conftest.py) is overridden per test."""
|
||||
return Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
relevance_threshold=threshold,
|
||||
lexical_support_floor=lex_floor,
|
||||
source_usefulness_floor=source_floor,
|
||||
related_max_docs=related_cap,
|
||||
top_n_docs=top_n,
|
||||
)
|
||||
|
||||
|
||||
def test_plan_turn_high_tiers_strong_plus_weak() -> None:
|
||||
"""Grounded turn: the bar-clearing doc is cited (and in the prompt),
|
||||
the weak 2nd doc loses its citation slot and lands in related_docs —
|
||||
the recurring incident's fix at the plan level."""
|
||||
strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT")
|
||||
weak = _doc("Backup Strategy", "WEAK_DOC_CONTENT")
|
||||
chunks = [_chunk(strong, 0.90, cosine=0.80), _chunk(weak, 0.80, cosine=0.20)]
|
||||
plan = chat_api.plan_turn(chunks, _bar_settings())
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Kubernetes Homelab Cluster"]
|
||||
assert [d.title for d in plan.related_docs] == ["Backup Strategy"]
|
||||
# The HIGH prompt carries the cited doc's content only.
|
||||
assert "STRONG_DOC_CONTENT" in plan.system_prompt
|
||||
assert "WEAK_DOC_CONTENT" not in plan.system_prompt
|
||||
|
||||
|
||||
def test_plan_turn_high_single_strong_doc_yields_one_cited() -> None:
|
||||
"""top_n_docs is a CEILING, not a quota: one strong doc ⇒ one cited doc,
|
||||
an empty related tier (LOCKED A2)."""
|
||||
strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT")
|
||||
plan = chat_api.plan_turn([_chunk(strong, 0.90, cosine=0.80)], _bar_settings())
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Kubernetes Homelab Cluster"]
|
||||
assert plan.related_docs == []
|
||||
|
||||
|
||||
def test_plan_turn_low_weak_hits_fall_to_related() -> None:
|
||||
"""Deflected turn: nothing clears the bar ⇒ the cited tier is empty
|
||||
and the weak hits fall to related_docs (the done frame's home for
|
||||
their visibility). The LOW prompt is unchanged (titles only)."""
|
||||
a = _doc("Alpha", "ALPHA_DOC_NEVER_SENT")
|
||||
b = _doc("Beta", "BETA_DOC_NEVER_SENT")
|
||||
chunks = [_chunk(a, 0.30, cosine=0.20), _chunk(b, 0.20, cosine=0.15)]
|
||||
plan = chat_api.plan_turn(chunks, _bar_settings())
|
||||
assert plan.deflected is True
|
||||
assert plan.docs == [] # no citation slot below the bar
|
||||
assert [d.title for d in plan.related_docs] == ["Alpha", "Beta"] # rank order
|
||||
assert "ALPHA_DOC_NEVER_SENT" not in plan.system_prompt
|
||||
assert "Beta" in plan.system_prompt # weak-hit titles still carried
|
||||
assert plan.suggestions # chips unchanged
|
||||
|
||||
|
||||
def test_plan_turn_related_cap_zero_kills_the_related_tier() -> None:
|
||||
"""related_max_docs=0 is the kill switch: weak docs are scored but
|
||||
neither cited nor related (the pre-phase-113 visibility, minus the
|
||||
false citation — a deflected turn cites nothing)."""
|
||||
a = _doc("Alpha", "AAA")
|
||||
b = _doc("Beta", "BBB")
|
||||
chunks = [_chunk(a, 0.30, cosine=0.20), _chunk(b, 0.20, cosine=0.15)]
|
||||
plan = chat_api.plan_turn(chunks, _bar_settings(related_cap=0))
|
||||
assert plan.deflected is True
|
||||
assert plan.docs == []
|
||||
assert plan.related_docs == []
|
||||
|
||||
|
||||
def test_plan_turn_floor_zero_keeps_legacy_cited_docs() -> None:
|
||||
"""source_usefulness_floor=0 disables the bar: plan.docs is the legacy
|
||||
rank-ordered top-N (any cosine, incl. 0.0 lexical-only) and the
|
||||
related tier is empty."""
|
||||
a = _doc("Alpha", "AAA")
|
||||
b = _doc("Beta", "BBB")
|
||||
chunks = [
|
||||
_chunk(a, 0.90, cosine=0.0, fts_hit=True), # lexical-only, rank 1
|
||||
_chunk(b, 0.80, cosine=0.10),
|
||||
]
|
||||
plan = chat_api.plan_turn(
|
||||
chunks, _bar_settings(source_floor=0.0, lex_floor=0.05)
|
||||
)
|
||||
assert plan.deflected is False # 0.10 + the fts hit clears the 0.05 lex floor
|
||||
assert [d.title for d in plan.docs] == ["Alpha", "Beta"] # legacy order
|
||||
assert plan.related_docs == []
|
||||
|
||||
|
||||
def test_plan_turn_lexically_grounded_below_source_floor_has_no_cited_docs() -> None:
|
||||
"""The degenerate operator config (citation bar STRICTER than the
|
||||
grounding bar): a turn grounded by a corroborated-lexical hit whose
|
||||
cosine sits between the two floors has an EMPTY cited tier — the HIGH
|
||||
prompt carries no document content (the tools remain the escape
|
||||
hatch). The bar is a citation filter, not a gate input."""
|
||||
doc = _doc("Static DNS", "DNS_DOC_CONTENT")
|
||||
plan = chat_api.plan_turn(
|
||||
[_chunk(doc, 0.50, cosine=0.35, fts_hit=True)],
|
||||
_bar_settings(threshold=0.62, lex_floor=0.30, source_floor=0.50),
|
||||
)
|
||||
assert plan.deflected is False # 0.35 >= lex floor 0.30, fts fired
|
||||
assert plan.docs == [] # 0.35 < source floor 0.50 — no citation slot
|
||||
assert "DNS_DOC_CONTENT" not in plan.system_prompt
|
||||
# The doc still SCORED — it rides the related tier (the "nearby docs"
|
||||
# row), it is not invisible.
|
||||
assert [d.title for d in plan.related_docs] == ["Static DNS"]
|
||||
|
||||
|
||||
def test_plan_turn_related_tier_capped_in_rank_order() -> None:
|
||||
"""Grounded turn, four bar-clearing docs, ceiling 2: cited = the top-2
|
||||
in rank order; related = the next two (the ceiling overflow, any
|
||||
cosine), capped at related_max_docs."""
|
||||
docs_in = [
|
||||
_doc(f"Doc {i}", f"DOC_CONTENT_{i}") for i in range(4)
|
||||
]
|
||||
chunks = [
|
||||
_chunk(d, 0.9 - 0.1 * i, cosine=0.8 - 0.05 * i) for i, d in enumerate(docs_in)
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _bar_settings(top_n=2, related_cap=2))
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Doc 0", "Doc 1"]
|
||||
assert [d.title for d in plan.related_docs] == ["Doc 2", "Doc 3"]
|
||||
|
||||
|
||||
# ---------- summary hits (phase 30: summary → full source document) ----------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user