phase: 113_source_chip_quality
Build and Push Containers / build-and-push-app (push) Successful in 2m2s
Build and Push Containers / build-and-push-db (push) Successful in 15s

All gates green — no defects found; this pass was verification only.

**Phase 113 final verification pass — report**

- Verified (no code changes needed): `select_documents_tiered` cited/related tiering + `select_documents` wrapper, `TurnPlan.related_docs`, `ChatDoneEvent.related` (additive, old payloads parse), `appendRelated` UI row (`.related-doc`, never `.source-chip`), done-frame + restore-path wiring, two settings with validators, `.env.example` entries
- `uv run pytest --cov=app --cov-report=term-missing` → 2422 passed, app/ coverage **99%** (>90% gate)
- `uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov` (isolated) → 2 passed
- Regression E2E `test_retrieval_quality.py` + `test_honest_deflection.py` + `test_chat_rag.py` + `test_sources_midstream_bug.py` → 17 passed
- `uv run ruff check . && uv run pyright` → clean (0 errors); `bash .agents/validate.sh` → "validation OK"

Completion criteria:
1. Single-doc question → exactly one `.source-chip` (E2E): ✅ passed
2. Weak 2nd doc only in de-emphasized related row, never `.source-chip` (unit + E2E): ✅ passed
3. Deflected turn → zero citation chips, weak hits in related row: ✅ passed
4. Full suite green, coverage >90%, isolated E2E green, lint/types clean: ✅ passed
5. `--no-gpg-sign` commit + phase dir move: left to harness per pass rules (task files already in `complete/`)

No deviations. Next pending phase: `114_embed_question_length`.
This commit is contained in:
2026-09-15 03:11:05 -04:00
parent 1374faf136
commit 97d663d16d
31 changed files with 2370 additions and 52 deletions
+169 -1
View File
@@ -12,7 +12,12 @@ from types import SimpleNamespace
import pytest
from app.models import Document
from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk, select_documents
from app.rag.retriever import (
TRUNCATION_MARKER,
RetrievedChunk,
select_documents,
select_documents_tiered,
)
def _doc(path: str, content: str, source: str = "Homelab", title: str | None = None) -> Document:
@@ -90,6 +95,169 @@ def test_empty_hits_yield_no_documents() -> None:
assert select_documents([], n=2) == []
# ---------------------------------------------------------------------------
# Phase 113 — the usefulness bar: cited vs related tiers
# ---------------------------------------------------------------------------
def _cos_chunk(doc: Document, score: float, cosine: float, position: int = 0) -> RetrievedChunk:
"""A candidate with *score* (fused rank key) and *cosine* (gate input) decoupled."""
return RetrievedChunk(
chunk_id=uuid.uuid4(),
position=position,
content=doc.content[:40],
score=score,
document=doc,
cosine=cosine,
)
def test_tiered_both_clear_floor_both_cited() -> None:
"""Both docs clear the bar → both cited, nothing related."""
a = _doc("a.md", "A" * 50)
b = _doc("b.md", "B" * 50)
chunks = [_cos_chunk(a, 0.9, 0.50), _cos_chunk(b, 0.8, 0.40)]
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
assert [d.path for d in cited] == ["a.md", "b.md"]
assert related == []
def test_tiered_strong_plus_weak_one_cited_one_related() -> None:
"""The recurring incident shape: a strong 1st doc and a weak 2nd — the
weak doc loses its citation slot and lands in the related tier."""
a = _doc("a.md", "A" * 50)
b = _doc("b.md", "B" * 50)
chunks = [_cos_chunk(a, 0.9, 0.50), _cos_chunk(b, 0.8, 0.10)]
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
assert [d.path for d in cited] == ["a.md"]
assert [d.path for d in related] == ["b.md"]
def test_tiered_both_weak_zero_cited_all_related() -> None:
"""Neither doc clears the bar → no citation slot at all (the bar
filters, it never backfills), the weak hits become the related tier."""
a = _doc("a.md", "A" * 50)
b = _doc("b.md", "B" * 50)
chunks = [_cos_chunk(a, 0.9, 0.20), _cos_chunk(b, 0.8, 0.15)]
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
assert cited == []
assert [d.path for d in related] == ["a.md", "b.md"] # rank order kept
def test_tiered_related_cap_respected() -> None:
"""The related tier is capped (related_max_docs) in rank order."""
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(3)]
chunks = [_cos_chunk(d, 0.9 - 0.1 * i, 0.10 - 0.02 * i) for i, d in enumerate(docs_in)]
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
assert cited == []
assert [d.path for d in related] == ["d0.md", "d1.md"]
def test_tiered_n_is_ceiling_not_quota() -> None:
"""A single strong doc yields ONE cited doc — the bar never pads the
cited tier up to ``n`` (LOCKED A2). And docs that clear the bar but
exceed the ceiling fall through to the related tier (the next docs in
rank order, never overlapping cited)."""
only = _doc("only.md", "O" * 50)
cited, related = select_documents_tiered(
[_cos_chunk(only, 0.9, 0.80)], n=2, floor=0.35, related_cap=2
)
assert [d.path for d in cited] == ["only.md"]
assert related == []
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(4)]
chunks = [_cos_chunk(d, 0.9 - 0.1 * i, 0.8 - 0.05 * i) for i, d in enumerate(docs_in)]
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
assert [d.path for d in cited] == ["d0.md", "d1.md"] # the ceiling
assert [d.path for d in related] == ["d2.md", "d3.md"] # the next in rank order
def test_tiered_bar_skips_weak_rank_one() -> None:
"""A weak rank-1 doc does not consume a citation slot: the next-ranked
bar-clearing docs take it (the bar filters, it does not backfill)."""
w = _doc("w.md", "W" * 50)
s1 = _doc("s1.md", "1" * 50)
s2 = _doc("s2.md", "2" * 50)
chunks = [
_cos_chunk(w, 0.9, 0.20), # rank 1 — below the bar
_cos_chunk(s1, 0.8, 0.90),
_cos_chunk(s2, 0.7, 0.80),
]
cited, related = select_documents_tiered(chunks, n=2, floor=0.5, related_cap=2)
assert [d.path for d in cited] == ["s1.md", "s2.md"]
assert [d.path for d in related] == ["w.md"]
def test_tiered_tracks_best_chunk_cosine_across_a_docs_chunks() -> None:
"""The bar is on the doc's BEST hit-chunk cosine — a weak first chunk
(rank 1) does not sink a doc whose later chunk is vector-strong."""
a = _doc("a.md", "A" * 50)
b = _doc("b.md", "B" * 50)
chunks = [
_cos_chunk(a, 0.9, 0.10, position=0), # a's weak chunk ranks first
_cos_chunk(a, 0.5, 0.90, position=2), # a's strong chunk
_cos_chunk(b, 0.4, 0.0), # lexical-only b
]
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
assert [d.path for d in cited] == ["a.md"]
assert [d.path for d in related] == ["b.md"]
def test_tiered_lexical_only_hit_goes_to_related_above_zero_floor() -> None:
"""A lexical-only doc (cosine 0.0 by construction) is vector-unsupported
by definition: above a zero floor it never earns a cited slot (LOCKED A2)."""
a = _doc("a.md", "A" * 50)
b = _doc("b.md", "B" * 50)
lexical_only = RetrievedChunk(
chunk_id=uuid.uuid4(),
position=0,
content="X" * 10,
score=0.9, # top fused rank (the FTS hit)
document=a,
cosine=0.0, # no vector rank — lexical-only
fts_hit=True,
)
chunks = [lexical_only, _cos_chunk(b, 0.8, 0.5)]
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
assert [d.path for d in cited] == ["b.md"]
assert [d.path for d in related] == ["a.md"]
def test_tiered_floor_zero_is_no_bar() -> None:
"""A zero floor admits every scored document — lexical-only (cosine
0.0) and weak alike — so the bar can be disabled per deployment."""
a = _doc("a.md", "A" * 50)
b = _doc("b.md", "B" * 50)
chunks = [_cos_chunk(a, 0.9, 0.0), _cos_chunk(b, 0.8, 0.1)]
cited, related = select_documents_tiered(chunks, n=2, floor=0.0, related_cap=2)
assert [d.path for d in cited] == ["a.md", "b.md"]
assert related == []
def test_tiered_empty_chunks_yield_empty_tiers() -> None:
assert select_documents_tiered([], n=2, floor=0.35, related_cap=2) == ([], [])
def test_select_documents_wrapper_is_legacy_tiering() -> None:
"""The wrapper (floor 0.0, cap 0) is the legacy "any score, top-N"
selection — byte-identical for the shapes the existing callers see:
rank order, dedupe, multi-chunk docs, ties, lexical-only hits."""
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(5)]
chunks = [
_chunk(docs_in[0], 0.5), # cosine 0.0 (lexical-only)
_cos_chunk(docs_in[0], 0.9, 0.4, position=1),
_chunk(docs_in[1], 0.8),
_cos_chunk(docs_in[1], 0.8, 0.2, position=1), # fused tie across docs
_cos_chunk(docs_in[2], 0.7, 0.0),
_cos_chunk(docs_in[3], 0.6, 0.1),
_chunk(docs_in[4], 0.1),
]
for n in (1, 2, 3, 10):
assert select_documents(chunks, n=n) == select_documents_tiered(
chunks, n, 0.0, 0
)[0]
# ---------------------------------------------------------------------------
# Hybrid retrieval (A7): RRF fusion + lexical tsquery
# ---------------------------------------------------------------------------