phase: 113_source_chip_quality
All gates green — no defects found; this pass was verification only. **Phase 113 final verification pass — report** - Verified (no code changes needed): `select_documents_tiered` cited/related tiering + `select_documents` wrapper, `TurnPlan.related_docs`, `ChatDoneEvent.related` (additive, old payloads parse), `appendRelated` UI row (`.related-doc`, never `.source-chip`), done-frame + restore-path wiring, two settings with validators, `.env.example` entries - `uv run pytest --cov=app --cov-report=term-missing` → 2422 passed, app/ coverage **99%** (>90% gate) - `uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov` (isolated) → 2 passed - Regression E2E `test_retrieval_quality.py` + `test_honest_deflection.py` + `test_chat_rag.py` + `test_sources_midstream_bug.py` → 17 passed - `uv run ruff check . && uv run pyright` → clean (0 errors); `bash .agents/validate.sh` → "validation OK" Completion criteria: 1. Single-doc question → exactly one `.source-chip` (E2E): ✅ passed 2. Weak 2nd doc only in de-emphasized related row, never `.source-chip` (unit + E2E): ✅ passed 3. Deflected turn → zero citation chips, weak hits in related row: ✅ passed 4. Full suite green, coverage >90%, isolated E2E green, lint/types clean: ✅ passed 5. `--no-gpg-sign` commit + phase dir move: left to harness per pass rules (task files already in `complete/`) No deviations. Next pending phase: `114_embed_question_length`.
This commit is contained in:
@@ -12,7 +12,12 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
|
||||
from app.models import Document
|
||||
from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk, select_documents
|
||||
from app.rag.retriever import (
|
||||
TRUNCATION_MARKER,
|
||||
RetrievedChunk,
|
||||
select_documents,
|
||||
select_documents_tiered,
|
||||
)
|
||||
|
||||
|
||||
def _doc(path: str, content: str, source: str = "Homelab", title: str | None = None) -> Document:
|
||||
@@ -90,6 +95,169 @@ def test_empty_hits_yield_no_documents() -> None:
|
||||
assert select_documents([], n=2) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 113 — the usefulness bar: cited vs related tiers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cos_chunk(doc: Document, score: float, cosine: float, position: int = 0) -> RetrievedChunk:
|
||||
"""A candidate with *score* (fused rank key) and *cosine* (gate input) decoupled."""
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=position,
|
||||
content=doc.content[:40],
|
||||
score=score,
|
||||
document=doc,
|
||||
cosine=cosine,
|
||||
)
|
||||
|
||||
|
||||
def test_tiered_both_clear_floor_both_cited() -> None:
|
||||
"""Both docs clear the bar → both cited, nothing related."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [_cos_chunk(a, 0.9, 0.50), _cos_chunk(b, 0.8, 0.40)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["a.md", "b.md"]
|
||||
assert related == []
|
||||
|
||||
|
||||
def test_tiered_strong_plus_weak_one_cited_one_related() -> None:
|
||||
"""The recurring incident shape: a strong 1st doc and a weak 2nd — the
|
||||
weak doc loses its citation slot and lands in the related tier."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [_cos_chunk(a, 0.9, 0.50), _cos_chunk(b, 0.8, 0.10)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["a.md"]
|
||||
assert [d.path for d in related] == ["b.md"]
|
||||
|
||||
|
||||
def test_tiered_both_weak_zero_cited_all_related() -> None:
|
||||
"""Neither doc clears the bar → no citation slot at all (the bar
|
||||
filters, it never backfills), the weak hits become the related tier."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [_cos_chunk(a, 0.9, 0.20), _cos_chunk(b, 0.8, 0.15)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert cited == []
|
||||
assert [d.path for d in related] == ["a.md", "b.md"] # rank order kept
|
||||
|
||||
|
||||
def test_tiered_related_cap_respected() -> None:
|
||||
"""The related tier is capped (related_max_docs) in rank order."""
|
||||
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(3)]
|
||||
chunks = [_cos_chunk(d, 0.9 - 0.1 * i, 0.10 - 0.02 * i) for i, d in enumerate(docs_in)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert cited == []
|
||||
assert [d.path for d in related] == ["d0.md", "d1.md"]
|
||||
|
||||
|
||||
def test_tiered_n_is_ceiling_not_quota() -> None:
|
||||
"""A single strong doc yields ONE cited doc — the bar never pads the
|
||||
cited tier up to ``n`` (LOCKED A2). And docs that clear the bar but
|
||||
exceed the ceiling fall through to the related tier (the next docs in
|
||||
rank order, never overlapping cited)."""
|
||||
only = _doc("only.md", "O" * 50)
|
||||
cited, related = select_documents_tiered(
|
||||
[_cos_chunk(only, 0.9, 0.80)], n=2, floor=0.35, related_cap=2
|
||||
)
|
||||
assert [d.path for d in cited] == ["only.md"]
|
||||
assert related == []
|
||||
|
||||
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(4)]
|
||||
chunks = [_cos_chunk(d, 0.9 - 0.1 * i, 0.8 - 0.05 * i) for i, d in enumerate(docs_in)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["d0.md", "d1.md"] # the ceiling
|
||||
assert [d.path for d in related] == ["d2.md", "d3.md"] # the next in rank order
|
||||
|
||||
|
||||
def test_tiered_bar_skips_weak_rank_one() -> None:
|
||||
"""A weak rank-1 doc does not consume a citation slot: the next-ranked
|
||||
bar-clearing docs take it (the bar filters, it does not backfill)."""
|
||||
w = _doc("w.md", "W" * 50)
|
||||
s1 = _doc("s1.md", "1" * 50)
|
||||
s2 = _doc("s2.md", "2" * 50)
|
||||
chunks = [
|
||||
_cos_chunk(w, 0.9, 0.20), # rank 1 — below the bar
|
||||
_cos_chunk(s1, 0.8, 0.90),
|
||||
_cos_chunk(s2, 0.7, 0.80),
|
||||
]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.5, related_cap=2)
|
||||
assert [d.path for d in cited] == ["s1.md", "s2.md"]
|
||||
assert [d.path for d in related] == ["w.md"]
|
||||
|
||||
|
||||
def test_tiered_tracks_best_chunk_cosine_across_a_docs_chunks() -> None:
|
||||
"""The bar is on the doc's BEST hit-chunk cosine — a weak first chunk
|
||||
(rank 1) does not sink a doc whose later chunk is vector-strong."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [
|
||||
_cos_chunk(a, 0.9, 0.10, position=0), # a's weak chunk ranks first
|
||||
_cos_chunk(a, 0.5, 0.90, position=2), # a's strong chunk
|
||||
_cos_chunk(b, 0.4, 0.0), # lexical-only b
|
||||
]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["a.md"]
|
||||
assert [d.path for d in related] == ["b.md"]
|
||||
|
||||
|
||||
def test_tiered_lexical_only_hit_goes_to_related_above_zero_floor() -> None:
|
||||
"""A lexical-only doc (cosine 0.0 by construction) is vector-unsupported
|
||||
by definition: above a zero floor it never earns a cited slot (LOCKED A2)."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
lexical_only = RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=0,
|
||||
content="X" * 10,
|
||||
score=0.9, # top fused rank (the FTS hit)
|
||||
document=a,
|
||||
cosine=0.0, # no vector rank — lexical-only
|
||||
fts_hit=True,
|
||||
)
|
||||
chunks = [lexical_only, _cos_chunk(b, 0.8, 0.5)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["b.md"]
|
||||
assert [d.path for d in related] == ["a.md"]
|
||||
|
||||
|
||||
def test_tiered_floor_zero_is_no_bar() -> None:
|
||||
"""A zero floor admits every scored document — lexical-only (cosine
|
||||
0.0) and weak alike — so the bar can be disabled per deployment."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [_cos_chunk(a, 0.9, 0.0), _cos_chunk(b, 0.8, 0.1)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.0, related_cap=2)
|
||||
assert [d.path for d in cited] == ["a.md", "b.md"]
|
||||
assert related == []
|
||||
|
||||
|
||||
def test_tiered_empty_chunks_yield_empty_tiers() -> None:
|
||||
assert select_documents_tiered([], n=2, floor=0.35, related_cap=2) == ([], [])
|
||||
|
||||
|
||||
def test_select_documents_wrapper_is_legacy_tiering() -> None:
|
||||
"""The wrapper (floor 0.0, cap 0) is the legacy "any score, top-N"
|
||||
selection — byte-identical for the shapes the existing callers see:
|
||||
rank order, dedupe, multi-chunk docs, ties, lexical-only hits."""
|
||||
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(5)]
|
||||
chunks = [
|
||||
_chunk(docs_in[0], 0.5), # cosine 0.0 (lexical-only)
|
||||
_cos_chunk(docs_in[0], 0.9, 0.4, position=1),
|
||||
_chunk(docs_in[1], 0.8),
|
||||
_cos_chunk(docs_in[1], 0.8, 0.2, position=1), # fused tie across docs
|
||||
_cos_chunk(docs_in[2], 0.7, 0.0),
|
||||
_cos_chunk(docs_in[3], 0.6, 0.1),
|
||||
_chunk(docs_in[4], 0.1),
|
||||
]
|
||||
for n in (1, 2, 3, 10):
|
||||
assert select_documents(chunks, n=n) == select_documents_tiered(
|
||||
chunks, n, 0.0, 0
|
||||
)[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hybrid retrieval (A7): RRF fusion + lexical tsquery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user