phase: 113_source_chip_quality
All gates green — no defects found; this pass was verification only. **Phase 113 final verification pass — report** - Verified (no code changes needed): `select_documents_tiered` cited/related tiering + `select_documents` wrapper, `TurnPlan.related_docs`, `ChatDoneEvent.related` (additive, old payloads parse), `appendRelated` UI row (`.related-doc`, never `.source-chip`), done-frame + restore-path wiring, two settings with validators, `.env.example` entries - `uv run pytest --cov=app --cov-report=term-missing` → 2422 passed, app/ coverage **99%** (>90% gate) - `uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov` (isolated) → 2 passed - Regression E2E `test_retrieval_quality.py` + `test_honest_deflection.py` + `test_chat_rag.py` + `test_sources_midstream_bug.py` → 17 passed - `uv run ruff check . && uv run pyright` → clean (0 errors); `bash .agents/validate.sh` → "validation OK" Completion criteria: 1. Single-doc question → exactly one `.source-chip` (E2E): ✅ passed 2. Weak 2nd doc only in de-emphasized related row, never `.source-chip` (unit + E2E): ✅ passed 3. Deflected turn → zero citation chips, weak hits in related row: ✅ passed 4. Full suite green, coverage >90%, isolated E2E green, lint/types clean: ✅ passed 5. `--no-gpg-sign` commit + phase dir move: left to harness per pass rules (task files already in `complete/`) No deviations. Next pending phase: `114_embed_question_length`.
This commit is contained in:
@@ -179,8 +179,11 @@ def test_gate_lexical_only_chunk_does_not_inflate_cosine() -> None:
|
||||
plan = chat_api.plan_turn(chunks, _settings(threshold=0.30))
|
||||
assert plan.top_score == pytest.approx(0.55)
|
||||
assert plan.deflected is False # 0.55 >= 0.30 anyway
|
||||
# ranking follows the fused score: Beta's doc is the top source
|
||||
assert plan.docs[0].title == "Beta"
|
||||
# Phase 113 (the usefulness bar): Beta's doc ranks first by fused
|
||||
# score, but a lexical-only doc (cosine 0.0 by construction) cannot
|
||||
# clear the bar — it lands in the RELATED tier, never the cited one.
|
||||
assert plan.docs[0].title == "Alpha"
|
||||
assert plan.related_docs[0].title == "Beta"
|
||||
|
||||
|
||||
# ---------- lexical support floor (A8 revised 2026-09-14) ----------
|
||||
@@ -370,6 +373,136 @@ def test_gate_zero_chunks_deflects_with_fallback_chips() -> None:
|
||||
assert 2 <= len(plan.suggestions) <= MAX_SUGGESTIONS
|
||||
|
||||
|
||||
# ---------- usefulness bar tiering (phase 113, LOCKED A2/A4) ----------
|
||||
|
||||
|
||||
def _bar_settings(
|
||||
threshold: float = 0.62,
|
||||
lex_floor: float = 0.35,
|
||||
source_floor: float = 0.35,
|
||||
related_cap: int = 2,
|
||||
top_n: int = 2,
|
||||
) -> Settings:
|
||||
"""Explicit code defaults (production calibration) — the env's mock-
|
||||
calibrated floor (tests/conftest.py) is overridden per test."""
|
||||
return Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
relevance_threshold=threshold,
|
||||
lexical_support_floor=lex_floor,
|
||||
source_usefulness_floor=source_floor,
|
||||
related_max_docs=related_cap,
|
||||
top_n_docs=top_n,
|
||||
)
|
||||
|
||||
|
||||
def test_plan_turn_high_tiers_strong_plus_weak() -> None:
|
||||
"""Grounded turn: the bar-clearing doc is cited (and in the prompt),
|
||||
the weak 2nd doc loses its citation slot and lands in related_docs —
|
||||
the recurring incident's fix at the plan level."""
|
||||
strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT")
|
||||
weak = _doc("Backup Strategy", "WEAK_DOC_CONTENT")
|
||||
chunks = [_chunk(strong, 0.90, cosine=0.80), _chunk(weak, 0.80, cosine=0.20)]
|
||||
plan = chat_api.plan_turn(chunks, _bar_settings())
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Kubernetes Homelab Cluster"]
|
||||
assert [d.title for d in plan.related_docs] == ["Backup Strategy"]
|
||||
# The HIGH prompt carries the cited doc's content only.
|
||||
assert "STRONG_DOC_CONTENT" in plan.system_prompt
|
||||
assert "WEAK_DOC_CONTENT" not in plan.system_prompt
|
||||
|
||||
|
||||
def test_plan_turn_high_single_strong_doc_yields_one_cited() -> None:
|
||||
"""top_n_docs is a CEILING, not a quota: one strong doc ⇒ one cited doc,
|
||||
an empty related tier (LOCKED A2)."""
|
||||
strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT")
|
||||
plan = chat_api.plan_turn([_chunk(strong, 0.90, cosine=0.80)], _bar_settings())
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Kubernetes Homelab Cluster"]
|
||||
assert plan.related_docs == []
|
||||
|
||||
|
||||
def test_plan_turn_low_weak_hits_fall_to_related() -> None:
|
||||
"""Deflected turn: nothing clears the bar ⇒ the cited tier is empty
|
||||
and the weak hits fall to related_docs (the done frame's home for
|
||||
their visibility). The LOW prompt is unchanged (titles only)."""
|
||||
a = _doc("Alpha", "ALPHA_DOC_NEVER_SENT")
|
||||
b = _doc("Beta", "BETA_DOC_NEVER_SENT")
|
||||
chunks = [_chunk(a, 0.30, cosine=0.20), _chunk(b, 0.20, cosine=0.15)]
|
||||
plan = chat_api.plan_turn(chunks, _bar_settings())
|
||||
assert plan.deflected is True
|
||||
assert plan.docs == [] # no citation slot below the bar
|
||||
assert [d.title for d in plan.related_docs] == ["Alpha", "Beta"] # rank order
|
||||
assert "ALPHA_DOC_NEVER_SENT" not in plan.system_prompt
|
||||
assert "Beta" in plan.system_prompt # weak-hit titles still carried
|
||||
assert plan.suggestions # chips unchanged
|
||||
|
||||
|
||||
def test_plan_turn_related_cap_zero_kills_the_related_tier() -> None:
|
||||
"""related_max_docs=0 is the kill switch: weak docs are scored but
|
||||
neither cited nor related (the pre-phase-113 visibility, minus the
|
||||
false citation — a deflected turn cites nothing)."""
|
||||
a = _doc("Alpha", "AAA")
|
||||
b = _doc("Beta", "BBB")
|
||||
chunks = [_chunk(a, 0.30, cosine=0.20), _chunk(b, 0.20, cosine=0.15)]
|
||||
plan = chat_api.plan_turn(chunks, _bar_settings(related_cap=0))
|
||||
assert plan.deflected is True
|
||||
assert plan.docs == []
|
||||
assert plan.related_docs == []
|
||||
|
||||
|
||||
def test_plan_turn_floor_zero_keeps_legacy_cited_docs() -> None:
|
||||
"""source_usefulness_floor=0 disables the bar: plan.docs is the legacy
|
||||
rank-ordered top-N (any cosine, incl. 0.0 lexical-only) and the
|
||||
related tier is empty."""
|
||||
a = _doc("Alpha", "AAA")
|
||||
b = _doc("Beta", "BBB")
|
||||
chunks = [
|
||||
_chunk(a, 0.90, cosine=0.0, fts_hit=True), # lexical-only, rank 1
|
||||
_chunk(b, 0.80, cosine=0.10),
|
||||
]
|
||||
plan = chat_api.plan_turn(
|
||||
chunks, _bar_settings(source_floor=0.0, lex_floor=0.05)
|
||||
)
|
||||
assert plan.deflected is False # 0.10 + the fts hit clears the 0.05 lex floor
|
||||
assert [d.title for d in plan.docs] == ["Alpha", "Beta"] # legacy order
|
||||
assert plan.related_docs == []
|
||||
|
||||
|
||||
def test_plan_turn_lexically_grounded_below_source_floor_has_no_cited_docs() -> None:
|
||||
"""The degenerate operator config (citation bar STRICTER than the
|
||||
grounding bar): a turn grounded by a corroborated-lexical hit whose
|
||||
cosine sits between the two floors has an EMPTY cited tier — the HIGH
|
||||
prompt carries no document content (the tools remain the escape
|
||||
hatch). The bar is a citation filter, not a gate input."""
|
||||
doc = _doc("Static DNS", "DNS_DOC_CONTENT")
|
||||
plan = chat_api.plan_turn(
|
||||
[_chunk(doc, 0.50, cosine=0.35, fts_hit=True)],
|
||||
_bar_settings(threshold=0.62, lex_floor=0.30, source_floor=0.50),
|
||||
)
|
||||
assert plan.deflected is False # 0.35 >= lex floor 0.30, fts fired
|
||||
assert plan.docs == [] # 0.35 < source floor 0.50 — no citation slot
|
||||
assert "DNS_DOC_CONTENT" not in plan.system_prompt
|
||||
# The doc still SCORED — it rides the related tier (the "nearby docs"
|
||||
# row), it is not invisible.
|
||||
assert [d.title for d in plan.related_docs] == ["Static DNS"]
|
||||
|
||||
|
||||
def test_plan_turn_related_tier_capped_in_rank_order() -> None:
|
||||
"""Grounded turn, four bar-clearing docs, ceiling 2: cited = the top-2
|
||||
in rank order; related = the next two (the ceiling overflow, any
|
||||
cosine), capped at related_max_docs."""
|
||||
docs_in = [
|
||||
_doc(f"Doc {i}", f"DOC_CONTENT_{i}") for i in range(4)
|
||||
]
|
||||
chunks = [
|
||||
_chunk(d, 0.9 - 0.1 * i, cosine=0.8 - 0.05 * i) for i, d in enumerate(docs_in)
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _bar_settings(top_n=2, related_cap=2))
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Doc 0", "Doc 1"]
|
||||
assert [d.title for d in plan.related_docs] == ["Doc 2", "Doc 3"]
|
||||
|
||||
|
||||
# ---------- summary hits (phase 30: summary → full source document) ----------
|
||||
|
||||
|
||||
|
||||
@@ -213,6 +213,75 @@ def test_agent_max_rounds_rejects_negative(monkeypatch: pytest.MonkeyPatch) -> N
|
||||
_settings()
|
||||
|
||||
|
||||
def test_source_usefulness_floor_default_and_env_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 113 (LOCKED A2): the citation-slot bar — default 0.35 (the
|
||||
same bar as the A8 lexical support floor), env-tunable, ``0`` = the
|
||||
no-bar kill switch."""
|
||||
monkeypatch.delenv("BOR_SOURCE_USEFULNESS_FLOOR", raising=False)
|
||||
# The test process pins the mock-calibrated threshold (0.30, see
|
||||
# tests/conftest.py) — clear it so the PRODUCTION default pair
|
||||
# (0.62 / 0.35) is what the validator sees.
|
||||
monkeypatch.delenv("BOR_RELEVANCE_THRESHOLD", raising=False)
|
||||
assert _settings().source_usefulness_floor == 0.35
|
||||
monkeypatch.setenv("BOR_SOURCE_USEFULNESS_FLOOR", "0.5")
|
||||
assert _settings().source_usefulness_floor == 0.5
|
||||
monkeypatch.setenv("BOR_SOURCE_USEFULNESS_FLOOR", "0")
|
||||
assert _settings().source_usefulness_floor == 0.0
|
||||
|
||||
|
||||
def test_source_usefulness_floor_at_threshold_is_legal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The validator bound is inclusive (>=): a bar exactly at the
|
||||
relevance threshold is legal — the gate and the bar agree on every
|
||||
grounded document."""
|
||||
monkeypatch.delenv("BOR_SOURCE_USEFULNESS_FLOOR", raising=False)
|
||||
monkeypatch.delenv("BOR_RELEVANCE_THRESHOLD", raising=False)
|
||||
s = _settings(relevance_threshold=0.62, source_usefulness_floor=0.62)
|
||||
assert s.source_usefulness_floor == 0.62
|
||||
|
||||
|
||||
def test_source_usefulness_floor_rejects_above_threshold(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A bar above the relevance threshold would demote to the related
|
||||
tier documents the gate itself calls grounded — a typo, so the
|
||||
validator fails loudly at startup (the lexical_support_floor guard)."""
|
||||
monkeypatch.delenv("BOR_SOURCE_USEFULNESS_FLOOR", raising=False)
|
||||
with pytest.raises(ValidationError, match="source_usefulness_floor"):
|
||||
_settings(relevance_threshold=0.62, source_usefulness_floor=0.70)
|
||||
|
||||
|
||||
def test_source_usefulness_floor_rejects_negative(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A negative bar is a typo (the agent_max_rounds pattern)."""
|
||||
with pytest.raises(ValidationError, match="source_usefulness_floor"):
|
||||
_settings(source_usefulness_floor=-0.1)
|
||||
|
||||
|
||||
def test_related_max_docs_default_and_env_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 113 (LOCKED A4): the related-doc tier cap — default 2,
|
||||
``0`` = the no-related-docs kill switch."""
|
||||
monkeypatch.delenv("BOR_RELATED_MAX_DOCS", raising=False)
|
||||
assert _settings().related_max_docs == 2
|
||||
monkeypatch.setenv("BOR_RELATED_MAX_DOCS", "0")
|
||||
assert _settings().related_max_docs == 0
|
||||
|
||||
|
||||
def test_related_max_docs_rejects_negative(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A negative cap is a typo (the agent_max_rounds pattern)."""
|
||||
monkeypatch.setenv("BOR_RELATED_MAX_DOCS", "-1")
|
||||
with pytest.raises(ValidationError, match="related_max_docs"):
|
||||
_settings()
|
||||
|
||||
|
||||
def test_read_max_chars_default_and_env_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
@@ -12,7 +12,12 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
|
||||
from app.models import Document
|
||||
from app.rag.retriever import TRUNCATION_MARKER, RetrievedChunk, select_documents
|
||||
from app.rag.retriever import (
|
||||
TRUNCATION_MARKER,
|
||||
RetrievedChunk,
|
||||
select_documents,
|
||||
select_documents_tiered,
|
||||
)
|
||||
|
||||
|
||||
def _doc(path: str, content: str, source: str = "Homelab", title: str | None = None) -> Document:
|
||||
@@ -90,6 +95,169 @@ def test_empty_hits_yield_no_documents() -> None:
|
||||
assert select_documents([], n=2) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 113 — the usefulness bar: cited vs related tiers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cos_chunk(doc: Document, score: float, cosine: float, position: int = 0) -> RetrievedChunk:
|
||||
"""A candidate with *score* (fused rank key) and *cosine* (gate input) decoupled."""
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=position,
|
||||
content=doc.content[:40],
|
||||
score=score,
|
||||
document=doc,
|
||||
cosine=cosine,
|
||||
)
|
||||
|
||||
|
||||
def test_tiered_both_clear_floor_both_cited() -> None:
|
||||
"""Both docs clear the bar → both cited, nothing related."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [_cos_chunk(a, 0.9, 0.50), _cos_chunk(b, 0.8, 0.40)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["a.md", "b.md"]
|
||||
assert related == []
|
||||
|
||||
|
||||
def test_tiered_strong_plus_weak_one_cited_one_related() -> None:
|
||||
"""The recurring incident shape: a strong 1st doc and a weak 2nd — the
|
||||
weak doc loses its citation slot and lands in the related tier."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [_cos_chunk(a, 0.9, 0.50), _cos_chunk(b, 0.8, 0.10)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["a.md"]
|
||||
assert [d.path for d in related] == ["b.md"]
|
||||
|
||||
|
||||
def test_tiered_both_weak_zero_cited_all_related() -> None:
|
||||
"""Neither doc clears the bar → no citation slot at all (the bar
|
||||
filters, it never backfills), the weak hits become the related tier."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [_cos_chunk(a, 0.9, 0.20), _cos_chunk(b, 0.8, 0.15)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert cited == []
|
||||
assert [d.path for d in related] == ["a.md", "b.md"] # rank order kept
|
||||
|
||||
|
||||
def test_tiered_related_cap_respected() -> None:
|
||||
"""The related tier is capped (related_max_docs) in rank order."""
|
||||
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(3)]
|
||||
chunks = [_cos_chunk(d, 0.9 - 0.1 * i, 0.10 - 0.02 * i) for i, d in enumerate(docs_in)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert cited == []
|
||||
assert [d.path for d in related] == ["d0.md", "d1.md"]
|
||||
|
||||
|
||||
def test_tiered_n_is_ceiling_not_quota() -> None:
|
||||
"""A single strong doc yields ONE cited doc — the bar never pads the
|
||||
cited tier up to ``n`` (LOCKED A2). And docs that clear the bar but
|
||||
exceed the ceiling fall through to the related tier (the next docs in
|
||||
rank order, never overlapping cited)."""
|
||||
only = _doc("only.md", "O" * 50)
|
||||
cited, related = select_documents_tiered(
|
||||
[_cos_chunk(only, 0.9, 0.80)], n=2, floor=0.35, related_cap=2
|
||||
)
|
||||
assert [d.path for d in cited] == ["only.md"]
|
||||
assert related == []
|
||||
|
||||
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(4)]
|
||||
chunks = [_cos_chunk(d, 0.9 - 0.1 * i, 0.8 - 0.05 * i) for i, d in enumerate(docs_in)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["d0.md", "d1.md"] # the ceiling
|
||||
assert [d.path for d in related] == ["d2.md", "d3.md"] # the next in rank order
|
||||
|
||||
|
||||
def test_tiered_bar_skips_weak_rank_one() -> None:
|
||||
"""A weak rank-1 doc does not consume a citation slot: the next-ranked
|
||||
bar-clearing docs take it (the bar filters, it does not backfill)."""
|
||||
w = _doc("w.md", "W" * 50)
|
||||
s1 = _doc("s1.md", "1" * 50)
|
||||
s2 = _doc("s2.md", "2" * 50)
|
||||
chunks = [
|
||||
_cos_chunk(w, 0.9, 0.20), # rank 1 — below the bar
|
||||
_cos_chunk(s1, 0.8, 0.90),
|
||||
_cos_chunk(s2, 0.7, 0.80),
|
||||
]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.5, related_cap=2)
|
||||
assert [d.path for d in cited] == ["s1.md", "s2.md"]
|
||||
assert [d.path for d in related] == ["w.md"]
|
||||
|
||||
|
||||
def test_tiered_tracks_best_chunk_cosine_across_a_docs_chunks() -> None:
|
||||
"""The bar is on the doc's BEST hit-chunk cosine — a weak first chunk
|
||||
(rank 1) does not sink a doc whose later chunk is vector-strong."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [
|
||||
_cos_chunk(a, 0.9, 0.10, position=0), # a's weak chunk ranks first
|
||||
_cos_chunk(a, 0.5, 0.90, position=2), # a's strong chunk
|
||||
_cos_chunk(b, 0.4, 0.0), # lexical-only b
|
||||
]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["a.md"]
|
||||
assert [d.path for d in related] == ["b.md"]
|
||||
|
||||
|
||||
def test_tiered_lexical_only_hit_goes_to_related_above_zero_floor() -> None:
|
||||
"""A lexical-only doc (cosine 0.0 by construction) is vector-unsupported
|
||||
by definition: above a zero floor it never earns a cited slot (LOCKED A2)."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
lexical_only = RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=0,
|
||||
content="X" * 10,
|
||||
score=0.9, # top fused rank (the FTS hit)
|
||||
document=a,
|
||||
cosine=0.0, # no vector rank — lexical-only
|
||||
fts_hit=True,
|
||||
)
|
||||
chunks = [lexical_only, _cos_chunk(b, 0.8, 0.5)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.35, related_cap=2)
|
||||
assert [d.path for d in cited] == ["b.md"]
|
||||
assert [d.path for d in related] == ["a.md"]
|
||||
|
||||
|
||||
def test_tiered_floor_zero_is_no_bar() -> None:
|
||||
"""A zero floor admits every scored document — lexical-only (cosine
|
||||
0.0) and weak alike — so the bar can be disabled per deployment."""
|
||||
a = _doc("a.md", "A" * 50)
|
||||
b = _doc("b.md", "B" * 50)
|
||||
chunks = [_cos_chunk(a, 0.9, 0.0), _cos_chunk(b, 0.8, 0.1)]
|
||||
cited, related = select_documents_tiered(chunks, n=2, floor=0.0, related_cap=2)
|
||||
assert [d.path for d in cited] == ["a.md", "b.md"]
|
||||
assert related == []
|
||||
|
||||
|
||||
def test_tiered_empty_chunks_yield_empty_tiers() -> None:
|
||||
assert select_documents_tiered([], n=2, floor=0.35, related_cap=2) == ([], [])
|
||||
|
||||
|
||||
def test_select_documents_wrapper_is_legacy_tiering() -> None:
|
||||
"""The wrapper (floor 0.0, cap 0) is the legacy "any score, top-N"
|
||||
selection — byte-identical for the shapes the existing callers see:
|
||||
rank order, dedupe, multi-chunk docs, ties, lexical-only hits."""
|
||||
docs_in = [_doc(f"d{i}.md", "X" * 20) for i in range(5)]
|
||||
chunks = [
|
||||
_chunk(docs_in[0], 0.5), # cosine 0.0 (lexical-only)
|
||||
_cos_chunk(docs_in[0], 0.9, 0.4, position=1),
|
||||
_chunk(docs_in[1], 0.8),
|
||||
_cos_chunk(docs_in[1], 0.8, 0.2, position=1), # fused tie across docs
|
||||
_cos_chunk(docs_in[2], 0.7, 0.0),
|
||||
_cos_chunk(docs_in[3], 0.6, 0.1),
|
||||
_chunk(docs_in[4], 0.1),
|
||||
]
|
||||
for n in (1, 2, 3, 10):
|
||||
assert select_documents(chunks, n=n) == select_documents_tiered(
|
||||
chunks, n, 0.0, 0
|
||||
)[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hybrid retrieval (A7): RRF fusion + lexical tsquery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,825 @@
|
||||
"""Unit: the phase-113 source-chip-quality contract (TODO L5 + L2c —
|
||||
"the 2nd chip is often noise the answer never used").
|
||||
|
||||
Phase 113 demotes sub-floor hits out of the citation surface: the
|
||||
done frame carries the cited tier in ``sources`` (rendered by
|
||||
``appendSources`` as ``.source-chip`` pills, UNCHANGED) and the
|
||||
related tier in ``related`` (rendered by the NEW ``appendRelated`` as
|
||||
the de-emphasized labeled row — ``.related-doc`` links, never
|
||||
``.source-chip``). A deflected turn carries ``sources: []`` → zero
|
||||
chips; its weak hits live in the related row only.
|
||||
|
||||
This module pins the STATIC SOURCES the UI contract stands on, in the
|
||||
house source-pin pattern (the test_chip_sizing_question_cap.py
|
||||
``_rule`` style):
|
||||
|
||||
* task 02 — the frontend: ``appendRelated`` exists and never builds a
|
||||
``source-chip``; the row renders only when ``related`` is non-empty;
|
||||
the label copy is present; the done-frame handler and the restore path
|
||||
both call it (the related tier persists with the turn, so a reload
|
||||
re-renders the row exactly as it looked live); the CSS row is clearly
|
||||
secondary (dashed border, muted ink, flat hover);
|
||||
* task 03 — the acceptance pin (TODO L5): the FOUR OBSERVED LIVE SHAPES
|
||||
(L110–123), each modeled as a ``plan_turn`` fixture with controlled
|
||||
cosine/``fts_hit``/fused ``score`` under the production calibration
|
||||
(the code defaults — the shapes were observed live):
|
||||
|
||||
1. **both docs weak** ("What is the capital of Mongolia?" →
|
||||
``Trooper_Nagraz.pl`` + ``Trooper_Begzei.pl``, both unrelated) —
|
||||
cited tier empty, the weak hits ride the related tier, capped at
|
||||
``related_max_docs``; the FTS hit without vector corroboration
|
||||
stays LOW (the A8-revised "Mongolia" case);
|
||||
2. **one strong + one weak** (the phase-gate question answered from
|
||||
``brain-of-reese/.agents/validate.sh``; the 2nd chip
|
||||
``ServMon/README.md`` unused) — exactly ONE cited ref, the weak
|
||||
doc in ``related``;
|
||||
3. **the Nagraz case** (``Trooper_Nagraz.pl`` strong,
|
||||
``Trooper_Byzin.pl`` weak — same shape, different fixtures);
|
||||
4. **the meta/history question** (no doc clears the bar, the agent
|
||||
reads nothing — chips ``app/api/suggestions.py`` +
|
||||
``108_history_wire_check/00_phase.md``, neither used) — pinned on
|
||||
the DONE FRAME (endpoint-level, fake retriever/LLM/session): the
|
||||
frame is row-only — ``sources: []`` (the UI's chip list — zero
|
||||
chips) + the weak hits in ``related``;
|
||||
5. **the agent-read exemption** (LOCKED A2): a below-floor doc the
|
||||
agent ``read`` via the tool joins ``sources`` (cited, last) and is
|
||||
excluded from ``related``.
|
||||
|
||||
The browser behavior (chip counts on a single-source question, zero
|
||||
chips on a deflected turn) is E2E-gated by
|
||||
tests/e2e/test_source_chip_quality.py (task 03).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api import chat as chat_api
|
||||
from app.config import Settings
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Document, QueryLog
|
||||
from app.rag import agent
|
||||
from app.rag.llm import StreamPiece, ToolCallPiece
|
||||
from app.rag.retriever import RetrievedChunk
|
||||
from tests.conftest import ADMIN_PASSWORD
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.rag.scaffolding import ScaffoldingFilter
|
||||
|
||||
FRONTEND = Path(__file__).resolve().parents[2] / "frontend"
|
||||
STYLES_CSS = FRONTEND / "assets" / "styles.css"
|
||||
APP_JS = FRONTEND / "assets" / "app.js"
|
||||
|
||||
|
||||
def _css() -> str:
|
||||
assert STYLES_CSS.is_file(), f"missing {STYLES_CSS}"
|
||||
return STYLES_CSS.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _app_js() -> str:
|
||||
assert APP_JS.is_file(), f"missing {APP_JS}"
|
||||
return APP_JS.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _function_body(js: str, header: str) -> str:
|
||||
"""The full text of the function whose header is ``header`` — from
|
||||
the header to its brace-matched closing ``}``. The naive brace
|
||||
count is safe for the pinned functions: their template literals
|
||||
carry balanced ``${…}`` pairs and no string literal holds a stray
|
||||
brace."""
|
||||
start = js.index(header)
|
||||
body_open = js.index("{", start)
|
||||
depth = 0
|
||||
for j in range(body_open, len(js)):
|
||||
if js[j] == "{":
|
||||
depth += 1
|
||||
elif js[j] == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return js[start : j + 1]
|
||||
raise AssertionError(f"unbalanced braces in {header!r}")
|
||||
|
||||
|
||||
def _rule(css: str, selector: str) -> str:
|
||||
"""The body of the rule whose selector line is exactly ``selector``
|
||||
(multi-line block)."""
|
||||
block = re.search(rf"^{re.escape(selector)} \{{\n([\s\S]*?)\n\}}", css, re.MULTILINE)
|
||||
assert block, f"styles.css must carry a `{selector} {{ … }}` rule"
|
||||
return block.group(1)
|
||||
|
||||
|
||||
def _done_branch(js: str) -> str:
|
||||
"""The SSE ``done`` branch of the stream handler — from the
|
||||
``ev.type === "done"`` test to the next ``else if`` (the
|
||||
``tool_result`` branch). The branch is a flat block (no nested
|
||||
else-if chain), so a slice between the two branch markers is
|
||||
exact."""
|
||||
start = js.index('ev.type === "done"')
|
||||
end = js.index('ev.type === "tool_result"', start)
|
||||
branch = js[start:end]
|
||||
assert "appendSources(wrap, ev.sources);" in branch, (
|
||||
"the done branch must keep appending the cited tier (phase 113 "
|
||||
"demotes to a row — it never removed the citation surface)"
|
||||
)
|
||||
return branch
|
||||
|
||||
|
||||
# ---------- task 02: appendRelated — the secondary row, never a chip ----------
|
||||
|
||||
|
||||
def test_append_related_exists_and_never_uses_source_chip() -> None:
|
||||
"""``appendRelated`` exists and builds ONLY ``.related-doc`` links —
|
||||
the string ``source-chip`` must NOT appear anywhere in its body
|
||||
(the acceptance criterion: a weak doc renders only as
|
||||
``.related-doc``, never as ``.source-chip``). It reuses the chip
|
||||
behavior for navigation: the same ``documentUrl(s.source, s.path,
|
||||
"/")`` href (the /document.html no-JS escape hatch) and the same
|
||||
left-click → ``openDocumentModal`` (phase 26, the same-page modal).
|
||||
Each link carries the full path in ``title`` AND ``aria-label``
|
||||
(the accessible name never depends on the visible text fitting)."""
|
||||
body = _function_body(_app_js(), "function appendRelated(wrap, related) {")
|
||||
assert "link.className = \"related-doc\";" in body, (
|
||||
"the related links must carry the .related-doc class"
|
||||
)
|
||||
assert "source-chip" not in body, (
|
||||
"appendRelated must NEVER build a citation chip — the related "
|
||||
"row is not a citation surface (phase 113 LOCKED A1/A2)"
|
||||
)
|
||||
assert "link.href = documentUrl(s.source, s.path, \"/\");" in body, (
|
||||
"each related link keeps the chip's /document.html href — the "
|
||||
"no-JS / context-menu escape hatch (back → the chat page)"
|
||||
)
|
||||
assert "openDocumentModal(s.source, s.path, link);" in body, (
|
||||
"left-click opens the same-page document modal exactly like the "
|
||||
"chips (phase 26 contract)"
|
||||
)
|
||||
assert "e.preventDefault();" in body, (
|
||||
"the click must prevent default navigation — the modal takes "
|
||||
"over, no new tab (the chip pattern)"
|
||||
)
|
||||
assert "link.title = docLabel;" in body, (
|
||||
"the native tooltip carries the FULL path (the chip pattern)"
|
||||
)
|
||||
assert 'link.setAttribute("aria-label", docLabel);' in body, (
|
||||
"the accessible name is the full path, always"
|
||||
)
|
||||
|
||||
|
||||
def test_append_related_renders_only_when_related_is_non_empty() -> None:
|
||||
"""The row renders ONLY when ``related`` is non-empty: an
|
||||
``undefined``/``null``/``[]`` input (every pre-phase turn, every
|
||||
turn with nothing under the bar) early-returns with NO DOM — the
|
||||
bubble reads exactly as it did before phase 113. The pin is the
|
||||
house guard, verbatim, as the FIRST statement of the body."""
|
||||
body = _function_body(_app_js(), "function appendRelated(wrap, related) {")
|
||||
first_stmt = body.split("{", 1)[1].lstrip()
|
||||
assert first_stmt.startswith("if (!related || !related.length) return;"), (
|
||||
"appendRelated must early-return on !related || !related.length — "
|
||||
"an empty related tier adds zero DOM (the pre-phase look stays "
|
||||
"byte-identical for those turns)"
|
||||
)
|
||||
|
||||
|
||||
def test_append_related_row_contract() -> None:
|
||||
"""The row itself: a ``.msg-meta.related-docs`` div (it joins the
|
||||
bubble's meta family — the .msg-body flex gap stacks it below the
|
||||
citation row with the existing gap) that is an accessible list
|
||||
(``role="list"`` + ``aria-label="Nearby docs, in case"``) headed
|
||||
by the visible small-caps label ``Nearby docs, in case:`` (the
|
||||
TODO's suggested wording, trimmed) with the
|
||||
``.related-docs-label`` class, and one ``role="listitem"`` link
|
||||
per doc."""
|
||||
body = _function_body(_app_js(), "function appendRelated(wrap, related) {")
|
||||
assert "row.className = \"msg-meta related-docs\";" in body, (
|
||||
"the row is a .msg-meta row (the family the bubble's meta rows "
|
||||
"already form) with the .related-docs marker"
|
||||
)
|
||||
assert 'row.setAttribute("role", "list");' in body
|
||||
assert 'row.setAttribute("aria-label", "Nearby docs, in case");' in body, (
|
||||
"the row is an accessible list named 'Nearby docs, in case'"
|
||||
)
|
||||
assert "label.className = \"related-docs-label\";" in body
|
||||
assert "label.textContent = \"Nearby docs, in case:\";" in body, (
|
||||
"the visible label carries the TODO's suggested wording (trimmed)"
|
||||
)
|
||||
assert 'link.setAttribute("role", "listitem");' in body, (
|
||||
"each link is a listitem of the row's list (ARIA stays valid)"
|
||||
)
|
||||
assert "for (const s of related) {" in body, (
|
||||
"one link per related doc"
|
||||
)
|
||||
|
||||
|
||||
# ---------- task 02: wiring — the done frame and the restore path ----------
|
||||
|
||||
|
||||
def test_done_handler_calls_append_related_last() -> None:
|
||||
"""The done-frame handler calls ``appendRelated(wrap, ev.related)``
|
||||
— and appends the row LAST among the bubble's meta rows: AFTER
|
||||
``markLastRetryable()`` (the appendTuneButton / appendSaveAsDoc /
|
||||
appendRetryButton claimers all take the FIRST ``.msg-meta`` row,
|
||||
so a deflected turn with related docs — no citation row — gets its
|
||||
OWN meta row for the buttons instead of actions joining the
|
||||
related row). The call sits after the Retry claim in the branch."""
|
||||
branch = _done_branch(_app_js())
|
||||
assert "appendRelated(wrap, ev.related);" in branch, (
|
||||
"the done handler must render the related tier (it arrives in "
|
||||
"ev.related — on a deflected turn ev.sources is empty and the "
|
||||
"weak hits live here, row only, zero chips)"
|
||||
)
|
||||
assert branch.index("appendRelated(wrap, ev.related);") > branch.index(
|
||||
"markLastRetryable();"
|
||||
), (
|
||||
"appendRelated must run AFTER the meta-action claimers (last "
|
||||
"meta row) — a deflected turn's tune/retry buttons must land in "
|
||||
"their own row, never in the related row"
|
||||
)
|
||||
|
||||
|
||||
def test_done_handler_persists_related_with_the_turn() -> None:
|
||||
"""The related tier PERSISTS with the turn (the done handler's
|
||||
``rememberBrainTurn`` meta), using the house optional-meta pattern
|
||||
(``undefined`` drops the key from the JSON — no ``related: []``
|
||||
noise on turns with nothing related). Without this the restore
|
||||
path could never re-render the row after a reload."""
|
||||
branch = _done_branch(_app_js())
|
||||
pattern = (
|
||||
r"rememberBrainTurn\(finalText \|\| acc, \{([\s\S]*?)\}"
|
||||
r"\s*,\s*leavePartialIndex\);"
|
||||
)
|
||||
persist = re.search(pattern, branch)
|
||||
assert persist, "the done handler must persist the turn through rememberBrainTurn"
|
||||
meta = persist.group(1)
|
||||
assert "related: ev.related?.length ? ev.related : undefined," in meta, (
|
||||
"the done frame's related tier must persist (undefined drops "
|
||||
"the key — the house optional-meta pattern, cf. `tools`)"
|
||||
)
|
||||
|
||||
|
||||
def test_restore_path_calls_append_related() -> None:
|
||||
"""The phase-14 restore path (``renderStoredMessage`` — used by
|
||||
BOTH the localStorage restore and the /?chat=<id> saved-chat boot
|
||||
load) re-renders the related row from the stored payload when it
|
||||
carries ``related``; a pre-phase record without the field
|
||||
restores exactly as today (appendRelated no-ops — no row). The
|
||||
call sits after the meta-row claimers, exactly like the live
|
||||
done path, so restored buttons never join the related row."""
|
||||
body = _function_body(_app_js(), "function renderStoredMessage(m) {")
|
||||
assert "appendRelated(wrap, m.related);" in body, (
|
||||
"the restore path must render the related tier from the stored "
|
||||
"payload (pre-phase records carry no `related` → no row, "
|
||||
"graceful)"
|
||||
)
|
||||
assert body.index("appendRelated(wrap, m.related);") > body.index(
|
||||
"appendStoppedNote(wrap);"
|
||||
), (
|
||||
"the related row appends LAST (after the claimers) on the "
|
||||
"restore path too — identical order to the live done path"
|
||||
)
|
||||
|
||||
|
||||
def test_citation_surface_untouched() -> None:
|
||||
"""The citation chip component is NOT touched (phase 113 "NOT
|
||||
touched" list): ``appendSources`` still builds ``.source-chip``
|
||||
pills and still early-returns on an empty list — which is exactly
|
||||
what a deflected turn (``ev.sources === []``) hits: zero
|
||||
``.source-chip`` elements under the bubble. The suggestion chips
|
||||
(``appendMaybeTry`` / ``.suggestion-chip``) are a different
|
||||
surface and stay as they were."""
|
||||
js = _app_js()
|
||||
body = _function_body(js, "function appendSources(wrap, sources) {")
|
||||
assert "chip.className = \"source-chip\";" in body
|
||||
assert body.lstrip().startswith("function appendSources"), "sanity"
|
||||
assert "if (!sources || !sources.length) return;" in body, (
|
||||
"appendSources still no-ops on an empty list — a deflected "
|
||||
"turn's empty ev.sources renders ZERO citation chips (the "
|
||||
"weak hits arrive in ev.related → the row only)"
|
||||
)
|
||||
assert js.count("function appendRelated(wrap, related) {") == 1, (
|
||||
"exactly ONE appendRelated — no second related renderer"
|
||||
)
|
||||
|
||||
|
||||
# ---------- task 02: the CSS — clearly secondary, AA, theme-neutral ----------
|
||||
|
||||
|
||||
def test_related_doc_rule_is_the_secondary_look() -> None:
|
||||
"""The ``.related-doc`` link: dashed border (the citation chip's
|
||||
solid 1px ``--line`` pill is the citation look — the dash is the
|
||||
visual split), transparent fill (no ``--brand-soft``), muted
|
||||
``--ink-soft`` text (8.6:1 on ``--bg`` — verified ≥4.5:1, WCAG
|
||||
2.1 AA; the ratio is recorded in the rule's provenance comment),
|
||||
smaller mono than the chip (0.7rem < 0.72rem), and the
|
||||
single-line ellipsis set (same overflow contract as the chip).
|
||||
Palette variables only — zero new literals (phase-92 invariant),
|
||||
so the monochrome theme grays the row automatically."""
|
||||
body = _rule(_css(), ".related-doc")
|
||||
assert "border: 1px dashed var(--line);" in body, (
|
||||
"the related link is DASHED — the chip's solid border is the "
|
||||
"citation look, the dash is the 'not a citation' signal"
|
||||
)
|
||||
assert "background: transparent;" in body, (
|
||||
"no brand-soft fill — that surface is the citation pill's"
|
||||
)
|
||||
assert "color: var(--ink-soft);" in body, (
|
||||
"the link text is the muted ink — AA on the page bg (8.6:1)"
|
||||
)
|
||||
assert "font-family: var(--mono);" in body, (
|
||||
"source/path reads mono like the chips (same data, secondary "
|
||||
"weight)"
|
||||
)
|
||||
assert "font-size: 0.7rem;" in body, (
|
||||
"smaller than the chip's 0.72rem — visually secondary"
|
||||
)
|
||||
for prop in (
|
||||
"white-space: nowrap;",
|
||||
"overflow: hidden;",
|
||||
"text-overflow: ellipsis;",
|
||||
"max-width: 100%;",
|
||||
"min-width: 0;",
|
||||
):
|
||||
assert prop in body, f"the related link keeps the chip's single-line ellipsis set ({prop})"
|
||||
comment = re.search(r"(/\*[^*]*?\*/)\s*\.related-docs-label \{", _css())
|
||||
assert comment, "the related-docs rules must carry their provenance comment"
|
||||
note = comment.group(1)
|
||||
assert "8.6:1" in note and "4.5:1" in note, (
|
||||
"the --ink-soft on --bg ratio must be recorded (verified "
|
||||
"8.6:1 ≥ 4.5:1, WCAG AA — house style)"
|
||||
)
|
||||
assert "#" not in note.replace("--", ""), (
|
||||
"the comment is theme-variable language — no literal colors "
|
||||
"(phase-92 zero-literal invariant)"
|
||||
)
|
||||
|
||||
|
||||
def test_related_doc_hover_is_flat() -> None:
|
||||
"""The ``.related-doc:hover`` rule is deliberately FLAT: no
|
||||
background swap (the chip's ``background: var(--brand-soft)``
|
||||
hover is the citation affordance — 'no hover elevation of the
|
||||
citation chips'), only the ink-soft → ink step-up (16.7:1 on
|
||||
``--bg``) plus the underline. The global 3px ``:focus-visible``
|
||||
outline rule covers the focus ring (no per-rule ring needed, the
|
||||
chip precedent)."""
|
||||
body = _rule(_css(), ".related-doc:hover")
|
||||
assert "background" not in body, (
|
||||
"the hover must NOT change the background — that surface swap "
|
||||
"is the citation chip's affordance"
|
||||
)
|
||||
assert "box-shadow" not in body, "no hover elevation (no shadow gain)"
|
||||
assert "color: var(--ink);" in body, (
|
||||
"the hover is the flat ink step-up (16.7:1 on --bg, AA)"
|
||||
)
|
||||
assert "text-decoration: underline;" in body
|
||||
|
||||
|
||||
def test_related_docs_label_is_small_caps_muted() -> None:
|
||||
"""The ``.related-docs-label``: small muted uppercase text (the
|
||||
'small caps or muted small text' from the task) — smaller than the
|
||||
row's 0.75rem ``.msg-meta`` base, the same AA-safe ``--ink-soft``
|
||||
(8.6:1 on ``--bg``) as the links, letterspaced like every other
|
||||
overline in the app."""
|
||||
body = _rule(_css(), ".related-docs-label")
|
||||
assert "text-transform: uppercase;" in body
|
||||
assert "color: var(--ink-soft);" in body, (
|
||||
"the label uses the AA-safe muted ink (8.6:1 on --bg)"
|
||||
)
|
||||
size = re.search(r"font-size: ([0-9.]+)rem;", body)
|
||||
assert size, "the label must set its own (smaller) font size"
|
||||
assert float(size.group(1)) < 0.75, (
|
||||
"the label is smaller than the row's 0.75rem base — it is a "
|
||||
"whisper, not a heading"
|
||||
)
|
||||
|
||||
|
||||
# ---------- task 03: the four observed shapes (TODO L110–123) ----------
|
||||
|
||||
#: The canned answer the endpoint-level fakes stream (the gate-suite
|
||||
#: convention — byte-stable, assertable against the wire).
|
||||
ANSWER = "I haven't done anything like that — try one of these instead!"
|
||||
|
||||
|
||||
def _shape_settings() -> Settings:
|
||||
"""The PRODUCTION calibration (the code defaults, explicit) — the
|
||||
four shapes were observed LIVE under this threshold/floor pair.
|
||||
``_env_file=None`` keeps the mock-calibrated values from
|
||||
``tests/conftest.py`` (and any local ``.env``) out of the pin."""
|
||||
return Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
relevance_threshold=0.62,
|
||||
lexical_support_floor=0.35,
|
||||
source_usefulness_floor=0.35, # LOCKED A2 default
|
||||
related_max_docs=2, # LOCKED A4 default
|
||||
top_n_docs=2, # the ceiling — never a quota (LOCKED A2)
|
||||
)
|
||||
|
||||
|
||||
def _doc(source: str, path: str, title: str, content: str) -> Document:
|
||||
return Document(
|
||||
id=uuid.uuid4(),
|
||||
source=source,
|
||||
path=path,
|
||||
full_path=f"/{source}/{path}",
|
||||
title=title,
|
||||
content=content,
|
||||
content_hash="0" * 64,
|
||||
# Phase 106, D5: the HIGH block / the ``read`` result's date
|
||||
# line format the row's created_at — a fixed value keeps the
|
||||
# fixtures deterministic.
|
||||
created_at=datetime(2024, 6, 15, 12, 0, 0, tzinfo=UTC),
|
||||
)
|
||||
|
||||
|
||||
def _chunk(
|
||||
doc: Document,
|
||||
score: float,
|
||||
cosine: float,
|
||||
fts_hit: bool = False,
|
||||
) -> RetrievedChunk:
|
||||
"""One fake retrieval candidate: *score* is the RRF fused rank key,
|
||||
*cosine* the vector similarity (the bar's input — independent of
|
||||
*score* on purpose: the bar is on the cosine, LOCKED A2)."""
|
||||
return RetrievedChunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
position=0,
|
||||
content=doc.content[:32],
|
||||
score=score,
|
||||
document=doc,
|
||||
cosine=cosine,
|
||||
fts_hit=fts_hit,
|
||||
)
|
||||
|
||||
|
||||
def test_shape_1_mongolia_both_docs_weak_cite_nothing() -> None:
|
||||
"""Observed shape 1 (TODO L110–113): "What is the capital of
|
||||
Mongolia?" → chips ``Trooper_Nagraz.pl`` + ``Trooper_Begzei.pl``,
|
||||
BOTH unrelated. Both below the bar: the cited tier is EMPTY (zero
|
||||
citation chips) and the weak hits ride the related tier — in rank
|
||||
order, capped at ``related_max_docs`` (the 3rd weak doc drops out).
|
||||
The FTS hit without vector corroboration (0.20 < the 0.35 lexical
|
||||
floor) stays LOW — the A8-revised "Mongolia" case; the weak content
|
||||
never reaches the LLM."""
|
||||
nagraz = _doc("scripts", "scripts/Trooper_Nagraz.pl", "Trooper_Nagraz.pl",
|
||||
"NAGRAZ_PL_CONTENT")
|
||||
begzei = _doc("scripts", "scripts/Trooper_Begzei.pl", "Trooper_Begzei.pl",
|
||||
"BEGZEI_PL_CONTENT")
|
||||
third = _doc("scripts", "scripts/Trooper_Third.pl", "Trooper_Third.pl",
|
||||
"THIRD_PL_CONTENT")
|
||||
chunks = [
|
||||
_chunk(nagraz, 0.033, cosine=0.20, fts_hit=True), # rank 1, lexical hit
|
||||
_chunk(begzei, 0.031, cosine=0.12),
|
||||
_chunk(third, 0.030, cosine=0.10), # below the cap — related drops it
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _shape_settings())
|
||||
assert plan.deflected is True # 0.20 < 0.62 AND 0.20 < the 0.35 lex floor
|
||||
assert plan.docs == [] # NO citation slot below the bar
|
||||
assert [d.title for d in plan.related_docs] == [
|
||||
"Trooper_Nagraz.pl",
|
||||
"Trooper_Begzei.pl",
|
||||
] # rank order, capped at related_max_docs (2)
|
||||
assert len(plan.related_docs) <= 2
|
||||
# The LOW prompt is titles only — none of the weak content is sent.
|
||||
assert "NAGRAZ_PL_CONTENT" not in plan.system_prompt
|
||||
assert "Trooper_Nagraz.pl" in plan.system_prompt # weak-hit titles carried
|
||||
assert plan.suggestions # the "Maybe try" chips are unchanged
|
||||
|
||||
|
||||
def test_shape_2_validate_sh_strong_plus_unused_second_chip() -> None:
|
||||
"""Observed shape 2 (TODO L114–116): the phase-gate question is
|
||||
answered from ``brain-of-reese/.agents/validate.sh`` — the 2nd chip
|
||||
``ServMon/README.md`` was NEVER used. The strong doc clears the bar
|
||||
and takes the only cited slot (top_n_docs is a ceiling, not a
|
||||
quota); the weak 2nd doc demotes to related — never a citation.
|
||||
The HIGH prompt carries the cited content only."""
|
||||
validate = _doc("brain-of-reese", ".agents/validate.sh", "validate.sh",
|
||||
"VALIDATE_SH_CONTENT")
|
||||
servmon = _doc("ServMon", "README.md", "ServMon README",
|
||||
"SERVMON_README_CONTENT")
|
||||
chunks = [
|
||||
_chunk(validate, 0.90, cosine=0.70), # clears threshold AND bar
|
||||
_chunk(servmon, 0.80, cosine=0.20), # high fused rank, weak cosine
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _shape_settings())
|
||||
assert plan.deflected is False # 0.70 >= 0.62
|
||||
assert [d.title for d in plan.docs] == ["validate.sh"] # exactly ONE cited
|
||||
assert [d.title for d in plan.related_docs] == ["ServMon README"]
|
||||
assert "VALIDATE_SH_CONTENT" in plan.system_prompt
|
||||
assert "SERVMON_README_CONTENT" not in plan.system_prompt
|
||||
|
||||
|
||||
def test_shape_3_nagraz_answered_by_own_doc_byzin_uncited() -> None:
|
||||
"""Observed shape 3 (TODO L117–119): the Trooper_Nagraz question is
|
||||
answered from ``Trooper_Nagraz.pl`` — the 2nd chip
|
||||
``Trooper_Byzin.pl`` uncited. The SAME shape as shape 2 with
|
||||
different fixtures — the bar filters the 2nd chip; it is not a
|
||||
coincidence of the validate.sh pair."""
|
||||
nagraz = _doc("scripts", "scripts/Trooper_Nagraz.pl", "Trooper_Nagraz.pl",
|
||||
"NAGRAZ_PL_CONTENT")
|
||||
byzin = _doc("scripts", "scripts/Trooper_Byzin.pl", "Trooper_Byzin.pl",
|
||||
"BYZIN_PL_CONTENT")
|
||||
chunks = [
|
||||
_chunk(nagraz, 0.85, cosine=0.70),
|
||||
_chunk(byzin, 0.75, cosine=0.15), # below the bar
|
||||
]
|
||||
plan = chat_api.plan_turn(chunks, _shape_settings())
|
||||
assert plan.deflected is False
|
||||
assert [d.title for d in plan.docs] == ["Trooper_Nagraz.pl"] # 1 cited
|
||||
assert [d.title for d in plan.related_docs] == ["Trooper_Byzin.pl"] # 1 related
|
||||
assert "BYZIN_PL_CONTENT" not in plan.system_prompt
|
||||
|
||||
|
||||
# ---------- task 03: done-frame wire (endpoint-level fakes, no stack) ----------
|
||||
|
||||
|
||||
class _CannedLLM:
|
||||
"""Records the requests; streams the canned *answer*.
|
||||
|
||||
Without *read_path* it never emits tool calls (the single-request
|
||||
shape). With *read_path*, the first tools-offering request that
|
||||
carries no tool result yet emits ONE ``read`` call on the combined
|
||||
path; the follow-up request (carrying the tool result) streams the
|
||||
answer — the phase-37 single-read shape, stateless (the e2e mock's
|
||||
convention)."""
|
||||
|
||||
def __init__(self, answer: str = ANSWER, read_path: str | None = None) -> None:
|
||||
self.settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue]
|
||||
self.answer = answer
|
||||
self.read_path = read_path
|
||||
self.seen: list[list[dict[str, Any]]] = []
|
||||
self.seen_tools: list[list[dict[str, Any]] | None] = []
|
||||
|
||||
async def embed_one(self, _text: str) -> list[float]:
|
||||
return [0.0] * 768
|
||||
|
||||
async def chat_stream(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
scaffolding: ScaffoldingFilter | None = None,
|
||||
):
|
||||
self.seen.append(messages)
|
||||
self.seen_tools.append(tools)
|
||||
if (
|
||||
self.read_path is not None
|
||||
and tools is not None
|
||||
and not any(m.get("role") == "tool" for m in messages)
|
||||
):
|
||||
yield ToolCallPiece(
|
||||
id="call_1", name="read", arguments={"path": self.read_path}
|
||||
)
|
||||
return
|
||||
for i in range(0, len(self.answer), 12):
|
||||
yield StreamPiece("content", self.answer[i : i + 12])
|
||||
|
||||
|
||||
class _FakeSteeringResult:
|
||||
"""Empty steering-note result (no stored notes in these tests)."""
|
||||
|
||||
def all(self) -> list[Any]:
|
||||
return []
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Stands in for the DB session (the gate-suite pattern):
|
||||
records the ``QueryLog`` row, yields no steering notes, no KB
|
||||
overview (``get`` → ``None``). Tool execution's ``find_document``
|
||||
is monkeypatched separately (the agent module's, not the
|
||||
session's)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.added: list[Any] = []
|
||||
self.commits = 0
|
||||
|
||||
def __enter__(self) -> _FakeSession:
|
||||
return self
|
||||
|
||||
def __exit__(self, *args: Any) -> None:
|
||||
pass
|
||||
|
||||
def add(self, obj: Any) -> None:
|
||||
self.added.append(obj)
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def scalars(self, _stmt: Any) -> _FakeSteeringResult:
|
||||
return _FakeSteeringResult()
|
||||
|
||||
def get(self, _model: Any, _pk: Any) -> Any:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _admin_signed_in(client: TestClient) -> None:
|
||||
"""Phase 79: ``POST /api/chat`` is user-gated — the endpoint-level
|
||||
tests run as the signed-in ADMIN (the gate-suite pattern)."""
|
||||
r = client.post("/api/login", json={"password": ADMIN_PASSWORD})
|
||||
assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def chip_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> Iterator[tuple[_FakeSession, _CannedLLM]]:
|
||||
"""``POST /api/chat`` with retriever, session, and LLM all faked —
|
||||
the production calibration (``_shape_settings``) in force."""
|
||||
monkeypatch.setattr(chat_api, "db_available", lambda: True)
|
||||
session = _FakeSession()
|
||||
llm = _CannedLLM()
|
||||
monkeypatch.setattr(chat_api, "SessionLocal", lambda: session)
|
||||
monkeypatch.setitem(fastapi_app.dependency_overrides, chat_api.get_llm, lambda: llm)
|
||||
monkeypatch.setattr(chat_api, "get_settings", _shape_settings)
|
||||
yield session, llm
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def _ask(client: TestClient, message: str) -> list[dict[str, Any]]:
|
||||
with client.stream("POST", "/api/chat", json={"message": message}) as r:
|
||||
assert r.status_code == 200
|
||||
frames: list[dict[str, Any]] = []
|
||||
buf = ""
|
||||
for part in r.iter_text():
|
||||
buf += part
|
||||
while "\n\n" in buf:
|
||||
frame, buf = buf.split("\n\n", 1)
|
||||
frame = frame.strip()
|
||||
if frame.startswith("data:"):
|
||||
frames.append(json.loads(frame.removeprefix("data:").strip()))
|
||||
assert buf.strip() == ""
|
||||
return frames
|
||||
|
||||
|
||||
def _fake_retriever(chunks: list[RetrievedChunk]) -> Any:
|
||||
def retrieve(_db: Any, _question: str, _vec: list[float]) -> list[RetrievedChunk]:
|
||||
return chunks
|
||||
|
||||
return retrieve
|
||||
|
||||
|
||||
def test_shape_4_meta_question_deflected_frame_is_row_only(
|
||||
client: TestClient,
|
||||
chip_env: tuple[_FakeSession, _CannedLLM],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Observed shape 4 (TODO L120–123), pinned on the DONE FRAME: a
|
||||
meta question about the conversation's own history → chips
|
||||
``app/api/suggestions.py`` + ``108_history_wire_check/00_phase.md``,
|
||||
neither used. No doc clears the bar and the agent reads nothing —
|
||||
the frame is ROW-ONLY: ``sources: []`` (the UI chips every source
|
||||
entry — zero chips) with the weak hits in ``related`` (rank order,
|
||||
≤ ``related_max_docs``) — the de-emphasized row's links (the row's
|
||||
rendering itself is pinned by task 02's source tests + the E2E).
|
||||
The weak retrieval stays durably recorded (LOCKED A3); the weak
|
||||
content never reaches the LLM (LOW prompt, titles only)."""
|
||||
session, llm = chip_env
|
||||
suggestions = _doc("brain-of-reese", "app/api/suggestions.py",
|
||||
"suggestions.py", "SUGGESTIONS_PY_CONTENT")
|
||||
phase_md = _doc("brain-of-reese",
|
||||
".agents/108_history_wire_check/00_phase.md", "00_phase.md",
|
||||
"PHASE_MD_CONTENT")
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"retrieve",
|
||||
_fake_retriever(
|
||||
[
|
||||
_chunk(suggestions, 0.033, cosine=0.25),
|
||||
_chunk(phase_md, 0.031, cosine=0.10),
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
frames = _ask(client, "What have we covered in this conversation so far?")
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done"
|
||||
assert done["deflected"] is True
|
||||
assert done["sources"] == [] # zero citation chips on the wire
|
||||
related = done["related"]
|
||||
assert [(s["source"], s["path"]) for s in related] == [
|
||||
("brain-of-reese", "app/api/suggestions.py"),
|
||||
("brain-of-reese", ".agents/108_history_wire_check/00_phase.md"),
|
||||
] # rank order
|
||||
assert len(related) <= 2 # related_max_docs
|
||||
assert all(s["title"] for s in related) # the row's links carry the identity
|
||||
assert done["suggestions"] # the "Maybe try" chips are unchanged
|
||||
|
||||
(system, _user) = llm.seen[0][0], llm.seen[0][1]
|
||||
assert "SUGGESTIONS_PY_CONTENT" not in system["content"]
|
||||
assert "PHASE_MD_CONTENT" not in system["content"]
|
||||
|
||||
(row,) = session.added
|
||||
assert isinstance(row, QueryLog)
|
||||
assert row.deflected is True
|
||||
# LOCKED A3: the weak retrieval stays recorded (observability).
|
||||
assert "app/api/suggestions.py" in row.sources
|
||||
assert "108_history_wire_check/00_phase.md" in row.sources
|
||||
|
||||
|
||||
def test_done_frame_single_cited_ref_strong_plus_weak(
|
||||
client: TestClient,
|
||||
chip_env: tuple[_FakeSession, _CannedLLM],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Shape 2 on the wire — the single-document question's input to
|
||||
"exactly one citation chip" (the E2E asserts the rendered chip):
|
||||
the bar-clearing doc is the ONLY ``sources`` ref; the weak 2nd doc
|
||||
rides ``related``; the tiers are disjoint (the done frame's dedupe).
|
||||
The durable record keeps the FULL retrieval (LOCKED A3)."""
|
||||
session, _llm = chip_env
|
||||
validate = _doc("brain-of-reese", ".agents/validate.sh", "validate.sh",
|
||||
"VALIDATE_SH_CONTENT")
|
||||
servmon = _doc("ServMon", "README.md", "ServMon README",
|
||||
"SERVMON_README_CONTENT")
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"retrieve",
|
||||
_fake_retriever(
|
||||
[
|
||||
_chunk(validate, 0.90, cosine=0.70),
|
||||
_chunk(servmon, 0.80, cosine=0.20),
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
frames = _ask(client, "How does the phase gate decide to validate?")
|
||||
done = frames[-1]
|
||||
assert done["deflected"] is False
|
||||
assert [(s["source"], s["path"]) for s in done["sources"]] == [
|
||||
("brain-of-reese", ".agents/validate.sh"),
|
||||
] # EXACTLY one citation chip on the wire
|
||||
assert [(s["source"], s["path"]) for s in done["related"]] == [
|
||||
("ServMon", "README.md"),
|
||||
]
|
||||
cited = {(s["source"], s["path"]) for s in done["sources"]}
|
||||
related = {(s["source"], s["path"]) for s in done["related"]}
|
||||
assert cited.isdisjoint(related)
|
||||
|
||||
(row,) = session.added
|
||||
assert isinstance(row, QueryLog)
|
||||
assert row.deflected is False
|
||||
# The durable record keeps BOTH docs (retrieval, not citations — A3).
|
||||
assert ".agents/validate.sh" in row.sources
|
||||
assert "ServMon/README.md" in row.sources
|
||||
|
||||
|
||||
def test_agent_read_below_floor_doc_joins_sources(
|
||||
client: TestClient,
|
||||
chip_env: tuple[_FakeSession, _CannedLLM],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The agent-read exemption (LOCKED A2): a doc UNDER the bar that
|
||||
the agent ``read`` via the tool is cited by definition — the model
|
||||
read it, so it was used. It joins ``sources`` (after the retrieved
|
||||
cited docs, deduped) and is EXCLUDED from ``related`` (a used doc
|
||||
must never read as "nearby"); the other below-floor doc stays in
|
||||
the tier. The read content reached the model (the tool result in
|
||||
the follow-up request)."""
|
||||
session, _default_llm = chip_env
|
||||
strong = _doc("docs", "strong.md", "Strong", "STRONG_DOC_CONTENT")
|
||||
weak_b = _doc("docs", "weak-b.md", "Weak B", "WEAK_B_READ_BY_AGENT")
|
||||
weak_c = _doc("docs", "weak-c.md", "Weak C", "WEAK_C_CONTENT")
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"retrieve",
|
||||
_fake_retriever(
|
||||
[
|
||||
_chunk(strong, 0.90, cosine=0.70), # clears the bar
|
||||
_chunk(weak_b, 0.80, cosine=0.20), # below the bar — read by the agent
|
||||
_chunk(weak_c, 0.70, cosine=0.10), # below the bar — nobody reads it
|
||||
]
|
||||
),
|
||||
)
|
||||
read_llm = _CannedLLM(read_path="docs/weak-b.md")
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: read_llm
|
||||
|
||||
def _find_document(_db: Any, source: str, path: str) -> Document | None:
|
||||
return weak_b if (source, path) == ("docs", "weak-b.md") else None
|
||||
|
||||
monkeypatch.setattr(agent, "find_document", _find_document)
|
||||
|
||||
frames = _ask(client, "What does the weak B document say?")
|
||||
done = frames[-1]
|
||||
assert done["deflected"] is False
|
||||
# The tool ran (one ``tool`` frame) and the read doc reached the
|
||||
# model's follow-up request.
|
||||
assert any(f["type"] == "tool" for f in frames)
|
||||
tool_msgs = [m for m in read_llm.seen[1] if m.get("role") == "tool"]
|
||||
assert len(tool_msgs) == 1
|
||||
assert "WEAK_B_READ_BY_AGENT" in tool_msgs[0]["content"]
|
||||
|
||||
sources = [(s["source"], s["path"]) for s in done["sources"]]
|
||||
assert sources == [("docs", "strong.md"), ("docs", "weak-b.md")] # read ⇒ cited, last
|
||||
related = [(s["source"], s["path"]) for s in done["related"]]
|
||||
assert related == [("docs", "weak-c.md")] # the read doc is not "nearby"
|
||||
assert set(sources).isdisjoint(set(related))
|
||||
|
||||
(row,) = session.added
|
||||
assert isinstance(row, QueryLog)
|
||||
assert "weak-b.md" in row.sources # the full retrieval is recorded (A3)
|
||||
Reference in New Issue
Block a user