phase: 118_summary_seed_context
**Phase 118 final verification pass — complete.** All criteria verified; 4 pre-existing defects found and fixed.
- **Verified:** summary-seed wiring (`select_suggested` top-5 no-floor → summary blocks, no full text in HIGH prompt), all-doc markdown summaries + NULL backfill (`summary_backfilled`, no `sources_meta` bump), `read` adds full text with `read_docs`-only dedupe, `done.sources` = suggested+read / durable record = suggested+related+read + `suggested=N` log line (seen live in E2E), byte-locked PERSONA/LOW/TOOLS_SECTION, battery gate PASS recorded in `TOOL_CALLING_TESTING.md` §10 (turbo 2026-09-16: 1/2/4 GREEN, cond-3 reported 9/10 per A7, contract 21/21, caps 0).
- **Defects fixed (all pre-existing, none phase-118):** ① `ChatMessage` schema missing the phase-113 `related` key → `extra="forbid"` 422'd every done-time auto-save of grounded turns with a related tier, leaving `message_count=1` (root cause of `test_share_chat` 3F; browser-level instrumentation proved the PUT 422) — added the field + unit/integration pins; ② `test_theme_semantic_completion` pins stale vs phase-117 debox (border/chip removed) — re-targeted to assert border/chip *absence*; ③ `test_header_consistency` `<26`px pin red on 26.125px native date-input line — bound relaxed to `<34` (wrap-detection intent kept); ④ `test_navbar_refresh` bor.chat.v1 key set updated for `related`.
- **Test/lint/coverage:** `uv run pytest --cov=app --cov-report=term-missing` → **2506 passed, app/ 99%** (>90%); `uv run ruff check . && uv run pyright` → clean, 0 errors.
- **E2E:** new story suite in isolation → **2 passed**; full 103-suite matrix sweep (each isolated) → **all 103 green** after the fixes; `test_share_chat` 4 passed, `test_theme_semantic_completion` 8 passed, `test_header_consistency` 3 passed, `test_navbar_refresh` 7 passed.
- **Deviations:** none from LOCKED decisions. Note: orphaned diagnostic uvicorn processes briefly made E2E sessions exercise stale code — killed and re-verified; a sweep-regenerated tracked screenshot was restored. No commits made (harness commits).
- **Completion criteria:** all 7 ✅ (commit/phase-move is the harness's step).
- **Next pending phase:** none — `todo/` holds only this phase's overview pending the harness move.
This commit is contained in:
@@ -282,17 +282,28 @@ def test_chat_streams_deltas_then_done_with_sources(client, db, seeded_kb: FakeR
|
||||
assert done[0]["suggestions"] == []
|
||||
sources = done[0]["sources"]
|
||||
assert sources, "done must carry the cited sources"
|
||||
# Phase 118 (A4): the citation surface is the suggested tier (top-5,
|
||||
# no floor) + the agent's reads (none on this turn) — deduped.
|
||||
assert len(sources) == get_settings().suggested_docs
|
||||
assert sources[0]["path"] == "homelab/kubernetes.md"
|
||||
assert sources[0]["source"] == "docs"
|
||||
assert sources[0]["title"] == "Kubernetes Homelab Cluster"
|
||||
|
||||
# The LLM received the locked HIGH prompt with the FULL document text.
|
||||
# The LLM received the locked HIGH prompt — the ``<documents>`` block
|
||||
# seeds the document's stored SUMMARY (phase 118, LOCKED A6: summary
|
||||
# seeding re-revises the pre-phase full-text contract; the full text
|
||||
# reaches the context only through the capped ``read`` tool). The
|
||||
# summarizer's code-appended pointer line proves the summary block is
|
||||
# present; the doc's full body is no longer seeded.
|
||||
(system, user) = seeded_kb.seen_messages[0][0], seeded_kb.seen_messages[0][1]
|
||||
assert user["content"] == QUESTION
|
||||
assert "<relevance>HIGH</relevance>" in system["content"]
|
||||
assert "DEFLECT_MODE" not in system["content"]
|
||||
assert "<documents>" in system["content"]
|
||||
assert "Talos Linux" in system["content"] # full doc, not just the chunk
|
||||
section = system["content"].split("<documents>", 1)[1].split("</documents>", 1)[0]
|
||||
assert "Summary of" in section # the fake lite model's summary text
|
||||
assert "Source: docs/homelab/kubernetes.md" in section # code-appended pointer
|
||||
assert "Talos Linux" not in section # full doc no longer seeded (A6)
|
||||
assert "HONESTY GATE" in system["content"]
|
||||
|
||||
|
||||
@@ -382,7 +393,19 @@ def test_chat_writes_query_log_row(client, db, seeded_kb: FakeRagLLM) -> None:
|
||||
assert 1 <= row.chunk_hits <= total_chunks
|
||||
assert row.top_score > 0.0 # genuine token-overlap cosine, best hit
|
||||
assert row.top_score <= 1.0
|
||||
assert "docs/homelab/kubernetes.md" in row.sources
|
||||
# Phase 118 (LOCKED A3): the durable record is the FULL retrieval —
|
||||
# the suggested tier (ranks 1–5) + the related tier (ranks 6–7) +
|
||||
# the agent's reads (none on this turn), for this question.
|
||||
for path in (
|
||||
"docs/homelab/kubernetes.md", # rank 1
|
||||
"docs/homelab/templates/deploy.j2", # rank 2
|
||||
"docs/homelab/ssh/ssh_aliases.txt", # rank 3
|
||||
"docs/homelab/container_gitlab/gitlab.md", # rank 4
|
||||
"docs/deployments/new-service.md", # rank 5
|
||||
"docs/homelab/quadlet/cache.volume", # rank 6 (related)
|
||||
"docs/homelab/quadlet/compose.container", # rank 7 (related)
|
||||
):
|
||||
assert path in row.sources
|
||||
assert row.latency_ms >= 0
|
||||
# Why the gate answered (A8 revised): cosine over the threshold OR a
|
||||
# lexical hit. The mock-calibrated threshold (0.30, see tests/conftest.py)
|
||||
@@ -446,10 +469,12 @@ def test_off_topic_question_deflects_honestly(client, db, seeded_kb: FakeRagLLM)
|
||||
def test_done_frame_carries_related_tier_on_grounded_turn(
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
"""Phase 113 (LOCKED A4): a grounded turn's done frame carries the
|
||||
related tier — the ranked docs beyond the cited ceiling, capped at
|
||||
``related_max_docs`` (2), disjoint from the cited list. The durable
|
||||
record keeps the FULL retrieval (cited + related, LOCKED A3)."""
|
||||
"""Phase 118 (LOCKED A3/A4): a grounded turn's done frame carries the
|
||||
suggested tier in ``sources`` (top-5, no floor) and the related
|
||||
tier — the ranked docs from rank 6+ after the suggested set, capped
|
||||
at ``related_max_docs`` (2) — in ``related``, disjoint from the
|
||||
citation surface. The durable record keeps the FULL retrieval
|
||||
(suggested + related + read, LOCKED A3)."""
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
@@ -458,49 +483,54 @@ def test_done_frame_carries_related_tier_on_grounded_turn(
|
||||
|
||||
done = frames[-1]
|
||||
assert done["deflected"] is False
|
||||
sources = {(s["source"], s["path"]) for s in done["sources"]}
|
||||
# A4: the citation surface is the suggested tier (5, no read on this
|
||||
# turn) — ranks 1–5 for the Kubernetes question.
|
||||
sources = [(s["source"], s["path"]) for s in done["sources"]]
|
||||
assert len(sources) == get_settings().suggested_docs
|
||||
assert sources[0] == ("docs", "homelab/kubernetes.md")
|
||||
related = done["related"]
|
||||
assert related, "the 2nd-and-lower scored docs ride the related tier"
|
||||
# Rank 6–7 for the Kubernetes question (after the top-5 suggested
|
||||
# set), capped at related_max_docs.
|
||||
assert [(s["source"], s["path"]) for s in related] == [
|
||||
("docs", "homelab/quadlet/cache.volume"),
|
||||
("docs", "homelab/quadlet/compose.container"),
|
||||
]
|
||||
assert len(related) <= get_settings().related_max_docs
|
||||
# The related tier never overlaps the cited list (the dedupe is by
|
||||
# (source, path) — the same pattern as the cited docs).
|
||||
# The related tier never overlaps the citation surface (the dedupe is
|
||||
# by (source, path) — the same pattern as the cited docs).
|
||||
related_keys = {(s["source"], s["path"]) for s in related}
|
||||
assert sources.isdisjoint(related_keys)
|
||||
# Rank order: the cited top-2 are the kubernetes doc and the template;
|
||||
# the next ranked doc is the ssh aliases file.
|
||||
assert related[0]["path"] == "homelab/ssh/ssh_aliases.txt"
|
||||
assert set(sources).isdisjoint(related_keys)
|
||||
# Every ref carries the chip identity fields (the UI row reuses them).
|
||||
assert all(s["title"] for s in related)
|
||||
|
||||
# Durable record: the full retrieval (cited + related) is logged.
|
||||
# Durable record: the full retrieval (suggested + related) is logged.
|
||||
row = db.scalars(select(QueryLog)).one()
|
||||
assert "docs/homelab/ssh/ssh_aliases.txt" in row.sources
|
||||
assert "docs/homelab/quadlet/cache.volume" in row.sources
|
||||
assert "docs/homelab/kubernetes.md" in row.sources
|
||||
|
||||
|
||||
def test_deflected_done_frame_carries_weak_hits_in_related(
|
||||
client, db, seeded_kb: FakeRagLLM, monkeypatch: pytest.MonkeyPatch
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
"""Phase 113: on a deflected turn nothing clears the bar — the cited
|
||||
tier is empty (done.sources stays [], the phase-112 contract) and the
|
||||
weak hits fall to the related tier (their visibility home). The
|
||||
durable record still carries the retrieval (LOCKED A3)."""
|
||||
monkeypatch.setenv("BOR_SOURCE_USEFULNESS_FLOOR", "0.20")
|
||||
get_settings.cache_clear()
|
||||
"""Phase 118: on a deflected turn done.sources stays [] (the
|
||||
phase-112 contract — a deflected answer cites nothing) and
|
||||
done.related carries rank 6+ after the suggested set (capped at
|
||||
``related_max_docs``) — the weak hits' visibility home; the weak
|
||||
hits themselves are the suggested tier (no floor, A3). The durable
|
||||
record still carries the retrieval (LOCKED A3)."""
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: seeded_kb
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, OFF_TOPIC)
|
||||
done = frames[-1]
|
||||
assert done["deflected"] is True
|
||||
assert done["sources"] == [] # a deflected answer cites nothing
|
||||
# The weak hits (the sourdough question's best mock cosines are
|
||||
# ~0.11/0.04 — both below the 0.20 bar) ride the related tier, in
|
||||
# rank order, capped at related_max_docs.
|
||||
# Rank 6–7 for the sourdough question (after the top-5 suggested
|
||||
# set), capped at related_max_docs.
|
||||
related = done["related"]
|
||||
assert len(related) <= get_settings().related_max_docs
|
||||
assert [s["path"] for s in related][:2] == [
|
||||
"deployments/new-service.md",
|
||||
"homelab/quadlet/lan.network",
|
||||
assert [s["path"] for s in related] == [
|
||||
"homelab/backups.md",
|
||||
"homelab/container_gitlab/gitlab-compose.yaml",
|
||||
]
|
||||
assert all(s["title"] for s in related)
|
||||
assert done["suggestions"] # the "Maybe try" chips are unchanged
|
||||
@@ -510,19 +540,16 @@ def test_deflected_done_frame_carries_weak_hits_in_related(
|
||||
assert row.deflected is True
|
||||
assert row.sources # the weak-hit paths, for threshold tuning
|
||||
finally:
|
||||
# The cache clear is LAST — an assertion that calls get_settings()
|
||||
# after the clear would re-populate the lru_cache with the
|
||||
# monkeypatched value and leak it into the next test.
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_related_doc_read_by_agent_is_cited_not_related(
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
"""Phase 113 × phase 37: an agent-read doc is a citation by definition
|
||||
— when the agent ``read``s a doc that would otherwise ride the related
|
||||
tier, it joins done.sources (deduped, last) and is EXCLUDED from
|
||||
"""Phase 118 × phase 37: an agent-read doc is a citation by definition
|
||||
(LOCKED A4) — when the agent ``read``s a rank-6+ doc (the related
|
||||
tier, "nearby docs"), it joins done.sources (deduped, last — it was
|
||||
not suggested, so the read appends it) and is EXCLUDED from
|
||||
done.related (a "nearby doc" that was actually used must not read as
|
||||
nearby)."""
|
||||
scripted = FakeRagLLM(
|
||||
@@ -531,7 +558,7 @@ def test_related_doc_read_by_agent_is_cited_not_related(
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="read",
|
||||
arguments={"path": "docs/homelab/ssh/ssh_aliases.txt"},
|
||||
arguments={"path": "docs/homelab/quadlet/cache.volume"},
|
||||
)
|
||||
]
|
||||
]
|
||||
@@ -545,12 +572,14 @@ def test_related_doc_read_by_agent_is_cited_not_related(
|
||||
done = frames[-1]
|
||||
assert done["deflected"] is False
|
||||
sources = [(s["source"], s["path"]) for s in done["sources"]]
|
||||
assert sources[-1] == ("docs", "homelab/ssh/ssh_aliases.txt") # read ⇒ cited
|
||||
# A4: suggested (5) + the read doc (last).
|
||||
assert len(sources) == get_settings().suggested_docs + 1
|
||||
assert sources[-1] == ("docs", "homelab/quadlet/cache.volume") # read ⇒ cited
|
||||
related = [(s["source"], s["path"]) for s in done["related"]]
|
||||
assert ("docs", "homelab/ssh/ssh_aliases.txt") not in related
|
||||
assert ("docs", "homelab/quadlet/cache.volume") not in related
|
||||
assert set(sources).isdisjoint(set(related))
|
||||
# The OTHER related-tier doc (gitlab, rank 4) stays in the tier.
|
||||
assert ("docs", "homelab/container_gitlab/gitlab.md") in related
|
||||
# The OTHER related-tier doc (compose.container, rank 7) stays in the tier.
|
||||
assert ("docs", "homelab/quadlet/compose.container") in related
|
||||
|
||||
|
||||
def test_keyword_question_grounded_by_lexical_hit_despite_weak_cosine(
|
||||
@@ -1341,6 +1370,11 @@ def test_deflected_turn_stays_byte_identical_without_tools(
|
||||
direct-``chat_stream`` output even for a fake scripted to call tools
|
||||
(its script is never consumed). The LLM was called once, without a
|
||||
``tools`` key."""
|
||||
# The scripted read targets a doc OUTSIDE the OFF_TOPIC retrieval
|
||||
# top-7 (tables.md ranks 11th — not suggested, not rank 6+ related),
|
||||
# so "never read" stays distinguishable from "retrieved" in the
|
||||
# durable record below (phase 118: backups.md — the pre-phase read
|
||||
# target — now rides the rank-6+ related tier, durably recorded).
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[ToolCallPiece(id="call_1", name="ls", arguments={})],
|
||||
@@ -1348,7 +1382,7 @@ def test_deflected_turn_stays_byte_identical_without_tools(
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="read",
|
||||
arguments={"path": "docs/homelab/backups.md"},
|
||||
arguments={"path": "docs/homelab/tables.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "never used — the agent never runs")],
|
||||
@@ -1380,7 +1414,7 @@ def test_deflected_turn_stays_byte_identical_without_tools(
|
||||
if r.question == OFF_TOPIC
|
||||
][-1:]
|
||||
assert row.deflected is True
|
||||
assert "backups.md" not in row.sources
|
||||
assert "tables.md" not in row.sources
|
||||
|
||||
|
||||
def test_zero_max_rounds_reproduce_pre_phase_single_request(
|
||||
|
||||
Reference in New Issue
Block a user