phase: 119_name_signal_read_chips
All verification complete. Final report: **Phase 119 final verification pass — all criteria verified, one stale pin fixed.** - Verified implementation of all 6 tasks: D1 component name-hit rule (`name_hit` flag, titles never matched, retired length tie-break), D2 `BOR_NAME_HIT_BONUS` (0.005 default, 0 = byte-identical kill switch, negative fails startup, selection-layer only, `eval_retrieval` `suggested:` line), D3 suggested-folder lines (after `SUGGEST_INTRO`, before first block), D4 cite-discipline `SUGGEST_INTRO` sentence (PERSONA/LOW/`TOOLS_SECTION` byte-pins intact), D5 `done.sources` = read docs only (frontend no-op on empty confirmed), D6 mock `repeat your folder map` echo + new suite + telemetry. - Battery (replica restored per skill, fingerprint docs=1000/chunks=8866 verified, `eval_retrieval --from-file tests/fixtures/retrieval_battery.txt` re-run): **GATE PASS** — gitea README #4 in suggested top-5, forgejo 5/5 (README #1), gateway README in top-5 (#4), qwen3.8-27b quadlets top-5, Mongolia HIGH/fts=5 unchanged. - New E2E in isolation: `4 passed` ×2 (deterministic). All 27 modified E2E suites in isolation: 26 green; **1 stale pin fixed** — `test_source_chip_quality.py` durable-record order pin pre-dated the D1 re-rank (`aliases` stem sub-component name-hits `ssh_aliases.txt`, deterministically lifting `backups.md` over `kubernetes.md`; probe-verified 0.016277 vs 0.016036, 4/4 stable) — re-pinned with the phase-119 rationale; suite green ×2. - Gates: `uv run pytest --cov=app --cov-report=term-missing` → **2547 passed, app coverage 99%** (>90%); `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors. - Completion criteria: 1 ✅ (battery, recorded), 2 ✅ (folder lines; block/LOW byte-identical pins green), 3 ✅ (read-only chips, zero-read chips nothing, related row + durable record untouched — unit+E2E agree), 4 ✅ (all green), 5 → commit/phase-move left to the harness per pass rules (nothing committed). - Deviations: battery output + real-model telemetry recorded in `.agents/reports/119_name_signal_read_chips/task06_battery_and_e2e.md` and `TOOL_CALLING_TESTING.md` §11 (task files in `complete/` are immutable to this pass); gateway canonical doc at #4 vs overview's #3 was already documented at task 06 (containment gate met). - Next pending phase: **none** — `todo/` holds only phase 119.
This commit is contained in:
+237
-14
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterator, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
@@ -22,9 +22,10 @@ from app.api import chat as chat_api
|
||||
from app.config import Settings
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Document, KbOverview, QueryLog
|
||||
from app.rag import agent as rag_agent
|
||||
from app.rag.agent import AGENT_TOOLS
|
||||
from app.rag.llm import StreamPiece
|
||||
from app.rag.prompts import build_deflect_prompt
|
||||
from app.rag.llm import StreamPiece, ToolCallPiece
|
||||
from app.rag.prompts import SUGGEST_INTRO, build_deflect_prompt
|
||||
from app.rag.retriever import RetrievedChunk, weak_hit_titles
|
||||
from app.rag.suggestions import MAX_SUGGESTIONS, derive_suggestions
|
||||
from tests.conftest import ADMIN_PASSWORD
|
||||
@@ -443,6 +444,51 @@ def test_plan_turn_high_seeds_top5_suggested_related_is_rank6plus() -> None:
|
||||
assert f"FULL_CONTENT_SENTINEL_{i}" not in plan.system_prompt
|
||||
|
||||
|
||||
def test_plan_turn_folder_lines_ride_the_high_prompt() -> None:
|
||||
"""Phase 119 (D3, LOCKED A4): plan_turn passes *folder_lines* through
|
||||
to the HIGH prompt — after ``SUGGEST_INTRO``, before the first
|
||||
``<document>`` block (each on its own line)."""
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
|
||||
lines = ["Homelab/: kubernetes-homelab-cluster.md (2 docs)"]
|
||||
plan = chat_api.plan_turn(
|
||||
[_chunk(doc, 0.90)], _settings(threshold=0.30), folder_lines=lines
|
||||
)
|
||||
assert plan.deflected is False
|
||||
prompt = plan.system_prompt
|
||||
i_open = prompt.index("<documents>")
|
||||
i_block = prompt.index("<document ")
|
||||
assert prompt[i_open:i_block] == f"<documents>\n{SUGGEST_INTRO}\n{lines[0]}\n\n"
|
||||
|
||||
|
||||
def test_plan_turn_folder_lines_default_keeps_phase_118_high_prompt() -> None:
|
||||
"""Omitted *folder_lines* (the default ``()``) ⇒ the HIGH prompt is
|
||||
the phase-118 shape, byte-identical."""
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
|
||||
plan = chat_api.plan_turn([_chunk(doc, 0.90)], _settings(threshold=0.30))
|
||||
assert plan.deflected is False
|
||||
prompt = plan.system_prompt
|
||||
i_open = prompt.index("<documents>")
|
||||
i_block = prompt.index("<document ")
|
||||
assert prompt[i_open:i_block] == f"<documents>\n{SUGGEST_INTRO}\n\n"
|
||||
|
||||
|
||||
def test_plan_turn_folder_lines_ignored_on_the_low_branch() -> None:
|
||||
"""The LOW (deflected) branch IGNORES *folder_lines* — the
|
||||
deflected prompt stays byte-identical (LOCKED A4)."""
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_CONTENT")
|
||||
low_with = chat_api.plan_turn(
|
||||
[_chunk(doc, 0.05, cosine=0.05)],
|
||||
_settings(threshold=0.30),
|
||||
folder_lines=["Homelab/: x.md"],
|
||||
)
|
||||
low_without = chat_api.plan_turn(
|
||||
[_chunk(doc, 0.05, cosine=0.05)], _settings(threshold=0.30)
|
||||
)
|
||||
assert low_with.deflected is True
|
||||
assert low_with.system_prompt == low_without.system_prompt
|
||||
assert "Homelab/: x.md" not in low_with.system_prompt
|
||||
|
||||
|
||||
def test_plan_turn_high_single_strong_doc_yields_one_suggested() -> None:
|
||||
"""The suggested cap is a CEILING, not a quota: one doc ⇒ one
|
||||
suggested doc, an empty related tier (nothing beyond rank 1)."""
|
||||
@@ -458,10 +504,11 @@ def test_plan_turn_high_single_strong_doc_yields_one_suggested() -> None:
|
||||
|
||||
def test_plan_turn_high_suggests_strong_and_weak_no_floor() -> None:
|
||||
"""The recurring incident under phase 118 (A3): the weak 2nd doc no
|
||||
longer loses a citation slot to a bar — the floor never filters, so
|
||||
BOTH docs are suggested (rank order) and ride the citation surface
|
||||
(A4); the HIGH prompt seeds both summaries (the A5 fallback carries
|
||||
the short fixture content whole)."""
|
||||
longer loses a seeding slot to a bar — the floor never filters, so
|
||||
BOTH docs are suggested (rank order) and ride the durable record
|
||||
(118-A3 — the phase-119 A1 citation surface is the read docs only);
|
||||
the HIGH prompt seeds both summaries (the A5 fallback carries the
|
||||
short fixture content whole)."""
|
||||
strong = _doc("Kubernetes Homelab Cluster", "STRONG_DOC_CONTENT")
|
||||
weak = _doc("Backup Strategy", "WEAK_DOC_CONTENT")
|
||||
chunks = [_chunk(strong, 0.90, cosine=0.80), _chunk(weak, 0.80, cosine=0.20)]
|
||||
@@ -845,15 +892,21 @@ def test_suggestions_empty_input_yields_fallback_only() -> None:
|
||||
class _CannedLLM:
|
||||
"""Records the messages it is given; streams a canned answer.
|
||||
|
||||
Never emits tool calls, so a grounded turn through the phase-37 agent
|
||||
loop ends after the single (tools-offered) request; *seen_tools*
|
||||
records each request's ``tools`` value for the phase-37 wiring pins.
|
||||
Without *read_paths* it never emits tool calls, so a grounded turn
|
||||
through the phase-37 agent loop ends after the single (tools-offered)
|
||||
request. With *read_paths*, the request whose conversation carries
|
||||
*i* tool results (i < len(read_paths)) emits ``read(read_paths[i])``
|
||||
(phase 119: the read-only done-sources pins drive multi-read turns),
|
||||
and once the list is exhausted the request streams the answer.
|
||||
*seen_tools* records each request's ``tools`` value for the phase-37
|
||||
wiring pins.
|
||||
"""
|
||||
|
||||
def __init__(self, answer: str = ANSWER) -> None:
|
||||
def __init__(self, answer: str = ANSWER, read_paths: Sequence[str] = ()) -> None:
|
||||
self.settings = Settings(_env_file=None) # pyright: ignore[reportCallIssue]
|
||||
self.embed_batches = 0
|
||||
self.answer = answer
|
||||
self.read_paths = list(read_paths)
|
||||
self.seen: list[list[dict[str, str]]] = []
|
||||
self.seen_tools: list[list[dict[str, Any]] | None] = []
|
||||
|
||||
@@ -868,6 +921,15 @@ class _CannedLLM:
|
||||
):
|
||||
self.seen.append(messages)
|
||||
self.seen_tools.append(tools)
|
||||
if tools is not None and self.read_paths:
|
||||
tool_results = sum(1 for m in messages if m.get("role") == "tool")
|
||||
if tool_results < len(self.read_paths):
|
||||
yield ToolCallPiece(
|
||||
id=f"call_{tool_results + 1}",
|
||||
name="read",
|
||||
arguments={"path": self.read_paths[tool_results]},
|
||||
)
|
||||
return
|
||||
for i in range(0, len(self.answer), 12):
|
||||
yield StreamPiece("content", self.answer[i : i + 12])
|
||||
|
||||
@@ -913,6 +975,15 @@ class _FakeSession:
|
||||
return KbOverview(id=1, content=self.kb_overview)
|
||||
return None
|
||||
|
||||
def execute(self, *args: Any, **kwargs: Any) -> list[Any]:
|
||||
"""Phase 119 (D3): the endpoint's suggested-folder lines run the
|
||||
``ls`` catalog fetchers (``_source_document_rows`` /
|
||||
``_source_folder_summaries``) on this session — the fake catalog
|
||||
is empty, so each suggested doc's line is its header alone
|
||||
(``Homelab/:``), keeping the prompt builds deterministic in
|
||||
these gate tests."""
|
||||
return []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _admin_signed_in(client: TestClient) -> None:
|
||||
@@ -972,6 +1043,16 @@ def _fake_retriever(chunks: list[RetrievedChunk]) -> Any:
|
||||
return retrieve
|
||||
|
||||
|
||||
def _find_docs(docs: dict[tuple[str, str], Document]) -> Any:
|
||||
"""A fake ``find_document`` resolving *docs* by ``(source, path)``
|
||||
(the agent module's hook — the fake sessions here have no catalog)."""
|
||||
|
||||
def find_document(_db: Any, source: str, path: str) -> Document | None:
|
||||
return docs.get((source, path))
|
||||
|
||||
return find_document
|
||||
|
||||
|
||||
def test_endpoint_just_below_threshold_deflects(
|
||||
client: TestClient,
|
||||
gate_env: tuple[_FakeSession, _CannedLLM],
|
||||
@@ -1073,11 +1154,18 @@ def test_endpoint_deflected_turn_never_offers_tools(
|
||||
assert "You may extend your context with three tools" not in system["content"]
|
||||
|
||||
|
||||
def test_endpoint_score_at_threshold_answers(
|
||||
def test_endpoint_score_at_threshold_answers_zero_read_grounded_turn_chips_nothing(
|
||||
client: TestClient,
|
||||
gate_env: tuple[_FakeSession, _CannedLLM],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 119 (LOCKED A1 — the phase-118 "always chips" pin
|
||||
re-targeted): a grounded turn on which the agent reads NOTHING (the
|
||||
summary-only fast path — the canned LLM never emits a tool call)
|
||||
chips nothing: ``done.sources`` is the READ DOCS only, so the
|
||||
never-read suggested doc appears NOWHERE in it (the explicit contrast
|
||||
against the retired A4 union). The durable record keeps the retrieval
|
||||
(118-A3 stands)."""
|
||||
session, llm = gate_env
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.30)]))
|
||||
@@ -1088,17 +1176,152 @@ def test_endpoint_score_at_threshold_answers(
|
||||
assert done["type"] == "done"
|
||||
assert done["deflected"] is False
|
||||
assert done["suggestions"] == []
|
||||
assert done["sources"] and done["sources"][0]["title"] == "Kubernetes Homelab Cluster"
|
||||
# A1: read docs only — nothing was read ⇒ no chips. The suggested
|
||||
# doc (rank 1) appears NOWHERE in done.sources (the retired A4 union
|
||||
# would have carried it).
|
||||
assert done["sources"] == []
|
||||
|
||||
(system, _user) = llm.seen[0][0], llm.seen[0][1]
|
||||
assert "<relevance>HIGH</relevance>" in system["content"]
|
||||
assert "DEFLECT_MODE" not in system["content"]
|
||||
assert "TALOS_DOC_SENT" in system["content"]
|
||||
|
||||
# The durable record still records the retrieval (118-A3 stands).
|
||||
(row,) = session.added
|
||||
assert isinstance(row, QueryLog)
|
||||
assert row.deflected is False
|
||||
assert row.top_score == pytest.approx(0.30)
|
||||
assert "kubernetes-homelab-cluster.md" in row.sources
|
||||
|
||||
|
||||
def test_endpoint_done_sources_are_the_read_docs_in_read_order(
|
||||
client: TestClient,
|
||||
gate_env: tuple[_FakeSession, _CannedLLM],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 119 (LOCKED A1): the done frame's ``sources`` is exactly
|
||||
the agent's READ documents — deduped by (source, path), READ order
|
||||
(not the suggested tier's rank order). Suggested docs that were
|
||||
never read appear NOWHERE in the frame (the contrast pin against
|
||||
the retired A4 union); the durable record keeps the full retrieval
|
||||
(118-A3 stands)."""
|
||||
session, _llm = gate_env
|
||||
a = _doc("Alpha", "ALPHA_DOC_CONTENT")
|
||||
b = _doc("Beta", "BETA_DOC_CONTENT")
|
||||
c = _doc("Gamma", "GAMMA_DOC_CONTENT")
|
||||
chunks = [_chunk(a, 0.90), _chunk(b, 0.80), _chunk(c, 0.70)]
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever(chunks))
|
||||
llm = _CannedLLM(read_paths=[f"Homelab/{b.path}", f"Homelab/{a.path}"])
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: llm
|
||||
monkeypatch.setattr(
|
||||
rag_agent,
|
||||
"find_document",
|
||||
_find_docs({(a.source, a.path): a, (b.source, b.path): b}),
|
||||
)
|
||||
|
||||
frames = _ask(client, "Which of these documents do I have?")
|
||||
|
||||
done = frames[-1]
|
||||
assert done["deflected"] is False
|
||||
# Both reads streamed as tool frames (the agent loop ran).
|
||||
reads = [f for f in frames if f["type"] == "tool"]
|
||||
assert [f["argument"] for f in reads] == [
|
||||
f"Homelab/{b.path}",
|
||||
f"Homelab/{a.path}",
|
||||
]
|
||||
# A1: read docs only, read order — B first (it was read first), A
|
||||
# second; the never-read suggested doc C is NOWHERE in the frame, and
|
||||
# the suggested tier's rank order (A, B, C) is NOT the chip order.
|
||||
assert [(s["source"], s["path"]) for s in done["sources"]] == [
|
||||
(b.source, b.path),
|
||||
(a.source, a.path),
|
||||
]
|
||||
assert not any(s["path"] == c.path for s in done["sources"])
|
||||
assert len(done["sources"]) == len({(s["source"], s["path"]) for s in done["sources"]})
|
||||
assert done["related"] == [] # nothing beyond rank 3 for 3 docs
|
||||
|
||||
# The durable record keeps the full retrieval (118-A3 stands).
|
||||
(row,) = session.added
|
||||
assert isinstance(row, QueryLog)
|
||||
for doc in (a, b, c):
|
||||
assert f"Homelab/{doc.path}" in row.sources
|
||||
|
||||
|
||||
def test_endpoint_read_related_doc_is_cited_not_related(
|
||||
client: TestClient,
|
||||
gate_env: tuple[_FakeSession, _CannedLLM],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 119 (A1) × phase 118 (A3): a related-tier doc (rank 6+) that
|
||||
the agent ``read``s is a CHIP (in ``done.sources``) and is EXCLUDED
|
||||
from ``done.related`` — a "nearby doc" that was actually used must
|
||||
not read as nearby (unchanged intent, keyed on read docs since
|
||||
phase 119). The suggested docs, never read, are absent from the
|
||||
frame; the other related doc stays in the tier."""
|
||||
session, _llm = gate_env
|
||||
docs = [_doc(f"Doc {i}", f"CONTENT_{i}") for i in range(7)]
|
||||
chunks = [
|
||||
_chunk(d, 0.9 - 0.1 * i, cosine=0.8 - 0.05 * i) for i, d in enumerate(docs)
|
||||
]
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever(chunks))
|
||||
read_doc = docs[5] # rank 6 — the related tier (rank 7 = docs[6])
|
||||
llm = _CannedLLM(read_paths=[f"Homelab/{read_doc.path}"])
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: llm
|
||||
monkeypatch.setattr(
|
||||
rag_agent,
|
||||
"find_document",
|
||||
_find_docs({(read_doc.source, read_doc.path): read_doc}),
|
||||
)
|
||||
|
||||
frames = _ask(client, "What does the sixth ranked document say?")
|
||||
|
||||
done = frames[-1]
|
||||
assert done["deflected"] is False
|
||||
# A1: read docs only — the read related doc is the ONLY chip (the
|
||||
# never-read suggested docs are nowhere in the frame).
|
||||
assert [(s["source"], s["path"]) for s in done["sources"]] == [
|
||||
(read_doc.source, read_doc.path)
|
||||
]
|
||||
# The read related doc is NOT "nearby"; the other related doc is.
|
||||
related = [(s["source"], s["path"]) for s in done["related"]]
|
||||
assert related == [(docs[6].source, docs[6].path)]
|
||||
assert not any((d.source, d.path) in related for d in docs[:5])
|
||||
|
||||
# The durable record keeps the full retrieval (118-A3 stands).
|
||||
(row,) = session.added
|
||||
assert isinstance(row, QueryLog)
|
||||
assert row.deflected is False
|
||||
for d in docs:
|
||||
assert f"Homelab/{d.path}" in row.sources
|
||||
|
||||
|
||||
def test_endpoint_suggested_folder_lines_reach_the_high_prompt(
|
||||
client: TestClient,
|
||||
gate_env: tuple[_FakeSession, _CannedLLM],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 119 (D3, LOCKED A4): the endpoint computes the
|
||||
suggested-folder lines BEFORE plan_turn (the same deterministic
|
||||
suggested walk, over the short-lived step session) and they ride
|
||||
the grounded HIGH prompt — after ``SUGGEST_INTRO``, before the
|
||||
first ``<document>`` block."""
|
||||
_session, llm = gate_env
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.30)]))
|
||||
line = "Homelab/: kubernetes-homelab-cluster.md (2 docs)"
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"suggested_folder_lines",
|
||||
lambda db, suggested: [line],
|
||||
)
|
||||
frames = _ask(client, "How is my Kubernetes cluster set up?")
|
||||
assert frames[-1]["deflected"] is False
|
||||
(system, _user) = llm.seen[0][0], llm.seen[0][1]
|
||||
i_open = system["content"].index("<documents>")
|
||||
i_block = system["content"].index("<document ")
|
||||
assert (
|
||||
system["content"][i_open:i_block]
|
||||
== f"<documents>\n{SUGGEST_INTRO}\n{line}\n\n"
|
||||
)
|
||||
|
||||
|
||||
# ---------- endpoint: KB overview row (phase 31) ----------
|
||||
|
||||
Reference in New Issue
Block a user