phase: 119_name_signal_read_chips
All verification complete. Final report: **Phase 119 final verification pass — all criteria verified, one stale pin fixed.** - Verified implementation of all 6 tasks: D1 component name-hit rule (`name_hit` flag, titles never matched, retired length tie-break), D2 `BOR_NAME_HIT_BONUS` (0.005 default, 0 = byte-identical kill switch, negative fails startup, selection-layer only, `eval_retrieval` `suggested:` line), D3 suggested-folder lines (after `SUGGEST_INTRO`, before first block), D4 cite-discipline `SUGGEST_INTRO` sentence (PERSONA/LOW/`TOOLS_SECTION` byte-pins intact), D5 `done.sources` = read docs only (frontend no-op on empty confirmed), D6 mock `repeat your folder map` echo + new suite + telemetry. - Battery (replica restored per skill, fingerprint docs=1000/chunks=8866 verified, `eval_retrieval --from-file tests/fixtures/retrieval_battery.txt` re-run): **GATE PASS** — gitea README #4 in suggested top-5, forgejo 5/5 (README #1), gateway README in top-5 (#4), qwen3.8-27b quadlets top-5, Mongolia HIGH/fts=5 unchanged. - New E2E in isolation: `4 passed` ×2 (deterministic). All 27 modified E2E suites in isolation: 26 green; **1 stale pin fixed** — `test_source_chip_quality.py` durable-record order pin pre-dated the D1 re-rank (`aliases` stem sub-component name-hits `ssh_aliases.txt`, deterministically lifting `backups.md` over `kubernetes.md`; probe-verified 0.016277 vs 0.016036, 4/4 stable) — re-pinned with the phase-119 rationale; suite green ×2. - Gates: `uv run pytest --cov=app --cov-report=term-missing` → **2547 passed, app coverage 99%** (>90%); `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors. - Completion criteria: 1 ✅ (battery, recorded), 2 ✅ (folder lines; block/LOW byte-identical pins green), 3 ✅ (read-only chips, zero-read chips nothing, related row + durable record untouched — unit+E2E agree), 4 ✅ (all green), 5 → commit/phase-move left to the harness per pass rules (nothing committed). - Deviations: battery output + real-model telemetry recorded in `.agents/reports/119_name_signal_read_chips/task06_battery_and_e2e.md` and `TOOL_CALLING_TESTING.md` §11 (task files in `complete/` are immutable to this pass); gateway canonical doc at #4 vs overview's #3 was already documented at task 06 (containment gate met). - Next pending phase: **none** — `todo/` holds only phase 119.
This commit is contained in:
@@ -24,6 +24,13 @@ document is refused — while the ``ls``/``grep`` clauses and the
|
||||
discipline rules stay byte-identical): the teaching refusals in
|
||||
:mod:`app.rag.agent` re-state the same contract; the ``<tools>``
|
||||
marker keying (HIGH only) is unchanged.
|
||||
|
||||
And the phase-119 cite discipline (task 04, LOCKED A5): the intro's
|
||||
final sentence ("cite the document(s) you used, by path", phase 118)
|
||||
is REPLACED — the HIGH prompt carries the discipline sentence exactly
|
||||
once, inside ``<documents>`` after the intro's start-here framing;
|
||||
the LOW prompt and ``PERSONA`` / ``TOOLS_SECTION`` stay byte-identical
|
||||
(the full sha pins live in :mod:`tests.unit.test_prompt_lock`).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -447,6 +454,155 @@ def test_documents_section_without_blocks_has_no_intro() -> None:
|
||||
)
|
||||
|
||||
|
||||
# ---------- phase 119 (D3, LOCKED A4): the suggested-folder lines ----------
|
||||
|
||||
|
||||
def test_high_prompt_folder_lines_after_intro_before_first_block() -> None:
|
||||
"""The folder lines ride the ``<documents>`` section immediately
|
||||
AFTER the ``SUGGEST_INTRO`` line — each on its own line — then a
|
||||
blank line, then the first ``<document>`` block. Plain lines: no
|
||||
new markup/tag anywhere (the E2E mock keys off the ``<documents>``
|
||||
marker and the LAST block's tail)."""
|
||||
doc = _doc(
|
||||
"deploy/Deployments/reeseapps/gitea/app/gitea-web.env.j2",
|
||||
"FULL_CONTENT_SENTINEL_119",
|
||||
"Gitea Web Env",
|
||||
summary="Gitea web env file.",
|
||||
)
|
||||
lines = [
|
||||
"Homelab/deploy/Deployments/reeseapps/gitea/: app/ (5 docs), README.md",
|
||||
"Homelab/deploy/Deployments/reeseapps/: gateway/ (3 docs), gitea/ (8 docs)",
|
||||
]
|
||||
prompt = build_high_prompt([doc], folder_lines=lines)
|
||||
i_open = prompt.index("<documents>")
|
||||
i_block = prompt.index("<document ")
|
||||
assert prompt[i_open:i_block] == f"<documents>\n{SUGGEST_INTRO}\n{lines[0]}\n{lines[1]}\n\n"
|
||||
# Each line occurs exactly once, inside the section, before the
|
||||
# first block (never after a summary — the E2E tail echo is safe).
|
||||
for line in lines:
|
||||
assert prompt.count(line) == 1
|
||||
assert prompt.index("<documents>") < prompt.index(line) < prompt.index("<document ")
|
||||
assert prompt.index(line) < prompt.index("</documents>")
|
||||
# The <document> block markup AND body stay byte-identical: from the
|
||||
# first block on, the prompt equals the no-folder-line build.
|
||||
plain = build_high_prompt([doc])
|
||||
assert prompt[prompt.index("<document ") :] == plain[plain.index("<document ") :]
|
||||
# No new markup: the section still opens/closes exactly once (the
|
||||
# ``<documents>`` MENTION in the TOOLS_SECTION copy is plain text —
|
||||
# count the tag + newline, not the bare substring).
|
||||
assert prompt.count("<documents>\n") == 1
|
||||
assert prompt.count("</documents>") == 1
|
||||
assert prompt.count("<document ") == 1
|
||||
|
||||
|
||||
def test_high_prompt_single_folder_line_exact_shape() -> None:
|
||||
"""One folder line: the exact slice between the section open and the
|
||||
first block is ``<documents>\n`` + intro + ``\n`` + line +
|
||||
``\n\n``."""
|
||||
doc = _doc("a.md", "CONTENT", "Title A", summary="Summary A.")
|
||||
line = "Homelab/: a.md (2 docs), b/ (1 doc)"
|
||||
prompt = build_high_prompt([doc], folder_lines=[line])
|
||||
i_open = prompt.index("<documents>")
|
||||
i_block = prompt.index("<document ")
|
||||
assert prompt[i_open:i_block] == f"<documents>\n{SUGGEST_INTRO}\n{line}\n\n"
|
||||
|
||||
|
||||
def test_high_prompt_empty_folder_lines_byte_identical_to_phase_118() -> None:
|
||||
"""LOCKED A4: empty ``folder_lines`` (the default) ⇒ the phase-118
|
||||
build, byte-identical — the existing pins keep passing and this
|
||||
pins the default explicitly (both an explicit ``()`` and the
|
||||
omitted parameter)."""
|
||||
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
|
||||
block = (
|
||||
'<document source="Homelab" path="kubernetes.md" '
|
||||
'title="Kubernetes Homelab Cluster" date="2024-06-15">\n'
|
||||
"Talos Linux on three nodes.\n"
|
||||
"</document>"
|
||||
)
|
||||
expected = (
|
||||
_base("HIGH")
|
||||
+ "\n<documents>\n"
|
||||
+ SUGGEST_INTRO
|
||||
+ "\n\n"
|
||||
+ block
|
||||
+ "\n</documents>"
|
||||
+ "\n"
|
||||
+ TOOLS_SECTION
|
||||
)
|
||||
# An explicit empty tuple AND the omitted parameter (the default):
|
||||
assert build_high_prompt([doc], folder_lines=()) == expected
|
||||
assert build_high_prompt([doc]) == expected
|
||||
# The LOW (deflected) prompt has no folder_lines parameter at all —
|
||||
# the deflected build stays byte-identical (the phase-118 pin in
|
||||
# test_zero_note_prompt_is_byte_identical_to_pre_steering stands).
|
||||
|
||||
|
||||
def test_folder_lines_ride_only_on_present_blocks() -> None:
|
||||
"""Like the intro, the folder lines ride on present blocks ONLY: an
|
||||
empty ``<documents>`` section is unchanged (no lines, no blocks)."""
|
||||
prompt = build_high_prompt([], folder_lines=["X/: y.md"])
|
||||
assert prompt == build_high_prompt([])
|
||||
assert "X/: y.md" not in prompt
|
||||
|
||||
|
||||
# ---------- phase 119 (D4, LOCKED A5): the cite-discipline sentence ----------
|
||||
|
||||
#: The LOCKED A5 sentence (phase 119, task 04) — the exact replacement
|
||||
#: for the retired phase-118 final sentence of :data:`SUGGEST_INTRO`
|
||||
#: ("Cite the document(s) you used, by path."). Closes the live
|
||||
#: confabulation: the answer's "Docs used:" line cited a file the
|
||||
#: agent never read.
|
||||
CITE_DISCIPLINE = (
|
||||
"Cite only the document(s) you read — or, if you answered from a "
|
||||
"suggested summary without reading it, cite that suggested "
|
||||
"document — never a document you neither read nor used."
|
||||
)
|
||||
|
||||
|
||||
def test_high_prompt_carries_cite_discipline_exactly_once_in_documents() -> None:
|
||||
"""LOCKED A5: the HIGH prompt carries the discipline sentence
|
||||
EXACTLY ONCE — it IS the intro's final sentence, so it sits inside
|
||||
``<documents>``, AFTER the intro's start-here framing and BEFORE
|
||||
the first block (both the plain build and the folder-line build;
|
||||
the sentence rides the intro line, which never moves)."""
|
||||
doc = _doc(
|
||||
"kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster", summary="K8S"
|
||||
)
|
||||
assert CITE_DISCIPLINE in SUGGEST_INTRO
|
||||
assert SUGGEST_INTRO.endswith(CITE_DISCIPLINE) # it is the final sentence
|
||||
for high in (build_high_prompt([doc]), build_high_prompt([doc], folder_lines=["X/: y.md"])):
|
||||
assert high.count(CITE_DISCIPLINE) == 1
|
||||
i_docs = high.index("<documents>")
|
||||
i_sentence = high.index(CITE_DISCIPLINE)
|
||||
i_block = high.index("<document ")
|
||||
i_close = high.index("</documents>")
|
||||
assert i_docs < i_sentence < i_close
|
||||
assert i_sentence > high.index("start here if one seems right")
|
||||
assert i_sentence < i_block # before the first block, never after a summary
|
||||
# The retired phase-118 sentence is gone from the prompt (it is
|
||||
# gone from the constant — pinned in test_prompt_lock too).
|
||||
high = build_high_prompt([doc])
|
||||
assert "Cite the document(s) you used, by path." not in high
|
||||
# The sentence rides on present blocks ONLY: an empty ``<documents>``
|
||||
# section (no intro) carries none of it.
|
||||
assert CITE_DISCIPLINE not in build_high_prompt([])
|
||||
|
||||
|
||||
def test_cite_discipline_absent_from_low_prompt() -> None:
|
||||
"""LOCKED A5: the discipline sentence belongs to ``SUGGEST_INTRO``
|
||||
(the HIGH path) — it never leaks into the LOW (deflected) prompt,
|
||||
whose byte-identity is pinned separately (the LOW anchors in
|
||||
``test_prompt_lock`` pass unchanged)."""
|
||||
for prompt in (
|
||||
build_deflect_prompt(["T1", "T2"]),
|
||||
build_deflect_prompt(["T1"], notes=["be concise"], kb_overview=OVERVIEW),
|
||||
build_deflect_prompt([]),
|
||||
):
|
||||
assert CITE_DISCIPLINE not in prompt
|
||||
assert "Cite only the document(s) you read" not in prompt
|
||||
assert "Cite the document(s) you used, by path." not in prompt
|
||||
|
||||
|
||||
def test_tools_section_old_names_and_budget_copy_gone() -> None:
|
||||
"""The phase-37/68 tool names and the phase-37 per-tool budget line
|
||||
(phase 45: the round cap is the bound — the prompt does not
|
||||
|
||||
Reference in New Issue
Block a user