feat(agent): strip raw tool-scaffolding from streamed answers — deterministic filter with one bounded recovery

This commit is contained in:
2026-09-03 13:39:15 -04:00
parent 801639efcc
commit 575d6c88d0
38 changed files with 2793 additions and 50 deletions
+77
View File
@@ -5,6 +5,11 @@ lite-generated KB outline): its own builder contract (empty → ``""``,
char budget + ``[…truncated…]`` marker, pathological budgets), its
placement between ``<relevance>`` and ``<tuning>`` in both modes, and
the byte-identical-when-absent convention (phase 15 precedent).
And the phase-71 deflection plain-text line (owner-permitted
2026-09-03): the LOW prompt = pre-phase text + exactly the one new
line; the ``DEFLECT_MODE`` marker-keying contract is unchanged and
the line never leaks into the HIGH prompt.
"""
from __future__ import annotations
@@ -129,11 +134,15 @@ def test_zero_note_prompt_is_byte_identical_to_pre_steering() -> None:
assert build_high_prompt([doc]) == (
_base("HIGH") + "\n<documents>\n" + block + "\n</documents>" + "\n" + TOOLS_SECTION
)
# Phase 71: the LOW prompt carries the owner-permitted plain-text
# line after the DEFLECT_MODE sentence (the marker-keying contract
# is unchanged — the E2E mock keys on the marker's presence).
assert build_deflect_prompt(["T1", "T2"]) == (
_base("LOW")
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
"your notes come to the question. They are titles only; do not pretend "
"they answer it. Use them to propose 2-3 alternative questions.\n"
"Reply in plain text only — you have no tools in this mode.\n"
+ "- T1\n- T2"
)
assert "<tuning>" not in build_high_prompt([doc])
@@ -299,11 +308,14 @@ def test_no_overview_prompt_is_byte_identical_to_pre_phase() -> None:
docs_block = "\n<documents>\n" + block + "\n</documents>" + "\n" + TOOLS_SECTION
high_plain = _base("HIGH") + docs_block
high_steered = _base("HIGH") + "\n" + build_steering_section(["be concise"]) + docs_block
# Phase 71: the owner-permitted plain-text line is part of the
# DEFLECT_MODE body in every LOW build (with or without steering).
low_plain = (
_base("LOW")
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
"your notes come to the question. They are titles only; do not pretend "
"they answer it. Use them to propose 2-3 alternative questions.\n"
"Reply in plain text only — you have no tools in this mode.\n"
+ "- T1\n- T2"
)
low_steered = (
@@ -313,6 +325,7 @@ def test_no_overview_prompt_is_byte_identical_to_pre_phase() -> None:
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
"your notes come to the question. They are titles only; do not pretend "
"they answer it. Use them to propose 2-3 alternative questions.\n"
"Reply in plain text only — you have no tools in this mode.\n"
+ "- T1\n- T2"
)
for kb in (None, "", " \n\t "):
@@ -399,3 +412,67 @@ def test_prompt_kb_section_over_settings_budget_capped_with_marker(
close = section.index("</knowledge_base>")
section = section[: close + len("</knowledge_base>")]
assert len(section) <= 200
# ---------- phase 71: the deflection plain-text line (prevention) ----------
#: The owner-permitted (2026-09-03) line appended to the ``DEFLECT_MODE``
#: body — the LOW prompt's only phase-71 change. The E2E mock keys on
#: the ``DEFLECT_MODE`` marker's *presence*, not the wording, so the
#: marker-keying contract is unchanged by the appended line.
PLAIN_TEXT_ONLY_LINE = "Reply in plain text only — you have no tools in this mode."
def _pre_phase71_low_body() -> str:
"""The ``DEFLECT_MODE`` body exactly as it was before phase 71."""
return (
"DEFLECT_MODE: retrieval was weak — the titles below are the closest "
"your notes come to the question. They are titles only; do not pretend "
"they answer it. Use them to propose 2-3 alternative questions.\n"
)
def test_low_prompt_is_pre_phase_plus_exactly_the_plain_text_line() -> None:
"""Diff pin: the LOW prompt = pre-phase text + exactly the one new
line, appended to the ``DEFLECT_MODE`` body; the weak-hit title list
follows exactly as before (and the line occurs exactly once)."""
prompt = build_deflect_prompt(["T1", "T2"])
assert prompt == (
_base("LOW")
+ "\n"
+ _pre_phase71_low_body()
+ PLAIN_TEXT_ONLY_LINE
+ "\n"
+ "- T1\n- T2"
)
assert prompt.count(PLAIN_TEXT_ONLY_LINE) == 1
assert prompt.endswith("- T1\n- T2") # the title list is untouched
def test_low_prompt_carries_the_line_and_keeps_the_mock_marker() -> None:
"""The new line is present in the LOW prompt (inside the
``DEFLECT_MODE`` body, after the marker) and the ``DEFLECT_MODE``
marker the E2E mock keys on stays put."""
for titles, tail in ((["T1"], "- T1"), ([], "(nothing close at all)")):
prompt = build_deflect_prompt(titles)
assert "DEFLECT_MODE" in prompt
assert PLAIN_TEXT_ONLY_LINE in prompt
assert prompt.index("DEFLECT_MODE") < prompt.index(PLAIN_TEXT_ONLY_LINE)
# The title list (or the no-titles fallback) follows the line
# exactly as before.
assert prompt.endswith(tail)
def test_plain_text_line_never_leaks_into_high_prompt() -> None:
"""The line is the LOW prompt's: every HIGH build (with/without
steering/overview) is unchanged and carries none of it."""
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
for notes, kb in (
(None, None),
(["be concise"], None),
(None, OVERVIEW),
(["be concise"], OVERVIEW),
):
high = build_high_prompt([doc], notes=notes, kb_overview=kb)
assert PLAIN_TEXT_ONLY_LINE not in high
assert "you have no tools" not in high