feat(agent): strip raw tool-scaffolding from streamed answers — deterministic filter with one bounded recovery
This commit is contained in:
@@ -5,6 +5,11 @@ lite-generated KB outline): its own builder contract (empty → ``""``,
|
||||
char budget + ``[…truncated…]`` marker, pathological budgets), its
|
||||
placement between ``<relevance>`` and ``<tuning>`` in both modes, and
|
||||
the byte-identical-when-absent convention (phase 15 precedent).
|
||||
|
||||
And the phase-71 deflection plain-text line (owner-permitted
|
||||
2026-09-03): the LOW prompt = pre-phase text + exactly the one new
|
||||
line; the ``DEFLECT_MODE`` marker-keying contract is unchanged and
|
||||
the line never leaks into the HIGH prompt.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -129,11 +134,15 @@ def test_zero_note_prompt_is_byte_identical_to_pre_steering() -> None:
|
||||
assert build_high_prompt([doc]) == (
|
||||
_base("HIGH") + "\n<documents>\n" + block + "\n</documents>" + "\n" + TOOLS_SECTION
|
||||
)
|
||||
# Phase 71: the LOW prompt carries the owner-permitted plain-text
|
||||
# line after the DEFLECT_MODE sentence (the marker-keying contract
|
||||
# is unchanged — the E2E mock keys on the marker's presence).
|
||||
assert build_deflect_prompt(["T1", "T2"]) == (
|
||||
_base("LOW")
|
||||
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
||||
"your notes come to the question. They are titles only; do not pretend "
|
||||
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
||||
"Reply in plain text only — you have no tools in this mode.\n"
|
||||
+ "- T1\n- T2"
|
||||
)
|
||||
assert "<tuning>" not in build_high_prompt([doc])
|
||||
@@ -299,11 +308,14 @@ def test_no_overview_prompt_is_byte_identical_to_pre_phase() -> None:
|
||||
docs_block = "\n<documents>\n" + block + "\n</documents>" + "\n" + TOOLS_SECTION
|
||||
high_plain = _base("HIGH") + docs_block
|
||||
high_steered = _base("HIGH") + "\n" + build_steering_section(["be concise"]) + docs_block
|
||||
# Phase 71: the owner-permitted plain-text line is part of the
|
||||
# DEFLECT_MODE body in every LOW build (with or without steering).
|
||||
low_plain = (
|
||||
_base("LOW")
|
||||
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
||||
"your notes come to the question. They are titles only; do not pretend "
|
||||
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
||||
"Reply in plain text only — you have no tools in this mode.\n"
|
||||
+ "- T1\n- T2"
|
||||
)
|
||||
low_steered = (
|
||||
@@ -313,6 +325,7 @@ def test_no_overview_prompt_is_byte_identical_to_pre_phase() -> None:
|
||||
+ "\nDEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
||||
"your notes come to the question. They are titles only; do not pretend "
|
||||
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
||||
"Reply in plain text only — you have no tools in this mode.\n"
|
||||
+ "- T1\n- T2"
|
||||
)
|
||||
for kb in (None, "", " \n\t "):
|
||||
@@ -399,3 +412,67 @@ def test_prompt_kb_section_over_settings_budget_capped_with_marker(
|
||||
close = section.index("</knowledge_base>")
|
||||
section = section[: close + len("</knowledge_base>")]
|
||||
assert len(section) <= 200
|
||||
|
||||
|
||||
# ---------- phase 71: the deflection plain-text line (prevention) ----------
|
||||
|
||||
#: The owner-permitted (2026-09-03) line appended to the ``DEFLECT_MODE``
|
||||
#: body — the LOW prompt's only phase-71 change. The E2E mock keys on
|
||||
#: the ``DEFLECT_MODE`` marker's *presence*, not the wording, so the
|
||||
#: marker-keying contract is unchanged by the appended line.
|
||||
PLAIN_TEXT_ONLY_LINE = "Reply in plain text only — you have no tools in this mode."
|
||||
|
||||
|
||||
def _pre_phase71_low_body() -> str:
|
||||
"""The ``DEFLECT_MODE`` body exactly as it was before phase 71."""
|
||||
return (
|
||||
"DEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
||||
"your notes come to the question. They are titles only; do not pretend "
|
||||
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
||||
)
|
||||
|
||||
|
||||
def test_low_prompt_is_pre_phase_plus_exactly_the_plain_text_line() -> None:
|
||||
"""Diff pin: the LOW prompt = pre-phase text + exactly the one new
|
||||
line, appended to the ``DEFLECT_MODE`` body; the weak-hit title list
|
||||
follows exactly as before (and the line occurs exactly once)."""
|
||||
prompt = build_deflect_prompt(["T1", "T2"])
|
||||
assert prompt == (
|
||||
_base("LOW")
|
||||
+ "\n"
|
||||
+ _pre_phase71_low_body()
|
||||
+ PLAIN_TEXT_ONLY_LINE
|
||||
+ "\n"
|
||||
+ "- T1\n- T2"
|
||||
)
|
||||
assert prompt.count(PLAIN_TEXT_ONLY_LINE) == 1
|
||||
assert prompt.endswith("- T1\n- T2") # the title list is untouched
|
||||
|
||||
|
||||
def test_low_prompt_carries_the_line_and_keeps_the_mock_marker() -> None:
|
||||
"""The new line is present in the LOW prompt (inside the
|
||||
``DEFLECT_MODE`` body, after the marker) and the ``DEFLECT_MODE``
|
||||
marker the E2E mock keys on stays put."""
|
||||
for titles, tail in ((["T1"], "- T1"), ([], "(nothing close at all)")):
|
||||
prompt = build_deflect_prompt(titles)
|
||||
assert "DEFLECT_MODE" in prompt
|
||||
assert PLAIN_TEXT_ONLY_LINE in prompt
|
||||
assert prompt.index("DEFLECT_MODE") < prompt.index(PLAIN_TEXT_ONLY_LINE)
|
||||
# The title list (or the no-titles fallback) follows the line
|
||||
# exactly as before.
|
||||
assert prompt.endswith(tail)
|
||||
|
||||
|
||||
def test_plain_text_line_never_leaks_into_high_prompt() -> None:
|
||||
"""The line is the LOW prompt's: every HIGH build (with/without
|
||||
steering/overview) is unchanged and carries none of it."""
|
||||
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
|
||||
for notes, kb in (
|
||||
(None, None),
|
||||
(["be concise"], None),
|
||||
(None, OVERVIEW),
|
||||
(["be concise"], OVERVIEW),
|
||||
):
|
||||
high = build_high_prompt([doc], notes=notes, kb_overview=kb)
|
||||
assert PLAIN_TEXT_ONLY_LINE not in high
|
||||
assert "you have no tools" not in high
|
||||
|
||||
Reference in New Issue
Block a user