feat(agent): align the document tools with the harness-trained shape — ls, read(path), grep(pattern, path?)

This commit is contained in:
2026-09-03 11:17:47 -04:00
parent 16f1cfbcaf
commit 801639efcc
55 changed files with 4031 additions and 1466 deletions
+839 -812
View File
File diff suppressed because it is too large Load Diff
+10 -1
View File
@@ -578,10 +578,16 @@ def test_endpoint_grounded_turn_runs_agent_loop_with_tools(
assert not any(f["type"] == "tool" for f in frames)
assert len(llm.seen) == 1
assert llm.seen_tools == [AGENT_TOOLS] # one request, tools offered
# The system prompt is the HIGH prompt with the <tools> instructions.
# The system prompt is the HIGH prompt with the <tools> instructions
# (phase 70: the harness-aligned ls/read/grep copy — new names in,
# old phase-37/68 names out).
(system, _user) = llm.seen[0][0], llm.seen[0][1]
assert "<relevance>HIGH</relevance>" in system["content"]
assert "<tools>" in system["content"]
for tool in ("`ls`", "`grep`", "`read`"):
assert tool in system["content"]
for old in ("list_documents", "read_document", "search_documents"):
assert old not in system["content"]
def test_endpoint_deflected_turn_never_offers_tools(
@@ -605,6 +611,9 @@ def test_endpoint_deflected_turn_never_offers_tools(
assert llm.seen_tools == [None]
(system, _user) = llm.seen[0][0], llm.seen[0][1]
assert "<tools>" not in system["content"] # the LOW prompt never carries it
# Phase 70: the rewritten <tools> copy stays out of the deflected path
# (the LOW prompt is byte-identical to the pre-phase text).
assert "You may extend your context with three tools" not in system["content"]
def test_endpoint_score_at_threshold_answers(
+97 -16
View File
@@ -6,7 +6,12 @@ suite (task 06). Like the other frontend-adjacent unit files, this module
pins the JS/CSS markers the story depends on, so a silent regression in
the tool branch, the persistence shape, or the tool-line styling is
catched without a browser. Phase 68 extends the pins with the
``search_documents`` status/line contract.
``search_documents`` status/line contract. Phase 70 extends the pins to
the harness-aligned names (``ls`` / ``read`` / ``grep``) in both
``app.js`` and the shared page's local copy (``shared.js``) — the legacy
names (``list_documents`` / ``read_document`` / ``search_documents``)
must keep rendering exactly as before for persisted turns (no
migration).
"""
from __future__ import annotations
@@ -15,6 +20,7 @@ from pathlib import Path
FRONTEND = Path(__file__).resolve().parents[2] / "frontend"
APP_JS = FRONTEND / "assets" / "app.js"
SHARED_JS = FRONTEND / "assets" / "shared.js"
STYLES_CSS = FRONTEND / "assets" / "styles.css"
@@ -22,6 +28,10 @@ def _js() -> str:
return APP_JS.read_text(encoding="utf-8")
def _shared_js() -> str:
return SHARED_JS.read_text(encoding="utf-8")
def _css() -> str:
return STYLES_CSS.read_text(encoding="utf-8")
@@ -66,7 +76,10 @@ def test_calling_tool_label_strings() -> None:
status lives in #send-status + the typing-indicator aria-label only.
Phase 39 centralizes the brand prefix: the name resolves from
window.BOR_BRAND at call time via brand() (the default name renders
the same bytes)."""
the same bytes). Phase 70: the ternary keys off the harness-aligned
names (read / grep / ls) and still carries the legacy names
(read_document / search_documents) — a pre-remap label stays
accurate."""
js = _js()
tool_idx = js.find('ev.type === "tool"')
delta_idx = js.find('ev.type === "delta"')
@@ -74,12 +87,19 @@ def test_calling_tool_label_strings() -> None:
assert "sendLabel" not in branch, "phase 48: the button keeps its Stop label"
assert "`${brand()} is listing documents`" in branch
assert "`${brand()} is reading ${argument}`" in branch
# Phase 68: the search status — locked name+argument gate, sitting
# BETWEEN the read branch and the listing fallback in the ternary.
assert "name === \"search_documents\" && argument" in branch, (
"the search status requires the name AND a string argument"
# Phase 70: the read status — new + legacy name, locked
# name+argument gate, first in the ternary.
assert 'name === "read" || name === "read_document") && argument' in branch, (
"the read status requires the name (new or legacy) AND a string argument"
)
# The search status — new + legacy name, sitting BETWEEN the read
# branch and the listing fallback in the ternary.
assert 'name === "grep" || name === "search_documents") && argument' in branch
assert "`${brand()} is searching for ${argument}`" in branch
# Phase 70: the scoped ls status mirrors the scoped tool line; the
# unscoped listing stays the final fallback.
assert 'name === "ls" && argument' in branch
assert "`${brand()} is listing documents in ${argument}`" in branch
read = branch.find("is reading")
search = branch.find("is searching for")
listing = branch.find("is listing documents")
@@ -120,21 +140,41 @@ def test_tool_lines_render_into_the_bubble_wrap() -> None:
assert "code.textContent = argument" in body, (
"the path is data — textContent, never innerHTML"
)
assert "name === \"read_document\" && argument" in body
# Phase 68: the search branch mirrors the read branch — the same
# name+argument gate, a <code> element, and the pattern through
# textContent (never markup); the listing stays the final else.
assert "name === \"search_documents\" && argument" in body
# Phase 70: the harness-aligned names key the branches, with the
# legacy names kept — a persisted turn from before the remap
# (read_document / search_documents / list_documents) renders
# unchanged (no migration).
assert '(name === "read" || name === "read_document") && argument' in body, (
"read (new) and read_document (legacy) both render the Reading line"
)
assert '(name === "grep" || name === "search_documents") && argument' in body, (
"grep (new) and search_documents (legacy) both render the Searching line"
)
assert 'line.textContent = "🔎 Searching for "' in body
search_part = body.split('name === "search_documents"', 1)[1]
assert 'document.createElement("code")' in search_part, (
grep_part = body.split('name === "grep"', 1)[1]
assert 'document.createElement("code")' in grep_part, (
"the pattern gets the same <code> treatment as the read path"
)
assert "code.textContent = argument" in search_part, (
assert "code.textContent = argument" in grep_part, (
"the pattern is data — textContent, never innerHTML"
)
assert 'line.textContent = "🔎 Listing documents"' in search_part, (
"the listing fallback remains the final else"
# Phase 70: the scoped ls line — the scope through textContent, and
# the unscoped "Listing documents" stays the final else (legacy
# list_documents, and a nameless/unknown frame, land there too).
assert 'name === "ls" && argument' in body
assert 'line.textContent = "🔎 Listing documents in "' in body
ls_part = body.split('name === "ls" && argument', 1)[1]
assert 'document.createElement("code")' in ls_part, (
"the scope gets the same <code> treatment as the read path"
)
assert "code.textContent = argument" in ls_part, (
"the scope is data — textContent, never innerHTML"
)
assert 'line.textContent = "🔎 Listing documents"' in ls_part, (
"the unscoped listing fallback remains the final else"
)
assert "innerHTML" not in body, (
"no HTML injection surface on tool lines — textContent only"
)
@@ -231,6 +271,47 @@ def test_tool_call_style_is_accent_and_contrast_safe() -> None:
)
def test_shared_page_tool_lines_cover_new_and_legacy_names() -> None:
"""Phase 70: the shared page's local copy (``addToolLines``) renders
the harness-aligned names — read → Reading, grep → Searching for,
ls → Listing documents, scoped ls → Listing documents in <scope> —
and keeps the legacy branches (read_document / search_documents), so
a conversation saved before the remap renders exactly as before (no
migration). Every argument through textContent; the lines carry no
innerHTML at all."""
js = _shared_js()
fn = js.find("function addToolLines")
assert fn != -1, "addToolLines must exist in shared.js"
body = js[fn : js.find("\n}\n", fn)]
assert '(t.name === "read" || t.name === "read_document") && argument' in body, (
"read (new) and read_document (legacy) both render the Reading line"
)
assert '(t.name === "grep" || t.name === "search_documents") && argument' in body, (
"grep (new) and search_documents (legacy) both render the Searching line"
)
assert 'line.textContent = "📄 Reading "' in body
assert 'line.textContent = "🔎 Searching for "' in body
assert 't.name === "ls" && argument' in body
assert 'line.textContent = "🔎 Listing documents in "' in body
ls_part = body.split('t.name === "ls" && argument', 1)[1]
assert 'document.createElement("code")' in ls_part, (
"the scope gets the same <code> treatment as the read path"
)
assert "code.textContent = argument" in ls_part, (
"the scope is data — textContent, never innerHTML"
)
assert 'line.textContent = "🔎 Listing documents"' in ls_part, (
"the unscoped listing fallback remains the final else (legacy"
" list_documents renders unchanged)"
)
assert body.count("code.textContent = argument") == 3, (
"all three argument-bearing lines (read / grep / ls) are textContent-only"
)
assert "innerHTML" not in body, (
"no HTML injection surface on shared tool lines — textContent only"
)
def test_no_cdn_added() -> None:
"""AGENTS.md rule 6: the tool state adds no external script/link."""
index = (FRONTEND / "index.html").read_text(encoding="utf-8")
+30 -32
View File
@@ -500,29 +500,27 @@ def test_chat_stream_llm_error_passes_through_unwrapped() -> None:
# ---------- tool-call streaming (phase 37, task 02) ----------
#: The agent's tool list (phase 37) — the exact wire shape AGENT_TOOLS will
#: pass through (the names are whatever the caller's tools list names).
#: The agent's tool list (phase 70: the harness-aligned surface) — the
#: exact wire shape AGENT_TOOLS passes through (the names are whatever
#: the caller's tools list names).
_AGENT_TOOLS: list[dict[str, Any]] = [
{
"type": "function",
"function": {
"name": "list_documents",
"name": "ls",
"description": "List the indexed documents.",
"parameters": {"type": "object", "properties": {}},
"parameters": {"type": "object", "properties": {}, "required": []},
},
},
{
"type": "function",
"function": {
"name": "read_document",
"name": "read",
"description": "Add one indexed document's full text to the context.",
"parameters": {
"type": "object",
"properties": {
"source": {"type": "string"},
"path": {"type": "string"},
},
"required": ["source", "path"],
"properties": {"path": {"type": "string"}},
"required": ["path"],
},
},
},
@@ -557,12 +555,12 @@ def test_chat_stream_accumulates_tool_call_across_chunk_partials() -> None:
_tool_call(
0,
id="call_abc",
name="read_document",
arguments='{"source": "Homelab", "pa',
name="read",
arguments='{"path": "Homelab/ku',
)
],
),
_chunk(None, tool_calls=[_tool_call(0, arguments='th": "kubernetes.md"}')]),
_chunk(None, tool_calls=[_tool_call(0, arguments='bernetes.md"}')]),
_chunk(None, finish_reason="tool_calls"),
]
)
@@ -572,8 +570,8 @@ def test_chat_stream_accumulates_tool_call_across_chunk_partials() -> None:
assert pieces == [
ToolCallPiece(
id="call_abc",
name="read_document",
arguments={"source": "Homelab", "path": "kubernetes.md"},
name="read",
arguments={"path": "Homelab/kubernetes.md"},
)
]
@@ -586,14 +584,14 @@ def test_chat_stream_two_tool_calls_yielded_in_index_order() -> None:
_chunk(
None,
tool_calls=[
_tool_call(1, id="call_b", name="read_document", arguments='{"sou')
_tool_call(1, id="call_b", name="read", arguments='{"pa')
],
),
_chunk(
None,
tool_calls=[
_tool_call(0, id="call_a", name="list_documents"),
_tool_call(1, arguments='rce": "Homelab", "path": "a.md"}')
_tool_call(0, id="call_a", name="ls"),
_tool_call(1, arguments='th": "Homelab/a.md"}')
],
),
_chunk(None, finish_reason="tool_calls"),
@@ -603,11 +601,11 @@ def test_chat_stream_two_tool_calls_yielded_in_index_order() -> None:
llm, [{"role": "user", "content": "q"}], _AGENT_TOOLS
)
assert pieces == [
ToolCallPiece(id="call_a", name="list_documents", arguments={}),
ToolCallPiece(id="call_a", name="ls", arguments={}),
ToolCallPiece(
id="call_b",
name="read_document",
arguments={"source": "Homelab", "path": "a.md"},
name="read",
arguments={"path": "Homelab/a.md"},
),
]
@@ -619,21 +617,21 @@ def test_chat_stream_tool_calls_yielded_at_stream_end_without_finish_reason() ->
[
_chunk(
None,
tool_calls=[_tool_call(0, id="call_z", name="list_documents")],
tool_calls=[_tool_call(0, id="call_z", name="ls")],
)
]
)
pieces = _collect_with_tools(
llm, [{"role": "user", "content": "q"}], _AGENT_TOOLS
)
assert pieces == [ToolCallPiece(id="call_z", name="list_documents", arguments={})]
assert pieces == [ToolCallPiece(id="call_z", name="ls", arguments={})]
def test_chat_stream_synthesizes_call_id_when_absent() -> None:
"""Wire never carried the call id ⇒ synthesized "call_<index>"."""
llm, _ = _make_stream_client(
[
_chunk(None, tool_calls=[_tool_call(2, name="read_document", arguments="{}")]),
_chunk(None, tool_calls=[_tool_call(2, name="read", arguments="{}")]),
_chunk(None, finish_reason="tool_calls"),
]
)
@@ -643,7 +641,7 @@ def test_chat_stream_synthesizes_call_id_when_absent() -> None:
assert pieces == [
ToolCallPiece(
id="call_2",
name="read_document",
name="read",
arguments={},
)
]
@@ -656,7 +654,7 @@ def test_chat_stream_null_arguments_become_empty_dict() -> None:
_chunk(
None,
tool_calls=[
_tool_call(0, id="call_n", name="list_documents", arguments="null")
_tool_call(0, id="call_n", name="ls", arguments="null")
],
),
_chunk(None, finish_reason="tool_calls"),
@@ -665,7 +663,7 @@ def test_chat_stream_null_arguments_become_empty_dict() -> None:
pieces = _collect_with_tools(
llm, [{"role": "user", "content": "q"}], _AGENT_TOOLS
)
assert pieces == [ToolCallPiece(id="call_n", name="list_documents", arguments={})]
assert pieces == [ToolCallPiece(id="call_n", name="ls", arguments={})]
def test_chat_stream_malformed_tool_arguments_raise_llm_error() -> None:
@@ -679,8 +677,8 @@ def test_chat_stream_malformed_tool_arguments_raise_llm_error() -> None:
_tool_call(
0,
id="call_x",
name="read_document",
arguments='{"source": "Homelab",',
name="read",
arguments='{"path": "Homelab",',
)
],
),
@@ -706,7 +704,7 @@ def test_chat_stream_non_object_tool_arguments_raise_llm_error() -> None:
_chunk(
None,
tool_calls=[
_tool_call(0, id="call_y", name="read_document", arguments='[1, 2]')
_tool_call(0, id="call_y", name="grep", arguments='[1, 2]')
],
),
_chunk(None, finish_reason="tool_calls"),
@@ -987,7 +985,7 @@ def test_retried_healthy_stream_is_untouched(
a healthy turn is byte-identical to the plain chat_stream."""
answer = [
StreamPiece("thinking", "hmm"),
ToolCallPiece(id="call_1", name="list_documents", arguments={}),
ToolCallPiece(id="call_1", name="ls", arguments={}),
StreamPiece("content", "Talos."),
]
client = _ScriptedClient([(answer, None)])
@@ -995,7 +993,7 @@ def test_retried_healthy_stream_is_untouched(
tools = [
{
"type": "function",
"function": {"name": "list_documents", "parameters": {}},
"function": {"name": "ls", "parameters": {}},
}
]
pieces = _collect_retried(
+2 -2
View File
@@ -39,7 +39,7 @@ def _chunk(content: str) -> SimpleNamespace:
def _tool_chunk() -> SimpleNamespace:
"""One chunk carrying a malformed-arguments tool call (index 0)."""
fn = SimpleNamespace(name="read_document", arguments='{"source": "Homelab",')
fn = SimpleNamespace(name="read", arguments='{"path": "Homelab",')
tc = SimpleNamespace(index=0, id="call_x", function=fn)
delta = SimpleNamespace(content=None, tool_calls=[tc])
return SimpleNamespace(choices=[SimpleNamespace(delta=delta)])
@@ -194,7 +194,7 @@ def test_llm_error_materialization_passes_through_and_closes() -> None:
async def drain() -> None:
async for _ in llm.chat_stream(
[{"role": "user", "content": "q"}],
tools=[{"type": "function", "function": {"name": "read_document"}}],
tools=[{"type": "function", "function": {"name": "read"}}],
):
pass
+7 -5
View File
@@ -32,10 +32,11 @@ from tests.e2e.mock_llm import (
SYSTEM_HIGH = "<relevance>HIGH</relevance>\n<documents>\n</documents>\n<tools>\n…\n</tools>"
SYSTEM_LOW = "<relevance>LOW</relevance>\n"
#: A minimal truthy ``tools`` parameter (the mock only checks presence).
TOOLS = [{"type": "function", "function": {"name": "list_documents"}}]
#: A minimal truthy ``tools`` parameter (the mock only checks presence;
#: the phase-70 harness-aligned names).
TOOLS = [{"type": "function", "function": {"name": "ls"}}]
#: The agent's ``list_documents`` output for a two-document KB
#: The agent's ``ls`` output for a two-document KB
#: (``app/rag/agent.py`` ``_execute_tool``): one
#: ``source: X | path: Y | title: Z`` line per document (phase 63: labeled,
#: unambiguous fields), ``(source, path)`` order.
@@ -103,7 +104,7 @@ def _body(
{
"id": f"call_{i}",
"type": "function",
"function": {"name": "list_documents", "arguments": "{}"},
"function": {"name": "ls", "arguments": "{}"},
}
],
}
@@ -274,7 +275,8 @@ SEARCH_USER = (
assert SEARCH_TRIGGER in SEARCH_USER.lower()
assert TOOLS_TRIGGER not in SEARCH_USER.lower()
#: The agent's ``search_documents`` result for the e2e fixture
#: The agent's ``grep`` result for the e2e fixture (phase 70 renamed
#: the phase-68 tool; the line format is unchanged)
#: (``app/rag/agent.py`` ``_execute_tool``): one ``source/path:LINE: text``
#: match line (the sentinel line, 200-char-capped server-side).
SEARCH_RESULT = (
+62
View File
@@ -138,6 +138,68 @@ def test_zero_note_prompt_is_byte_identical_to_pre_steering() -> None:
)
assert "<tuning>" not in build_high_prompt([doc])
assert "<tuning>" not in build_deflect_prompt([])
# Phase 70: the rewritten <tools> copy stays out of the LOW path —
# the byte-identical equality above already proves it; this names
# the contract (no <tools>, no new copy) on both empty/non-empty LOW
# builds.
for low in (build_deflect_prompt(["T1"]), build_deflect_prompt([])):
assert "<tools>" not in low
assert TOOLS_SECTION not in low
# ---------- <tools> section copy (phase 70: ls / read / grep) ----------
def test_tools_section_markers_and_new_tool_names() -> None:
"""Phase 70: the section keeps the ``<tools>``/``</tools>`` markers
the E2E mock keys on and teaches the harness-aligned tool names
(backticked, exactly as the ``AGENT_TOOLS`` schemas name them)."""
assert TOOLS_SECTION.startswith("<tools>\n")
assert TOOLS_SECTION.rstrip().endswith("</tools>")
for tool in ("`ls`", "`grep`", "`read`"):
assert tool in TOOLS_SECTION
def test_tools_section_teaches_the_harness_shapes() -> None:
"""Copy pins: ``ls``'s phase-63 catalog-line format (and its
optional one-source scope), ``grep``'s case-insensitive exact-string
locator contract (up to 20 ``source/path:line: text`` lines, a
locator not a context-adder), and ``read``'s combined
``source/path`` + full content."""
assert "source: X | path: Y | title: Z" in TOOLS_SECTION
assert "pass a source name as `path`" in TOOLS_SECTION
assert "case-insensitive" in TOOLS_SECTION
assert "up to 20" in TOOLS_SECTION
assert "source/path:line: text" in TOOLS_SECTION
assert "locator, not a context-adder" in TOOLS_SECTION
assert "combined `source/path`" in TOOLS_SECTION
assert "full content" in TOOLS_SECTION
assert "Answer as soon as you have what you need" in TOOLS_SECTION
def test_tools_section_old_names_and_budget_copy_gone() -> None:
"""The phase-37/68 tool names and the phase-37 per-tool budget line
(phase 45: the round cap is the bound — the prompt does not
re-state it) are out of the copy."""
for old in ("list_documents", "read_document", "search_documents"):
assert old not in TOOLS_SECTION
assert "more than one" not in TOOLS_SECTION
assert "extra document" not in TOOLS_SECTION
def test_high_prompt_still_ends_with_tools_section() -> None:
"""Mock keying intact: the HIGH prompt still ends with the
``<tools>`` section after ``</documents>``, now in the phase-70
copy — new names in, old names out."""
doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster")
prompt = build_high_prompt([doc])
assert TOOLS_SECTION in prompt
assert prompt.index("</documents>") < prompt.index("<tools>")
assert prompt.rstrip().endswith("</tools>")
for tool in ("`ls`", "`grep`", "`read`"):
assert tool in prompt
for old in ("list_documents", "read_document", "search_documents"):
assert old not in prompt
def test_relevance_placeholder_rejected_for_garbage() -> None:
+9 -8
View File
@@ -88,21 +88,22 @@ def test_tool_frame_serializes_exactly() -> None:
``{type: "tool", name: str, argument: str | null}`` — one per
model-requested document tool call, streamed ahead of the ``delta``
frames of the answer."""
frame = sse_event(ChatToolEvent(name="read_document", argument="S/p.md").model_dump())
assert frame == 'data: {"type": "tool", "name": "read_document", "argument": "S/p.md"}\n\n'
assert _payload(frame) == {"type": "tool", "name": "read_document", "argument": "S/p.md"}
frame = sse_event(ChatToolEvent(name="read", argument="S/p.md").model_dump())
assert frame == 'data: {"type": "tool", "name": "read", "argument": "S/p.md"}\n\n'
assert _payload(frame) == {"type": "tool", "name": "read", "argument": "S/p.md"}
def test_tool_frame_argument_is_null_for_parameterless_tools() -> None:
"""``list_documents`` takes no parameters, so its frame's ``argument``
serializes as JSON null (the client renders the name alone)."""
dumped = ChatToolEvent(name="list_documents").model_dump()
assert dumped == {"type": "tool", "name": "list_documents", "argument": None}
"""``ls`` (unscoped) carries no string argument, so its frame's
``argument`` serializes as JSON null (the client renders the name
alone)."""
dumped = ChatToolEvent(name="ls").model_dump()
assert dumped == {"type": "tool", "name": "ls", "argument": None}
assert _payload(sse_event(dumped))["argument"] is None
def test_tool_event_shape_is_type_name_argument_only() -> None:
dumped = ChatToolEvent(name="read_document", argument="S/p.md").model_dump()
dumped = ChatToolEvent(name="read", argument="S/p.md").model_dump()
assert set(dumped.keys()) == {"type", "name", "argument"}
assert dumped["type"] == "tool" # default — call sites never spell it out