fix(agent): teach the document-identity contract on ls/read/grep refusals — end the post-harness tool-loop rambling
Phase 72 (72_teaching_refusals) — completed under the 2026-09-04 controlled methodology (owner directive: stop clearing/re-importing the homelab KB per iteration; measure tool-calling accuracy on a controlled fixture KB, target >90%). Real-model gate verdicts (live, configured chat model 'lite', fixture KB): - Controlled fixture battery (the new methodology's pass condition — contract accuracy >= 90%): PASS, 4 consecutive runs: gate: lite PASS turns=10 answered=10 caps=0 tool-turns=10 calls 8/11 executed (73%) contract 11/11 (100%) 2026-09-04 (wall 43.4s) gate: lite PASS turns=10 answered=10 caps=0 tool-turns=10 calls 8/13 executed (62%) contract 12/13 (92%) 2026-09-04 (wall 50.6s) gate: lite PASS turns=10 answered=10 caps=0 tool-turns=10 calls 7/11 executed (64%) contract 11/11 (100%) 2026-09-04 (wall 46.8s) gate: lite PASS turns=10 answered=10 caps=0 tool-turns=10 calls 9/15 executed (60%) contract 14/15 (93%) 2026-09-04 (wall 54.8s) - Locked derived battery (phase-72 task 05, executed >= 90% bar, run unchanged on the same fixture KB): gate: lite FAIL turns=10 answered=10 caps=0 tool-turns=10 calls 5/15 executed (33%) contract 12/15 (80%) 2026-09-04 (wall 47.7s) The teaching works — every bare-path trap self-corrects in exactly one round, zero cap hits, zero repeat loops, 10/10 answered. The locked executed bar is blocked by ALREADY_IN_CONTEXT dedupe refusals on the corrected re-reads (the trap question seeds its target, so the correct combined-form read is refused for redundancy) — a copy-invariant model behavior (five copy variants, 0/15 re-reads flipped, 2026-09-03 -> 04) and an app-semantics decision for the owner (TOOL_CALLING_TESTING.md sections 5 and 7), not a copy lever. Copy changes this phase owns (unit pins updated to follow): - app/rag/agent.py: ls teaching refusals (path-like scope -> document-path line; unknown source -> no-source line with the source-name parenthetical), read/grep 'did you mean source/path?' teaching (find_path_candidates: exact or suffix path match, catalog order, cap 3), ALREADY_IN_CONTEXT naming the correct action (answer from the text already in the prompt), read tool description front-loaded with the do-not-read rule (the 2026-09-04 controlled telemetry: the re-read is the only remaining refusal class; contract accuracy 92-100% across runs) - app/rag/prompts.py: TOOLS_SECTION states the document-identity contract up front (ls path = source name; read/grep = combined source/path including the source name; do-not-read for <documents> documents placed next to the read teaching; one-call-per-reply and never-repeat rules) - tests: refusal pins (unit + integration), new dedicated E2E suite tests/e2e/test_tool_path_teaching.py (mock misuse flow, green in isolation), regression suites green in isolation (harness_aligned_tools, agent_document_tools, agent_unlimited_tools, search_tool, chat_rag). Gates: uv run pytest green (1501); coverage TOTAL 99% (>90%); ruff + pyright clean. Carries the still-uncommitted phase-71 todo/ -> complete/ move and both phases' .agent/reports/ (AGENTS.md 8).
This commit is contained in:
@@ -185,22 +185,48 @@ def _settings(**kwargs: Any) -> Settings:
|
||||
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
class ScriptedToolCallsLLM:
|
||||
"""N scripted tool-call rounds (one ``ToolCallPiece`` each), then one
|
||||
canned answer; records every request's messages and tools (the
|
||||
phase-72 task-02 two-round self-correction cases: the refusal round,
|
||||
then the corrected call)."""
|
||||
|
||||
def __init__(self, calls: list[ToolCallPiece]) -> None:
|
||||
self.calls = calls
|
||||
self.requests: list[
|
||||
tuple[list[dict[str, Any]], list[dict[str, Any]] | None]
|
||||
] = []
|
||||
|
||||
async def chat_stream(
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
scaffolding: ScaffoldingFilter | None = None,
|
||||
) -> AsyncIterator[StreamPiece | ToolCallPiece]:
|
||||
self.requests.append((deepcopy(messages), deepcopy(tools)))
|
||||
index = len(self.requests) - 1
|
||||
if index < len(self.calls):
|
||||
yield self.calls[index]
|
||||
else:
|
||||
yield StreamPiece("content", "ans")
|
||||
|
||||
|
||||
def _run_call(
|
||||
db: Session, name: str, arguments: dict[str, Any]
|
||||
) -> tuple[AgentHolder, ScriptedToolLLM]:
|
||||
"""Drive one scripted tool call through ``run_agent``."""
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedToolLLM(ToolCallPiece(id="call_1", name=name, arguments=arguments))
|
||||
asyncio.run(_consume(llm, db, holder))
|
||||
asyncio.run(_consume(cast("LLMClient", llm), db, holder))
|
||||
return holder, llm
|
||||
|
||||
|
||||
async def _consume(
|
||||
llm: ScriptedToolLLM, db: Session, holder: AgentHolder
|
||||
llm: LLMClient, db: Session, holder: AgentHolder
|
||||
) -> list[StreamPiece | ToolCallPiece | RetryPiece]:
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece] = []
|
||||
async for piece in run_agent(
|
||||
cast("LLMClient", llm),
|
||||
llm,
|
||||
db,
|
||||
system_prompt="SYSTEM_PROMPT",
|
||||
user_message="QUESTION",
|
||||
@@ -233,17 +259,39 @@ def test_ls_scoped_to_registered_source_through_run_agent(kb, src, db) -> None:
|
||||
|
||||
|
||||
def test_ls_scoped_unknown_source_refused_through_run_agent(kb, src, db) -> None:
|
||||
"""Phase 72: the no-source refusal now carries the teaching
|
||||
parenthetical — the prefix byte-identical to the pre-phase-72 line;
|
||||
still not counted."""
|
||||
_doc(db, "Homelab", "a.md", "A", "A-CONTENT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Ghost"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"] == "No source named 'Ghost' — check the ls output."
|
||||
llm.requests[1][0][3]["content"]
|
||||
== agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="Ghost")
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
|
||||
def test_ls_path_like_scope_teaching_refusal_through_run_agent(kb, src, db) -> None:
|
||||
"""Phase 72: a ``/``-containing ``path`` is a document path, not a
|
||||
source name — the ``LS_PATH_NOT_A_SOURCE`` teaching line (no
|
||||
registry lookup needed), not counted, the tools stay offered on the
|
||||
next request."""
|
||||
_doc(db, "Homelab", "a.md", "A", "A-CONTENT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "app/rag/importer.py"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== agent.LS_PATH_NOT_A_SOURCE.format(path="app/rag/importer.py")
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
|
||||
# ---------- read (the canonical combined source/path form) ----------
|
||||
|
||||
|
||||
@@ -280,9 +328,10 @@ def test_read_bare_source_name_refused_through_run_agent(kb, db) -> None:
|
||||
|
||||
|
||||
def test_read_unknown_combined_path_refused_through_run_agent(kb, db) -> None:
|
||||
"""A combined identity that matches nothing gets the no-document
|
||||
refusal (the argument echoed as passed — the model sees its own
|
||||
form)."""
|
||||
"""A combined identity that matches NOTHING — not a document and not
|
||||
any indexed document's ``path`` (zero candidates) — gets today's
|
||||
no-document refusal byte-identical (the argument echoed as passed —
|
||||
the model sees its own form)."""
|
||||
_doc(db, "Alpha", "x.md", "X", "X-CONTENT")
|
||||
db.commit()
|
||||
|
||||
@@ -295,6 +344,79 @@ def test_read_unknown_combined_path_refused_through_run_agent(kb, db) -> None:
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
|
||||
def test_read_bare_path_single_source_suggestion_then_corrected_read(kb, db) -> None:
|
||||
"""Phase 72, task 02: a bare path under ONE source (exact ``path``
|
||||
match, the source prefix missing) → the single-identity suggestion
|
||||
(a refusal — not counted); the scripted corrected call (round 2, the
|
||||
suggested combined identity) then succeeds against real Postgres —
|
||||
the two-round self-correction."""
|
||||
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
|
||||
db.commit()
|
||||
|
||||
llm = ScriptedToolCallsLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="read", arguments={"path": "deep/nested/doc.md"}
|
||||
),
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="read",
|
||||
arguments={"path": "Alpha/deep/nested/doc.md"},
|
||||
),
|
||||
]
|
||||
)
|
||||
holder = AgentHolder()
|
||||
asyncio.run(_consume(cast("LLMClient", llm), db, holder))
|
||||
|
||||
# Round 1: the bare path resolves to no combined identity, but it IS
|
||||
# the indexed document's path — the refusal names the one combined
|
||||
# identity to use (not counted, the tools stay offered).
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No document at 'deep/nested/doc.md' — "
|
||||
"did you mean 'Alpha/deep/nested/doc.md'?"
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
# Round 2: the corrected combined identity succeeds — the full
|
||||
# content, the holder records the row, and it counts.
|
||||
assert llm.requests[2][0][5]["content"] == (
|
||||
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
|
||||
)
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
assert holder.read_docs == [created]
|
||||
assert holder.tool_calls == 1 # only the corrected read executed
|
||||
|
||||
|
||||
def test_read_bare_path_two_sources_one_of_suggestion_then_corrected_read(
|
||||
kb, db,
|
||||
) -> None:
|
||||
"""Phase 72, task 02: the same bare path under TWO sources → the
|
||||
``one of`` line (up to ``SUGGESTION_LIMIT`` identities, catalog
|
||||
order — Alpha before Beta); the scripted corrected call (round 2,
|
||||
the first suggested identity) then succeeds."""
|
||||
a = _doc(db, "Alpha", "shared/x.md", "Alpha X", "A-TEXT")
|
||||
_doc(db, "Beta", "shared/x.md", "Beta X", "B-TEXT")
|
||||
db.commit()
|
||||
|
||||
llm = ScriptedToolCallsLLM(
|
||||
[
|
||||
ToolCallPiece(id="call_1", name="read", arguments={"path": "shared/x.md"}),
|
||||
ToolCallPiece(
|
||||
id="call_2", name="read", arguments={"path": "Alpha/shared/x.md"}
|
||||
),
|
||||
]
|
||||
)
|
||||
holder = AgentHolder()
|
||||
asyncio.run(_consume(cast("LLMClient", llm), db, holder))
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No document at 'shared/x.md' — did you mean one of: "
|
||||
"'Alpha/shared/x.md', 'Beta/shared/x.md'?"
|
||||
)
|
||||
assert llm.requests[2][0][5]["content"] == "Document Alpha/shared/x.md:\nA-TEXT"
|
||||
assert holder.read_docs == [a]
|
||||
assert holder.tool_calls == 1 # only the corrected read executed
|
||||
|
||||
|
||||
# ---------- grep (the phase-68 A5 contract under the new name) ----------
|
||||
|
||||
|
||||
@@ -345,6 +467,9 @@ def test_grep_scoped_through_run_agent(kb, db) -> None:
|
||||
|
||||
|
||||
def test_grep_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:
|
||||
"""A scoped ``grep`` miss that matches no indexed document's ``path``
|
||||
(zero candidates) keeps today's line byte-identical — a refusal,
|
||||
not counted."""
|
||||
_doc(db, "Alpha", "a/one.md", "One", "nothing")
|
||||
db.commit()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user