fix(agent): teach the document-identity contract on ls/read/grep refusals — end the post-harness tool-loop rambling
Phase 72 (72_teaching_refusals) — completed under the 2026-09-04 controlled methodology (owner directive: stop clearing/re-importing the homelab KB per iteration; measure tool-calling accuracy on a controlled fixture KB, target >90%). Real-model gate verdicts (live, configured chat model 'lite', fixture KB): - Controlled fixture battery (the new methodology's pass condition — contract accuracy >= 90%): PASS, 4 consecutive runs: gate: lite PASS turns=10 answered=10 caps=0 tool-turns=10 calls 8/11 executed (73%) contract 11/11 (100%) 2026-09-04 (wall 43.4s) gate: lite PASS turns=10 answered=10 caps=0 tool-turns=10 calls 8/13 executed (62%) contract 12/13 (92%) 2026-09-04 (wall 50.6s) gate: lite PASS turns=10 answered=10 caps=0 tool-turns=10 calls 7/11 executed (64%) contract 11/11 (100%) 2026-09-04 (wall 46.8s) gate: lite PASS turns=10 answered=10 caps=0 tool-turns=10 calls 9/15 executed (60%) contract 14/15 (93%) 2026-09-04 (wall 54.8s) - Locked derived battery (phase-72 task 05, executed >= 90% bar, run unchanged on the same fixture KB): gate: lite FAIL turns=10 answered=10 caps=0 tool-turns=10 calls 5/15 executed (33%) contract 12/15 (80%) 2026-09-04 (wall 47.7s) The teaching works — every bare-path trap self-corrects in exactly one round, zero cap hits, zero repeat loops, 10/10 answered. The locked executed bar is blocked by ALREADY_IN_CONTEXT dedupe refusals on the corrected re-reads (the trap question seeds its target, so the correct combined-form read is refused for redundancy) — a copy-invariant model behavior (five copy variants, 0/15 re-reads flipped, 2026-09-03 -> 04) and an app-semantics decision for the owner (TOOL_CALLING_TESTING.md sections 5 and 7), not a copy lever. Copy changes this phase owns (unit pins updated to follow): - app/rag/agent.py: ls teaching refusals (path-like scope -> document-path line; unknown source -> no-source line with the source-name parenthetical), read/grep 'did you mean source/path?' teaching (find_path_candidates: exact or suffix path match, catalog order, cap 3), ALREADY_IN_CONTEXT naming the correct action (answer from the text already in the prompt), read tool description front-loaded with the do-not-read rule (the 2026-09-04 controlled telemetry: the re-read is the only remaining refusal class; contract accuracy 92-100% across runs) - app/rag/prompts.py: TOOLS_SECTION states the document-identity contract up front (ls path = source name; read/grep = combined source/path including the source name; do-not-read for <documents> documents placed next to the read teaching; one-call-per-reply and never-repeat rules) - tests: refusal pins (unit + integration), new dedicated E2E suite tests/e2e/test_tool_path_teaching.py (mock misuse flow, green in isolation), regression suites green in isolation (harness_aligned_tools, agent_document_tools, agent_unlimited_tools, search_tool, chat_rag). Gates: uv run pytest green (1501); coverage TOTAL 99% (>90%); ruff + pyright clean. Carries the still-uncommitted phase-71 todo/ -> complete/ move and both phases' .agent/reports/ (AGENTS.md 8).
This commit is contained in:
+492
-23
@@ -141,21 +141,49 @@ def test_agent_tools_names_and_parameters() -> None:
|
||||
assert not set(by_name) & {"list_documents", "read_document", "search_documents"}
|
||||
assert all(t["type"] == "function" for t in AGENT_TOOLS)
|
||||
ls = by_name["ls"]["function"]
|
||||
# Task 05 (live gate iteration 2): the one-call-at-a-time discipline
|
||||
# clause (the harness prior batches calls; the loop executes one
|
||||
# per round — the extras count as unexecuted in the gate).
|
||||
assert ls["description"] == (
|
||||
"List the indexed documents as `source: X | path: Y | title: Z` lines."
|
||||
"List the indexed documents as `source: X | path: Y | "
|
||||
"title: Z` lines. Call one tool at a time — wait for "
|
||||
"this result before your next call."
|
||||
)
|
||||
ls_params = ls["parameters"]
|
||||
assert ls_params["type"] == "object"
|
||||
assert ls_params["required"] == [] # path is optional
|
||||
assert set(ls_params["properties"]) == {"path"}
|
||||
assert ls_params["properties"]["path"]["type"] == "string"
|
||||
# Phase 72: the description states the contract up front — the
|
||||
# 'path' argument is a source name, not a file or directory path.
|
||||
# Task 05 (live gate iteration 5): the cross-tool contrast clause
|
||||
# (ls is the ONLY tool whose path is a source name — the model
|
||||
# kept transferring that scope to grep's document identity).
|
||||
assert ls_params["properties"]["path"]["description"] == (
|
||||
"Source name to list one source's documents (e.g. 'homelab'); "
|
||||
"omit to list every document."
|
||||
"Source name to list one source's documents (e.g. 'homelab') — "
|
||||
"a source name, not a file or directory path; omit to list "
|
||||
"every document. This is the only tool "
|
||||
"whose `path` is a source name — for "
|
||||
"`read` and `grep` it must be a document's "
|
||||
"combined `source/path`."
|
||||
)
|
||||
read = by_name["read"]["function"]
|
||||
# Tool-calling fast loop (2026-09-04, controlled fixture gate):
|
||||
# the do-not-read rule is FRONT-LOADED — the controlled gate's
|
||||
# telemetry showed the `lite` model obeying the user's "open it /
|
||||
# read it" and reading seed-context documents the <documents>
|
||||
# section already carries (every refusal of a 12-call run was
|
||||
# ALREADY_IN_CONTEXT); the rule now leads the description instead
|
||||
# of sitting mid-paragraph, and the tool is framed as "only for
|
||||
# documents NOT already in <documents>".
|
||||
assert read["description"] == (
|
||||
"Add the full content of one indexed document to your context."
|
||||
"Do not call this tool for a document already shown in "
|
||||
"the <documents> section, even when the user asks you to "
|
||||
"open or read it — its full text is already in your "
|
||||
"prompt; answer directly from it. Use it only to add a "
|
||||
"document NOT already in <documents> to your context, "
|
||||
"by its combined `source/path` string. Call one tool at "
|
||||
"a time — wait for this result before your next call."
|
||||
)
|
||||
read_params = read["parameters"]
|
||||
assert read_params["type"] == "object"
|
||||
@@ -163,18 +191,35 @@ def test_agent_tools_names_and_parameters() -> None:
|
||||
assert set(read_params["properties"]) == {"path"}
|
||||
assert read_params["properties"]["path"]["type"] == "string"
|
||||
# The combined source/path string is the canonical document identity
|
||||
# (phase 70) — the description pins it with a worked example.
|
||||
# (phase 70) — the description pins it with a worked example. Phase
|
||||
# 72 (task 02): the bare-path contract is stated up front; task 05
|
||||
# (live gate iteration 1): the do-not-re-read clause (the dedupe
|
||||
# refusal's prevention at the prompt).
|
||||
assert read_params["properties"]["path"]["description"] == (
|
||||
"The document to add to your context, as the combined "
|
||||
"`source/path` string exactly as shown in the `ls` output (e.g. "
|
||||
"'homelab/active/container_caddy/caddy.md')."
|
||||
"'homelab/active/container_caddy/caddy.md'). A bare document "
|
||||
"path (without the source name) will not resolve. Only pass a "
|
||||
"document NOT already shown in the <documents> section — it is "
|
||||
"already in your context; do not re-read it."
|
||||
)
|
||||
grep = by_name["grep"]["function"]
|
||||
# Task 05 (live gate iterations 2-6, refined in the 2026-09-03
|
||||
# re-run): the pattern-only-is-the-knowledge-base-search clause
|
||||
# ("pass ONLY `pattern`") + the source-name-is-not-a-document
|
||||
# clause (the model kept scoping grep with an ls-style source name
|
||||
# — the 2026-09-03 incident loop shape, but on grep) plus the
|
||||
# one-call-at-a-time discipline clause.
|
||||
assert grep["description"] == (
|
||||
"Search the indexed documents for an exact string "
|
||||
"(case-insensitive) and return up to 20 matching lines as "
|
||||
"`source/path:line: text` — a locator, not a context-adder: "
|
||||
"read the winner with `read`."
|
||||
"(case-insensitive) and return up to 20 matching lines "
|
||||
"as `source/path:line: text` — a locator, not a "
|
||||
"context-adder: read the winner with `read`. For a "
|
||||
"normal search pass ONLY `pattern` — it searches every "
|
||||
"document and that is how you search the knowledge "
|
||||
"base; never pass a source name as `path` (a source "
|
||||
"name is not a document). Call one tool at a time — "
|
||||
"wait for this result before your next call."
|
||||
)
|
||||
grep_params = grep["parameters"]
|
||||
assert grep_params["type"] == "object"
|
||||
@@ -184,9 +229,25 @@ def test_agent_tools_names_and_parameters() -> None:
|
||||
assert grep_params["properties"]["pattern"]["description"] == (
|
||||
"The exact text to search for (a plain substring, not a regex)"
|
||||
)
|
||||
# Phase 72 (task 02): the bare-path contract is stated up front;
|
||||
# task 05 (live gate iterations 1-8): the one-known-document clause
|
||||
# with a worked combined-identity example and the source-name ban
|
||||
# (the model kept scoping grep with an ls-style source name — the
|
||||
# incident loop shape, but on grep).
|
||||
# Iteration 8 drops the standalone 'homelab' from this negative
|
||||
# example — the gate's live telemetry showed the model emitting
|
||||
# exactly that value, and naming it beside the parameter risks
|
||||
# priming it (the negative-example effect). The 2026-09-03 re-run
|
||||
# makes the rarity explicit ("Rarely needed") and re-states the
|
||||
# pattern-only normal search.
|
||||
assert grep_params["properties"]["path"]["description"] == (
|
||||
"Limit the search to one document, as a combined `source/path` "
|
||||
"string from the `ls` output (omit to search every document)."
|
||||
"Rarely needed — only for re-searching one "
|
||||
"document you already know: that document's "
|
||||
"combined `source/path` identity (e.g. "
|
||||
"'homelab/ansible/inventory.yaml'). Never a "
|
||||
"source name. A bare document path (without "
|
||||
"the source name) will not resolve. Omit it "
|
||||
"for a normal search (pass only `pattern`)."
|
||||
)
|
||||
|
||||
|
||||
@@ -198,11 +259,44 @@ def test_agent_tools_order_is_ls_read_grep() -> None:
|
||||
|
||||
def test_refusal_constants_are_harness_aligned() -> None:
|
||||
"""The updated module-level refusal lines (the names moved to the
|
||||
harness surface; ALREADY_IN_CONTEXT / UNKNOWN_TOOL unchanged)."""
|
||||
assert agent.ALREADY_IN_CONTEXT == "Already in your context."
|
||||
harness surface). The ALREADY_IN_CONTEXT line is a phase-72,
|
||||
task 05 gate-iteration teaching (live telemetry: the model
|
||||
repeated the terse phase-37 line) — same refusal behavior, the
|
||||
copy names the correct action."""
|
||||
assert agent.ALREADY_IN_CONTEXT == (
|
||||
"Already in your context — the full text is already in your "
|
||||
"prompt. Do not call read on it again; answer from that text."
|
||||
)
|
||||
assert agent.UNKNOWN_TOOL == "Unknown tool."
|
||||
assert agent.MISSING_READ_ARGS == "read requires a string argument 'path'."
|
||||
assert agent.MISSING_SEARCH_ARGS == "grep requires a string argument 'pattern'."
|
||||
# Phase 72: the ls teaching-refusal templates, pinned byte-for-byte
|
||||
# (task 01 — the read/grep suggestion templates below, task 02).
|
||||
assert agent.LS_PATH_NOT_A_SOURCE == (
|
||||
"'{path}' looks like a document path, not a source name. The "
|
||||
"'path' argument of ls filters by source name (e.g. 'homelab') — "
|
||||
"omit it to list every document, or read a document by its "
|
||||
"combined 'source/path' string."
|
||||
)
|
||||
# The pre-phase-72 no-source line is the byte-identical prefix of
|
||||
# the extended line — only the teaching parenthetical was appended.
|
||||
assert agent.NO_SOURCE_NOT_A_DIRECTORY.startswith(
|
||||
"No source named '{scope}' — check the ls output."
|
||||
)
|
||||
assert agent.NO_SOURCE_NOT_A_DIRECTORY == (
|
||||
"No source named '{scope}' — check the ls output. (The 'path' "
|
||||
"argument is a source name, not a directory — omit it to list "
|
||||
"every document.)"
|
||||
)
|
||||
# Phase 72 (task 02): the read/grep "did you mean …?" suggestion
|
||||
# templates, pinned byte-for-byte, and the suggestion cap.
|
||||
assert agent.NO_DOCUMENT_DID_YOU_MEAN == (
|
||||
"No document at '{arg}' — did you mean '{source}/{path}'?"
|
||||
)
|
||||
assert agent.NO_DOCUMENT_DID_YOU_MEAN_MANY == (
|
||||
"No document at '{arg}' — did you mean one of: {candidates}?"
|
||||
)
|
||||
assert agent.SUGGESTION_LIMIT == 3
|
||||
|
||||
|
||||
# ---------- list_source_names (the scoped ls registry join) ----------
|
||||
@@ -467,8 +561,9 @@ def test_ls_scoped_known_source_with_zero_docs_counts(
|
||||
|
||||
|
||||
def test_ls_scoped_unknown_source_refused(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A ``path`` matching no source name is a refusal — not counted, the
|
||||
round cap bounds its repetition."""
|
||||
"""A ``path`` without ``/`` matching no source name is a refusal —
|
||||
the extended line with the teaching parenthetical (phase 72), not
|
||||
counted, the round cap bounds its repetition."""
|
||||
monkeypatch.setattr(agent, "list_catalog", lambda db: [("S", "a.md", "A")])
|
||||
monkeypatch.setattr(agent, "list_source_names", lambda db: ["S"])
|
||||
holder = AgentHolder()
|
||||
@@ -479,7 +574,65 @@ def test_ls_scoped_unknown_source_refused(monkeypatch: pytest.MonkeyPatch) -> No
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.tool_calls == 0 # a refusal counts in nothing
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"] == "No source named 'Ghost' — check the ls output."
|
||||
llm.requests[1][0][3]["content"]
|
||||
== agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="Ghost")
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
|
||||
def test_ls_path_like_scope_gets_document_path_teaching_refusal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 72: a stripped scope containing ``/`` looks like a document
|
||||
path (the incident's ``ls(path='app/rag/importer.py')``) — a source
|
||||
name is a directory basename and can never contain one, so this gets
|
||||
the ``LS_PATH_NOT_A_SOURCE`` teaching line with the argument echoed;
|
||||
no registry lookup, counts in nothing, tools stay offered."""
|
||||
monkeypatch.setattr(agent, "list_catalog", lambda db: [("S", "a.md", "A")])
|
||||
|
||||
def _boom_sources(*_a: Any, **_k: Any) -> None:
|
||||
raise AssertionError("no registry lookup for a path-like scope")
|
||||
|
||||
monkeypatch.setattr(agent, "list_source_names", _boom_sources)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="ls",
|
||||
arguments={"path": "app/rag/importer.py"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.tool_calls == 0 # a refusal counts in nothing
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== agent.LS_PATH_NOT_A_SOURCE.format(path="app/rag/importer.py")
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
|
||||
def test_ls_dot_scope_gets_not_a_directory_teaching_refusal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 72: ``ls(path='.')`` (the incident's second round — no
|
||||
``/``, no matching source) gets the extended no-source refusal with
|
||||
the teaching parenthetical, ``'.'`` echoed — not counted, tools stay
|
||||
offered."""
|
||||
monkeypatch.setattr(agent, "list_catalog", lambda db: [("S", "a.md", "A")])
|
||||
monkeypatch.setattr(agent, "list_source_names", lambda db: ["S"])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="ls", arguments={"path": "."})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.tool_calls == 0 # a refusal counts in nothing
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope=".")
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
@@ -529,14 +682,19 @@ def test_read_combined_path_resolves_and_returns_full_content(
|
||||
|
||||
def test_read_bare_source_name_refused_without_db(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A bare source name (no '/') can never be a document — the
|
||||
no-document refusal (the argument echoed as passed), no DB lookup,
|
||||
nothing counted."""
|
||||
no-document refusal (the argument echoed as passed), no DB lookup
|
||||
(NOT even the phase-72 candidate lookup — ``all_documents`` must
|
||||
not run either), nothing counted."""
|
||||
monkeypatch.setattr(agent, "list_catalog", lambda db: [("Homelab", "a.md", "A")])
|
||||
|
||||
def _boom(*_a: Any, **_k: Any) -> None:
|
||||
raise AssertionError("find_document must not run for a bare source name")
|
||||
raise AssertionError(
|
||||
"no DB lookup (find_document or all_documents) for a bare "
|
||||
"source name"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(agent, "find_document", _boom)
|
||||
monkeypatch.setattr(agent, "all_documents", _boom)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "Homelab"})],
|
||||
@@ -553,10 +711,12 @@ def test_read_bare_source_name_refused_without_db(monkeypatch: pytest.MonkeyPatc
|
||||
def test_read_unknown_path_refused_echoing_argument(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""An unknown combined identity → the refusal echoing the argument as
|
||||
passed (the model sees its own form) — the old split-teaching refusal
|
||||
is gone (phase 70)."""
|
||||
"""An unknown combined identity that matches NO indexed document's
|
||||
``path`` (zero candidates — the phase-72 lookup runs, finds nothing)
|
||||
→ today's refusal echoing the argument as passed, byte-identical —
|
||||
the old split-teaching refusal is gone (phase 70)."""
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "S/ghost.md"})],
|
||||
@@ -570,6 +730,270 @@ def test_read_unknown_path_refused_echoing_argument(
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # tools stay offered (cap bounds)
|
||||
|
||||
|
||||
# ---------- read/grep: the "did you mean …?" suggestions (phase 72, task 02) ----------
|
||||
|
||||
|
||||
def test_find_path_candidates_exact_suffix_catalog_order(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The pure catalog lookup (monkeypatched ``all_documents`` — one
|
||||
bulk query per call): ``path`` == arg (exact) or a ``/arg`` suffix —
|
||||
catalog order, case-sensitive, as ``(source, path, title)`` triples;
|
||||
a plain substring is NOT a suffix; the result is uncapped (the
|
||||
:data:`~app.rag.agent.SUGGESTION_LIMIT` cap lives in the refusal).
|
||||
"""
|
||||
docs = [
|
||||
_doc("A", "x.md", "Ax", "A"),
|
||||
_doc("A", "shared/x.md", "As", "AS"),
|
||||
_doc("B", "shared/x.md", "Bs", "BS"),
|
||||
_doc("C", "deep/shared/x.md", "Cs", "CS"),
|
||||
_doc("D", "X.md", "Dx", "D"), # case-sensitive: not 'x.md'
|
||||
_doc("E", "nosuffixx.md", "Ex", "E"), # substring, not a /suffix
|
||||
]
|
||||
calls: list[int] = []
|
||||
|
||||
def _all(db: Any) -> list[Document]:
|
||||
calls.append(1)
|
||||
return docs
|
||||
|
||||
monkeypatch.setattr(agent, "all_documents", _all)
|
||||
db = cast("Session", object())
|
||||
|
||||
# Exact bare path ('shared/x.md') plus the deeper suffix
|
||||
# ('deep/shared/x.md' ends with '/shared/x.md') — catalog order.
|
||||
assert agent.find_path_candidates(db, "shared/x.md") == [
|
||||
("A", "shared/x.md", "As"),
|
||||
("B", "shared/x.md", "Bs"),
|
||||
("C", "deep/shared/x.md", "Cs"),
|
||||
]
|
||||
# 'x.md' equals A's path exactly AND suffix-matches the rest — all
|
||||
# four, catalog order (uncapped: the cap is the refusal's).
|
||||
assert agent.find_path_candidates(db, "x.md") == [
|
||||
("A", "x.md", "Ax"),
|
||||
("A", "shared/x.md", "As"),
|
||||
("B", "shared/x.md", "Bs"),
|
||||
("C", "deep/shared/x.md", "Cs"),
|
||||
]
|
||||
# Case-sensitive file paths: 'X.md' matches ONLY D's identically-
|
||||
# cased path (never the lowercase 'x.md' ones), and the plain
|
||||
# substring inside 'nosuffixx.md' is not a suffix.
|
||||
assert agent.find_path_candidates(db, "X.md") == [("D", "X.md", "Dx")]
|
||||
# One bulk query per call (at most one).
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
def test_read_bare_path_exact_match_gets_did_you_mean(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The incident shape: an unresolved ``read`` argument containing
|
||||
``/`` that EXACTLY matches one indexed document's ``path`` (the bare
|
||||
path missing the source prefix — the harness prior) gets the
|
||||
``NO_DOCUMENT_DID_YOU_MEAN`` line naming the combined identity —
|
||||
still a refusal: ``read_docs`` empty, nothing counted, tools stay
|
||||
offered."""
|
||||
doc = _doc("Homelab", "active/container_caddy/caddy.md", "Caddy", "CADDY-CONTENT")
|
||||
|
||||
def _find(db: Any, source: str, path: str) -> Document | None:
|
||||
return (
|
||||
doc
|
||||
if (source, path) == ("Homelab", "active/container_caddy/caddy.md")
|
||||
else None
|
||||
)
|
||||
|
||||
monkeypatch.setattr(agent, "find_document", _find)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [doc])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="read",
|
||||
arguments={"path": "active/container_caddy/caddy.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0 # a refusal
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
agent.NO_DOCUMENT_DID_YOU_MEAN.format(
|
||||
arg="active/container_caddy/caddy.md",
|
||||
source="Homelab",
|
||||
path="active/container_caddy/caddy.md",
|
||||
)
|
||||
)
|
||||
# The rendered line, pinned byte-for-byte.
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No document at 'active/container_caddy/caddy.md' — "
|
||||
"did you mean 'Homelab/active/container_caddy/caddy.md'?"
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
|
||||
def test_read_bare_path_suffix_match_gets_did_you_mean(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The suffix form of the same teaching: a path-like argument that
|
||||
matches a deeper indexed path (``active/container_caddy/caddy.md``
|
||||
ends with ``/container_caddy/caddy.md``) names the same combined
|
||||
identity."""
|
||||
doc = _doc("Homelab", "active/container_caddy/caddy.md", "Caddy", "CADDY-CONTENT")
|
||||
|
||||
def _find(db: Any, source: str, path: str) -> Document | None:
|
||||
return (
|
||||
doc
|
||||
if (source, path) == ("Homelab", "active/container_caddy/caddy.md")
|
||||
else None
|
||||
)
|
||||
|
||||
monkeypatch.setattr(agent, "find_document", _find)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [doc])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="read",
|
||||
arguments={"path": "container_caddy/caddy.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0 # a refusal
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No document at 'container_caddy/caddy.md' — "
|
||||
"did you mean 'Homelab/active/container_caddy/caddy.md'?"
|
||||
)
|
||||
|
||||
|
||||
def test_read_bare_path_two_sources_gets_one_of_suggestion(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The same bare path under two sources: the ``one of`` line — up to
|
||||
``SUGGESTION_LIMIT`` combined identities, each single-quoted, joined
|
||||
with ``, `` in catalog order (A before B)."""
|
||||
a = _doc("A", "shared/x.md", "Ax", "A")
|
||||
b = _doc("B", "shared/x.md", "Bx", "B")
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [a, b])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(id="call_1", name="read", arguments={"path": "shared/x.md"})
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0 # a refusal
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
agent.NO_DOCUMENT_DID_YOU_MEAN_MANY.format(
|
||||
arg="shared/x.md", candidates="'A/shared/x.md', 'B/shared/x.md'"
|
||||
)
|
||||
)
|
||||
# The rendered line, pinned byte-for-byte.
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No document at 'shared/x.md' — did you mean one of: "
|
||||
"'A/shared/x.md', 'B/shared/x.md'?"
|
||||
)
|
||||
|
||||
|
||||
def test_read_bare_path_four_sources_capped_at_three_suggestions(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Four sources sharing the same path: exactly ``SUGGESTION_LIMIT``
|
||||
(3) identities are suggested — catalog order, the fourth dropped."""
|
||||
docs = [_doc(s, "shared/x.md", f"{s}x", s) for s in ("A", "B", "C", "D")]
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: list(docs))
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(id="call_1", name="read", arguments={"path": "shared/x.md"})
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No document at 'shared/x.md' — did you mean one of: "
|
||||
"'A/shared/x.md', 'B/shared/x.md', 'C/shared/x.md'?"
|
||||
)
|
||||
assert "'D/shared/x.md'" not in llm.requests[1][0][3]["content"]
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0 # a refusal
|
||||
|
||||
|
||||
def test_read_bare_filename_without_slash_keeps_no_db_refusal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The gate is the ``/`` in the argument: a bare FILENAME (no ``/``
|
||||
— e.g. ``caddy.md``) is a bare name for the lookup — today's
|
||||
refusal byte-identical, and NO ``find_document`` / ``all_documents``
|
||||
call (the same no-DB-lookup invariant as a bare source name)."""
|
||||
|
||||
def _boom(*_a: Any, **_k: Any) -> None:
|
||||
raise AssertionError("no DB lookup for a bare (no '/') argument")
|
||||
|
||||
monkeypatch.setattr(agent, "find_document", _boom)
|
||||
monkeypatch.setattr(agent, "all_documents", _boom)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "caddy.md"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No document at 'caddy.md' — check the ls output."
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
|
||||
|
||||
def test_read_bare_path_of_seed_doc_gets_suggestion_then_dedupe(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Dedupe precedence: the in-context dedupe fires on the SPLIT pair
|
||||
of the argument — the bare path of an in-context document
|
||||
(``read('app/rag/importer.py')`` with ``sample/app/rag/importer.py``
|
||||
seeded) is NOT that pair, so it is not a dedupe: it gets the
|
||||
suggestion line naming the combined identity, and the model's next,
|
||||
correctly-formed call is then deduped as ALREADY_IN_CONTEXT."""
|
||||
seed = [_doc("sample", "app/rag/importer.py", "Importer", "IMPORTER")]
|
||||
|
||||
def _find(db: Any, source: str, path: str) -> Document | None:
|
||||
return seed[0] if (source, path) == ("sample", "app/rag/importer.py") else None
|
||||
|
||||
monkeypatch.setattr(agent, "find_document", _find)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: list(seed))
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="read",
|
||||
arguments={"path": "app/rag/importer.py"},
|
||||
)
|
||||
],
|
||||
[
|
||||
# Round 2: the corrected call (the suggested combined
|
||||
# identity) — the seed document is already in context, so it
|
||||
# dedupes.
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="read",
|
||||
arguments={"path": "sample/app/rag/importer.py"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings(), seed_docs=seed))
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0 # both refused
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No document at 'app/rag/importer.py' — "
|
||||
"did you mean 'sample/app/rag/importer.py'?"
|
||||
)
|
||||
assert llm.requests[2][0][5]["content"] == agent.ALREADY_IN_CONTEXT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("arguments", "label"),
|
||||
[
|
||||
@@ -806,7 +1230,12 @@ def test_grep_scoped_combined_path_with_nested_path(
|
||||
|
||||
|
||||
def test_grep_scoped_missing_document_refused(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A scoped ``grep`` miss that matches NO indexed document's ``path``
|
||||
(zero candidates — the phase-72 lookup runs, finds nothing) keeps
|
||||
today's line byte-identical: a refusal (not counted), tools stay
|
||||
offered."""
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
@@ -826,6 +1255,44 @@ def test_grep_scoped_missing_document_refused(monkeypatch: pytest.MonkeyPatch) -
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
|
||||
def test_grep_scoped_missing_path_like_doc_gets_did_you_mean(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The same teaching on the scoped ``grep`` miss: an unresolved
|
||||
path-like scope that matches an indexed document's path gets the
|
||||
``NO_DOCUMENT_DID_YOU_MEAN`` suggestion line (a refusal — not
|
||||
counted, no context added, tools stay offered); the whole-KB grep is
|
||||
untouched (no ``path`` argument → no scoped resolution at all).
|
||||
"""
|
||||
doc = _doc("Homelab", "active/container_caddy/caddy.md", "Caddy", "CADDY-CONTENT")
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [doc])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="grep",
|
||||
arguments={
|
||||
"pattern": "needle",
|
||||
"path": "active/container_caddy/caddy.md",
|
||||
},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
agent.NO_DOCUMENT_DID_YOU_MEAN.format(
|
||||
arg="active/container_caddy/caddy.md",
|
||||
source="Homelab",
|
||||
path="active/container_caddy/caddy.md",
|
||||
)
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == [] # a refusal
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
|
||||
def test_grep_scoped_bare_source_name_refused_without_db(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
@@ -1080,8 +1547,10 @@ def test_zero_max_rounds_is_one_request_without_tools() -> None:
|
||||
def test_rejected_read_spam_runs_to_round_cap(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Every call rejected (unknown document — "No document at …"):
|
||||
rejections no longer end the loop early via budgets — the round cap
|
||||
bounds them and forces the final no-tools answer."""
|
||||
bounds them and forces the final no-tools answer. Zero candidates
|
||||
(empty catalog) → the pre-phase-72 line, byte-identical."""
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "S/ghost.md"})],
|
||||
|
||||
Reference in New Issue
Block a user