Experimental/brain-of-reese
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(agent): make read_document robust to combined source/path arguments

Browse Source 
The model treated the combined 'source/path' string (as printed in
search result lines, read-result headers and refusals) as the
document's identity and passed it as 'source' — e.g.
source='homelab/active/container_caddy/caddy.md' instead of
source='homelab', path='active/container_caddy/caddy.md'.

- Rewrite the read_document description with the split rule (source =
  before the FIRST '/', path = after it) and a worked example; share
  the source/path parameter descriptions between read_document and
  search_documents; map search result lines back onto the split.
- New _resolve_document: on a lookup miss with a '/' in source, retry
  at the first slash (source names are directory basenames and can
  never contain '/'), plus a continuation candidate for a split at a
  later slash; a self-corrected combined form for an already-in-context
  document is still rejected as ALREADY_IN_CONTEXT.
- A slash-carrying source that matches nothing gets an educational
  refusal naming the corrected arguments instead of the generic line
  that repeated the combined form.

Verified live against aipi (lite) + the imported homelab KB: A/B on
the exact failure scenario (5 runs each, right after a
combined-source search result) — old descriptions 5/5 combined, new
descriptions 5/5 clean; two live UI turns (Playwright) produced only
clean split arguments, including a multi-hop read of
install_caddy_deskwork.yaml that landed in done.sources. Full suite:
1376 passed, app coverage 99% (agent.py 100%), ruff + pyright clean,
agent/search E2E green in isolation.
This commit is contained in:
ducoterra
2026-09-02 17:42:57 -04:00
parent 137d5fa1a5
commit 0bf96f22e1
3 changed files with 402 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_agent.py
+219 -24
View File
@@ -112,32 +112,49 @@ def test_agent_tools_names_and_parameters() -> None:
read_params = by_name["read_document"]["function"]["parameters"]
assert read_params["required"] == ["source", "path"]
assert set(read_params["properties"]) == {"source", "path"}
# Phase 45: the per-tool budgets are gone — "exactly one more"
# dropped out of the read_document description.
# The model repeatedly conflated the two fields — passing the
# combined 'source/path' string as 'source' — so the read_document
# description pins the split rule with a worked example.
assert by_name["read_document"]["function"]["description"] == (
"Add the full content of one more indexed document to your context"
"Add the full content of one more indexed document to your "
"context. A document is identified by the (source, path) pair "
"exactly as shown in the list_documents output: 'source' is "
"the top-level source name only (e.g. 'homelab'), 'path' is "
"the file path inside that source (e.g. "
"'active/container_caddy/caddy.md'). If you only have a "
"combined 'source/path' string (as in search_documents "
"results), split it at the FIRST '/': the part before is the "
"source, the part after is the path. Example: "
"read_document(source='homelab', "
"path='active/container_caddy/caddy.md')."
)
# Phase 63 (A2): the parameter descriptions point the LLM at the
# labeled `source:` / `path:` fields of the list_documents output
# (the example was dropped by the phase-68 description fix — the
# wording stays pinned, the model saw invented paths in calls).
# The parameter descriptions define the split: source = before the
# first '/', path = after it.
assert read_params["properties"]["source"]["description"] == (
"The document's source, as shown after 'source: ' in the "
"list_documents output."
"Top-level source name only (e.g. 'homelab') — the part BEFORE "
"the first '/' of a combined 'source/path' string, exactly as "
"shown after 'source: ' in the list_documents output. Must not "
"contain '/' itself — do not pass the full source/path here."
)
assert read_params["properties"]["path"]["description"] == (
"The document's path, as shown after 'path: ' in the "
"list_documents output."
"File path relative to the source directory (e.g. "
"'active/container_caddy/caddy.md') — the part AFTER the first "
"'/' of a combined 'source/path' string, exactly as shown "
"after 'path: ' in the list_documents output. Must not start "
"with the source name."
)
# Phase 68: search_documents — the third tool, a locator (locked A5).
# Phase 68: search_documents — the third tool, a locator (locked
# A5); its description maps result lines back onto the split.
search = by_name["search_documents"]["function"]
assert search["description"] == (
"Search every indexed document for an exact string "
"(case-insensitive) and return up to 20 matching lines as "
"'source/path:line: text' — use this to locate content, "
"then read_document the winner. Optionally pass 'source' "
"and 'path' (as shown in list_documents) to search one "
"document only."
"then read_document the winner (each result line's "
"'source/path' splits at the first '/': the part before "
"is the source, the part after is the path). Optionally "
"pass 'source' and 'path' (as shown in list_documents) "
"to search one document only."
)
search_params = search["parameters"]
assert search_params["type"] == "object"
@@ -146,15 +163,10 @@ def test_agent_tools_names_and_parameters() -> None:
assert search_params["properties"]["pattern"]["description"] == (
"The exact text to search for (a plain substring, not a regex)"
)
# Phase 63 labeled-field wording, same as read_document's parameters.
assert search_params["properties"]["source"]["description"] == (
"The document's source, as shown after 'source: ' in the "
"list_documents output."
)
assert search_params["properties"]["path"]["description"] == (
"The document's path, as shown after 'path: ' in the "
"list_documents output."
)
# Shared constants: search's source/path params ARE read_document's
# (one definition, no drift between the two tools).
assert search_params["properties"]["source"] is read_params["properties"]["source"]
assert search_params["properties"]["path"] is read_params["properties"]["path"]
# ---------- happy path: list → read → answer ----------
@@ -574,6 +586,189 @@ def test_read_document_missing_arguments_refused(
assert llm.requests[1][1] == AGENT_TOOLS
# ---------- combined 'source/path' self-correction ----------
# The model treats the combined 'source/path' string (search result
# lines, read-result headers, refusals) as the document's identity and
# sometimes passes it as 'source'. _resolve_document splits it at the
# first slash (source names are directory basenames — they can never
# contain '/'); a refusal for a still-unknown split teaches the split.
def test_read_combined_source_is_split_and_read(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""source='S/a/b.md' (the combined form) + path='a/b.md': the exact
lookup misses, the first-slash split hits — the read executes, the
holder records the document, and the result header carries the TRUE
source/path (not the model's raw arguments)."""
doc = _doc("S", "a/b.md", "B", "B-CONTENT")
calls: list[tuple[str, str]] = []
def _find(db: Any, source: str, path: str) -> Document | None:
calls.append((source, path))
return doc if (source, path) == ("S", "a/b.md") else None
monkeypatch.setattr(agent, "find_document", _find)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="read_document",
arguments={"source": "S/a/b.md", "path": "a/b.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
# Exact pair first, then the first-slash split (no third attempt).
assert calls == [("S/a/b.md", "a/b.md"), ("S", "a/b.md")]
assert holder.read_docs == [doc]
assert holder.tool_calls == 1
assert llm.requests[1][0][3]["content"] == "Document S/a/b.md:\nB-CONTENT"
def test_read_combined_source_split_at_later_slash(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""source='S/a' + path='b.md' — a split at a LATER slash (source
carried source + leading directory, path the remainder): the exact
lookup and the first-slash split miss, the continuation candidate
(source, split/path) hits."""
doc = _doc("S", "a/b.md", "B", "B-CONTENT")
calls: list[tuple[str, str]] = []
def _find(db: Any, source: str, path: str) -> Document | None:
calls.append((source, path))
return doc if (source, path) == ("S", "a/b.md") else None
monkeypatch.setattr(agent, "find_document", _find)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="read_document",
arguments={"source": "S/a", "path": "b.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert calls == [("S/a", "b.md"), ("S", "a"), ("S", "a/b.md")]
assert holder.read_docs == [doc]
assert holder.tool_calls == 1
assert llm.requests[1][0][3]["content"] == "Document S/a/b.md:\nB-CONTENT"
def test_read_combined_source_unknown_teaches_the_split(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A combined source that matches nothing — even split — gets the
EDUCATIONAL refusal: it names the corrected arguments instead of
repeating the combined form (the old generic line reinforced the
mistake)."""
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="read_document",
arguments={"source": "S/a/b.md", "path": "a/b.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert holder.read_docs == [] and holder.tool_calls == 0
assert llm.requests[1][0][3]["content"] == (
"source must not contain '/': for 'S/a/b.md' call "
"read_document(source='S', path='a/b.md')."
)
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
def test_read_combined_source_for_seed_doc_is_already_in_context(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The combined form of a document ALREADY in context: the raw pair
cannot match the dedupe set, so the split resolves it — and it is
still rejected as already-in-context (no duplicate read_docs entry,
no re-read into the context)."""
seed = [_doc("S", "a.md", "A", "A-CONTENT")]
def _find(db: Any, source: str, path: str) -> Document | None:
return seed[0] if (source, path) == ("S", "a.md") else None
monkeypatch.setattr(agent, "find_document", _find)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="read_document",
arguments={"source": "S/a.md", "path": "a.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings(), seed_docs=seed))
assert holder.read_docs == [] and holder.tool_calls == 0
assert llm.requests[1][0][3]["content"] == agent.ALREADY_IN_CONTEXT
def test_search_scoped_combined_source_is_split(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A scoped search whose 'source' carries the combined form resolves
through the split — the search runs on the right document."""
doc = _doc("S", "a.md", "A", "needle here")
def _find(db: Any, source: str, path: str) -> Document | None:
return doc if (source, path) == ("S", "a.md") else None
monkeypatch.setattr(agent, "find_document", _find)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="search_documents",
arguments={"pattern": "needle", "source": "S/a.md", "path": "a.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here"
assert holder.tool_calls == 1
assert holder.read_docs == [] # searched doc did not enter the context
def test_search_scoped_combined_source_unknown_teaches_the_split(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="search_documents",
arguments={"pattern": "x", "source": "S/ghost.md", "path": "ghost.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"source must not contain '/': for 'S/ghost.md' use "
"source='S', path='ghost.md'."
)
assert holder.tool_calls == 0 and holder.read_docs == [] # a refusal
# ---------- search_documents (phase 68, locked A5/A6) ----------