feat(agent): align the document tools with the harness-trained shape — ls, read(path), grep(pattern, path?)

This commit is contained in:
2026-09-03 11:17:47 -04:00
parent 16f1cfbcaf
commit 801639efcc
55 changed files with 4031 additions and 1466 deletions
+182 -135
View File
@@ -1,17 +1,19 @@
"""Integration: the agent DB accessors against real Postgres (phase 37).
"""Integration: the agent DB accessors against real Postgres (phase 37;
the harness-aligned ``ls``/``read``/``grep`` surface, phase 70).
``list_catalog`` must order rows by ``(source, path)`` — the same order as
``GET /api/docs`` — and ``find_document`` must resolve a hit to the full
document row (content included, for the never-truncated read) and return
``None`` for unknown ``source``/``path`` pairs. Phase 68: the
``search_documents`` tool is pinned here too — its locked parameter
shape in ``AGENT_TOOLS``, and a scripted ``ToolCallPiece`` executed
through ``run_agent`` against the real DB (``all_documents`` for a
whole-KB search, ``find_document`` for a scoped one). The
combined-form self-correction (a ``source`` argument carrying
``source/path``) is pinned here as well, through ``run_agent``:
the split read executes against the real table, and a still-unknown
split gets the educational refusal.
``list_catalog`` must order rows by ``(source, path)`` — the same order
as ``GET /api/docs`` — ``list_source_names`` must resolve the
registered source names (the scoped ``ls`` join), and ``find_document``
must resolve a hit to the full document row (content included, for the
never-truncated read) and return ``None`` for unknown pairs. Phase 70:
the ``ls``/``read``/``grep`` tools are pinned here too — the locked
parameter shape in ``AGENT_TOOLS``, and scripted ``ToolCallPiece``s
executed through ``run_agent`` against the real DB: ``ls`` scoped to a
registered source name (unknown name → refusal), ``read`` on the
canonical combined ``source/path`` form (first-slash split; a bare
source name and an unknown identity get the no-document refusal), and
``grep`` (``all_documents`` for a whole-KB search, ``find_document`` for
a scoped one).
Requires: podman compose up -d db
"""
@@ -24,11 +26,11 @@ from copy import deepcopy
from typing import Any, cast
import pytest
from sqlalchemy import text
from sqlalchemy import delete, text
from sqlalchemy.orm import Session
from app.config import Settings
from app.models import Document
from app.models import Document, GitSource
from app.rag import agent
from app.rag.agent import AGENT_TOOLS, AgentHolder, run_agent
from app.rag.llm import LLMClient, RetryPiece, StreamPiece, ToolCallPiece
@@ -58,6 +60,19 @@ def kb(db) -> Iterator[None]:
db.commit()
@pytest.fixture()
def src(db) -> Iterator[GitSource]:
"""One registered git source — the scoped ``ls`` source-name check
reads the real registry, so the row is inserted and deleted around
the tests (``repo_name`` resolves the URL to ``Homelab``)."""
row = GitSource(url="https://github.com/reese/Homelab.git", kind="git")
db.add(row)
db.commit()
yield row
db.execute(delete(GitSource).where(GitSource.id == row.id))
db.commit()
def test_list_catalog_orders_by_source_then_path(kb, db) -> None:
_doc(db, "Zeta", "b/second.md", "Zeta B", "ZB")
_doc(db, "Zeta", "a/first.md", "Zeta A", "ZA")
@@ -75,6 +90,23 @@ def test_list_catalog_is_empty_without_rows(kb, db) -> None:
assert agent.list_catalog(db) == []
def test_list_source_names_resolves_registry_rows(db) -> None:
"""The real registry: git names resolve through the import pipeline's
``repo_name`` (trailing ``.git`` stripped); a second row resolving to
the same name (the phase-69 sibling case) is listed once."""
a = GitSource(url="https://github.com/reese/Homelab.git", kind="git")
b = GitSource(url="https://github.com/reese/Homelab", kind="git") # sibling
c = GitSource(url="https://e.com/deployments", kind="git")
db.add_all([a, b, c])
db.commit()
try:
assert agent.list_source_names(db).count("Homelab") == 1 # deduped
assert "deployments" in agent.list_source_names(db)
finally:
db.execute(delete(GitSource).where(GitSource.id.in_([a.id, b.id, c.id])))
db.commit()
def test_find_document_hit_returns_full_row(kb, db) -> None:
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
db.commit()
@@ -97,36 +129,29 @@ def test_find_document_none_for_unknown_pairs(kb, db) -> None:
assert agent.find_document(db, "nope", "nope.md") is None # nothing at all
# ---------- search_documents (phase 68) ----------
# ---------- AGENT_TOOLS surface (phase 70: ls / read / grep) ----------
def test_all_documents_orders_by_source_then_path(kb, db) -> None:
_doc(db, "Zeta", "b/second.md", "Zeta B", "ZB")
_doc(db, "Zeta", "a/first.md", "Zeta A", "ZA")
_doc(db, "Alpha", "c/third.md", "Alpha C", "AC")
db.commit()
docs = agent.all_documents(db)
assert [(d.source, d.path) for d in docs] == [
("Alpha", "c/third.md"),
("Zeta", "a/first.md"),
("Zeta", "b/second.md"),
]
assert [d.content for d in docs] == ["AC", "ZA", "ZB"] # full rows
def test_agent_tools_offers_search_documents_with_locked_shape() -> None:
def test_agent_tools_offers_the_harness_aligned_surface() -> None:
by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
assert list(by_name) == [ # the third tool, in order
"list_documents",
"read_document",
"search_documents",
assert list(by_name) == [ # the harness order, phase 70
"ls",
"read",
"grep",
]
search = by_name["search_documents"]["function"]["parameters"]
assert search["type"] == "object"
assert search["required"] == ["pattern"]
assert set(search["properties"]) == {"pattern", "source", "path"}
assert all(p["type"] == "string" for p in search["properties"].values())
ls = by_name["ls"]["function"]["parameters"]
assert ls["type"] == "object"
assert ls["required"] == [] # path is optional
assert set(ls["properties"]) == {"path"}
read = by_name["read"]["function"]["parameters"]
assert read["type"] == "object"
assert read["required"] == ["path"]
assert set(read["properties"]) == {"path"}
grep = by_name["grep"]["function"]["parameters"]
assert grep["type"] == "object"
assert grep["required"] == ["pattern"]
assert set(grep["properties"]) == {"pattern", "path"}
assert all(p["type"] == "string" for p in grep["properties"].values())
class ScriptedToolLLM:
@@ -156,26 +181,12 @@ def _settings(**kwargs: Any) -> Settings:
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
def _run_search(
db: Session, arguments: dict[str, Any]
def _run_call(
db: Session, name: str, arguments: dict[str, Any]
) -> tuple[AgentHolder, ScriptedToolLLM]:
"""Drive one scripted ``search_documents`` call through ``run_agent``."""
"""Drive one scripted tool call through ``run_agent``."""
holder = AgentHolder()
llm = ScriptedToolLLM(
ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)
)
asyncio.run(_consume(llm, db, holder))
return holder, llm
def _run_read(
db: Session, arguments: dict[str, Any]
) -> tuple[AgentHolder, ScriptedToolLLM]:
"""Drive one scripted ``read_document`` call through ``run_agent``."""
holder = AgentHolder()
llm = ScriptedToolLLM(
ToolCallPiece(id="call_1", name="read_document", arguments=arguments)
)
llm = ScriptedToolLLM(ToolCallPiece(id="call_1", name=name, arguments=arguments))
asyncio.run(_consume(llm, db, holder))
return holder, llm
@@ -197,12 +208,113 @@ async def _consume(
return out
def test_search_whole_kb_through_run_agent(kb, db) -> None:
# ---------- ls (scoped through the real registry) ----------
def test_ls_scoped_to_registered_source_through_run_agent(kb, src, db) -> None:
_doc(db, "Homelab", "a.md", "A", "A-CONTENT")
_doc(db, "Other", "b.md", "B", "B-CONTENT")
db.commit()
holder, llm = _run_call(db, "ls", {"path": "Homelab"})
# Offered: the first request carries AGENT_TOOLS (the 3-tool list).
assert llm.requests[0][1] == AGENT_TOOLS
# Executed against the real DB: the listing filtered to the source.
assert llm.requests[1][0][3]["content"] == (
"1 documents:\nsource: Homelab | path: a.md | title: A"
)
assert holder.tool_calls == 1
assert holder.read_docs == []
def test_ls_scoped_unknown_source_refused_through_run_agent(kb, src, db) -> None:
_doc(db, "Homelab", "a.md", "A", "A-CONTENT")
db.commit()
holder, llm = _run_call(db, "ls", {"path": "Ghost"})
assert (
llm.requests[1][0][3]["content"] == "No source named 'Ghost' — check the ls output."
)
assert holder.tool_calls == 0 and holder.read_docs == []
# ---------- read (the canonical combined source/path form) ----------
def test_read_combined_path_through_run_agent(kb, db) -> None:
"""The combined ``source/path`` identity resolves at the FIRST slash
against the REAL table (a path with further slashes included): the
read executes, the holder records the row, the result header carries
the true source/path."""
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
db.commit()
holder, llm = _run_call(db, "read", {"path": "Alpha/deep/nested/doc.md"})
assert llm.requests[1][0][3]["content"] == (
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
)
assert holder.tool_calls == 1
assert holder.read_docs == [created]
def test_read_bare_source_name_refused_through_run_agent(kb, db) -> None:
"""A bare source name (no '/') can never be a document — the
no-document refusal echoing the argument as passed; the old
split-teaching refusal is gone (phase 70)."""
_doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
db.commit()
holder, llm = _run_call(db, "read", {"path": "Alpha"})
assert (
llm.requests[1][0][3]["content"] == "No document at 'Alpha' — check the ls output."
)
assert holder.tool_calls == 0 and holder.read_docs == []
def test_read_unknown_combined_path_refused_through_run_agent(kb, db) -> None:
"""A combined identity that matches nothing gets the no-document
refusal (the argument echoed as passed — the model sees its own
form)."""
_doc(db, "Alpha", "x.md", "X", "X-CONTENT")
db.commit()
holder, llm = _run_call(db, "read", {"path": "Alpha/nope/deep.md"})
assert (
llm.requests[1][0][3]["content"]
== "No document at 'Alpha/nope/deep.md' — check the ls output."
)
assert holder.tool_calls == 0 and holder.read_docs == []
# ---------- grep (the phase-68 A5 contract under the new name) ----------
def test_all_documents_orders_by_source_then_path(kb, db) -> None:
_doc(db, "Zeta", "b/second.md", "Zeta B", "ZB")
_doc(db, "Zeta", "a/first.md", "Zeta A", "ZA")
_doc(db, "Alpha", "c/third.md", "Alpha C", "AC")
db.commit()
docs = agent.all_documents(db)
assert [(d.source, d.path) for d in docs] == [
("Alpha", "c/third.md"),
("Zeta", "a/first.md"),
("Zeta", "b/second.md"),
]
assert [d.content for d in docs] == ["AC", "ZA", "ZB"] # full rows
def test_grep_whole_kb_through_run_agent(kb, db) -> None:
_doc(db, "Beta", "b/two.md", "Two", "no hit\nNEEDLE in two\nlast")
_doc(db, "Alpha", "a/one.md", "One", "first\nneedle in one\nthird")
db.commit()
holder, llm = _run_search(db, {"pattern": "needle"})
holder, llm = _run_call(db, "grep", {"pattern": "needle"})
# Offered: the first request carries AGENT_TOOLS (the 3-tool list).
assert llm.requests[0][1] == AGENT_TOOLS
@@ -212,17 +324,15 @@ def test_search_whole_kb_through_run_agent(kb, db) -> None:
"Beta/b/two.md:2: NEEDLE in two"
)
assert holder.tool_calls == 1
assert holder.read_docs == [] # locked A5: search adds no context
assert holder.read_docs == [] # locked A5: grep adds no context
def test_search_scoped_through_run_agent(kb, db) -> None:
def test_grep_scoped_through_run_agent(kb, db) -> None:
_doc(db, "Alpha", "a/one.md", "One", "first\nNeedle here\nthird")
_doc(db, "Beta", "b/two.md", "Two", "NEEDLE too")
db.commit()
holder, llm = _run_search(
db, {"pattern": "needle", "source": "Alpha", "path": "a/one.md"}
)
holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/a/one.md"})
# Only the named document is searched — the other one's hit is absent.
assert llm.requests[1][0][3]["content"] == "Alpha/a/one.md:2: Needle here"
@@ -230,90 +340,27 @@ def test_search_scoped_through_run_agent(kb, db) -> None:
assert holder.read_docs == []
def test_search_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:
def test_grep_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:
_doc(db, "Alpha", "a/one.md", "One", "nothing")
db.commit()
holder, llm = _run_search(
db, {"pattern": "needle", "source": "Alpha", "path": "ghost.md"}
)
holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/ghost.md"})
assert (
llm.requests[1][0][3]["content"]
== "No document at Alpha/ghost.md — check the list_documents output."
== "No document at 'Alpha/ghost.md' — check the ls output."
)
assert holder.tool_calls == 0 and holder.read_docs == []
# ---------- combined 'source/path' self-correction (read_document) ----------
def test_read_combined_source_self_corrects_through_run_agent(kb, db) -> None:
"""The model's combined 'source' ('Alpha/deep/nested/doc.md') resolves
through the first-slash split against the REAL table: the read
executes, the holder records the row, the result header carries the
true source/path."""
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
db.commit()
holder, llm = _run_read(
db,
{
"source": "Alpha/deep/nested/doc.md",
"path": "deep/nested/doc.md",
},
)
assert llm.requests[1][0][3]["content"] == (
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
)
assert holder.tool_calls == 1
assert holder.read_docs == [created]
def test_read_combined_source_later_slash_split_through_run_agent(kb, db) -> None:
"""source='Alpha/deep' + path='nested/doc.md' (a split at a LATER
slash) resolves via the continuation candidate against the real
table."""
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
db.commit()
holder, llm = _run_read(
db, {"source": "Alpha/deep", "path": "nested/doc.md"}
)
assert llm.requests[1][0][3]["content"] == (
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
)
assert holder.tool_calls == 1
assert holder.read_docs == [created]
def test_read_combined_source_refusal_teaches_split(kb, db) -> None:
"""A combined source that matches nothing (even split) gets the
educational refusal naming the corrected arguments."""
_doc(db, "Alpha", "x.md", "X", "X-CONTENT")
db.commit()
holder, llm = _run_read(
db, {"source": "Alpha/nope/deep.md", "path": "nope/deep.md"}
)
assert llm.requests[1][0][3]["content"] == (
"source must not contain '/': for 'Alpha/nope/deep.md' call "
"read_document(source='Alpha', path='nope/deep.md')."
)
assert holder.tool_calls == 0 and holder.read_docs == []
def test_search_no_matches_through_run_agent(kb, db) -> None:
def test_grep_no_matches_through_run_agent(kb, db) -> None:
_doc(db, "Alpha", "a/one.md", "One", "nothing matching")
db.commit()
holder, llm = _run_search(db, {"pattern": "zebra"})
holder, llm = _run_call(db, "grep", {"pattern": "zebra"})
assert llm.requests[1][0][3]["content"] == (
"No matches for 'zebra' in the knowledge base."
)
assert holder.tool_calls == 1 # an executed search with zero hits
assert holder.tool_calls == 1 # an executed grep with zero hits
assert holder.read_docs == []
+7 -5
View File
@@ -300,11 +300,12 @@ def test_ui_chrome_has_no_emoji(client, path: str) -> None:
Phase 37 revision (owner permission 2026-08-26, PLAN §4): the agent's
``.tool-call`` line carries the CONTENT marks — 🔎 (list) and 📄
(read) — the only emoji in the whole frontend, and only as the exact
tool-line template strings in app.js. Phase 68 revision: the
``search_documents`` tool line adds the third template literal
("🔎 Searching for "). The guard strips precisely those three
literals; any other emoji, or those marks anywhere else, still
fails."""
tool-line template strings in app.js. Phase 68 revision: the search
tool line (the ``grep`` tool, phase 70) adds the third template
literal ("🔎 Searching for "). Phase 70 revision: the scoped ``ls``
tool line adds the fourth ("🔎 Listing documents in "). The guard strips
precisely those four literals; any other emoji, or those marks
anywhere else, still fails."""
r = client.get(path)
assert r.status_code == 200
text = r.text
@@ -312,6 +313,7 @@ def test_ui_chrome_has_no_emoji(client, path: str) -> None:
text = text.replace('"🔎 Listing documents"', "")
text = text.replace('"📄 Reading "', "")
text = text.replace('"🔎 Searching for "', "")
text = text.replace('"🔎 Listing documents in "', "")
assert _find_emoji(text) == [], f"emoji found in {path}: {_find_emoji(text)!r}"
+94 -31
View File
@@ -22,12 +22,12 @@ from typing import Any
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import func, select, text
from sqlalchemy import delete, func, select, text
from app.api import chat as chat_api
from app.config import Settings, get_settings
from app.main import app as fastapi_app
from app.models import Chunk, QueryLog
from app.models import Chunk, GitSource, QueryLog
from app.rag import agent
from app.rag.agent import AGENT_TOOLS
from app.rag.importer import import_sources
@@ -550,13 +550,13 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
tool_script=[
[
StreamPiece("thinking", "Let me list what is indexed…"),
ToolCallPiece(id="call_1", name="list_documents", arguments={}),
ToolCallPiece(id="call_1", name="ls", arguments={}),
],
[
ToolCallPiece(
id="call_2",
name="read_document",
arguments={"source": "docs", "path": "homelab/backups.md"},
name="read",
arguments={"path": "docs/homelab/backups.md"},
)
],
# the answer request still carries the tools (2 rounds < the
@@ -580,10 +580,12 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
list_frame, read_frame = frames[1], frames[2]
assert set(list_frame) == {"type", "name", "argument"}
assert list_frame["name"] == "list_documents"
assert list_frame["argument"] is None # the tool takes no parameters
assert list_frame["name"] == "ls"
assert list_frame["argument"] is None # no ``path`` argument was passed
assert set(read_frame) == {"type", "name", "argument"}
assert read_frame["name"] == "read_document"
assert read_frame["name"] == "read"
# Phase 70: the frame's argument is the single string the model
# passed — the combined ``source/path``.
assert read_frame["argument"] == "docs/homelab/backups.md"
deltas = [f for f in frames if f["type"] == "delta"]
@@ -621,28 +623,24 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
assert "'docs/homelab/backups.md'" in lines[-1]
def test_grounded_turn_streams_search_tool_frames(
def test_grounded_turn_streams_grep_tool_frames(
client, db, seeded_kb: FakeRagLLM
) -> None:
"""Phase 68: a scripted ``search_documents`` call streams as
``{type: "tool", name: "search_documents", argument: <pattern>}`` —
"""Phase 68 (renamed ``grep`` in phase 70): a scripted ``grep`` call
streams as ``{type: "tool", name: "grep", argument: <pattern>}`` —
the raw pattern is the frame's ``argument`` (the UI renders the
"searching for" line from it). A non-string pattern — a model error
the backend refuses — yields ``argument: null``. A search adds no
the backend refuses — yields ``argument: null``. A grep adds no
source: ``done.sources`` stays the retrieval docs (locked A5)."""
scripted = FakeRagLLM(
tool_script=[
[
ToolCallPiece(
id="call_1",
name="search_documents",
arguments={"pattern": "Cilium"},
),
ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "Cilium"}),
],
[
ToolCallPiece(
id="call_2",
name="search_documents",
name="grep",
arguments={"pattern": 42}, # model error: non-string
),
],
@@ -659,25 +657,90 @@ def test_grounded_turn_streams_search_tool_frames(
types = [f["type"] for f in frames]
assert "error" not in types
assert len(scripted.seen_tools) == 3 # both searches executed (rounds)
assert len(scripted.seen_tools) == 3 # both greps executed (rounds)
tool_frames = [f for f in frames if f["type"] == "tool"]
assert len(tool_frames) == 2
first, second = tool_frames
assert set(first) == {"type", "name", "argument"}
assert first["name"] == "search_documents"
assert first["name"] == "grep"
assert first["argument"] == "Cilium" # the raw pattern
assert set(second) == {"type", "name", "argument"}
assert second["name"] == "search_documents"
assert second["name"] == "grep"
assert second["argument"] is None # the non-string pattern → null
# The searches still answered: deltas, then a grounded done.
# The greps still answered: deltas, then a grounded done.
assert [f for f in frames if f["type"] == "delta"]
done = frames[-1]
assert done["type"] == "done" and done["deflected"] is False
paths = [s["path"] for s in done["sources"]]
assert "homelab/kubernetes.md" in paths # retrieval docs, unchanged
assert "homelab/backups.md" not in paths # a search adds no source
assert "homelab/backups.md" not in paths # a grep adds no source
def test_tool_frames_carry_the_model_arguments_regardless_of_execution(
client, db, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
) -> None:
"""Phase 70 pins: the frame's ``argument`` is the single string
argument the model passed — an ``ls`` frame carries the scope when
the model gave one (null only when it is omitted, pinned above) —
and frame emission is execution-independent: a rejected call (an
unknown ``read`` path) still streams its frame with the model's
argument as-is. The rejected read adds no source (``done.sources``
stays the retrieval docs), and rejected calls count nothing
(``tool_calls=1`` — only the executed scoped ``ls``)."""
# The scoped ``ls`` source-name check reads the registry — insert a
# row resolving to ``docs`` (the fixture's source name) and delete
# it again afterwards.
src = GitSource(url="https://github.com/reese/docs.git", kind="git")
db.add(src)
db.commit()
try:
scripted = FakeRagLLM(
tool_script=[
[ToolCallPiece(id="call_1", name="ls", arguments={"path": "docs"})],
[
ToolCallPiece(
id="call_2", name="read", arguments={"path": "docs/homelab/nope.md"}
)
],
]
)
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
try:
caplog.set_level(logging.INFO, logger="app.chat")
_, _, frames = _stream_chat(client, QUESTION)
finally:
fastapi_app.dependency_overrides.clear()
finally:
db.execute(delete(GitSource).where(GitSource.id == src.id))
db.commit()
types = [f["type"] for f in frames]
assert "error" not in types
# Both calls stream a frame — the rejected read included.
tool_frames = [f for f in frames if f["type"] == "tool"]
assert len(tool_frames) == 2
ls_frame, read_frame = tool_frames
assert set(ls_frame) == {"type", "name", "argument"}
assert ls_frame["name"] == "ls"
assert ls_frame["argument"] == "docs" # the model's scope, as passed
assert set(read_frame) == {"type", "name", "argument"}
assert read_frame["name"] == "read"
# The rejected call's frame still carries the model's argument as
# passed — frame emission is execution-independent.
assert read_frame["argument"] == "docs/homelab/nope.md"
# The rejected read adds no source — done.sources stays retrieval.
done = frames[-1]
assert done["type"] == "done" and done["deflected"] is False
paths = [s["path"] for s in done["sources"]]
assert "homelab/kubernetes.md" in paths # retrieval docs, unchanged
assert "homelab/nope.md" not in paths # the refused read cites nothing
# The rejected call counts nothing — only the executed scoped ls.
lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
assert lines and "tool_calls=1" in lines[-1]
def test_deflected_turn_stays_byte_identical_without_tools(
@@ -690,12 +753,12 @@ def test_deflected_turn_stays_byte_identical_without_tools(
``tools`` key."""
scripted = FakeRagLLM(
tool_script=[
[ToolCallPiece(id="call_1", name="list_documents", arguments={})],
[ToolCallPiece(id="call_1", name="ls", arguments={})],
[
ToolCallPiece(
id="call_2",
name="read_document",
arguments={"source": "docs", "path": "homelab/backups.md"},
name="read",
arguments={"path": "docs/homelab/backups.md"},
)
],
[StreamPiece("content", "never used — the agent never runs")],
@@ -743,12 +806,12 @@ def test_zero_max_rounds_reproduce_pre_phase_single_request(
the kill switch survives the phase-45 budget removal."""
scripted = FakeRagLLM(
tool_script=[
[ToolCallPiece(id="call_1", name="list_documents", arguments={})],
[ToolCallPiece(id="call_1", name="ls", arguments={})],
[
ToolCallPiece(
id="call_2",
name="read_document",
arguments={"source": "docs", "path": "homelab/backups.md"},
name="read",
arguments={"path": "docs/homelab/backups.md"},
)
],
]
@@ -799,7 +862,7 @@ def test_tool_execution_db_failure_yields_error_event(
``error`` event as the pre-stream retrieval path — never a severed
stream (the "never stale" contract, PLAN §7.4)."""
scripted = FakeRagLLM(
tool_script=[[ToolCallPiece(id="call_1", name="list_documents", arguments={})]]
tool_script=[[ToolCallPiece(id="call_1", name="ls", arguments={})]]
)
def boom(*_a: Any, **_k: Any) -> Any:
@@ -815,7 +878,7 @@ def test_tool_execution_db_failure_yields_error_event(
# The ``tool`` frame went out first (the model requested the call);
# the failed execution ends the turn with the structured error event.
assert [f["type"] for f in frames] == ["tool", "error"]
assert frames[0]["name"] == "list_documents"
assert frames[0]["name"] == "ls"
assert "offline mid-question" in frames[1]["detail"]
assert db.scalars(select(QueryLog)).all() == [] # no row for a failed turn
+6 -2
View File
@@ -64,8 +64,12 @@ FULL_BRAIN: dict[str, Any] = {
"suggestions": ["What ports does Traefik expose?"],
"thinking": "The kubernetes doc covers the cluster layout…",
"tools": [
{"name": "read_document", "argument": "Homelab/kubernetes.md"},
{"name": "list_documents", "argument": None},
{"name": "read", "argument": "Homelab/kubernetes.md"},
{"name": "ls", "argument": None},
# Saved chats persisting the pre-phase-70 tool names still
# validate — ``name`` is opaque to the API (no migration,
# locked: old chats render fine).
{"name": "read_document", "argument": "Homelab/legacy-notes.md"},
],
"stopped": False,
}