feat(agent): align the document tools with the harness-trained shape — ls, read(path), grep(pattern, path?)
This commit is contained in:
@@ -1,17 +1,19 @@
|
||||
"""Integration: the agent DB accessors against real Postgres (phase 37).
|
||||
"""Integration: the agent DB accessors against real Postgres (phase 37;
|
||||
the harness-aligned ``ls``/``read``/``grep`` surface, phase 70).
|
||||
|
||||
``list_catalog`` must order rows by ``(source, path)`` — the same order as
|
||||
``GET /api/docs`` — and ``find_document`` must resolve a hit to the full
|
||||
document row (content included, for the never-truncated read) and return
|
||||
``None`` for unknown ``source``/``path`` pairs. Phase 68: the
|
||||
``search_documents`` tool is pinned here too — its locked parameter
|
||||
shape in ``AGENT_TOOLS``, and a scripted ``ToolCallPiece`` executed
|
||||
through ``run_agent`` against the real DB (``all_documents`` for a
|
||||
whole-KB search, ``find_document`` for a scoped one). The
|
||||
combined-form self-correction (a ``source`` argument carrying
|
||||
``source/path``) is pinned here as well, through ``run_agent``:
|
||||
the split read executes against the real table, and a still-unknown
|
||||
split gets the educational refusal.
|
||||
``list_catalog`` must order rows by ``(source, path)`` — the same order
|
||||
as ``GET /api/docs`` — ``list_source_names`` must resolve the
|
||||
registered source names (the scoped ``ls`` join), and ``find_document``
|
||||
must resolve a hit to the full document row (content included, for the
|
||||
never-truncated read) and return ``None`` for unknown pairs. Phase 70:
|
||||
the ``ls``/``read``/``grep`` tools are pinned here too — the locked
|
||||
parameter shape in ``AGENT_TOOLS``, and scripted ``ToolCallPiece``s
|
||||
executed through ``run_agent`` against the real DB: ``ls`` scoped to a
|
||||
registered source name (unknown name → refusal), ``read`` on the
|
||||
canonical combined ``source/path`` form (first-slash split; a bare
|
||||
source name and an unknown identity get the no-document refusal), and
|
||||
``grep`` (``all_documents`` for a whole-KB search, ``find_document`` for
|
||||
a scoped one).
|
||||
|
||||
Requires: podman compose up -d db
|
||||
"""
|
||||
@@ -24,11 +26,11 @@ from copy import deepcopy
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import delete, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import Document
|
||||
from app.models import Document, GitSource
|
||||
from app.rag import agent
|
||||
from app.rag.agent import AGENT_TOOLS, AgentHolder, run_agent
|
||||
from app.rag.llm import LLMClient, RetryPiece, StreamPiece, ToolCallPiece
|
||||
@@ -58,6 +60,19 @@ def kb(db) -> Iterator[None]:
|
||||
db.commit()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def src(db) -> Iterator[GitSource]:
|
||||
"""One registered git source — the scoped ``ls`` source-name check
|
||||
reads the real registry, so the row is inserted and deleted around
|
||||
the tests (``repo_name`` resolves the URL to ``Homelab``)."""
|
||||
row = GitSource(url="https://github.com/reese/Homelab.git", kind="git")
|
||||
db.add(row)
|
||||
db.commit()
|
||||
yield row
|
||||
db.execute(delete(GitSource).where(GitSource.id == row.id))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_list_catalog_orders_by_source_then_path(kb, db) -> None:
|
||||
_doc(db, "Zeta", "b/second.md", "Zeta B", "ZB")
|
||||
_doc(db, "Zeta", "a/first.md", "Zeta A", "ZA")
|
||||
@@ -75,6 +90,23 @@ def test_list_catalog_is_empty_without_rows(kb, db) -> None:
|
||||
assert agent.list_catalog(db) == []
|
||||
|
||||
|
||||
def test_list_source_names_resolves_registry_rows(db) -> None:
|
||||
"""The real registry: git names resolve through the import pipeline's
|
||||
``repo_name`` (trailing ``.git`` stripped); a second row resolving to
|
||||
the same name (the phase-69 sibling case) is listed once."""
|
||||
a = GitSource(url="https://github.com/reese/Homelab.git", kind="git")
|
||||
b = GitSource(url="https://github.com/reese/Homelab", kind="git") # sibling
|
||||
c = GitSource(url="https://e.com/deployments", kind="git")
|
||||
db.add_all([a, b, c])
|
||||
db.commit()
|
||||
try:
|
||||
assert agent.list_source_names(db).count("Homelab") == 1 # deduped
|
||||
assert "deployments" in agent.list_source_names(db)
|
||||
finally:
|
||||
db.execute(delete(GitSource).where(GitSource.id.in_([a.id, b.id, c.id])))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_find_document_hit_returns_full_row(kb, db) -> None:
|
||||
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
|
||||
db.commit()
|
||||
@@ -97,36 +129,29 @@ def test_find_document_none_for_unknown_pairs(kb, db) -> None:
|
||||
assert agent.find_document(db, "nope", "nope.md") is None # nothing at all
|
||||
|
||||
|
||||
# ---------- search_documents (phase 68) ----------
|
||||
# ---------- AGENT_TOOLS surface (phase 70: ls / read / grep) ----------
|
||||
|
||||
|
||||
def test_all_documents_orders_by_source_then_path(kb, db) -> None:
|
||||
_doc(db, "Zeta", "b/second.md", "Zeta B", "ZB")
|
||||
_doc(db, "Zeta", "a/first.md", "Zeta A", "ZA")
|
||||
_doc(db, "Alpha", "c/third.md", "Alpha C", "AC")
|
||||
db.commit()
|
||||
|
||||
docs = agent.all_documents(db)
|
||||
assert [(d.source, d.path) for d in docs] == [
|
||||
("Alpha", "c/third.md"),
|
||||
("Zeta", "a/first.md"),
|
||||
("Zeta", "b/second.md"),
|
||||
]
|
||||
assert [d.content for d in docs] == ["AC", "ZA", "ZB"] # full rows
|
||||
|
||||
|
||||
def test_agent_tools_offers_search_documents_with_locked_shape() -> None:
|
||||
def test_agent_tools_offers_the_harness_aligned_surface() -> None:
|
||||
by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
|
||||
assert list(by_name) == [ # the third tool, in order
|
||||
"list_documents",
|
||||
"read_document",
|
||||
"search_documents",
|
||||
assert list(by_name) == [ # the harness order, phase 70
|
||||
"ls",
|
||||
"read",
|
||||
"grep",
|
||||
]
|
||||
search = by_name["search_documents"]["function"]["parameters"]
|
||||
assert search["type"] == "object"
|
||||
assert search["required"] == ["pattern"]
|
||||
assert set(search["properties"]) == {"pattern", "source", "path"}
|
||||
assert all(p["type"] == "string" for p in search["properties"].values())
|
||||
ls = by_name["ls"]["function"]["parameters"]
|
||||
assert ls["type"] == "object"
|
||||
assert ls["required"] == [] # path is optional
|
||||
assert set(ls["properties"]) == {"path"}
|
||||
read = by_name["read"]["function"]["parameters"]
|
||||
assert read["type"] == "object"
|
||||
assert read["required"] == ["path"]
|
||||
assert set(read["properties"]) == {"path"}
|
||||
grep = by_name["grep"]["function"]["parameters"]
|
||||
assert grep["type"] == "object"
|
||||
assert grep["required"] == ["pattern"]
|
||||
assert set(grep["properties"]) == {"pattern", "path"}
|
||||
assert all(p["type"] == "string" for p in grep["properties"].values())
|
||||
|
||||
|
||||
class ScriptedToolLLM:
|
||||
@@ -156,26 +181,12 @@ def _settings(**kwargs: Any) -> Settings:
|
||||
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _run_search(
|
||||
db: Session, arguments: dict[str, Any]
|
||||
def _run_call(
|
||||
db: Session, name: str, arguments: dict[str, Any]
|
||||
) -> tuple[AgentHolder, ScriptedToolLLM]:
|
||||
"""Drive one scripted ``search_documents`` call through ``run_agent``."""
|
||||
"""Drive one scripted tool call through ``run_agent``."""
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedToolLLM(
|
||||
ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)
|
||||
)
|
||||
asyncio.run(_consume(llm, db, holder))
|
||||
return holder, llm
|
||||
|
||||
|
||||
def _run_read(
|
||||
db: Session, arguments: dict[str, Any]
|
||||
) -> tuple[AgentHolder, ScriptedToolLLM]:
|
||||
"""Drive one scripted ``read_document`` call through ``run_agent``."""
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedToolLLM(
|
||||
ToolCallPiece(id="call_1", name="read_document", arguments=arguments)
|
||||
)
|
||||
llm = ScriptedToolLLM(ToolCallPiece(id="call_1", name=name, arguments=arguments))
|
||||
asyncio.run(_consume(llm, db, holder))
|
||||
return holder, llm
|
||||
|
||||
@@ -197,12 +208,113 @@ async def _consume(
|
||||
return out
|
||||
|
||||
|
||||
def test_search_whole_kb_through_run_agent(kb, db) -> None:
|
||||
# ---------- ls (scoped through the real registry) ----------
|
||||
|
||||
|
||||
def test_ls_scoped_to_registered_source_through_run_agent(kb, src, db) -> None:
|
||||
_doc(db, "Homelab", "a.md", "A", "A-CONTENT")
|
||||
_doc(db, "Other", "b.md", "B", "B-CONTENT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Homelab"})
|
||||
|
||||
# Offered: the first request carries AGENT_TOOLS (the 3-tool list).
|
||||
assert llm.requests[0][1] == AGENT_TOOLS
|
||||
# Executed against the real DB: the listing filtered to the source.
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"1 documents:\nsource: Homelab | path: a.md | title: A"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
|
||||
def test_ls_scoped_unknown_source_refused_through_run_agent(kb, src, db) -> None:
|
||||
_doc(db, "Homelab", "a.md", "A", "A-CONTENT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Ghost"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"] == "No source named 'Ghost' — check the ls output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
|
||||
# ---------- read (the canonical combined source/path form) ----------
|
||||
|
||||
|
||||
def test_read_combined_path_through_run_agent(kb, db) -> None:
|
||||
"""The combined ``source/path`` identity resolves at the FIRST slash
|
||||
against the REAL table (a path with further slashes included): the
|
||||
read executes, the holder records the row, the result header carries
|
||||
the true source/path."""
|
||||
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "read", {"path": "Alpha/deep/nested/doc.md"})
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [created]
|
||||
|
||||
|
||||
def test_read_bare_source_name_refused_through_run_agent(kb, db) -> None:
|
||||
"""A bare source name (no '/') can never be a document — the
|
||||
no-document refusal echoing the argument as passed; the old
|
||||
split-teaching refusal is gone (phase 70)."""
|
||||
_doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "read", {"path": "Alpha"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"] == "No document at 'Alpha' — check the ls output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
|
||||
def test_read_unknown_combined_path_refused_through_run_agent(kb, db) -> None:
|
||||
"""A combined identity that matches nothing gets the no-document
|
||||
refusal (the argument echoed as passed — the model sees its own
|
||||
form)."""
|
||||
_doc(db, "Alpha", "x.md", "X", "X-CONTENT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "read", {"path": "Alpha/nope/deep.md"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== "No document at 'Alpha/nope/deep.md' — check the ls output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
|
||||
# ---------- grep (the phase-68 A5 contract under the new name) ----------
|
||||
|
||||
|
||||
def test_all_documents_orders_by_source_then_path(kb, db) -> None:
|
||||
_doc(db, "Zeta", "b/second.md", "Zeta B", "ZB")
|
||||
_doc(db, "Zeta", "a/first.md", "Zeta A", "ZA")
|
||||
_doc(db, "Alpha", "c/third.md", "Alpha C", "AC")
|
||||
db.commit()
|
||||
|
||||
docs = agent.all_documents(db)
|
||||
assert [(d.source, d.path) for d in docs] == [
|
||||
("Alpha", "c/third.md"),
|
||||
("Zeta", "a/first.md"),
|
||||
("Zeta", "b/second.md"),
|
||||
]
|
||||
assert [d.content for d in docs] == ["AC", "ZA", "ZB"] # full rows
|
||||
|
||||
|
||||
def test_grep_whole_kb_through_run_agent(kb, db) -> None:
|
||||
_doc(db, "Beta", "b/two.md", "Two", "no hit\nNEEDLE in two\nlast")
|
||||
_doc(db, "Alpha", "a/one.md", "One", "first\nneedle in one\nthird")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_search(db, {"pattern": "needle"})
|
||||
holder, llm = _run_call(db, "grep", {"pattern": "needle"})
|
||||
|
||||
# Offered: the first request carries AGENT_TOOLS (the 3-tool list).
|
||||
assert llm.requests[0][1] == AGENT_TOOLS
|
||||
@@ -212,17 +324,15 @@ def test_search_whole_kb_through_run_agent(kb, db) -> None:
|
||||
"Beta/b/two.md:2: NEEDLE in two"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [] # locked A5: search adds no context
|
||||
assert holder.read_docs == [] # locked A5: grep adds no context
|
||||
|
||||
|
||||
def test_search_scoped_through_run_agent(kb, db) -> None:
|
||||
def test_grep_scoped_through_run_agent(kb, db) -> None:
|
||||
_doc(db, "Alpha", "a/one.md", "One", "first\nNeedle here\nthird")
|
||||
_doc(db, "Beta", "b/two.md", "Two", "NEEDLE too")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_search(
|
||||
db, {"pattern": "needle", "source": "Alpha", "path": "a/one.md"}
|
||||
)
|
||||
holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/a/one.md"})
|
||||
|
||||
# Only the named document is searched — the other one's hit is absent.
|
||||
assert llm.requests[1][0][3]["content"] == "Alpha/a/one.md:2: Needle here"
|
||||
@@ -230,90 +340,27 @@ def test_search_scoped_through_run_agent(kb, db) -> None:
|
||||
assert holder.read_docs == []
|
||||
|
||||
|
||||
def test_search_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:
|
||||
def test_grep_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:
|
||||
_doc(db, "Alpha", "a/one.md", "One", "nothing")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_search(
|
||||
db, {"pattern": "needle", "source": "Alpha", "path": "ghost.md"}
|
||||
)
|
||||
holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/ghost.md"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== "No document at Alpha/ghost.md — check the list_documents output."
|
||||
== "No document at 'Alpha/ghost.md' — check the ls output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
|
||||
# ---------- combined 'source/path' self-correction (read_document) ----------
|
||||
|
||||
|
||||
def test_read_combined_source_self_corrects_through_run_agent(kb, db) -> None:
|
||||
"""The model's combined 'source' ('Alpha/deep/nested/doc.md') resolves
|
||||
through the first-slash split against the REAL table: the read
|
||||
executes, the holder records the row, the result header carries the
|
||||
true source/path."""
|
||||
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_read(
|
||||
db,
|
||||
{
|
||||
"source": "Alpha/deep/nested/doc.md",
|
||||
"path": "deep/nested/doc.md",
|
||||
},
|
||||
)
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [created]
|
||||
|
||||
|
||||
def test_read_combined_source_later_slash_split_through_run_agent(kb, db) -> None:
|
||||
"""source='Alpha/deep' + path='nested/doc.md' (a split at a LATER
|
||||
slash) resolves via the continuation candidate against the real
|
||||
table."""
|
||||
created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_read(
|
||||
db, {"source": "Alpha/deep", "path": "nested/doc.md"}
|
||||
)
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"Document Alpha/deep/nested/doc.md:\nFULL-TEXT"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [created]
|
||||
|
||||
|
||||
def test_read_combined_source_refusal_teaches_split(kb, db) -> None:
|
||||
"""A combined source that matches nothing (even split) gets the
|
||||
educational refusal naming the corrected arguments."""
|
||||
_doc(db, "Alpha", "x.md", "X", "X-CONTENT")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_read(
|
||||
db, {"source": "Alpha/nope/deep.md", "path": "nope/deep.md"}
|
||||
)
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"source must not contain '/': for 'Alpha/nope/deep.md' call "
|
||||
"read_document(source='Alpha', path='nope/deep.md')."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
|
||||
def test_search_no_matches_through_run_agent(kb, db) -> None:
|
||||
def test_grep_no_matches_through_run_agent(kb, db) -> None:
|
||||
_doc(db, "Alpha", "a/one.md", "One", "nothing matching")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_search(db, {"pattern": "zebra"})
|
||||
holder, llm = _run_call(db, "grep", {"pattern": "zebra"})
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No matches for 'zebra' in the knowledge base."
|
||||
)
|
||||
assert holder.tool_calls == 1 # an executed search with zero hits
|
||||
assert holder.tool_calls == 1 # an executed grep with zero hits
|
||||
assert holder.read_docs == []
|
||||
|
||||
@@ -300,11 +300,12 @@ def test_ui_chrome_has_no_emoji(client, path: str) -> None:
|
||||
Phase 37 revision (owner permission 2026-08-26, PLAN §4): the agent's
|
||||
``.tool-call`` line carries the CONTENT marks — 🔎 (list) and 📄
|
||||
(read) — the only emoji in the whole frontend, and only as the exact
|
||||
tool-line template strings in app.js. Phase 68 revision: the
|
||||
``search_documents`` tool line adds the third template literal
|
||||
("🔎 Searching for "). The guard strips precisely those three
|
||||
literals; any other emoji, or those marks anywhere else, still
|
||||
fails."""
|
||||
tool-line template strings in app.js. Phase 68 revision: the search
|
||||
tool line (the ``grep`` tool, phase 70) adds the third template
|
||||
literal ("🔎 Searching for "). Phase 70 revision: the scoped ``ls``
|
||||
tool line adds the fourth ("🔎 Listing documents in "). The guard strips
|
||||
precisely those four literals; any other emoji, or those marks
|
||||
anywhere else, still fails."""
|
||||
r = client.get(path)
|
||||
assert r.status_code == 200
|
||||
text = r.text
|
||||
@@ -312,6 +313,7 @@ def test_ui_chrome_has_no_emoji(client, path: str) -> None:
|
||||
text = text.replace('"🔎 Listing documents"', "")
|
||||
text = text.replace('"📄 Reading "', "")
|
||||
text = text.replace('"🔎 Searching for "', "")
|
||||
text = text.replace('"🔎 Listing documents in "', "")
|
||||
assert _find_emoji(text) == [], f"emoji found in {path}: {_find_emoji(text)!r}"
|
||||
|
||||
|
||||
|
||||
@@ -22,12 +22,12 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy import delete, func, select, text
|
||||
|
||||
from app.api import chat as chat_api
|
||||
from app.config import Settings, get_settings
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Chunk, QueryLog
|
||||
from app.models import Chunk, GitSource, QueryLog
|
||||
from app.rag import agent
|
||||
from app.rag.agent import AGENT_TOOLS
|
||||
from app.rag.importer import import_sources
|
||||
@@ -550,13 +550,13 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
|
||||
tool_script=[
|
||||
[
|
||||
StreamPiece("thinking", "Let me list what is indexed…"),
|
||||
ToolCallPiece(id="call_1", name="list_documents", arguments={}),
|
||||
ToolCallPiece(id="call_1", name="ls", arguments={}),
|
||||
],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="read_document",
|
||||
arguments={"source": "docs", "path": "homelab/backups.md"},
|
||||
name="read",
|
||||
arguments={"path": "docs/homelab/backups.md"},
|
||||
)
|
||||
],
|
||||
# the answer request still carries the tools (2 rounds < the
|
||||
@@ -580,10 +580,12 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
|
||||
|
||||
list_frame, read_frame = frames[1], frames[2]
|
||||
assert set(list_frame) == {"type", "name", "argument"}
|
||||
assert list_frame["name"] == "list_documents"
|
||||
assert list_frame["argument"] is None # the tool takes no parameters
|
||||
assert list_frame["name"] == "ls"
|
||||
assert list_frame["argument"] is None # no ``path`` argument was passed
|
||||
assert set(read_frame) == {"type", "name", "argument"}
|
||||
assert read_frame["name"] == "read_document"
|
||||
assert read_frame["name"] == "read"
|
||||
# Phase 70: the frame's argument is the single string the model
|
||||
# passed — the combined ``source/path``.
|
||||
assert read_frame["argument"] == "docs/homelab/backups.md"
|
||||
|
||||
deltas = [f for f in frames if f["type"] == "delta"]
|
||||
@@ -621,28 +623,24 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
|
||||
assert "'docs/homelab/backups.md'" in lines[-1]
|
||||
|
||||
|
||||
def test_grounded_turn_streams_search_tool_frames(
|
||||
def test_grounded_turn_streams_grep_tool_frames(
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
"""Phase 68: a scripted ``search_documents`` call streams as
|
||||
``{type: "tool", name: "search_documents", argument: <pattern>}`` —
|
||||
"""Phase 68 (renamed ``grep`` in phase 70): a scripted ``grep`` call
|
||||
streams as ``{type: "tool", name: "grep", argument: <pattern>}`` —
|
||||
the raw pattern is the frame's ``argument`` (the UI renders the
|
||||
"searching for" line from it). A non-string pattern — a model error
|
||||
the backend refuses — yields ``argument: null``. A search adds no
|
||||
the backend refuses — yields ``argument: null``. A grep adds no
|
||||
source: ``done.sources`` stays the retrieval docs (locked A5)."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="search_documents",
|
||||
arguments={"pattern": "Cilium"},
|
||||
),
|
||||
ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "Cilium"}),
|
||||
],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="search_documents",
|
||||
name="grep",
|
||||
arguments={"pattern": 42}, # model error: non-string
|
||||
),
|
||||
],
|
||||
@@ -659,25 +657,90 @@ def test_grounded_turn_streams_search_tool_frames(
|
||||
|
||||
types = [f["type"] for f in frames]
|
||||
assert "error" not in types
|
||||
assert len(scripted.seen_tools) == 3 # both searches executed (rounds)
|
||||
assert len(scripted.seen_tools) == 3 # both greps executed (rounds)
|
||||
|
||||
tool_frames = [f for f in frames if f["type"] == "tool"]
|
||||
assert len(tool_frames) == 2
|
||||
first, second = tool_frames
|
||||
assert set(first) == {"type", "name", "argument"}
|
||||
assert first["name"] == "search_documents"
|
||||
assert first["name"] == "grep"
|
||||
assert first["argument"] == "Cilium" # the raw pattern
|
||||
assert set(second) == {"type", "name", "argument"}
|
||||
assert second["name"] == "search_documents"
|
||||
assert second["name"] == "grep"
|
||||
assert second["argument"] is None # the non-string pattern → null
|
||||
|
||||
# The searches still answered: deltas, then a grounded done.
|
||||
# The greps still answered: deltas, then a grounded done.
|
||||
assert [f for f in frames if f["type"] == "delta"]
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done" and done["deflected"] is False
|
||||
paths = [s["path"] for s in done["sources"]]
|
||||
assert "homelab/kubernetes.md" in paths # retrieval docs, unchanged
|
||||
assert "homelab/backups.md" not in paths # a search adds no source
|
||||
assert "homelab/backups.md" not in paths # a grep adds no source
|
||||
|
||||
|
||||
def test_tool_frames_carry_the_model_arguments_regardless_of_execution(
|
||||
client, db, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""Phase 70 pins: the frame's ``argument`` is the single string
|
||||
argument the model passed — an ``ls`` frame carries the scope when
|
||||
the model gave one (null only when it is omitted, pinned above) —
|
||||
and frame emission is execution-independent: a rejected call (an
|
||||
unknown ``read`` path) still streams its frame with the model's
|
||||
argument as-is. The rejected read adds no source (``done.sources``
|
||||
stays the retrieval docs), and rejected calls count nothing
|
||||
(``tool_calls=1`` — only the executed scoped ``ls``)."""
|
||||
# The scoped ``ls`` source-name check reads the registry — insert a
|
||||
# row resolving to ``docs`` (the fixture's source name) and delete
|
||||
# it again afterwards.
|
||||
src = GitSource(url="https://github.com/reese/docs.git", kind="git")
|
||||
db.add(src)
|
||||
db.commit()
|
||||
try:
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[ToolCallPiece(id="call_1", name="ls", arguments={"path": "docs"})],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_2", name="read", arguments={"path": "docs/homelab/nope.md"}
|
||||
)
|
||||
],
|
||||
]
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
caplog.set_level(logging.INFO, logger="app.chat")
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
finally:
|
||||
db.execute(delete(GitSource).where(GitSource.id == src.id))
|
||||
db.commit()
|
||||
|
||||
types = [f["type"] for f in frames]
|
||||
assert "error" not in types
|
||||
# Both calls stream a frame — the rejected read included.
|
||||
tool_frames = [f for f in frames if f["type"] == "tool"]
|
||||
assert len(tool_frames) == 2
|
||||
ls_frame, read_frame = tool_frames
|
||||
assert set(ls_frame) == {"type", "name", "argument"}
|
||||
assert ls_frame["name"] == "ls"
|
||||
assert ls_frame["argument"] == "docs" # the model's scope, as passed
|
||||
assert set(read_frame) == {"type", "name", "argument"}
|
||||
assert read_frame["name"] == "read"
|
||||
# The rejected call's frame still carries the model's argument as
|
||||
# passed — frame emission is execution-independent.
|
||||
assert read_frame["argument"] == "docs/homelab/nope.md"
|
||||
|
||||
# The rejected read adds no source — done.sources stays retrieval.
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done" and done["deflected"] is False
|
||||
paths = [s["path"] for s in done["sources"]]
|
||||
assert "homelab/kubernetes.md" in paths # retrieval docs, unchanged
|
||||
assert "homelab/nope.md" not in paths # the refused read cites nothing
|
||||
|
||||
# The rejected call counts nothing — only the executed scoped ls.
|
||||
lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
|
||||
assert lines and "tool_calls=1" in lines[-1]
|
||||
|
||||
|
||||
def test_deflected_turn_stays_byte_identical_without_tools(
|
||||
@@ -690,12 +753,12 @@ def test_deflected_turn_stays_byte_identical_without_tools(
|
||||
``tools`` key."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[ToolCallPiece(id="call_1", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_1", name="ls", arguments={})],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="read_document",
|
||||
arguments={"source": "docs", "path": "homelab/backups.md"},
|
||||
name="read",
|
||||
arguments={"path": "docs/homelab/backups.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "never used — the agent never runs")],
|
||||
@@ -743,12 +806,12 @@ def test_zero_max_rounds_reproduce_pre_phase_single_request(
|
||||
the kill switch survives the phase-45 budget removal."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[ToolCallPiece(id="call_1", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_1", name="ls", arguments={})],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="read_document",
|
||||
arguments={"source": "docs", "path": "homelab/backups.md"},
|
||||
name="read",
|
||||
arguments={"path": "docs/homelab/backups.md"},
|
||||
)
|
||||
],
|
||||
]
|
||||
@@ -799,7 +862,7 @@ def test_tool_execution_db_failure_yields_error_event(
|
||||
``error`` event as the pre-stream retrieval path — never a severed
|
||||
stream (the "never stale" contract, PLAN §7.4)."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[[ToolCallPiece(id="call_1", name="list_documents", arguments={})]]
|
||||
tool_script=[[ToolCallPiece(id="call_1", name="ls", arguments={})]]
|
||||
)
|
||||
|
||||
def boom(*_a: Any, **_k: Any) -> Any:
|
||||
@@ -815,7 +878,7 @@ def test_tool_execution_db_failure_yields_error_event(
|
||||
# The ``tool`` frame went out first (the model requested the call);
|
||||
# the failed execution ends the turn with the structured error event.
|
||||
assert [f["type"] for f in frames] == ["tool", "error"]
|
||||
assert frames[0]["name"] == "list_documents"
|
||||
assert frames[0]["name"] == "ls"
|
||||
assert "offline mid-question" in frames[1]["detail"]
|
||||
assert db.scalars(select(QueryLog)).all() == [] # no row for a failed turn
|
||||
|
||||
|
||||
@@ -64,8 +64,12 @@ FULL_BRAIN: dict[str, Any] = {
|
||||
"suggestions": ["What ports does Traefik expose?"],
|
||||
"thinking": "The kubernetes doc covers the cluster layout…",
|
||||
"tools": [
|
||||
{"name": "read_document", "argument": "Homelab/kubernetes.md"},
|
||||
{"name": "list_documents", "argument": None},
|
||||
{"name": "read", "argument": "Homelab/kubernetes.md"},
|
||||
{"name": "ls", "argument": None},
|
||||
# Saved chats persisting the pre-phase-70 tool names still
|
||||
# validate — ``name`` is opaque to the API (no migration,
|
||||
# locked: old chats render fine).
|
||||
{"name": "read_document", "argument": "Homelab/legacy-notes.md"},
|
||||
],
|
||||
"stopped": False,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user