feat(agent): search_documents tool — the model can grep the indexed documents for an exact string
Build and Push Containers / build-and-push-db (push) Canceled after 0s
Build and Push Containers / build-and-push-app (push) Canceled after 1m11s

This commit is contained in:
2026-09-02 12:04:34 -04:00
parent 88293ed02f
commit 8cf3a827ee
26 changed files with 1780 additions and 85 deletions
+118
View File
@@ -100,6 +100,24 @@ Implements just enough of the aipi surface:
offered — e.g. ``agent_max_rounds=0``) behave exactly as today.
``E2E_REAL_LLM=1`` ignores the mock entirely (the real model does
what it does).
- user message containing ``search your documents``
(``SEARCH_TRIGGER``, phase 68, search tool) **and** the system
prompt carries the ``<tools>`` section -> the deterministic SEARCH
tool flow, discriminated statelessly from the messages (streaming
only):
* request 1 (``tools`` offered, no search result yet): stream
ONLY ``tool_calls`` deltas — ``search_documents`` with
``{"pattern": SEARCH_PATTERN}`` (id ``call_0``);
* request 2 (a ``tool``-role search result in the messages —
recognizable by its ``source/path:line: text`` match lines or
the sentinel in its content): the content answer, deterministic:
``Found <first matched line's content up to 80 chars>`` — so a
suite can assert the search result reached the model and landed
in the answer.
Checked BEFORE the plain ``use your tools`` flow (it is the more
specific phrase — same convention as ``think in paragraphs``); no
existing E2E question or fixture file contains the trigger, so
every other suite is unaffected.
- user message containing ``show me a table`` (phase 44, markdown
tables, TODO.md L6) -> the fixed table answer (``TABLE_ANSWER``):
a 3-column service table, an ``<img onerror>`` XSS probe line, and
@@ -264,6 +282,24 @@ TOOLS_TRIGGER = "use your tools"
#: so the 3-step flow is untouched.
MULTI_READ_TRIGGER = "read two documents"
#: Phase 68 (search tool, TODO.md L4): a user message containing this
#: substring (case-insensitive) — combined with the ``<tools>`` section
#: in the system prompt — drives the deterministic SEARCH tool flow
#: (search_documents for ``SEARCH_PATTERN`` → the "Found …" answer),
#: documented in the module docstring. Checked BEFORE ``TOOLS_TRIGGER``
#: (the more specific phrase wins — the same convention as
#: ``THINK_PARAS_TRIGGER``); verified 2026-09-01: no existing E2E
#: question or fixture file contains the phrase, so every other suite
#: is unaffected.
SEARCH_TRIGGER = "search your documents"
#: The sentinel the search flow greps for: the e2e fixture document
#: (``tests/fixtures/search_docs/reese-notes.md``) carries exactly one
#: line containing it, so the search result — and the "Found …" answer
#: that quotes its first matched line — is byte-stable (the sentinel
#: convention of ``END_OF_NOTES_TRIGGER``).
SEARCH_PATTERN = "reese-sentinel-42"
#: Phase 44 (markdown-tables story, TODO.md L6): a user message
#: containing this substring (case-insensitive) gets the fixed table
#: answer (``TABLE_ANSWER`` below) — a 3-column table, an XSS probe
@@ -424,6 +460,69 @@ def _catalog_docs(body: dict[str, Any]) -> list[tuple[str, str]]:
return docs
#: One line of the agent's ``search_documents`` output (app.rag.agent
#: ``_execute_tool``, phase 68): ``source/path:LINE: text``. The
#: non-greedy prefix keeps nested paths (``/`` in the path) intact.
_SEARCH_LINE_RE = re.compile(r"^(?P<sp>.+?):(?P<line>\d+): (?P<text>.*)$")
def _search_result_line(body: dict[str, Any]) -> str | None:
"""The first matched line's text of a search result in the messages.
A search result is a ``tool``-role message — never a read result
(those start with the agent's ``"Document "`` prefix) — that either
carries ``source/path:LINE: text`` match lines (the agent's
``search_documents`` output, phase 68) or the sentinel pattern
itself (its no-match line quotes the pattern). Returns the first
match line's ``text`` part (already 200-char-capped server-side),
or the message's first line in the sentinel-only shape, or ``None``
when no search result is in the messages yet.
"""
sentinel = SEARCH_PATTERN.lower()
for m in _messages(body):
if m.get("role") != "tool":
continue
content = str(m.get("content") or "")
if content.startswith(_READ_RESULT_PREFIX):
continue
for line in content.splitlines():
match = _SEARCH_LINE_RE.match(line)
if match:
return match.group("text")
if sentinel in content.lower():
lines = content.splitlines()
return lines[0] if lines else ""
return None
def _search_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
"""Classify a SEARCH_TRIGGER request into a step of the search flow.
* ``("search",)`` — ``tools`` are offered and no search result is
in the messages yet: the model greps the whole KB for
``SEARCH_PATTERN`` (id ``call_0``).
* ``("found", first_line)`` — a ``tool``-role search result is in
the messages: the model answers, quoting the first matched line
(``Found <first matched line's content up to 80 chars>``). Reached
regardless of the ``tools`` parameter (phase 45 keeps the tools
offered until the round cap).
* ``None`` — not the search flow: the trigger is absent, the
``<tools>`` section is missing (deflected turns never carry it),
or ``tools`` are not offered and no search result is in the
messages yet (e.g. ``agent_max_rounds=0``).
"""
if SEARCH_TRIGGER not in _user(body).lower():
return None
if "<tools>" not in _system(body):
return None
first_line = _search_result_line(body)
if first_line is not None:
return ("found", first_line)
if not body.get("tools"):
return None
return ("search",)
def _tool_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
"""Classify a marker request into one step of the tool flow.
@@ -948,6 +1047,25 @@ def chat_completions(body: dict[str, Any]) -> Any:
if _chat_dead(RETRY_TRIGGER, RETRY_DEAD_ATTEMPTS):
return _llm_500(RETRY_TRIGGER)
_fail_posts[RETRY_TRIGGER] = 0 # the answer streamed — restart
# Phase 68 (search tool): the deterministic search marker flow —
# checked BEFORE the phase-37 tool flow (the more specific
# trigger phrase wins, same convention as THINK_PARAS_TRIGGER).
search_flow = _search_flow(body)
if search_flow is not None:
if search_flow[0] == "search":
stream = _tool_call_stream(
"search_documents", {"pattern": SEARCH_PATTERN}, "call_0"
)
else: # "found" — quote the first matched line (80 chars)
answer = _apply_max_tokens(
f"Found {search_flow[1][:80]}", body.get("max_tokens")
)
stream = _sse_stream(answer, 0.0)
return StreamingResponse(
stream,
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)
flow = _tool_flow(body)
if flow is not None:
if flow[0] == "list":
+477
View File
@@ -0,0 +1,477 @@
"""Phase 68 E2E (Playwright, mock-only): the ``search_documents`` tool.
Story: n/a (TODO-derived — the owner roadmap confirmation 2026-09-01,
TODO.md L4: "Add a search tool that allows the LLM to grep through the
uploaded documents for a given string").
Run in isolation (DB must be up: ``podman compose up -d db``):
uv run pytest tests/e2e/test_search_tool.py -v --no-cov
MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the gate is the
deterministic SEARCH marker flow in ``tests/e2e/mock_llm.py`` (user
message contains ``search your documents`` (``SEARCH_TRIGGER``)
**and** the system prompt carries the ``<tools>`` section of the HIGH
prompt):
1. request 1 (``tools`` offered, no search result yet) → streams ONLY
``tool_calls`` deltas calling ``search_documents`` with
``{"pattern": SEARCH_PATTERN}`` (id ``call_0``);
2. request 2 (a ``tool``-role search result — the
``source/path:line: text`` match line) → the content answer
``Found <first matched line's content up to 80 chars>`` — so this
suite can assert the search result reached the model and landed in
the answer.
KB fixture — ONE document, imported through the real importer (the same
admin-import pipeline the admin Sources page drives) against
``tests/fixtures/search_docs/`` with the deterministic mock embeddings:
* ``search_docs/reese-notes.md`` — homelab kubernetes backup notes; the
marker question cosines ≈0.51 against it (mock bag-of-words, well
past the E2E 0.30 threshold, and it FTS-matches too) → grounded, so
the HIGH prompt carries the ``<tools>`` section. Its line 6 carries
the sentinel ``reese-sentinel-42`` exactly once — the deterministic
match of the mock's grep.
Regression-safe marker: ``search your documents`` appears in NO other
suite's question or fixture text (verified 2026-09-01 by repo grep;
``tests/unit/test_mock_tool_flow.py`` pins that the trigger does not
shadow the phase-37/45 flows and vice versa).
Test → phase mapping:
1. ``test_search_flow_searches_and_answers_from_match`` — the live
search flow: the SSE carries the ``tool`` frame
(``search_documents`` with ``argument = <sentinel>``, ahead of any
delta), #send-status recorded the transient "… is searching for
<sentinel>" state, the bubble shows ONE ``🔎 Searching for``
tool line with the sentinel in a ``<code>`` element, the answer
quotes the matched line (``Found …`` — the match reached the
model), and the turn settles to idle with no error banner.
2. ``test_search_adds_no_source_by_itself`` — context accounting
(locked A5): the search-only flow (no read) leaves
``done.sources`` / the source chips / ``query_log.sources`` at the
retrieval baseline — the search adds no source by itself.
3. ``test_search_tool_line_re_renders_after_reload`` — the persisted
record (phase 14 convention: the generic ``{name, argument}``
toolAcc) re-renders the search line through the same helper.
"""
from __future__ import annotations
import asyncio
import json
import re
import time
from pathlib import Path
from threading import Thread
from typing import Any
from playwright.sync_api import Page, expect
from sqlalchemy import select, text
from app.config import Settings
from app.db import SessionLocal
from app.models import QueryLog
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from tests.e2e.mock_llm import SEARCH_PATTERN, SEARCH_TRIGGER
REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "search_docs"
#: The one imported document (the importer derives ``source`` from the
#: fixture root's name and ``path`` from the file's relative position).
SEED_SOURCE = "search_docs"
SEED_PATH = "reese-notes.md"
SEED_SP = f"{SEED_SOURCE}/{SEED_PATH}"
#: The fixture's sentinel line (line 6) — the mock's grep matches it
#: exactly once, and its ``text`` part is what the "Found …" answer
#: quotes. Pinned against the fixture file itself below.
SENTINEL_LINE = f"The offsite vault passphrase marker is {SEARCH_PATTERN}."
FOUND_ANSWER = f"Found {SENTINEL_LINE[:80]}"
#: Carries ``SEARCH_TRIGGER`` and is on-topic (cosine ≈0.51 against the
#: fixture + FTS hits → HIGH gate, the ``<tools>`` section rides along).
SEARCH_QUESTION = (
"Search your documents for the vault passphrase marker in my homelab "
"kubernetes backup notes?"
)
assert SEARCH_TRIGGER in SEARCH_QUESTION.lower()
# The trigger must not collide with any other mock marker flow.
for other in (
"use your tools",
"read two documents",
"write a long answer",
"think in paragraphs",
"think out loud",
"show the end of your notes",
"show me a table",
"fail then answer",
"always fail",
"embed fail once",
"pretend to think slowly",
):
assert other not in SEARCH_QUESTION.lower(), other
def _pin_fixture() -> None:
"""The fixture carries the sentinel on line 6, exactly once."""
content = (FIXTURES / SEED_PATH).read_text(encoding="utf-8")
lines = content.split("\n")
assert lines[5] == SENTINEL_LINE, lines[5]
assert sum(SEARCH_PATTERN in line for line in lines) == 1
# --------------------------------------------------------------------------
# DB seeding (TRUNCATE-then-import via the real importer, cf.
# test_chat_rag.py — the same pipeline the admin Sources import drives)
# --------------------------------------------------------------------------
async def _import_fixtures(mock_port: int) -> ImportSummary:
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
return await import_sources([FIXTURES], LLMClient(settings))
def _run_in_thread(coro: Any) -> Any:
"""Run a coroutine on a worker thread.
Playwright's sync API keeps an asyncio loop running on the test
thread, so ``asyncio.run`` cannot be called directly from a test
body (the established house helper).
"""
box: dict[str, Any] = {}
def runner() -> None:
try:
box["value"] = asyncio.run(coro)
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
box["error"] = e
t = Thread(target=runner)
t.start()
t.join()
if "error" in box:
raise box["error"]
return box["value"]
def _reset_db(mock_port: int) -> ImportSummary:
"""Truncate the KB (plus the prompt-shaping tables), then re-import.
``steering_notes`` / ``kb_overview`` are truncated too, so the HIGH
prompt is exactly ``<relevance>`` + ``<documents>`` + ``<tools>``
regardless of leftovers from other suites — byte-stable prompts,
byte-stable answers."""
with SessionLocal() as db:
db.execute(
text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
)
db.commit()
summary = _run_in_thread(_import_fixtures(mock_port))
assert summary is not None and summary.added == 1, summary
return summary
def _last_query_log() -> QueryLog:
with SessionLocal() as db:
rows = db.scalars(select(QueryLog)).all()
assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
return rows[0]
# --------------------------------------------------------------------------
# Page helpers (the test_agent_document_tools.py pattern)
# --------------------------------------------------------------------------
#: Records every value #send-label takes during the turn (a
#: MutationObserver on the element), so the in-flight label state is
#: captured deterministically — no polling race. Phase 48: the label is
#: the Send↔Stop morph ("Stop" holds for the whole in-flight turn).
LABEL_RECORDER = """
() => {
if (window.__labelsInstalled) return;
window.__labelsInstalled = true;
window.__labels = [];
const el = document.querySelector('#send-label');
if (!el) return;
const rec = (v) => {
const l = window.__labels;
if (!l.length || l[l.length - 1] !== v) l.push(v);
};
rec(el.textContent);
new MutationObserver(() => rec(el.textContent)).observe(el, {
childList: true,
subtree: true,
});
}
"""
#: Records every value #send-status takes during the turn — the
#: transient "… is searching for <pattern>" calling-tool state is held
#: only from the first `tool` frame until the first answer delta, so the
#: pre-submit observer is the deterministic source of truth for it
#: (no polling race — the phase-44 task-03 flake fix).
STATUS_RECORDER = """
() => {
if (window.__statusesInstalled) return;
window.__statusesInstalled = true;
window.__statuses = [];
const el = document.querySelector('#send-status');
if (!el) return;
const rec = (v) => {
const l = window.__statuses;
if (!l.length || l[l.length - 1] !== v) l.push(v);
};
rec(el.textContent);
new MutationObserver(() => rec(el.textContent)).observe(el, {
childList: true,
subtree: true,
});
}
"""
#: Captures the raw SSE ``data:`` payloads of the /api/chat stream
#: (a response clone read in the background) — wire-level assertions
#: for the ``tool`` frames, independent of the UI rendering.
SSE_HOOK = """
() => {
if (window.__sseInstalled) return;
window.__sseInstalled = true;
window.__sseFrames = [];
const origFetch = window.fetch;
window.fetch = async function (...args) {
const res = await origFetch.apply(this, args);
try {
const url = typeof args[0] === 'string' ? args[0] : args[0].url;
if (url.includes('/api/chat')) {
res.clone().text().then((bodyText) => {
for (const block of bodyText.split('\\n\\n')) {
const line = block.trim();
if (line.startsWith('data: ')) {
window.__sseFrames.push(line.slice(6));
}
}
});
}
} catch (e) { /* non-clonable responses: ignored */ }
return res;
};
}
"""
def _install_page_hooks(page: Page) -> None:
"""Install all hooks on the loaded page (post-goto, pre-submit)."""
page.evaluate(SSE_HOOK)
page.evaluate(LABEL_RECORDER)
page.evaluate(STATUS_RECORDER)
def _frames(page: Page) -> list[dict]:
"""The captured SSE frames, once the hook's background read settles.
The hook reads ``res.clone().text()`` in a background promise that
resolves right after the stream closes — poll briefly until the
final ``done`` frame lands (fail loud if the hook captured nothing).
"""
deadline = time.monotonic() + 10.0
while True:
raw = page.evaluate("() => window.__sseFrames || []")
parsed = [json.loads(line) for line in raw if line]
if any(f.get("type") == "done" for f in parsed):
return parsed
if time.monotonic() > deadline:
raise AssertionError(
f"SSE hook captured no `done` frame (frames so far: "
f"{len(parsed)}) — hook install failed?"
)
time.sleep(0.05)
def _tool_frames(frames: list[dict]) -> list[dict]:
return [f for f in frames if f.get("type") == "tool"]
def _submit(page: Page, question: str) -> None:
page.fill("#message-input", question)
page.click("#send-btn")
# The user bubble lands synchronously with the submit handler.
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
def _wait_settled(page: Page) -> None:
"""The turn is complete: answer text in the bubble, button recovered.
Phase 48: the label assertion carries the settle wait with an
explicit timeout — the in-flight button is the enabled Stop control
(never disabled), so ``to_be_enabled`` no longer blocks until the
turn settles, and Playwright expect's default (5s) does not inherit
the page default."""
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
def _assert_no_error_banner(page: Page) -> None:
"""The turn must settle WITHOUT the terminal error banner (the red
role=alert error banner — the KB-offline banner is a separate,
non-error state)."""
banner = page.locator("#kb-banner")
expect(banner).to_be_hidden()
expect(banner).not_to_have_attribute("role", "alert")
expect(banner).not_to_have_class(re.compile(r"is-error"))
# --------------------------------------------------------------------------
# 1. The live search flow: search → matched line → "Found …" answer
# --------------------------------------------------------------------------
def test_search_flow_searches_and_answers_from_match(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_pin_fixture()
page.set_default_timeout(30_000)
_reset_db(mock_llm)
page.goto(app_url)
_install_page_hooks(page)
_submit(page, SEARCH_QUESTION)
# The "calling tool" STATUS window is transient: the first `tool`
# frame sets #send-status and it holds until the FIRST answer delta
# (the agent loop completes before the answer stream) — a polling
# expect can stride straight over that window (the phase-44 flake),
# so the pre-submit MutationObserver records below are the
# deterministic source of truth for the status transitions.
_wait_settled(page)
# Phase 48 (owner-locked): the in-flight button is the Stop control
# for the whole turn; #send-status walked "… is thinking" →
# "… is searching for <sentinel>", in order.
labels = page.evaluate("() => window.__labels")
assert "Stop" in labels, labels
statuses = page.evaluate("() => window.__statuses")
i_search = next(
(
i
for i, s in enumerate(statuses)
if f"is searching for {SEARCH_PATTERN}" in s
),
None,
)
assert i_search is not None, statuses
i_think = next(
(i for i, s in enumerate(statuses) if "is thinking" in s), None
)
assert i_think is not None and i_think < i_search, statuses
# Wire level: exactly ONE `tool` frame — search_documents carrying
# the PATTERN as its argument (phase 68 task 02) — ahead of the
# first `delta` frame.
frames = _frames(page)
assert _tool_frames(frames) == [
{"type": "tool", "name": "search_documents", "argument": SEARCH_PATTERN}
]
first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta")
assert all(
i < first_delta for i, f in enumerate(frames) if f.get("type") == "tool"
)
done = next(f for f in frames if f.get("type") == "done")
assert done["deflected"] is False
# ONE tool line above the answer: "🔎 Searching for " + the sentinel
# in a <code> element (the pattern is data, never markup).
lines = page.locator(".msg.brain .tool-call")
expect(lines).to_have_count(1)
expect(lines.nth(0)).to_contain_text("Searching for")
expect(lines.nth(0).locator("code")).to_have_text(SEARCH_PATTERN)
# The answer quotes the MATCHED LINE — the search result reached the
# model and landed in the answer (the mock's deterministic echo).
bubble = page.locator(".msg.brain .bubble").last
expect(bubble).to_contain_text(FOUND_ANSWER)
# The turn settled to idle WITHOUT the error banner, and the durable
# record is grounded with the retrieval doc only (see test 2 for the
# sources contract).
_assert_no_error_banner(page)
row = _last_query_log()
assert row.question == SEARCH_QUESTION
assert row.deflected is False
# --------------------------------------------------------------------------
# 2. Context accounting (locked A5): a search adds no source by itself
# --------------------------------------------------------------------------
def test_search_adds_no_source_by_itself(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_pin_fixture()
page.set_default_timeout(30_000)
_reset_db(mock_llm)
page.goto(app_url)
_install_page_hooks(page)
_submit(page, SEARCH_QUESTION)
_wait_settled(page)
# The search really ran (its wire frame is present) — yet the
# search-only flow (no read) leaves done.sources at the RETRIEVAL
# baseline: the one fixture doc, nothing added by the search.
frames = _frames(page)
assert _tool_frames(frames) == [
{"type": "tool", "name": "search_documents", "argument": SEARCH_PATTERN}
]
done = next(f for f in frames if f.get("type") == "done")
assert done["deflected"] is False
assert [(s["source"], s["path"]) for s in done["sources"]] == [
(SEED_SOURCE, SEED_PATH)
]
# UI: exactly one source chip — the retrieval doc (the search
# renders no chip of its own).
chips = page.locator(".msg.brain .source-chip")
expect(chips).to_have_count(1)
expect(chips.nth(0)).to_contain_text(SEED_SP)
# Durable record: the sources row is unchanged by the search alone.
row = _last_query_log()
assert row.deflected is False
assert row.sources == SEED_SP
# --------------------------------------------------------------------------
# 3. Persistence: the search tool line re-renders after a reload
# --------------------------------------------------------------------------
def test_search_tool_line_re_renders_after_reload(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_pin_fixture()
page.set_default_timeout(30_000)
_reset_db(mock_llm)
page.goto(app_url)
_submit(page, SEARCH_QUESTION)
_wait_settled(page)
expect(page.locator(".msg.brain .tool-call")).to_have_count(1)
page.reload()
expect(page.locator("#empty-state")).to_be_hidden()
# The persisted record (the generic {name, argument} toolAcc —
# phase 14 convention) re-renders the search line through the same
# append helper as the live frames: sentinel in a <code> element.
restored = page.locator(".msg.brain .tool-call")
expect(restored).to_have_count(1)
expect(restored.nth(0)).to_contain_text("Searching for")
expect(restored.nth(0).locator("code")).to_have_text(SEARCH_PATTERN)
# The answer (the matched-line echo) is intact after the restore.
bubble = page.locator(".msg.brain .bubble").last
expect(bubble).to_contain_text(FOUND_ANSWER)
+9
View File
@@ -0,0 +1,9 @@
# Homelab Backup Notes
The homelab kubernetes cluster's etcd volume is backed up nightly to the
offsite vault; the backup cron runs on homelab-gw at 03:00.
The offsite vault passphrase marker is reese-sentinel-42.
The restore procedure is in the homelab runbook; the vault key rotates
every ninety days.
+161 -2
View File
@@ -3,21 +3,31 @@
``list_catalog`` must order rows by ``(source, path)`` — the same order as
``GET /api/docs`` — and ``find_document`` must resolve a hit to the full
document row (content included, for the never-truncated read) and return
``None`` for unknown ``source``/``path`` pairs.
``None`` for unknown ``source``/``path`` pairs. Phase 68: the
``search_documents`` tool is pinned here too — its locked parameter
shape in ``AGENT_TOOLS``, and a scripted ``ToolCallPiece`` executed
through ``run_agent`` against the real DB (``all_documents`` for a
whole-KB search, ``find_document`` for a scoped one).
Requires: podman compose up -d db
"""
from __future__ import annotations
import asyncio
import uuid
from collections.abc import Iterator
from collections.abc import AsyncIterator, Iterator
from copy import deepcopy
from typing import Any, cast
import pytest
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.config import Settings
from app.models import Document
from app.rag import agent
from app.rag.agent import AGENT_TOOLS, AgentHolder, run_agent
from app.rag.llm import LLMClient, RetryPiece, StreamPiece, ToolCallPiece
def _doc(db: Session, source: str, path: str, title: str, content: str) -> Document:
@@ -81,3 +91,152 @@ def test_find_document_none_for_unknown_pairs(kb, db) -> None:
assert agent.find_document(db, "Alpha", "nope.md") is None # wrong path
assert agent.find_document(db, "Beta", "x.md") is None # wrong source
assert agent.find_document(db, "nope", "nope.md") is None # nothing at all
# ---------- search_documents (phase 68) ----------
def test_all_documents_orders_by_source_then_path(kb, db) -> None:
_doc(db, "Zeta", "b/second.md", "Zeta B", "ZB")
_doc(db, "Zeta", "a/first.md", "Zeta A", "ZA")
_doc(db, "Alpha", "c/third.md", "Alpha C", "AC")
db.commit()
docs = agent.all_documents(db)
assert [(d.source, d.path) for d in docs] == [
("Alpha", "c/third.md"),
("Zeta", "a/first.md"),
("Zeta", "b/second.md"),
]
assert [d.content for d in docs] == ["AC", "ZA", "ZB"] # full rows
def test_agent_tools_offers_search_documents_with_locked_shape() -> None:
by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
assert list(by_name) == [ # the third tool, in order
"list_documents",
"read_document",
"search_documents",
]
search = by_name["search_documents"]["function"]["parameters"]
assert search["type"] == "object"
assert search["required"] == ["pattern"]
assert set(search["properties"]) == {"pattern", "source", "path"}
assert all(p["type"] == "string" for p in search["properties"].values())
class ScriptedToolLLM:
"""One scripted tool-call stream, then one canned answer stream.
Records every ``chat_stream`` request's messages and tools."""
def __init__(self, call: ToolCallPiece) -> None:
self.call = call
self.requests: list[
tuple[list[dict[str, Any]], list[dict[str, Any]] | None]
] = []
async def chat_stream(
self,
messages: list[dict[str, str]],
tools: list[dict[str, Any]] | None = None,
) -> AsyncIterator[StreamPiece | ToolCallPiece]:
self.requests.append((deepcopy(messages), deepcopy(tools)))
if len(self.requests) == 1:
yield self.call
else:
yield StreamPiece("content", "ans")
def _settings(**kwargs: Any) -> Settings:
kwargs.setdefault("_env_file", None)
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
def _run_search(
db: Session, arguments: dict[str, Any]
) -> tuple[AgentHolder, ScriptedToolLLM]:
"""Drive one scripted ``search_documents`` call through ``run_agent``."""
holder = AgentHolder()
llm = ScriptedToolLLM(
ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)
)
asyncio.run(_consume(llm, db, holder))
return holder, llm
async def _consume(
llm: ScriptedToolLLM, db: Session, holder: AgentHolder
) -> list[StreamPiece | ToolCallPiece | RetryPiece]:
out: list[StreamPiece | ToolCallPiece | RetryPiece] = []
async for piece in run_agent(
cast("LLMClient", llm),
db,
system_prompt="SYSTEM_PROMPT",
user_message="QUESTION",
seed_docs=[],
settings=_settings(),
holder=holder,
):
out.append(piece)
return out
def test_search_whole_kb_through_run_agent(kb, db) -> None:
_doc(db, "Beta", "b/two.md", "Two", "no hit\nNEEDLE in two\nlast")
_doc(db, "Alpha", "a/one.md", "One", "first\nneedle in one\nthird")
db.commit()
holder, llm = _run_search(db, {"pattern": "needle"})
# Offered: the first request carries AGENT_TOOLS (the 3-tool list).
assert llm.requests[0][1] == AGENT_TOOLS
# Executed against the real DB: catalog order, grep-style lines.
assert llm.requests[1][0][3]["content"] == (
"Alpha/a/one.md:2: needle in one\n"
"Beta/b/two.md:2: NEEDLE in two"
)
assert holder.tool_calls == 1
assert holder.read_docs == [] # locked A5: search adds no context
def test_search_scoped_through_run_agent(kb, db) -> None:
_doc(db, "Alpha", "a/one.md", "One", "first\nNeedle here\nthird")
_doc(db, "Beta", "b/two.md", "Two", "NEEDLE too")
db.commit()
holder, llm = _run_search(
db, {"pattern": "needle", "source": "Alpha", "path": "a/one.md"}
)
# Only the named document is searched — the other one's hit is absent.
assert llm.requests[1][0][3]["content"] == "Alpha/a/one.md:2: Needle here"
assert holder.tool_calls == 1
assert holder.read_docs == []
def test_search_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:
_doc(db, "Alpha", "a/one.md", "One", "nothing")
db.commit()
holder, llm = _run_search(
db, {"pattern": "needle", "source": "Alpha", "path": "ghost.md"}
)
assert (
llm.requests[1][0][3]["content"]
== "No document at Alpha/ghost.md — check the list_documents output."
)
assert holder.tool_calls == 0 and holder.read_docs == []
def test_search_no_matches_through_run_agent(kb, db) -> None:
_doc(db, "Alpha", "a/one.md", "One", "nothing matching")
db.commit()
holder, llm = _run_search(db, {"pattern": "zebra"})
assert llm.requests[1][0][3]["content"] == (
"No matches for 'zebra' in the knowledge base."
)
assert holder.tool_calls == 1 # an executed search with zero hits
assert holder.read_docs == []
+7 -4
View File
@@ -298,17 +298,20 @@ def test_ui_chrome_has_no_emoji(client, path: str) -> None:
the JS that renders it, and the stylesheet — is emoji-free.
Phase 37 revision (owner permission 2026-08-26, PLAN §4): the agent's
``.tool-call`` line carries two CONTENT marks — 🔎 (list) and 📄
``.tool-call`` line carries the CONTENT marks — 🔎 (list) and 📄
(read) — the only emoji in the whole frontend, and only as the exact
tool-line template strings in app.js. The guard strips precisely
those two literals; any other emoji, or those marks anywhere else,
still fails."""
tool-line template strings in app.js. Phase 68 revision: the
``search_documents`` tool line adds the third template literal
("🔎 Searching for "). The guard strips precisely those three
literals; any other emoji, or those marks anywhere else, still
fails."""
r = client.get(path)
assert r.status_code == 200
text = r.text
if path in ("/assets/app.js", "/assets/shared.js"):
text = text.replace('"🔎 Listing documents"', "")
text = text.replace('"📄 Reading "', "")
text = text.replace('"🔎 Searching for "', "")
assert _find_emoji(text) == [], f"emoji found in {path}: {_find_emoji(text)!r}"
+59
View File
@@ -621,6 +621,65 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
assert "'docs/homelab/backups.md'" in lines[-1]
def test_grounded_turn_streams_search_tool_frames(
client, db, seeded_kb: FakeRagLLM
) -> None:
"""Phase 68: a scripted ``search_documents`` call streams as
``{type: "tool", name: "search_documents", argument: <pattern>}`` —
the raw pattern is the frame's ``argument`` (the UI renders the
"searching for" line from it). A non-string pattern — a model error
the backend refuses — yields ``argument: null``. A search adds no
source: ``done.sources`` stays the retrieval docs (locked A5)."""
scripted = FakeRagLLM(
tool_script=[
[
ToolCallPiece(
id="call_1",
name="search_documents",
arguments={"pattern": "Cilium"},
),
],
[
ToolCallPiece(
id="call_2",
name="search_documents",
arguments={"pattern": 42}, # model error: non-string
),
],
# the answer request still carries the tools (2 rounds < the
# default cap of 10); the fake's tool_script is exhausted, so
# it falls back to the thinking + answer stream
]
)
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
try:
_, _, frames = _stream_chat(client, QUESTION)
finally:
fastapi_app.dependency_overrides.clear()
types = [f["type"] for f in frames]
assert "error" not in types
assert len(scripted.seen_tools) == 3 # both searches executed (rounds)
tool_frames = [f for f in frames if f["type"] == "tool"]
assert len(tool_frames) == 2
first, second = tool_frames
assert set(first) == {"type", "name", "argument"}
assert first["name"] == "search_documents"
assert first["argument"] == "Cilium" # the raw pattern
assert set(second) == {"type", "name", "argument"}
assert second["name"] == "search_documents"
assert second["argument"] is None # the non-string pattern → null
# The searches still answered: deltas, then a grounded done.
assert [f for f in frames if f["type"] == "delta"]
done = frames[-1]
assert done["type"] == "done" and done["deflected"] is False
paths = [s["path"] for s in done["sources"]]
assert "homelab/kubernetes.md" in paths # retrieval docs, unchanged
assert "homelab/backups.md" not in paths # a search adds no source
def test_deflected_turn_stays_byte_identical_without_tools(
client, db, seeded_kb: FakeRagLLM
) -> None:
+318 -1
View File
@@ -103,7 +103,8 @@ async def _run(
def test_agent_tools_names_and_parameters() -> None:
by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
assert set(by_name) == {"list_documents", "read_document"}
assert len(AGENT_TOOLS) == 3 # list / read / search (phase 68)
assert set(by_name) == {"list_documents", "read_document", "search_documents"}
assert all(t["type"] == "function" for t in AGENT_TOOLS)
list_params = by_name["list_documents"]["function"]["parameters"]
assert list_params["type"] == "object"
@@ -128,6 +129,34 @@ def test_agent_tools_names_and_parameters() -> None:
"list_documents output (e.g. 'homelab/aws-route53.md' from "
"'source: Homelab | path: homelab/aws-route53.md')."
)
# Phase 68: search_documents — the third tool, a locator (locked A5).
search = by_name["search_documents"]["function"]
assert search["description"] == (
"Search every indexed document for an exact string "
"(case-insensitive) and return up to 20 matching lines as "
"'source/path:line: text' — use this to locate content, "
"then read_document the winner. Optionally pass 'source' "
"and 'path' (as shown in list_documents) to search one "
"document only."
)
search_params = search["parameters"]
assert search_params["type"] == "object"
assert search_params["required"] == ["pattern"]
assert set(search_params["properties"]) == {"pattern", "source", "path"}
assert search_params["properties"]["pattern"]["description"] == (
"The exact text to search for (a plain substring, not a regex)"
)
# Phase 63 labeled-field wording, same as read_document's parameters.
assert search_params["properties"]["source"]["description"] == (
"The document's source, as shown after 'source: ' in the "
"list_documents output (e.g. 'Homelab' from "
"'source: Homelab | path: homelab/aws-route53.md')."
)
assert search_params["properties"]["path"]["description"] == (
"The document's path, as shown after 'path: ' in the "
"list_documents output (e.g. 'homelab/aws-route53.md' from "
"'source: Homelab | path: homelab/aws-route53.md')."
)
# ---------- happy path: list → read → answer ----------
@@ -547,6 +576,294 @@ def test_read_document_missing_arguments_refused(
assert llm.requests[1][1] == AGENT_TOOLS
# ---------- search_documents (phase 68, locked A5/A6) ----------
def test_grep_document_case_insensitive_line_numbers() -> None:
"""Case-insensitive fixed substring, 1-based line numbers, file order,
repeated matches within a line collapse to one match (grep semantics)."""
content = "The NEEDLE is here\nno hit\nneedle again\nNEEDLE NEEDLE\n"
assert agent.grep_document(content, "NEEDLE") == [
(1, "The NEEDLE is here"),
(3, "needle again"),
(4, "NEEDLE NEEDLE"),
]
def test_grep_document_rstrips_lines_and_empty_content() -> None:
assert agent.grep_document("hello \t\nworld ", "WORLD") == [(2, "world")]
assert agent.grep_document("", "x") == []
assert agent.grep_document("no newlines", "NO") == [(1, "no newlines")]
assert agent.grep_document("a\nb\n", "MISSING") == []
def test_search_whole_kb_grep_style_output(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Whole-KB search: catalog order, `source/path:line: text` lines,
case-insensitive; the call counts in ``tool_calls`` and never touches
``read_docs``; the tools stay offered on the answer request."""
d1 = _doc("Alpha", "a/one.md", "One", "first\nNEEDLE in one\nlast")
d2 = _doc("Beta", "b/two.md", "Two", "no hit\nneedle in two\n")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2])
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1", name="search_documents", arguments={"pattern": "needle"}
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"Alpha/a/one.md:2: NEEDLE in one\n"
"Beta/b/two.md:2: needle in two"
)
assert holder.tool_calls == 1
assert holder.read_docs == [] # locked A5: a search adds no context
assert llm.requests[1][1] == AGENT_TOOLS # tools stay offered
def test_search_capped_at_20_matches_in_catalog_order(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The 20-match cap is GLOBAL across documents in catalog order, and
the scan stops once it is hit (a 35-match corpus yields exactly 20)."""
d1 = _doc("S", "a.md", "A", "\n".join(f"hit-{i}" for i in range(15)))
d2 = _doc("S", "b.md", "B", "\n".join(f"hit-{i}" for i in range(20)))
monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2])
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1", name="search_documents", arguments={"pattern": "hit-"}
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
lines = llm.requests[1][0][3]["content"].split("\n")
assert len(lines) == agent.SEARCH_MAX_MATCHES
assert lines[0] == "S/a.md:1: hit-0"
assert lines[14] == "S/a.md:15: hit-14" # all of a.md
assert lines[15] == "S/b.md:1: hit-0" # then b.md, in order
assert lines[19] == "S/b.md:5: hit-4" # cut at the global cap
assert holder.tool_calls == 1
def test_search_truncates_match_lines_at_200_chars(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A 300-char match line yields exactly 200 chars of it (no crash)."""
d1 = _doc("S", "a.md", "A", "top\n" + "x" * 300 + " NEEDLE tail")
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1", name="search_documents", arguments={"pattern": "needle"}
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert (
llm.requests[1][0][3]["content"] == f"S/a.md:2: {'x' * agent.SEARCH_LINE_LIMIT}"
)
assert holder.tool_calls == 1
def test_search_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None:
"""Scoped search: only the named document is loaded (find_document),
``all_documents`` never runs, and the match line carries its path."""
d1 = _doc("S", "a.md", "A", "needle here")
def _find(db: Any, source: str, path: str) -> Document | None:
if (source, path) == ("S", "a.md"):
return d1
raise AssertionError(
f"find_document({source}, {path}) — the scoped "
"search must not load any other document"
)
def _boom(*_a: Any, **_k: Any) -> None:
raise AssertionError("all_documents must not run for a scoped search")
monkeypatch.setattr(agent, "find_document", _find)
monkeypatch.setattr(agent, "all_documents", _boom)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="search_documents",
arguments={"pattern": "needle", "source": "S", "path": "a.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here"
assert holder.tool_calls == 1
assert holder.read_docs == [] # searched doc did not enter the context
def test_search_scoped_missing_document_refused(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="search_documents",
arguments={"pattern": "x", "source": "S", "path": "ghost.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert (
llm.requests[1][0][3]["content"]
== "No document at S/ghost.md — check the list_documents output."
)
assert holder.tool_calls == 0 and holder.read_docs == [] # a refusal
@pytest.mark.parametrize(
("arguments", "label"),
[
({}, "no arguments"),
({"pattern": ""}, "empty pattern"),
({"pattern": " "}, "whitespace pattern"),
({"pattern": 42}, "non-string pattern"),
({"pattern": None}, "null pattern"),
({"pattern": "x", "source": "S"}, "source without path"),
({"pattern": "x", "path": "a.md"}, "path without source"),
],
)
def test_search_missing_arguments_refused(
monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str
) -> None:
"""Unusable pattern OR a half-specified source/path pair → the
missing-args refusal, with no DB access at all."""
def _boom(*_a: Any, **_k: Any) -> None:
raise AssertionError(f"no DB access for a refused search ({label})")
monkeypatch.setattr(agent, "all_documents", _boom)
monkeypatch.setattr(agent, "find_document", _boom)
holder = AgentHolder()
llm = ScriptedLLM(
[ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == agent.MISSING_SEARCH_ARGS
assert holder.tool_calls == 0 and holder.read_docs == []
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
def test_search_no_matches_whole_kb(monkeypatch: pytest.MonkeyPatch) -> None:
"""Zero hits across the KB → the no-match line (pattern quoted); the
search still executed, so it counts — and never adds context."""
monkeypatch.setattr(
agent, "all_documents", lambda db: [_doc("S", "a.md", "A", "nothing here")]
)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1", name="search_documents", arguments={"pattern": "zebra"}
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
"No matches for 'zebra' in the knowledge base."
)
assert holder.tool_calls == 1
assert holder.read_docs == []
def test_search_no_matches_scoped(monkeypatch: pytest.MonkeyPatch) -> None:
doc = _doc("S", "a.md", "A", "nothing here")
monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="search_documents",
arguments={"pattern": "zebra", "source": "S", "path": "a.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == "No matches for 'zebra' in S/a.md."
assert holder.tool_calls == 1
assert holder.read_docs == []
def test_search_no_match_truncates_long_pattern(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A pattern longer than 100 chars is truncated in the no-match line
(kept short); the search itself still runs on the full pattern."""
monkeypatch.setattr(agent, "all_documents", lambda db: [])
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1",
name="search_documents",
arguments={"pattern": "p" * 150},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert llm.requests[1][0][3]["content"] == (
f"No matches for '{'p' * 100}' in the knowledge base."
)
assert holder.tool_calls == 1
def test_search_counts_but_never_adds_context(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The locate-then-read workflow: a search finds the document but does
NOT add it — the subsequent read_document does (and is not rejected as
already-in-context, because the search touched nothing)."""
doc = _doc("S", "a.md", "A", "needle here")
monkeypatch.setattr(agent, "all_documents", lambda db: [doc])
monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
holder = AgentHolder()
llm = ScriptedLLM(
[
ToolCallPiece(
id="call_1", name="search_documents", arguments={"pattern": "needle"}
)
],
[
ToolCallPiece(
id="call_2",
name="read_document",
arguments={"source": "S", "path": "a.md"},
)
],
[StreamPiece("content", "ans")],
)
asyncio.run(_run(llm, holder, _settings()))
assert holder.tool_calls == 2 # search + read, both executed
assert holder.read_docs == [doc] # only the read added context (A5)
assert llm.requests[2][0][5]["content"] == "Document S/a.md:\nneedle here"
# ---------- retries inside the agent loop (phase 67, locked A2) ----------
+32 -2
View File
@@ -5,7 +5,8 @@ No new Python app logic exists for this task — the behavior lives in
suite (task 06). Like the other frontend-adjacent unit files, this module
pins the JS/CSS markers the story depends on, so a silent regression in
the tool branch, the persistence shape, or the tool-line styling is
caught without a browser.
catched without a browser. Phase 68 extends the pins with the
``search_documents`` status/line contract.
"""
from __future__ import annotations
@@ -39,7 +40,11 @@ def test_tool_branch_is_a_first_class_turn_branch() -> None:
"the turn handler must branch on tool frames"
)
branch = js[tool_idx:delta_idx]
assert "toolAcc.push" in branch, "every tool frame is recorded for persistence"
assert "toolAcc.push({ name, argument })" in branch, (
"every tool frame is recorded for persistence — the record stays"
" {name, argument}-generic, no per-tool shape (phase 68: the"
" search tool rides the same accumulator)"
)
assert "clearTurnTimeout()" in branch, "a tool frame proves the stream is alive"
assert 'addMessage("brain", "")' in branch, "first frame creates the brain wrap"
assert "uiState === UI_STATE.thinking" in branch, (
@@ -69,6 +74,16 @@ def test_calling_tool_label_strings() -> None:
assert "sendLabel" not in branch, "phase 48: the button keeps its Stop label"
assert "`${brand()} is listing documents`" in branch
assert "`${brand()} is reading ${argument}`" in branch
# Phase 68: the search status — locked name+argument gate, sitting
# BETWEEN the read branch and the listing fallback in the ternary.
assert "name === \"search_documents\" && argument" in branch, (
"the search status requires the name AND a string argument"
)
assert "`${brand()} is searching for ${argument}`" in branch
read = branch.find("is reading")
search = branch.find("is searching for")
listing = branch.find("is listing documents")
assert -1 < read < search < listing, "ternary order: read → search → listing"
assert "sendStatus.textContent = toolStatus" in branch, (
"the #send-status live region announces what Brain is doing"
)
@@ -106,6 +121,21 @@ def test_tool_lines_render_into_the_bubble_wrap() -> None:
"the path is data — textContent, never innerHTML"
)
assert "name === \"read_document\" && argument" in body
# Phase 68: the search branch mirrors the read branch — the same
# name+argument gate, a <code> element, and the pattern through
# textContent (never markup); the listing stays the final else.
assert "name === \"search_documents\" && argument" in body
assert 'line.textContent = "🔎 Searching for "' in body
search_part = body.split('name === "search_documents"', 1)[1]
assert 'document.createElement("code")' in search_part, (
"the pattern gets the same <code> treatment as the read path"
)
assert "code.textContent = argument" in search_part, (
"the pattern is data — textContent, never innerHTML"
)
assert 'line.textContent = "🔎 Listing documents"' in search_part, (
"the listing fallback remains the final else"
)
def test_tool_branch_is_append_only_and_interleaving_safe() -> None:
+95
View File
@@ -17,7 +17,10 @@ from typing import Any
from tests.e2e.mock_llm import (
MULTI_READ_TRIGGER,
SEARCH_PATTERN,
SEARCH_TRIGGER,
TOOLS_TRIGGER,
_search_flow,
_tool_flow,
)
@@ -256,3 +259,95 @@ def test_multi_trigger_without_tools_trigger_is_none() -> None:
def test_multi_flow_requires_tools_section() -> None:
assert _tool_flow(_body(MULTI_USER, system=SYSTEM_LOW)) is None
# --------------------------------------------------------------------------
# Phase-68 search flow (task 03)
# --------------------------------------------------------------------------
#: Carries ONLY the search trigger (never ``use your tools`` — the
#: phase-68 suite's live question shape, regression-safe by assertion).
SEARCH_USER = (
"Search your documents for the vault passphrase marker in my homelab "
"kubernetes backup notes?"
)
assert SEARCH_TRIGGER in SEARCH_USER.lower()
assert TOOLS_TRIGGER not in SEARCH_USER.lower()
#: The agent's ``search_documents`` result for the e2e fixture
#: (``app/rag/agent.py`` ``_execute_tool``): one ``source/path:LINE: text``
#: match line (the sentinel line, 200-char-capped server-side).
SEARCH_RESULT = (
f"search_docs/reese-notes.md:6: The offsite vault passphrase marker "
f"is {SEARCH_PATTERN}."
)
#: The agent's no-match line quotes the pattern — the sentinel-only
#: shape ``_search_result_line`` also recognizes (degenerate path).
SEARCH_NO_MATCH = f"No matches for '{SEARCH_PATTERN}' in the knowledge base."
def test_search_flow_search_step() -> None:
# tools offered, no search result yet: the model greps.
assert _search_flow(_body(SEARCH_USER)) == ("search",)
def test_search_flow_search_step_requires_tools_offered() -> None:
# agent_max_rounds=0 path: trigger + <tools> prompt, but no tools
# and no search result — regular answer, not a flow.
assert _search_flow(_body(SEARCH_USER, tools=None)) is None
def test_search_flow_found_step_quotes_first_match_line() -> None:
flow = _search_flow(_body(SEARCH_USER, (SEARCH_RESULT,)))
assert flow == ("found", f"The offsite vault passphrase marker is {SEARCH_PATTERN}.")
def test_search_flow_found_step_with_nested_path() -> None:
# A nested path (``/`` in it) stays intact in the match-line parse.
result = f"search_docs/deep/nested-note.md:12: line with {SEARCH_PATTERN} inside"
flow = _search_flow(_body(SEARCH_USER, (result,)))
assert flow == ("found", f"line with {SEARCH_PATTERN} inside")
def test_search_flow_found_step_without_tools_offered() -> None:
# The answer is content, not a tool call — it must not be gated on
# the ``tools`` parameter (phase 45 keeps the tools offered until
# the round cap, but the no-tools final request must still answer).
flow = _search_flow(_body(SEARCH_USER, (SEARCH_RESULT,), tools=None))
assert flow == ("found", f"The offsite vault passphrase marker is {SEARCH_PATTERN}.")
def test_search_flow_ignores_catalog_and_read_results() -> None:
# A catalog (labeled lines) and a read result ("Document …" prefix)
# are NOT search results — the flow stays at the search step.
flow = _search_flow(_body(SEARCH_USER, (CATALOG_2, _read_result(DOC1_SP, DOC1_CONTENT))))
assert flow == ("search",)
def test_search_flow_sentinel_only_result_is_a_search_result() -> None:
# The no-match line quotes the pattern — sentinel-only recognition
# (degenerate path; the e2e fixture always matches).
flow = _search_flow(_body(SEARCH_USER, (SEARCH_NO_MATCH,)))
assert flow == ("found", SEARCH_NO_MATCH)
def test_search_flow_requires_tools_section() -> None:
# Deflected turns never carry the <tools> section.
assert _search_flow(_body(SEARCH_USER, system=SYSTEM_LOW)) is None
def test_search_flow_plain_question_is_none() -> None:
assert _search_flow(_body(PLAIN_USER)) is None
def test_search_trigger_does_not_shadow_the_tool_flow() -> None:
# The search question carries no ``use your tools`` — the phase-37
# classifier must stay inert on it (regression-safe marker).
assert _tool_flow(_body(SEARCH_USER)) is None
def test_tool_trigger_does_not_shadow_the_search_flow() -> None:
# The phase-37/45 questions carry no ``search your documents`` —
# the search classifier must stay inert on them.
assert _search_flow(_body(SINGLE_USER)) is None
assert _search_flow(_body(MULTI_USER)) is None