feat(agent): search_documents tool — the model can grep the indexed documents for an exact string
This commit is contained in:
@@ -100,6 +100,24 @@ Implements just enough of the aipi surface:
|
||||
offered — e.g. ``agent_max_rounds=0``) behave exactly as today.
|
||||
``E2E_REAL_LLM=1`` ignores the mock entirely (the real model does
|
||||
what it does).
|
||||
- user message containing ``search your documents``
|
||||
(``SEARCH_TRIGGER``, phase 68, search tool) **and** the system
|
||||
prompt carries the ``<tools>`` section -> the deterministic SEARCH
|
||||
tool flow, discriminated statelessly from the messages (streaming
|
||||
only):
|
||||
* request 1 (``tools`` offered, no search result yet): stream
|
||||
ONLY ``tool_calls`` deltas — ``search_documents`` with
|
||||
``{"pattern": SEARCH_PATTERN}`` (id ``call_0``);
|
||||
* request 2 (a ``tool``-role search result in the messages —
|
||||
recognizable by its ``source/path:line: text`` match lines or
|
||||
the sentinel in its content): the content answer, deterministic:
|
||||
``Found <first matched line's content up to 80 chars>`` — so a
|
||||
suite can assert the search result reached the model and landed
|
||||
in the answer.
|
||||
Checked BEFORE the plain ``use your tools`` flow (it is the more
|
||||
specific phrase — same convention as ``think in paragraphs``); no
|
||||
existing E2E question or fixture file contains the trigger, so
|
||||
every other suite is unaffected.
|
||||
- user message containing ``show me a table`` (phase 44, markdown
|
||||
tables, TODO.md L6) -> the fixed table answer (``TABLE_ANSWER``):
|
||||
a 3-column service table, an ``<img onerror>`` XSS probe line, and
|
||||
@@ -264,6 +282,24 @@ TOOLS_TRIGGER = "use your tools"
|
||||
#: so the 3-step flow is untouched.
|
||||
MULTI_READ_TRIGGER = "read two documents"
|
||||
|
||||
#: Phase 68 (search tool, TODO.md L4): a user message containing this
|
||||
#: substring (case-insensitive) — combined with the ``<tools>`` section
|
||||
#: in the system prompt — drives the deterministic SEARCH tool flow
|
||||
#: (search_documents for ``SEARCH_PATTERN`` → the "Found …" answer),
|
||||
#: documented in the module docstring. Checked BEFORE ``TOOLS_TRIGGER``
|
||||
#: (the more specific phrase wins — the same convention as
|
||||
#: ``THINK_PARAS_TRIGGER``); verified 2026-09-01: no existing E2E
|
||||
#: question or fixture file contains the phrase, so every other suite
|
||||
#: is unaffected.
|
||||
SEARCH_TRIGGER = "search your documents"
|
||||
|
||||
#: The sentinel the search flow greps for: the e2e fixture document
|
||||
#: (``tests/fixtures/search_docs/reese-notes.md``) carries exactly one
|
||||
#: line containing it, so the search result — and the "Found …" answer
|
||||
#: that quotes its first matched line — is byte-stable (the sentinel
|
||||
#: convention of ``END_OF_NOTES_TRIGGER``).
|
||||
SEARCH_PATTERN = "reese-sentinel-42"
|
||||
|
||||
#: Phase 44 (markdown-tables story, TODO.md L6): a user message
|
||||
#: containing this substring (case-insensitive) gets the fixed table
|
||||
#: answer (``TABLE_ANSWER`` below) — a 3-column table, an XSS probe
|
||||
@@ -424,6 +460,69 @@ def _catalog_docs(body: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
return docs
|
||||
|
||||
|
||||
#: One line of the agent's ``search_documents`` output (app.rag.agent
|
||||
#: ``_execute_tool``, phase 68): ``source/path:LINE: text``. The
|
||||
#: non-greedy prefix keeps nested paths (``/`` in the path) intact.
|
||||
_SEARCH_LINE_RE = re.compile(r"^(?P<sp>.+?):(?P<line>\d+): (?P<text>.*)$")
|
||||
|
||||
|
||||
def _search_result_line(body: dict[str, Any]) -> str | None:
|
||||
"""The first matched line's text of a search result in the messages.
|
||||
|
||||
A search result is a ``tool``-role message — never a read result
|
||||
(those start with the agent's ``"Document "`` prefix) — that either
|
||||
carries ``source/path:LINE: text`` match lines (the agent's
|
||||
``search_documents`` output, phase 68) or the sentinel pattern
|
||||
itself (its no-match line quotes the pattern). Returns the first
|
||||
match line's ``text`` part (already 200-char-capped server-side),
|
||||
or the message's first line in the sentinel-only shape, or ``None``
|
||||
when no search result is in the messages yet.
|
||||
"""
|
||||
sentinel = SEARCH_PATTERN.lower()
|
||||
for m in _messages(body):
|
||||
if m.get("role") != "tool":
|
||||
continue
|
||||
content = str(m.get("content") or "")
|
||||
if content.startswith(_READ_RESULT_PREFIX):
|
||||
continue
|
||||
for line in content.splitlines():
|
||||
match = _SEARCH_LINE_RE.match(line)
|
||||
if match:
|
||||
return match.group("text")
|
||||
if sentinel in content.lower():
|
||||
lines = content.splitlines()
|
||||
return lines[0] if lines else ""
|
||||
return None
|
||||
|
||||
|
||||
def _search_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
|
||||
"""Classify a SEARCH_TRIGGER request into a step of the search flow.
|
||||
|
||||
* ``("search",)`` — ``tools`` are offered and no search result is
|
||||
in the messages yet: the model greps the whole KB for
|
||||
``SEARCH_PATTERN`` (id ``call_0``).
|
||||
* ``("found", first_line)`` — a ``tool``-role search result is in
|
||||
the messages: the model answers, quoting the first matched line
|
||||
(``Found <first matched line's content up to 80 chars>``). Reached
|
||||
regardless of the ``tools`` parameter (phase 45 keeps the tools
|
||||
offered until the round cap).
|
||||
* ``None`` — not the search flow: the trigger is absent, the
|
||||
``<tools>`` section is missing (deflected turns never carry it),
|
||||
or ``tools`` are not offered and no search result is in the
|
||||
messages yet (e.g. ``agent_max_rounds=0``).
|
||||
"""
|
||||
if SEARCH_TRIGGER not in _user(body).lower():
|
||||
return None
|
||||
if "<tools>" not in _system(body):
|
||||
return None
|
||||
first_line = _search_result_line(body)
|
||||
if first_line is not None:
|
||||
return ("found", first_line)
|
||||
if not body.get("tools"):
|
||||
return None
|
||||
return ("search",)
|
||||
|
||||
|
||||
def _tool_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
|
||||
"""Classify a marker request into one step of the tool flow.
|
||||
|
||||
@@ -948,6 +1047,25 @@ def chat_completions(body: dict[str, Any]) -> Any:
|
||||
if _chat_dead(RETRY_TRIGGER, RETRY_DEAD_ATTEMPTS):
|
||||
return _llm_500(RETRY_TRIGGER)
|
||||
_fail_posts[RETRY_TRIGGER] = 0 # the answer streamed — restart
|
||||
# Phase 68 (search tool): the deterministic search marker flow —
|
||||
# checked BEFORE the phase-37 tool flow (the more specific
|
||||
# trigger phrase wins, same convention as THINK_PARAS_TRIGGER).
|
||||
search_flow = _search_flow(body)
|
||||
if search_flow is not None:
|
||||
if search_flow[0] == "search":
|
||||
stream = _tool_call_stream(
|
||||
"search_documents", {"pattern": SEARCH_PATTERN}, "call_0"
|
||||
)
|
||||
else: # "found" — quote the first matched line (80 chars)
|
||||
answer = _apply_max_tokens(
|
||||
f"Found {search_flow[1][:80]}", body.get("max_tokens")
|
||||
)
|
||||
stream = _sse_stream(answer, 0.0)
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
flow = _tool_flow(body)
|
||||
if flow is not None:
|
||||
if flow[0] == "list":
|
||||
|
||||
@@ -0,0 +1,477 @@
|
||||
"""Phase 68 E2E (Playwright, mock-only): the ``search_documents`` tool.
|
||||
|
||||
Story: n/a (TODO-derived — the owner roadmap confirmation 2026-09-01,
|
||||
TODO.md L4: "Add a search tool that allows the LLM to grep through the
|
||||
uploaded documents for a given string").
|
||||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/e2e/test_search_tool.py -v --no-cov
|
||||
|
||||
MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the gate is the
|
||||
deterministic SEARCH marker flow in ``tests/e2e/mock_llm.py`` (user
|
||||
message contains ``search your documents`` (``SEARCH_TRIGGER``)
|
||||
**and** the system prompt carries the ``<tools>`` section of the HIGH
|
||||
prompt):
|
||||
|
||||
1. request 1 (``tools`` offered, no search result yet) → streams ONLY
|
||||
``tool_calls`` deltas calling ``search_documents`` with
|
||||
``{"pattern": SEARCH_PATTERN}`` (id ``call_0``);
|
||||
2. request 2 (a ``tool``-role search result — the
|
||||
``source/path:line: text`` match line) → the content answer
|
||||
``Found <first matched line's content up to 80 chars>`` — so this
|
||||
suite can assert the search result reached the model and landed in
|
||||
the answer.
|
||||
|
||||
KB fixture — ONE document, imported through the real importer (the same
|
||||
admin-import pipeline the admin Sources page drives) against
|
||||
``tests/fixtures/search_docs/`` with the deterministic mock embeddings:
|
||||
|
||||
* ``search_docs/reese-notes.md`` — homelab kubernetes backup notes; the
|
||||
marker question cosines ≈0.51 against it (mock bag-of-words, well
|
||||
past the E2E 0.30 threshold, and it FTS-matches too) → grounded, so
|
||||
the HIGH prompt carries the ``<tools>`` section. Its line 6 carries
|
||||
the sentinel ``reese-sentinel-42`` exactly once — the deterministic
|
||||
match of the mock's grep.
|
||||
|
||||
Regression-safe marker: ``search your documents`` appears in NO other
|
||||
suite's question or fixture text (verified 2026-09-01 by repo grep;
|
||||
``tests/unit/test_mock_tool_flow.py`` pins that the trigger does not
|
||||
shadow the phase-37/45 flows and vice versa).
|
||||
|
||||
Test → phase mapping:
|
||||
1. ``test_search_flow_searches_and_answers_from_match`` — the live
|
||||
search flow: the SSE carries the ``tool`` frame
|
||||
(``search_documents`` with ``argument = <sentinel>``, ahead of any
|
||||
delta), #send-status recorded the transient "… is searching for
|
||||
<sentinel>" state, the bubble shows ONE ``🔎 Searching for``
|
||||
tool line with the sentinel in a ``<code>`` element, the answer
|
||||
quotes the matched line (``Found …`` — the match reached the
|
||||
model), and the turn settles to idle with no error banner.
|
||||
2. ``test_search_adds_no_source_by_itself`` — context accounting
|
||||
(locked A5): the search-only flow (no read) leaves
|
||||
``done.sources`` / the source chips / ``query_log.sources`` at the
|
||||
retrieval baseline — the search adds no source by itself.
|
||||
3. ``test_search_tool_line_re_renders_after_reload`` — the persisted
|
||||
record (phase 14 convention: the generic ``{name, argument}``
|
||||
toolAcc) re-renders the search line through the same helper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page, expect
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from app.config import Settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import QueryLog
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
from tests.e2e.mock_llm import SEARCH_PATTERN, SEARCH_TRIGGER
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "search_docs"
|
||||
|
||||
#: The one imported document (the importer derives ``source`` from the
|
||||
#: fixture root's name and ``path`` from the file's relative position).
|
||||
SEED_SOURCE = "search_docs"
|
||||
SEED_PATH = "reese-notes.md"
|
||||
SEED_SP = f"{SEED_SOURCE}/{SEED_PATH}"
|
||||
|
||||
#: The fixture's sentinel line (line 6) — the mock's grep matches it
|
||||
#: exactly once, and its ``text`` part is what the "Found …" answer
|
||||
#: quotes. Pinned against the fixture file itself below.
|
||||
SENTINEL_LINE = f"The offsite vault passphrase marker is {SEARCH_PATTERN}."
|
||||
FOUND_ANSWER = f"Found {SENTINEL_LINE[:80]}"
|
||||
|
||||
#: Carries ``SEARCH_TRIGGER`` and is on-topic (cosine ≈0.51 against the
|
||||
#: fixture + FTS hits → HIGH gate, the ``<tools>`` section rides along).
|
||||
SEARCH_QUESTION = (
|
||||
"Search your documents for the vault passphrase marker in my homelab "
|
||||
"kubernetes backup notes?"
|
||||
)
|
||||
assert SEARCH_TRIGGER in SEARCH_QUESTION.lower()
|
||||
# The trigger must not collide with any other mock marker flow.
|
||||
for other in (
|
||||
"use your tools",
|
||||
"read two documents",
|
||||
"write a long answer",
|
||||
"think in paragraphs",
|
||||
"think out loud",
|
||||
"show the end of your notes",
|
||||
"show me a table",
|
||||
"fail then answer",
|
||||
"always fail",
|
||||
"embed fail once",
|
||||
"pretend to think slowly",
|
||||
):
|
||||
assert other not in SEARCH_QUESTION.lower(), other
|
||||
|
||||
|
||||
def _pin_fixture() -> None:
|
||||
"""The fixture carries the sentinel on line 6, exactly once."""
|
||||
content = (FIXTURES / SEED_PATH).read_text(encoding="utf-8")
|
||||
lines = content.split("\n")
|
||||
assert lines[5] == SENTINEL_LINE, lines[5]
|
||||
assert sum(SEARCH_PATTERN in line for line in lines) == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# DB seeding (TRUNCATE-then-import via the real importer, cf.
|
||||
# test_chat_rag.py — the same pipeline the admin Sources import drives)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||||
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
|
||||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
return await import_sources([FIXTURES], LLMClient(settings))
|
||||
|
||||
|
||||
def _run_in_thread(coro: Any) -> Any:
|
||||
"""Run a coroutine on a worker thread.
|
||||
|
||||
Playwright's sync API keeps an asyncio loop running on the test
|
||||
thread, so ``asyncio.run`` cannot be called directly from a test
|
||||
body (the established house helper).
|
||||
"""
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(coro)
|
||||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||
box["error"] = e
|
||||
|
||||
t = Thread(target=runner)
|
||||
t.start()
|
||||
t.join()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _reset_db(mock_port: int) -> ImportSummary:
|
||||
"""Truncate the KB (plus the prompt-shaping tables), then re-import.
|
||||
|
||||
``steering_notes`` / ``kb_overview`` are truncated too, so the HIGH
|
||||
prompt is exactly ``<relevance>`` + ``<documents>`` + ``<tools>``
|
||||
regardless of leftovers from other suites — byte-stable prompts,
|
||||
byte-stable answers."""
|
||||
with SessionLocal() as db:
|
||||
db.execute(
|
||||
text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
|
||||
)
|
||||
db.commit()
|
||||
summary = _run_in_thread(_import_fixtures(mock_port))
|
||||
assert summary is not None and summary.added == 1, summary
|
||||
return summary
|
||||
|
||||
|
||||
def _last_query_log() -> QueryLog:
|
||||
with SessionLocal() as db:
|
||||
rows = db.scalars(select(QueryLog)).all()
|
||||
assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
|
||||
return rows[0]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Page helpers (the test_agent_document_tools.py pattern)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: Records every value #send-label takes during the turn (a
|
||||
#: MutationObserver on the element), so the in-flight label state is
|
||||
#: captured deterministically — no polling race. Phase 48: the label is
|
||||
#: the Send↔Stop morph ("Stop" holds for the whole in-flight turn).
|
||||
LABEL_RECORDER = """
|
||||
() => {
|
||||
if (window.__labelsInstalled) return;
|
||||
window.__labelsInstalled = true;
|
||||
window.__labels = [];
|
||||
const el = document.querySelector('#send-label');
|
||||
if (!el) return;
|
||||
const rec = (v) => {
|
||||
const l = window.__labels;
|
||||
if (!l.length || l[l.length - 1] !== v) l.push(v);
|
||||
};
|
||||
rec(el.textContent);
|
||||
new MutationObserver(() => rec(el.textContent)).observe(el, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
});
|
||||
}
|
||||
"""
|
||||
|
||||
#: Records every value #send-status takes during the turn — the
|
||||
#: transient "… is searching for <pattern>" calling-tool state is held
|
||||
#: only from the first `tool` frame until the first answer delta, so the
|
||||
#: pre-submit observer is the deterministic source of truth for it
|
||||
#: (no polling race — the phase-44 task-03 flake fix).
|
||||
STATUS_RECORDER = """
|
||||
() => {
|
||||
if (window.__statusesInstalled) return;
|
||||
window.__statusesInstalled = true;
|
||||
window.__statuses = [];
|
||||
const el = document.querySelector('#send-status');
|
||||
if (!el) return;
|
||||
const rec = (v) => {
|
||||
const l = window.__statuses;
|
||||
if (!l.length || l[l.length - 1] !== v) l.push(v);
|
||||
};
|
||||
rec(el.textContent);
|
||||
new MutationObserver(() => rec(el.textContent)).observe(el, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
});
|
||||
}
|
||||
"""
|
||||
|
||||
#: Captures the raw SSE ``data:`` payloads of the /api/chat stream
|
||||
#: (a response clone read in the background) — wire-level assertions
|
||||
#: for the ``tool`` frames, independent of the UI rendering.
|
||||
SSE_HOOK = """
|
||||
() => {
|
||||
if (window.__sseInstalled) return;
|
||||
window.__sseInstalled = true;
|
||||
window.__sseFrames = [];
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function (...args) {
|
||||
const res = await origFetch.apply(this, args);
|
||||
try {
|
||||
const url = typeof args[0] === 'string' ? args[0] : args[0].url;
|
||||
if (url.includes('/api/chat')) {
|
||||
res.clone().text().then((bodyText) => {
|
||||
for (const block of bodyText.split('\\n\\n')) {
|
||||
const line = block.trim();
|
||||
if (line.startsWith('data: ')) {
|
||||
window.__sseFrames.push(line.slice(6));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (e) { /* non-clonable responses: ignored */ }
|
||||
return res;
|
||||
};
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def _install_page_hooks(page: Page) -> None:
|
||||
"""Install all hooks on the loaded page (post-goto, pre-submit)."""
|
||||
page.evaluate(SSE_HOOK)
|
||||
page.evaluate(LABEL_RECORDER)
|
||||
page.evaluate(STATUS_RECORDER)
|
||||
|
||||
|
||||
def _frames(page: Page) -> list[dict]:
|
||||
"""The captured SSE frames, once the hook's background read settles.
|
||||
|
||||
The hook reads ``res.clone().text()`` in a background promise that
|
||||
resolves right after the stream closes — poll briefly until the
|
||||
final ``done`` frame lands (fail loud if the hook captured nothing).
|
||||
"""
|
||||
deadline = time.monotonic() + 10.0
|
||||
while True:
|
||||
raw = page.evaluate("() => window.__sseFrames || []")
|
||||
parsed = [json.loads(line) for line in raw if line]
|
||||
if any(f.get("type") == "done" for f in parsed):
|
||||
return parsed
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError(
|
||||
f"SSE hook captured no `done` frame (frames so far: "
|
||||
f"{len(parsed)}) — hook install failed?"
|
||||
)
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def _tool_frames(frames: list[dict]) -> list[dict]:
|
||||
return [f for f in frames if f.get("type") == "tool"]
|
||||
|
||||
|
||||
def _submit(page: Page, question: str) -> None:
|
||||
page.fill("#message-input", question)
|
||||
page.click("#send-btn")
|
||||
# The user bubble lands synchronously with the submit handler.
|
||||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||||
|
||||
|
||||
def _wait_settled(page: Page) -> None:
|
||||
"""The turn is complete: answer text in the bubble, button recovered.
|
||||
|
||||
Phase 48: the label assertion carries the settle wait with an
|
||||
explicit timeout — the in-flight button is the enabled Stop control
|
||||
(never disabled), so ``to_be_enabled`` no longer blocks until the
|
||||
turn settles, and Playwright expect's default (5s) does not inherit
|
||||
the page default."""
|
||||
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
|
||||
def _assert_no_error_banner(page: Page) -> None:
|
||||
"""The turn must settle WITHOUT the terminal error banner (the red
|
||||
role=alert error banner — the KB-offline banner is a separate,
|
||||
non-error state)."""
|
||||
banner = page.locator("#kb-banner")
|
||||
expect(banner).to_be_hidden()
|
||||
expect(banner).not_to_have_attribute("role", "alert")
|
||||
expect(banner).not_to_have_class(re.compile(r"is-error"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 1. The live search flow: search → matched line → "Found …" answer
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_search_flow_searches_and_answers_from_match(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_pin_fixture()
|
||||
page.set_default_timeout(30_000)
|
||||
_reset_db(mock_llm)
|
||||
page.goto(app_url)
|
||||
_install_page_hooks(page)
|
||||
|
||||
_submit(page, SEARCH_QUESTION)
|
||||
# The "calling tool" STATUS window is transient: the first `tool`
|
||||
# frame sets #send-status and it holds until the FIRST answer delta
|
||||
# (the agent loop completes before the answer stream) — a polling
|
||||
# expect can stride straight over that window (the phase-44 flake),
|
||||
# so the pre-submit MutationObserver records below are the
|
||||
# deterministic source of truth for the status transitions.
|
||||
_wait_settled(page)
|
||||
|
||||
# Phase 48 (owner-locked): the in-flight button is the Stop control
|
||||
# for the whole turn; #send-status walked "… is thinking" →
|
||||
# "… is searching for <sentinel>", in order.
|
||||
labels = page.evaluate("() => window.__labels")
|
||||
assert "Stop" in labels, labels
|
||||
statuses = page.evaluate("() => window.__statuses")
|
||||
i_search = next(
|
||||
(
|
||||
i
|
||||
for i, s in enumerate(statuses)
|
||||
if f"is searching for {SEARCH_PATTERN}" in s
|
||||
),
|
||||
None,
|
||||
)
|
||||
assert i_search is not None, statuses
|
||||
i_think = next(
|
||||
(i for i, s in enumerate(statuses) if "is thinking" in s), None
|
||||
)
|
||||
assert i_think is not None and i_think < i_search, statuses
|
||||
|
||||
# Wire level: exactly ONE `tool` frame — search_documents carrying
|
||||
# the PATTERN as its argument (phase 68 task 02) — ahead of the
|
||||
# first `delta` frame.
|
||||
frames = _frames(page)
|
||||
assert _tool_frames(frames) == [
|
||||
{"type": "tool", "name": "search_documents", "argument": SEARCH_PATTERN}
|
||||
]
|
||||
first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta")
|
||||
assert all(
|
||||
i < first_delta for i, f in enumerate(frames) if f.get("type") == "tool"
|
||||
)
|
||||
done = next(f for f in frames if f.get("type") == "done")
|
||||
assert done["deflected"] is False
|
||||
|
||||
# ONE tool line above the answer: "🔎 Searching for " + the sentinel
|
||||
# in a <code> element (the pattern is data, never markup).
|
||||
lines = page.locator(".msg.brain .tool-call")
|
||||
expect(lines).to_have_count(1)
|
||||
expect(lines.nth(0)).to_contain_text("Searching for")
|
||||
expect(lines.nth(0).locator("code")).to_have_text(SEARCH_PATTERN)
|
||||
|
||||
# The answer quotes the MATCHED LINE — the search result reached the
|
||||
# model and landed in the answer (the mock's deterministic echo).
|
||||
bubble = page.locator(".msg.brain .bubble").last
|
||||
expect(bubble).to_contain_text(FOUND_ANSWER)
|
||||
|
||||
# The turn settled to idle WITHOUT the error banner, and the durable
|
||||
# record is grounded with the retrieval doc only (see test 2 for the
|
||||
# sources contract).
|
||||
_assert_no_error_banner(page)
|
||||
row = _last_query_log()
|
||||
assert row.question == SEARCH_QUESTION
|
||||
assert row.deflected is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 2. Context accounting (locked A5): a search adds no source by itself
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_search_adds_no_source_by_itself(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_pin_fixture()
|
||||
page.set_default_timeout(30_000)
|
||||
_reset_db(mock_llm)
|
||||
page.goto(app_url)
|
||||
_install_page_hooks(page)
|
||||
|
||||
_submit(page, SEARCH_QUESTION)
|
||||
_wait_settled(page)
|
||||
|
||||
# The search really ran (its wire frame is present) — yet the
|
||||
# search-only flow (no read) leaves done.sources at the RETRIEVAL
|
||||
# baseline: the one fixture doc, nothing added by the search.
|
||||
frames = _frames(page)
|
||||
assert _tool_frames(frames) == [
|
||||
{"type": "tool", "name": "search_documents", "argument": SEARCH_PATTERN}
|
||||
]
|
||||
done = next(f for f in frames if f.get("type") == "done")
|
||||
assert done["deflected"] is False
|
||||
assert [(s["source"], s["path"]) for s in done["sources"]] == [
|
||||
(SEED_SOURCE, SEED_PATH)
|
||||
]
|
||||
|
||||
# UI: exactly one source chip — the retrieval doc (the search
|
||||
# renders no chip of its own).
|
||||
chips = page.locator(".msg.brain .source-chip")
|
||||
expect(chips).to_have_count(1)
|
||||
expect(chips.nth(0)).to_contain_text(SEED_SP)
|
||||
|
||||
# Durable record: the sources row is unchanged by the search alone.
|
||||
row = _last_query_log()
|
||||
assert row.deflected is False
|
||||
assert row.sources == SEED_SP
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 3. Persistence: the search tool line re-renders after a reload
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_search_tool_line_re_renders_after_reload(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_pin_fixture()
|
||||
page.set_default_timeout(30_000)
|
||||
_reset_db(mock_llm)
|
||||
page.goto(app_url)
|
||||
|
||||
_submit(page, SEARCH_QUESTION)
|
||||
_wait_settled(page)
|
||||
expect(page.locator(".msg.brain .tool-call")).to_have_count(1)
|
||||
|
||||
page.reload()
|
||||
expect(page.locator("#empty-state")).to_be_hidden()
|
||||
|
||||
# The persisted record (the generic {name, argument} toolAcc —
|
||||
# phase 14 convention) re-renders the search line through the same
|
||||
# append helper as the live frames: sentinel in a <code> element.
|
||||
restored = page.locator(".msg.brain .tool-call")
|
||||
expect(restored).to_have_count(1)
|
||||
expect(restored.nth(0)).to_contain_text("Searching for")
|
||||
expect(restored.nth(0).locator("code")).to_have_text(SEARCH_PATTERN)
|
||||
|
||||
# The answer (the matched-line echo) is intact after the restore.
|
||||
bubble = page.locator(".msg.brain .bubble").last
|
||||
expect(bubble).to_contain_text(FOUND_ANSWER)
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
# Homelab Backup Notes
|
||||
|
||||
The homelab kubernetes cluster's etcd volume is backed up nightly to the
|
||||
offsite vault; the backup cron runs on homelab-gw at 03:00.
|
||||
|
||||
The offsite vault passphrase marker is reese-sentinel-42.
|
||||
|
||||
The restore procedure is in the homelab runbook; the vault key rotates
|
||||
every ninety days.
|
||||
@@ -3,21 +3,31 @@
|
||||
``list_catalog`` must order rows by ``(source, path)`` — the same order as
|
||||
``GET /api/docs`` — and ``find_document`` must resolve a hit to the full
|
||||
document row (content included, for the never-truncated read) and return
|
||||
``None`` for unknown ``source``/``path`` pairs.
|
||||
``None`` for unknown ``source``/``path`` pairs. Phase 68: the
|
||||
``search_documents`` tool is pinned here too — its locked parameter
|
||||
shape in ``AGENT_TOOLS``, and a scripted ``ToolCallPiece`` executed
|
||||
through ``run_agent`` against the real DB (``all_documents`` for a
|
||||
whole-KB search, ``find_document`` for a scoped one).
|
||||
|
||||
Requires: podman compose up -d db
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from copy import deepcopy
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import Document
|
||||
from app.rag import agent
|
||||
from app.rag.agent import AGENT_TOOLS, AgentHolder, run_agent
|
||||
from app.rag.llm import LLMClient, RetryPiece, StreamPiece, ToolCallPiece
|
||||
|
||||
|
||||
def _doc(db: Session, source: str, path: str, title: str, content: str) -> Document:
|
||||
@@ -81,3 +91,152 @@ def test_find_document_none_for_unknown_pairs(kb, db) -> None:
|
||||
assert agent.find_document(db, "Alpha", "nope.md") is None # wrong path
|
||||
assert agent.find_document(db, "Beta", "x.md") is None # wrong source
|
||||
assert agent.find_document(db, "nope", "nope.md") is None # nothing at all
|
||||
|
||||
|
||||
# ---------- search_documents (phase 68) ----------
|
||||
|
||||
|
||||
def test_all_documents_orders_by_source_then_path(kb, db) -> None:
|
||||
_doc(db, "Zeta", "b/second.md", "Zeta B", "ZB")
|
||||
_doc(db, "Zeta", "a/first.md", "Zeta A", "ZA")
|
||||
_doc(db, "Alpha", "c/third.md", "Alpha C", "AC")
|
||||
db.commit()
|
||||
|
||||
docs = agent.all_documents(db)
|
||||
assert [(d.source, d.path) for d in docs] == [
|
||||
("Alpha", "c/third.md"),
|
||||
("Zeta", "a/first.md"),
|
||||
("Zeta", "b/second.md"),
|
||||
]
|
||||
assert [d.content for d in docs] == ["AC", "ZA", "ZB"] # full rows
|
||||
|
||||
|
||||
def test_agent_tools_offers_search_documents_with_locked_shape() -> None:
|
||||
by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
|
||||
assert list(by_name) == [ # the third tool, in order
|
||||
"list_documents",
|
||||
"read_document",
|
||||
"search_documents",
|
||||
]
|
||||
search = by_name["search_documents"]["function"]["parameters"]
|
||||
assert search["type"] == "object"
|
||||
assert search["required"] == ["pattern"]
|
||||
assert set(search["properties"]) == {"pattern", "source", "path"}
|
||||
assert all(p["type"] == "string" for p in search["properties"].values())
|
||||
|
||||
|
||||
class ScriptedToolLLM:
|
||||
"""One scripted tool-call stream, then one canned answer stream.
|
||||
Records every ``chat_stream`` request's messages and tools."""
|
||||
|
||||
def __init__(self, call: ToolCallPiece) -> None:
|
||||
self.call = call
|
||||
self.requests: list[
|
||||
tuple[list[dict[str, Any]], list[dict[str, Any]] | None]
|
||||
] = []
|
||||
|
||||
async def chat_stream(
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
) -> AsyncIterator[StreamPiece | ToolCallPiece]:
|
||||
self.requests.append((deepcopy(messages), deepcopy(tools)))
|
||||
if len(self.requests) == 1:
|
||||
yield self.call
|
||||
else:
|
||||
yield StreamPiece("content", "ans")
|
||||
|
||||
|
||||
def _settings(**kwargs: Any) -> Settings:
|
||||
kwargs.setdefault("_env_file", None)
|
||||
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _run_search(
|
||||
db: Session, arguments: dict[str, Any]
|
||||
) -> tuple[AgentHolder, ScriptedToolLLM]:
|
||||
"""Drive one scripted ``search_documents`` call through ``run_agent``."""
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedToolLLM(
|
||||
ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)
|
||||
)
|
||||
asyncio.run(_consume(llm, db, holder))
|
||||
return holder, llm
|
||||
|
||||
|
||||
async def _consume(
|
||||
llm: ScriptedToolLLM, db: Session, holder: AgentHolder
|
||||
) -> list[StreamPiece | ToolCallPiece | RetryPiece]:
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece] = []
|
||||
async for piece in run_agent(
|
||||
cast("LLMClient", llm),
|
||||
db,
|
||||
system_prompt="SYSTEM_PROMPT",
|
||||
user_message="QUESTION",
|
||||
seed_docs=[],
|
||||
settings=_settings(),
|
||||
holder=holder,
|
||||
):
|
||||
out.append(piece)
|
||||
return out
|
||||
|
||||
|
||||
def test_search_whole_kb_through_run_agent(kb, db) -> None:
|
||||
_doc(db, "Beta", "b/two.md", "Two", "no hit\nNEEDLE in two\nlast")
|
||||
_doc(db, "Alpha", "a/one.md", "One", "first\nneedle in one\nthird")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_search(db, {"pattern": "needle"})
|
||||
|
||||
# Offered: the first request carries AGENT_TOOLS (the 3-tool list).
|
||||
assert llm.requests[0][1] == AGENT_TOOLS
|
||||
# Executed against the real DB: catalog order, grep-style lines.
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"Alpha/a/one.md:2: needle in one\n"
|
||||
"Beta/b/two.md:2: NEEDLE in two"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [] # locked A5: search adds no context
|
||||
|
||||
|
||||
def test_search_scoped_through_run_agent(kb, db) -> None:
|
||||
_doc(db, "Alpha", "a/one.md", "One", "first\nNeedle here\nthird")
|
||||
_doc(db, "Beta", "b/two.md", "Two", "NEEDLE too")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_search(
|
||||
db, {"pattern": "needle", "source": "Alpha", "path": "a/one.md"}
|
||||
)
|
||||
|
||||
# Only the named document is searched — the other one's hit is absent.
|
||||
assert llm.requests[1][0][3]["content"] == "Alpha/a/one.md:2: Needle here"
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
|
||||
def test_search_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:
|
||||
_doc(db, "Alpha", "a/one.md", "One", "nothing")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_search(
|
||||
db, {"pattern": "needle", "source": "Alpha", "path": "ghost.md"}
|
||||
)
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== "No document at Alpha/ghost.md — check the list_documents output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
|
||||
def test_search_no_matches_through_run_agent(kb, db) -> None:
|
||||
_doc(db, "Alpha", "a/one.md", "One", "nothing matching")
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_search(db, {"pattern": "zebra"})
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No matches for 'zebra' in the knowledge base."
|
||||
)
|
||||
assert holder.tool_calls == 1 # an executed search with zero hits
|
||||
assert holder.read_docs == []
|
||||
|
||||
@@ -298,17 +298,20 @@ def test_ui_chrome_has_no_emoji(client, path: str) -> None:
|
||||
the JS that renders it, and the stylesheet — is emoji-free.
|
||||
|
||||
Phase 37 revision (owner permission 2026-08-26, PLAN §4): the agent's
|
||||
``.tool-call`` line carries two CONTENT marks — 🔎 (list) and 📄
|
||||
``.tool-call`` line carries the CONTENT marks — 🔎 (list) and 📄
|
||||
(read) — the only emoji in the whole frontend, and only as the exact
|
||||
tool-line template strings in app.js. The guard strips precisely
|
||||
those two literals; any other emoji, or those marks anywhere else,
|
||||
still fails."""
|
||||
tool-line template strings in app.js. Phase 68 revision: the
|
||||
``search_documents`` tool line adds the third template literal
|
||||
("🔎 Searching for "). The guard strips precisely those three
|
||||
literals; any other emoji, or those marks anywhere else, still
|
||||
fails."""
|
||||
r = client.get(path)
|
||||
assert r.status_code == 200
|
||||
text = r.text
|
||||
if path in ("/assets/app.js", "/assets/shared.js"):
|
||||
text = text.replace('"🔎 Listing documents"', "")
|
||||
text = text.replace('"📄 Reading "', "")
|
||||
text = text.replace('"🔎 Searching for "', "")
|
||||
assert _find_emoji(text) == [], f"emoji found in {path}: {_find_emoji(text)!r}"
|
||||
|
||||
|
||||
|
||||
@@ -621,6 +621,65 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
|
||||
assert "'docs/homelab/backups.md'" in lines[-1]
|
||||
|
||||
|
||||
def test_grounded_turn_streams_search_tool_frames(
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
"""Phase 68: a scripted ``search_documents`` call streams as
|
||||
``{type: "tool", name: "search_documents", argument: <pattern>}`` —
|
||||
the raw pattern is the frame's ``argument`` (the UI renders the
|
||||
"searching for" line from it). A non-string pattern — a model error
|
||||
the backend refuses — yields ``argument: null``. A search adds no
|
||||
source: ``done.sources`` stays the retrieval docs (locked A5)."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="search_documents",
|
||||
arguments={"pattern": "Cilium"},
|
||||
),
|
||||
],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="search_documents",
|
||||
arguments={"pattern": 42}, # model error: non-string
|
||||
),
|
||||
],
|
||||
# the answer request still carries the tools (2 rounds < the
|
||||
# default cap of 10); the fake's tool_script is exhausted, so
|
||||
# it falls back to the thinking + answer stream
|
||||
]
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
|
||||
types = [f["type"] for f in frames]
|
||||
assert "error" not in types
|
||||
assert len(scripted.seen_tools) == 3 # both searches executed (rounds)
|
||||
|
||||
tool_frames = [f for f in frames if f["type"] == "tool"]
|
||||
assert len(tool_frames) == 2
|
||||
first, second = tool_frames
|
||||
assert set(first) == {"type", "name", "argument"}
|
||||
assert first["name"] == "search_documents"
|
||||
assert first["argument"] == "Cilium" # the raw pattern
|
||||
assert set(second) == {"type", "name", "argument"}
|
||||
assert second["name"] == "search_documents"
|
||||
assert second["argument"] is None # the non-string pattern → null
|
||||
|
||||
# The searches still answered: deltas, then a grounded done.
|
||||
assert [f for f in frames if f["type"] == "delta"]
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done" and done["deflected"] is False
|
||||
paths = [s["path"] for s in done["sources"]]
|
||||
assert "homelab/kubernetes.md" in paths # retrieval docs, unchanged
|
||||
assert "homelab/backups.md" not in paths # a search adds no source
|
||||
|
||||
|
||||
def test_deflected_turn_stays_byte_identical_without_tools(
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
|
||||
+318
-1
@@ -103,7 +103,8 @@ async def _run(
|
||||
|
||||
def test_agent_tools_names_and_parameters() -> None:
|
||||
by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
|
||||
assert set(by_name) == {"list_documents", "read_document"}
|
||||
assert len(AGENT_TOOLS) == 3 # list / read / search (phase 68)
|
||||
assert set(by_name) == {"list_documents", "read_document", "search_documents"}
|
||||
assert all(t["type"] == "function" for t in AGENT_TOOLS)
|
||||
list_params = by_name["list_documents"]["function"]["parameters"]
|
||||
assert list_params["type"] == "object"
|
||||
@@ -128,6 +129,34 @@ def test_agent_tools_names_and_parameters() -> None:
|
||||
"list_documents output (e.g. 'homelab/aws-route53.md' from "
|
||||
"'source: Homelab | path: homelab/aws-route53.md')."
|
||||
)
|
||||
# Phase 68: search_documents — the third tool, a locator (locked A5).
|
||||
search = by_name["search_documents"]["function"]
|
||||
assert search["description"] == (
|
||||
"Search every indexed document for an exact string "
|
||||
"(case-insensitive) and return up to 20 matching lines as "
|
||||
"'source/path:line: text' — use this to locate content, "
|
||||
"then read_document the winner. Optionally pass 'source' "
|
||||
"and 'path' (as shown in list_documents) to search one "
|
||||
"document only."
|
||||
)
|
||||
search_params = search["parameters"]
|
||||
assert search_params["type"] == "object"
|
||||
assert search_params["required"] == ["pattern"]
|
||||
assert set(search_params["properties"]) == {"pattern", "source", "path"}
|
||||
assert search_params["properties"]["pattern"]["description"] == (
|
||||
"The exact text to search for (a plain substring, not a regex)"
|
||||
)
|
||||
# Phase 63 labeled-field wording, same as read_document's parameters.
|
||||
assert search_params["properties"]["source"]["description"] == (
|
||||
"The document's source, as shown after 'source: ' in the "
|
||||
"list_documents output (e.g. 'Homelab' from "
|
||||
"'source: Homelab | path: homelab/aws-route53.md')."
|
||||
)
|
||||
assert search_params["properties"]["path"]["description"] == (
|
||||
"The document's path, as shown after 'path: ' in the "
|
||||
"list_documents output (e.g. 'homelab/aws-route53.md' from "
|
||||
"'source: Homelab | path: homelab/aws-route53.md')."
|
||||
)
|
||||
|
||||
|
||||
# ---------- happy path: list → read → answer ----------
|
||||
@@ -547,6 +576,294 @@ def test_read_document_missing_arguments_refused(
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
|
||||
|
||||
# ---------- search_documents (phase 68, locked A5/A6) ----------
|
||||
|
||||
|
||||
def test_grep_document_case_insensitive_line_numbers() -> None:
|
||||
"""Case-insensitive fixed substring, 1-based line numbers, file order,
|
||||
repeated matches within a line collapse to one match (grep semantics)."""
|
||||
content = "The NEEDLE is here\nno hit\nneedle again\nNEEDLE NEEDLE\n"
|
||||
assert agent.grep_document(content, "NEEDLE") == [
|
||||
(1, "The NEEDLE is here"),
|
||||
(3, "needle again"),
|
||||
(4, "NEEDLE NEEDLE"),
|
||||
]
|
||||
|
||||
|
||||
def test_grep_document_rstrips_lines_and_empty_content() -> None:
|
||||
assert agent.grep_document("hello \t\nworld ", "WORLD") == [(2, "world")]
|
||||
assert agent.grep_document("", "x") == []
|
||||
assert agent.grep_document("no newlines", "NO") == [(1, "no newlines")]
|
||||
assert agent.grep_document("a\nb\n", "MISSING") == []
|
||||
|
||||
|
||||
def test_search_whole_kb_grep_style_output(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Whole-KB search: catalog order, `source/path:line: text` lines,
|
||||
case-insensitive; the call counts in ``tool_calls`` and never touches
|
||||
``read_docs``; the tools stay offered on the answer request."""
|
||||
d1 = _doc("Alpha", "a/one.md", "One", "first\nNEEDLE in one\nlast")
|
||||
d2 = _doc("Beta", "b/two.md", "Two", "no hit\nneedle in two\n")
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="search_documents", arguments={"pattern": "needle"}
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"Alpha/a/one.md:2: NEEDLE in one\n"
|
||||
"Beta/b/two.md:2: needle in two"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [] # locked A5: a search adds no context
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # tools stay offered
|
||||
|
||||
|
||||
def test_search_capped_at_20_matches_in_catalog_order(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The 20-match cap is GLOBAL across documents in catalog order, and
|
||||
the scan stops once it is hit (a 35-match corpus yields exactly 20)."""
|
||||
d1 = _doc("S", "a.md", "A", "\n".join(f"hit-{i}" for i in range(15)))
|
||||
d2 = _doc("S", "b.md", "B", "\n".join(f"hit-{i}" for i in range(20)))
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="search_documents", arguments={"pattern": "hit-"}
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
lines = llm.requests[1][0][3]["content"].split("\n")
|
||||
assert len(lines) == agent.SEARCH_MAX_MATCHES
|
||||
assert lines[0] == "S/a.md:1: hit-0"
|
||||
assert lines[14] == "S/a.md:15: hit-14" # all of a.md
|
||||
assert lines[15] == "S/b.md:1: hit-0" # then b.md, in order
|
||||
assert lines[19] == "S/b.md:5: hit-4" # cut at the global cap
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
def test_search_truncates_match_lines_at_200_chars(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A 300-char match line yields exactly 200 chars of it (no crash)."""
|
||||
d1 = _doc("S", "a.md", "A", "top\n" + "x" * 300 + " NEEDLE tail")
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="search_documents", arguments={"pattern": "needle"}
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"] == f"S/a.md:2: {'x' * agent.SEARCH_LINE_LIMIT}"
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
def test_search_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Scoped search: only the named document is loaded (find_document),
|
||||
``all_documents`` never runs, and the match line carries its path."""
|
||||
d1 = _doc("S", "a.md", "A", "needle here")
|
||||
|
||||
def _find(db: Any, source: str, path: str) -> Document | None:
|
||||
if (source, path) == ("S", "a.md"):
|
||||
return d1
|
||||
raise AssertionError(
|
||||
f"find_document({source}, {path}) — the scoped "
|
||||
"search must not load any other document"
|
||||
)
|
||||
|
||||
def _boom(*_a: Any, **_k: Any) -> None:
|
||||
raise AssertionError("all_documents must not run for a scoped search")
|
||||
|
||||
monkeypatch.setattr(agent, "find_document", _find)
|
||||
monkeypatch.setattr(agent, "all_documents", _boom)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="search_documents",
|
||||
arguments={"pattern": "needle", "source": "S", "path": "a.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here"
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [] # searched doc did not enter the context
|
||||
|
||||
|
||||
def test_search_scoped_missing_document_refused(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="search_documents",
|
||||
arguments={"pattern": "x", "source": "S", "path": "ghost.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== "No document at S/ghost.md — check the list_documents output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == [] # a refusal
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("arguments", "label"),
|
||||
[
|
||||
({}, "no arguments"),
|
||||
({"pattern": ""}, "empty pattern"),
|
||||
({"pattern": " "}, "whitespace pattern"),
|
||||
({"pattern": 42}, "non-string pattern"),
|
||||
({"pattern": None}, "null pattern"),
|
||||
({"pattern": "x", "source": "S"}, "source without path"),
|
||||
({"pattern": "x", "path": "a.md"}, "path without source"),
|
||||
],
|
||||
)
|
||||
def test_search_missing_arguments_refused(
|
||||
monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str
|
||||
) -> None:
|
||||
"""Unusable pattern OR a half-specified source/path pair → the
|
||||
missing-args refusal, with no DB access at all."""
|
||||
|
||||
def _boom(*_a: Any, **_k: Any) -> None:
|
||||
raise AssertionError(f"no DB access for a refused search ({label})")
|
||||
|
||||
monkeypatch.setattr(agent, "all_documents", _boom)
|
||||
monkeypatch.setattr(agent, "find_document", _boom)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == agent.MISSING_SEARCH_ARGS
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
|
||||
def test_search_no_matches_whole_kb(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Zero hits across the KB → the no-match line (pattern quoted); the
|
||||
search still executed, so it counts — and never adds context."""
|
||||
monkeypatch.setattr(
|
||||
agent, "all_documents", lambda db: [_doc("S", "a.md", "A", "nothing here")]
|
||||
)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="search_documents", arguments={"pattern": "zebra"}
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
"No matches for 'zebra' in the knowledge base."
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
|
||||
def test_search_no_matches_scoped(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
doc = _doc("S", "a.md", "A", "nothing here")
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="search_documents",
|
||||
arguments={"pattern": "zebra", "source": "S", "path": "a.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == "No matches for 'zebra' in S/a.md."
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
|
||||
def test_search_no_match_truncates_long_pattern(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A pattern longer than 100 chars is truncated in the no-match line
|
||||
(kept short); the search itself still runs on the full pattern."""
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="search_documents",
|
||||
arguments={"pattern": "p" * 150},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
f"No matches for '{'p' * 100}' in the knowledge base."
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
def test_search_counts_but_never_adds_context(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The locate-then-read workflow: a search finds the document but does
|
||||
NOT add it — the subsequent read_document does (and is not rejected as
|
||||
already-in-context, because the search touched nothing)."""
|
||||
doc = _doc("S", "a.md", "A", "needle here")
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [doc])
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="search_documents", arguments={"pattern": "needle"}
|
||||
)
|
||||
],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="read_document",
|
||||
arguments={"source": "S", "path": "a.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.tool_calls == 2 # search + read, both executed
|
||||
assert holder.read_docs == [doc] # only the read added context (A5)
|
||||
assert llm.requests[2][0][5]["content"] == "Document S/a.md:\nneedle here"
|
||||
|
||||
|
||||
# ---------- retries inside the agent loop (phase 67, locked A2) ----------
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,8 @@ No new Python app logic exists for this task — the behavior lives in
|
||||
suite (task 06). Like the other frontend-adjacent unit files, this module
|
||||
pins the JS/CSS markers the story depends on, so a silent regression in
|
||||
the tool branch, the persistence shape, or the tool-line styling is
|
||||
caught without a browser.
|
||||
catched without a browser. Phase 68 extends the pins with the
|
||||
``search_documents`` status/line contract.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -39,7 +40,11 @@ def test_tool_branch_is_a_first_class_turn_branch() -> None:
|
||||
"the turn handler must branch on tool frames"
|
||||
)
|
||||
branch = js[tool_idx:delta_idx]
|
||||
assert "toolAcc.push" in branch, "every tool frame is recorded for persistence"
|
||||
assert "toolAcc.push({ name, argument })" in branch, (
|
||||
"every tool frame is recorded for persistence — the record stays"
|
||||
" {name, argument}-generic, no per-tool shape (phase 68: the"
|
||||
" search tool rides the same accumulator)"
|
||||
)
|
||||
assert "clearTurnTimeout()" in branch, "a tool frame proves the stream is alive"
|
||||
assert 'addMessage("brain", "")' in branch, "first frame creates the brain wrap"
|
||||
assert "uiState === UI_STATE.thinking" in branch, (
|
||||
@@ -69,6 +74,16 @@ def test_calling_tool_label_strings() -> None:
|
||||
assert "sendLabel" not in branch, "phase 48: the button keeps its Stop label"
|
||||
assert "`${brand()} is listing documents`" in branch
|
||||
assert "`${brand()} is reading ${argument}`" in branch
|
||||
# Phase 68: the search status — locked name+argument gate, sitting
|
||||
# BETWEEN the read branch and the listing fallback in the ternary.
|
||||
assert "name === \"search_documents\" && argument" in branch, (
|
||||
"the search status requires the name AND a string argument"
|
||||
)
|
||||
assert "`${brand()} is searching for ${argument}`" in branch
|
||||
read = branch.find("is reading")
|
||||
search = branch.find("is searching for")
|
||||
listing = branch.find("is listing documents")
|
||||
assert -1 < read < search < listing, "ternary order: read → search → listing"
|
||||
assert "sendStatus.textContent = toolStatus" in branch, (
|
||||
"the #send-status live region announces what Brain is doing"
|
||||
)
|
||||
@@ -106,6 +121,21 @@ def test_tool_lines_render_into_the_bubble_wrap() -> None:
|
||||
"the path is data — textContent, never innerHTML"
|
||||
)
|
||||
assert "name === \"read_document\" && argument" in body
|
||||
# Phase 68: the search branch mirrors the read branch — the same
|
||||
# name+argument gate, a <code> element, and the pattern through
|
||||
# textContent (never markup); the listing stays the final else.
|
||||
assert "name === \"search_documents\" && argument" in body
|
||||
assert 'line.textContent = "🔎 Searching for "' in body
|
||||
search_part = body.split('name === "search_documents"', 1)[1]
|
||||
assert 'document.createElement("code")' in search_part, (
|
||||
"the pattern gets the same <code> treatment as the read path"
|
||||
)
|
||||
assert "code.textContent = argument" in search_part, (
|
||||
"the pattern is data — textContent, never innerHTML"
|
||||
)
|
||||
assert 'line.textContent = "🔎 Listing documents"' in search_part, (
|
||||
"the listing fallback remains the final else"
|
||||
)
|
||||
|
||||
|
||||
def test_tool_branch_is_append_only_and_interleaving_safe() -> None:
|
||||
|
||||
@@ -17,7 +17,10 @@ from typing import Any
|
||||
|
||||
from tests.e2e.mock_llm import (
|
||||
MULTI_READ_TRIGGER,
|
||||
SEARCH_PATTERN,
|
||||
SEARCH_TRIGGER,
|
||||
TOOLS_TRIGGER,
|
||||
_search_flow,
|
||||
_tool_flow,
|
||||
)
|
||||
|
||||
@@ -256,3 +259,95 @@ def test_multi_trigger_without_tools_trigger_is_none() -> None:
|
||||
|
||||
def test_multi_flow_requires_tools_section() -> None:
|
||||
assert _tool_flow(_body(MULTI_USER, system=SYSTEM_LOW)) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Phase-68 search flow (task 03)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: Carries ONLY the search trigger (never ``use your tools`` — the
|
||||
#: phase-68 suite's live question shape, regression-safe by assertion).
|
||||
SEARCH_USER = (
|
||||
"Search your documents for the vault passphrase marker in my homelab "
|
||||
"kubernetes backup notes?"
|
||||
)
|
||||
assert SEARCH_TRIGGER in SEARCH_USER.lower()
|
||||
assert TOOLS_TRIGGER not in SEARCH_USER.lower()
|
||||
|
||||
#: The agent's ``search_documents`` result for the e2e fixture
|
||||
#: (``app/rag/agent.py`` ``_execute_tool``): one ``source/path:LINE: text``
|
||||
#: match line (the sentinel line, 200-char-capped server-side).
|
||||
SEARCH_RESULT = (
|
||||
f"search_docs/reese-notes.md:6: The offsite vault passphrase marker "
|
||||
f"is {SEARCH_PATTERN}."
|
||||
)
|
||||
#: The agent's no-match line quotes the pattern — the sentinel-only
|
||||
#: shape ``_search_result_line`` also recognizes (degenerate path).
|
||||
SEARCH_NO_MATCH = f"No matches for '{SEARCH_PATTERN}' in the knowledge base."
|
||||
|
||||
|
||||
def test_search_flow_search_step() -> None:
|
||||
# tools offered, no search result yet: the model greps.
|
||||
assert _search_flow(_body(SEARCH_USER)) == ("search",)
|
||||
|
||||
|
||||
def test_search_flow_search_step_requires_tools_offered() -> None:
|
||||
# agent_max_rounds=0 path: trigger + <tools> prompt, but no tools
|
||||
# and no search result — regular answer, not a flow.
|
||||
assert _search_flow(_body(SEARCH_USER, tools=None)) is None
|
||||
|
||||
|
||||
def test_search_flow_found_step_quotes_first_match_line() -> None:
|
||||
flow = _search_flow(_body(SEARCH_USER, (SEARCH_RESULT,)))
|
||||
assert flow == ("found", f"The offsite vault passphrase marker is {SEARCH_PATTERN}.")
|
||||
|
||||
|
||||
def test_search_flow_found_step_with_nested_path() -> None:
|
||||
# A nested path (``/`` in it) stays intact in the match-line parse.
|
||||
result = f"search_docs/deep/nested-note.md:12: line with {SEARCH_PATTERN} inside"
|
||||
flow = _search_flow(_body(SEARCH_USER, (result,)))
|
||||
assert flow == ("found", f"line with {SEARCH_PATTERN} inside")
|
||||
|
||||
|
||||
def test_search_flow_found_step_without_tools_offered() -> None:
|
||||
# The answer is content, not a tool call — it must not be gated on
|
||||
# the ``tools`` parameter (phase 45 keeps the tools offered until
|
||||
# the round cap, but the no-tools final request must still answer).
|
||||
flow = _search_flow(_body(SEARCH_USER, (SEARCH_RESULT,), tools=None))
|
||||
assert flow == ("found", f"The offsite vault passphrase marker is {SEARCH_PATTERN}.")
|
||||
|
||||
|
||||
def test_search_flow_ignores_catalog_and_read_results() -> None:
|
||||
# A catalog (labeled lines) and a read result ("Document …" prefix)
|
||||
# are NOT search results — the flow stays at the search step.
|
||||
flow = _search_flow(_body(SEARCH_USER, (CATALOG_2, _read_result(DOC1_SP, DOC1_CONTENT))))
|
||||
assert flow == ("search",)
|
||||
|
||||
|
||||
def test_search_flow_sentinel_only_result_is_a_search_result() -> None:
|
||||
# The no-match line quotes the pattern — sentinel-only recognition
|
||||
# (degenerate path; the e2e fixture always matches).
|
||||
flow = _search_flow(_body(SEARCH_USER, (SEARCH_NO_MATCH,)))
|
||||
assert flow == ("found", SEARCH_NO_MATCH)
|
||||
|
||||
|
||||
def test_search_flow_requires_tools_section() -> None:
|
||||
# Deflected turns never carry the <tools> section.
|
||||
assert _search_flow(_body(SEARCH_USER, system=SYSTEM_LOW)) is None
|
||||
|
||||
|
||||
def test_search_flow_plain_question_is_none() -> None:
|
||||
assert _search_flow(_body(PLAIN_USER)) is None
|
||||
|
||||
|
||||
def test_search_trigger_does_not_shadow_the_tool_flow() -> None:
|
||||
# The search question carries no ``use your tools`` — the phase-37
|
||||
# classifier must stay inert on it (regression-safe marker).
|
||||
assert _tool_flow(_body(SEARCH_USER)) is None
|
||||
|
||||
|
||||
def test_tool_trigger_does_not_shadow_the_search_flow() -> None:
|
||||
# The phase-37/45 questions carry no ``search your documents`` —
|
||||
# the search classifier must stay inert on them.
|
||||
assert _search_flow(_body(SINGLE_USER)) is None
|
||||
assert _search_flow(_body(MULTI_USER)) is None
|
||||
|
||||
Reference in New Issue
Block a user