feat(agent): search_documents tool — the model can grep the indexed documents for an exact string
This commit is contained in:
@@ -100,6 +100,24 @@ Implements just enough of the aipi surface:
|
||||
offered — e.g. ``agent_max_rounds=0``) behave exactly as today.
|
||||
``E2E_REAL_LLM=1`` ignores the mock entirely (the real model does
|
||||
what it does).
|
||||
- user message containing ``search your documents``
|
||||
(``SEARCH_TRIGGER``, phase 68, search tool) **and** the system
|
||||
prompt carries the ``<tools>`` section -> the deterministic SEARCH
|
||||
tool flow, discriminated statelessly from the messages (streaming
|
||||
only):
|
||||
* request 1 (``tools`` offered, no search result yet): stream
|
||||
ONLY ``tool_calls`` deltas — ``search_documents`` with
|
||||
``{"pattern": SEARCH_PATTERN}`` (id ``call_0``);
|
||||
* request 2 (a ``tool``-role search result in the messages —
|
||||
recognizable by its ``source/path:line: text`` match lines or
|
||||
the sentinel in its content): the content answer, deterministic:
|
||||
``Found <first matched line's content up to 80 chars>`` — so a
|
||||
suite can assert the search result reached the model and landed
|
||||
in the answer.
|
||||
Checked BEFORE the plain ``use your tools`` flow (it is the more
|
||||
specific phrase — same convention as ``think in paragraphs``); no
|
||||
existing E2E question or fixture file contains the trigger, so
|
||||
every other suite is unaffected.
|
||||
- user message containing ``show me a table`` (phase 44, markdown
|
||||
tables, TODO.md L6) -> the fixed table answer (``TABLE_ANSWER``):
|
||||
a 3-column service table, an ``<img onerror>`` XSS probe line, and
|
||||
@@ -264,6 +282,24 @@ TOOLS_TRIGGER = "use your tools"
|
||||
#: so the 3-step flow is untouched.
|
||||
MULTI_READ_TRIGGER = "read two documents"
|
||||
|
||||
#: Phase 68 (search tool, TODO.md L4): a user message containing this
|
||||
#: substring (case-insensitive) — combined with the ``<tools>`` section
|
||||
#: in the system prompt — drives the deterministic SEARCH tool flow
|
||||
#: (search_documents for ``SEARCH_PATTERN`` → the "Found …" answer),
|
||||
#: documented in the module docstring. Checked BEFORE ``TOOLS_TRIGGER``
|
||||
#: (the more specific phrase wins — the same convention as
|
||||
#: ``THINK_PARAS_TRIGGER``); verified 2026-09-01: no existing E2E
|
||||
#: question or fixture file contains the phrase, so every other suite
|
||||
#: is unaffected.
|
||||
SEARCH_TRIGGER = "search your documents"
|
||||
|
||||
#: The sentinel the search flow greps for: the e2e fixture document
|
||||
#: (``tests/fixtures/search_docs/reese-notes.md``) carries exactly one
|
||||
#: line containing it, so the search result — and the "Found …" answer
|
||||
#: that quotes its first matched line — is byte-stable (the sentinel
|
||||
#: convention of ``END_OF_NOTES_TRIGGER``).
|
||||
SEARCH_PATTERN = "reese-sentinel-42"
|
||||
|
||||
#: Phase 44 (markdown-tables story, TODO.md L6): a user message
|
||||
#: containing this substring (case-insensitive) gets the fixed table
|
||||
#: answer (``TABLE_ANSWER`` below) — a 3-column table, an XSS probe
|
||||
@@ -424,6 +460,69 @@ def _catalog_docs(body: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
return docs
|
||||
|
||||
|
||||
#: One line of the agent's ``search_documents`` output (app.rag.agent
|
||||
#: ``_execute_tool``, phase 68): ``source/path:LINE: text``. The
|
||||
#: non-greedy prefix keeps nested paths (``/`` in the path) intact.
|
||||
_SEARCH_LINE_RE = re.compile(r"^(?P<sp>.+?):(?P<line>\d+): (?P<text>.*)$")
|
||||
|
||||
|
||||
def _search_result_line(body: dict[str, Any]) -> str | None:
|
||||
"""The first matched line's text of a search result in the messages.
|
||||
|
||||
A search result is a ``tool``-role message — never a read result
|
||||
(those start with the agent's ``"Document "`` prefix) — that either
|
||||
carries ``source/path:LINE: text`` match lines (the agent's
|
||||
``search_documents`` output, phase 68) or the sentinel pattern
|
||||
itself (its no-match line quotes the pattern). Returns the first
|
||||
match line's ``text`` part (already 200-char-capped server-side),
|
||||
or the message's first line in the sentinel-only shape, or ``None``
|
||||
when no search result is in the messages yet.
|
||||
"""
|
||||
sentinel = SEARCH_PATTERN.lower()
|
||||
for m in _messages(body):
|
||||
if m.get("role") != "tool":
|
||||
continue
|
||||
content = str(m.get("content") or "")
|
||||
if content.startswith(_READ_RESULT_PREFIX):
|
||||
continue
|
||||
for line in content.splitlines():
|
||||
match = _SEARCH_LINE_RE.match(line)
|
||||
if match:
|
||||
return match.group("text")
|
||||
if sentinel in content.lower():
|
||||
lines = content.splitlines()
|
||||
return lines[0] if lines else ""
|
||||
return None
|
||||
|
||||
|
||||
def _search_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
|
||||
"""Classify a SEARCH_TRIGGER request into a step of the search flow.
|
||||
|
||||
* ``("search",)`` — ``tools`` are offered and no search result is
|
||||
in the messages yet: the model greps the whole KB for
|
||||
``SEARCH_PATTERN`` (id ``call_0``).
|
||||
* ``("found", first_line)`` — a ``tool``-role search result is in
|
||||
the messages: the model answers, quoting the first matched line
|
||||
(``Found <first matched line's content up to 80 chars>``). Reached
|
||||
regardless of the ``tools`` parameter (phase 45 keeps the tools
|
||||
offered until the round cap).
|
||||
* ``None`` — not the search flow: the trigger is absent, the
|
||||
``<tools>`` section is missing (deflected turns never carry it),
|
||||
or ``tools`` are not offered and no search result is in the
|
||||
messages yet (e.g. ``agent_max_rounds=0``).
|
||||
"""
|
||||
if SEARCH_TRIGGER not in _user(body).lower():
|
||||
return None
|
||||
if "<tools>" not in _system(body):
|
||||
return None
|
||||
first_line = _search_result_line(body)
|
||||
if first_line is not None:
|
||||
return ("found", first_line)
|
||||
if not body.get("tools"):
|
||||
return None
|
||||
return ("search",)
|
||||
|
||||
|
||||
def _tool_flow(body: dict[str, Any]) -> tuple[str, ...] | None:
|
||||
"""Classify a marker request into one step of the tool flow.
|
||||
|
||||
@@ -948,6 +1047,25 @@ def chat_completions(body: dict[str, Any]) -> Any:
|
||||
if _chat_dead(RETRY_TRIGGER, RETRY_DEAD_ATTEMPTS):
|
||||
return _llm_500(RETRY_TRIGGER)
|
||||
_fail_posts[RETRY_TRIGGER] = 0 # the answer streamed — restart
|
||||
# Phase 68 (search tool): the deterministic search marker flow —
|
||||
# checked BEFORE the phase-37 tool flow (the more specific
|
||||
# trigger phrase wins, same convention as THINK_PARAS_TRIGGER).
|
||||
search_flow = _search_flow(body)
|
||||
if search_flow is not None:
|
||||
if search_flow[0] == "search":
|
||||
stream = _tool_call_stream(
|
||||
"search_documents", {"pattern": SEARCH_PATTERN}, "call_0"
|
||||
)
|
||||
else: # "found" — quote the first matched line (80 chars)
|
||||
answer = _apply_max_tokens(
|
||||
f"Found {search_flow[1][:80]}", body.get("max_tokens")
|
||||
)
|
||||
stream = _sse_stream(answer, 0.0)
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
flow = _tool_flow(body)
|
||||
if flow is not None:
|
||||
if flow[0] == "list":
|
||||
|
||||
@@ -0,0 +1,477 @@
|
||||
"""Phase 68 E2E (Playwright, mock-only): the ``search_documents`` tool.
|
||||
|
||||
Story: n/a (TODO-derived — the owner roadmap confirmation 2026-09-01,
|
||||
TODO.md L4: "Add a search tool that allows the LLM to grep through the
|
||||
uploaded documents for a given string").
|
||||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||||
|
||||
uv run pytest tests/e2e/test_search_tool.py -v --no-cov
|
||||
|
||||
MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the gate is the
|
||||
deterministic SEARCH marker flow in ``tests/e2e/mock_llm.py`` (user
|
||||
message contains ``search your documents`` (``SEARCH_TRIGGER``)
|
||||
**and** the system prompt carries the ``<tools>`` section of the HIGH
|
||||
prompt):
|
||||
|
||||
1. request 1 (``tools`` offered, no search result yet) → streams ONLY
|
||||
``tool_calls`` deltas calling ``search_documents`` with
|
||||
``{"pattern": SEARCH_PATTERN}`` (id ``call_0``);
|
||||
2. request 2 (a ``tool``-role search result — the
|
||||
``source/path:line: text`` match line) → the content answer
|
||||
``Found <first matched line's content up to 80 chars>`` — so this
|
||||
suite can assert the search result reached the model and landed in
|
||||
the answer.
|
||||
|
||||
KB fixture — ONE document, imported through the real importer (the same
|
||||
admin-import pipeline the admin Sources page drives) against
|
||||
``tests/fixtures/search_docs/`` with the deterministic mock embeddings:
|
||||
|
||||
* ``search_docs/reese-notes.md`` — homelab kubernetes backup notes; the
|
||||
marker question cosines ≈0.51 against it (mock bag-of-words, well
|
||||
past the E2E 0.30 threshold, and it FTS-matches too) → grounded, so
|
||||
the HIGH prompt carries the ``<tools>`` section. Its line 6 carries
|
||||
the sentinel ``reese-sentinel-42`` exactly once — the deterministic
|
||||
match of the mock's grep.
|
||||
|
||||
Regression-safe marker: ``search your documents`` appears in NO other
|
||||
suite's question or fixture text (verified 2026-09-01 by repo grep;
|
||||
``tests/unit/test_mock_tool_flow.py`` pins that the trigger does not
|
||||
shadow the phase-37/45 flows and vice versa).
|
||||
|
||||
Test → phase mapping:
|
||||
1. ``test_search_flow_searches_and_answers_from_match`` — the live
|
||||
search flow: the SSE carries the ``tool`` frame
|
||||
(``search_documents`` with ``argument = <sentinel>``, ahead of any
|
||||
delta), #send-status recorded the transient "… is searching for
|
||||
<sentinel>" state, the bubble shows ONE ``🔎 Searching for``
|
||||
tool line with the sentinel in a ``<code>`` element, the answer
|
||||
quotes the matched line (``Found …`` — the match reached the
|
||||
model), and the turn settles to idle with no error banner.
|
||||
2. ``test_search_adds_no_source_by_itself`` — context accounting
|
||||
(locked A5): the search-only flow (no read) leaves
|
||||
``done.sources`` / the source chips / ``query_log.sources`` at the
|
||||
retrieval baseline — the search adds no source by itself.
|
||||
3. ``test_search_tool_line_re_renders_after_reload`` — the persisted
|
||||
record (phase 14 convention: the generic ``{name, argument}``
|
||||
toolAcc) re-renders the search line through the same helper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Page, expect
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from app.config import Settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import QueryLog
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
from tests.e2e.mock_llm import SEARCH_PATTERN, SEARCH_TRIGGER
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "search_docs"
|
||||
|
||||
#: The one imported document (the importer derives ``source`` from the
|
||||
#: fixture root's name and ``path`` from the file's relative position).
|
||||
SEED_SOURCE = "search_docs"
|
||||
SEED_PATH = "reese-notes.md"
|
||||
SEED_SP = f"{SEED_SOURCE}/{SEED_PATH}"
|
||||
|
||||
#: The fixture's sentinel line (line 6) — the mock's grep matches it
|
||||
#: exactly once, and its ``text`` part is what the "Found …" answer
|
||||
#: quotes. Pinned against the fixture file itself below.
|
||||
SENTINEL_LINE = f"The offsite vault passphrase marker is {SEARCH_PATTERN}."
|
||||
FOUND_ANSWER = f"Found {SENTINEL_LINE[:80]}"
|
||||
|
||||
#: Carries ``SEARCH_TRIGGER`` and is on-topic (cosine ≈0.51 against the
|
||||
#: fixture + FTS hits → HIGH gate, the ``<tools>`` section rides along).
|
||||
SEARCH_QUESTION = (
|
||||
"Search your documents for the vault passphrase marker in my homelab "
|
||||
"kubernetes backup notes?"
|
||||
)
|
||||
assert SEARCH_TRIGGER in SEARCH_QUESTION.lower()
|
||||
# The trigger must not collide with any other mock marker flow.
|
||||
for other in (
|
||||
"use your tools",
|
||||
"read two documents",
|
||||
"write a long answer",
|
||||
"think in paragraphs",
|
||||
"think out loud",
|
||||
"show the end of your notes",
|
||||
"show me a table",
|
||||
"fail then answer",
|
||||
"always fail",
|
||||
"embed fail once",
|
||||
"pretend to think slowly",
|
||||
):
|
||||
assert other not in SEARCH_QUESTION.lower(), other
|
||||
|
||||
|
||||
def _pin_fixture() -> None:
|
||||
"""The fixture carries the sentinel on line 6, exactly once."""
|
||||
content = (FIXTURES / SEED_PATH).read_text(encoding="utf-8")
|
||||
lines = content.split("\n")
|
||||
assert lines[5] == SENTINEL_LINE, lines[5]
|
||||
assert sum(SEARCH_PATTERN in line for line in lines) == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# DB seeding (TRUNCATE-then-import via the real importer, cf.
|
||||
# test_chat_rag.py — the same pipeline the admin Sources import drives)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||||
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
|
||||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
return await import_sources([FIXTURES], LLMClient(settings))
|
||||
|
||||
|
||||
def _run_in_thread(coro: Any) -> Any:
|
||||
"""Run a coroutine on a worker thread.
|
||||
|
||||
Playwright's sync API keeps an asyncio loop running on the test
|
||||
thread, so ``asyncio.run`` cannot be called directly from a test
|
||||
body (the established house helper).
|
||||
"""
|
||||
box: dict[str, Any] = {}
|
||||
|
||||
def runner() -> None:
|
||||
try:
|
||||
box["value"] = asyncio.run(coro)
|
||||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||||
box["error"] = e
|
||||
|
||||
t = Thread(target=runner)
|
||||
t.start()
|
||||
t.join()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
return box["value"]
|
||||
|
||||
|
||||
def _reset_db(mock_port: int) -> ImportSummary:
|
||||
"""Truncate the KB (plus the prompt-shaping tables), then re-import.
|
||||
|
||||
``steering_notes`` / ``kb_overview`` are truncated too, so the HIGH
|
||||
prompt is exactly ``<relevance>`` + ``<documents>`` + ``<tools>``
|
||||
regardless of leftovers from other suites — byte-stable prompts,
|
||||
byte-stable answers."""
|
||||
with SessionLocal() as db:
|
||||
db.execute(
|
||||
text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
|
||||
)
|
||||
db.commit()
|
||||
summary = _run_in_thread(_import_fixtures(mock_port))
|
||||
assert summary is not None and summary.added == 1, summary
|
||||
return summary
|
||||
|
||||
|
||||
def _last_query_log() -> QueryLog:
|
||||
with SessionLocal() as db:
|
||||
rows = db.scalars(select(QueryLog)).all()
|
||||
assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
|
||||
return rows[0]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Page helpers (the test_agent_document_tools.py pattern)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: Records every value #send-label takes during the turn (a
|
||||
#: MutationObserver on the element), so the in-flight label state is
|
||||
#: captured deterministically — no polling race. Phase 48: the label is
|
||||
#: the Send↔Stop morph ("Stop" holds for the whole in-flight turn).
|
||||
LABEL_RECORDER = """
|
||||
() => {
|
||||
if (window.__labelsInstalled) return;
|
||||
window.__labelsInstalled = true;
|
||||
window.__labels = [];
|
||||
const el = document.querySelector('#send-label');
|
||||
if (!el) return;
|
||||
const rec = (v) => {
|
||||
const l = window.__labels;
|
||||
if (!l.length || l[l.length - 1] !== v) l.push(v);
|
||||
};
|
||||
rec(el.textContent);
|
||||
new MutationObserver(() => rec(el.textContent)).observe(el, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
});
|
||||
}
|
||||
"""
|
||||
|
||||
#: Records every value #send-status takes during the turn — the
|
||||
#: transient "… is searching for <pattern>" calling-tool state is held
|
||||
#: only from the first `tool` frame until the first answer delta, so the
|
||||
#: pre-submit observer is the deterministic source of truth for it
|
||||
#: (no polling race — the phase-44 task-03 flake fix).
|
||||
STATUS_RECORDER = """
|
||||
() => {
|
||||
if (window.__statusesInstalled) return;
|
||||
window.__statusesInstalled = true;
|
||||
window.__statuses = [];
|
||||
const el = document.querySelector('#send-status');
|
||||
if (!el) return;
|
||||
const rec = (v) => {
|
||||
const l = window.__statuses;
|
||||
if (!l.length || l[l.length - 1] !== v) l.push(v);
|
||||
};
|
||||
rec(el.textContent);
|
||||
new MutationObserver(() => rec(el.textContent)).observe(el, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
});
|
||||
}
|
||||
"""
|
||||
|
||||
#: Captures the raw SSE ``data:`` payloads of the /api/chat stream
|
||||
#: (a response clone read in the background) — wire-level assertions
|
||||
#: for the ``tool`` frames, independent of the UI rendering.
|
||||
SSE_HOOK = """
|
||||
() => {
|
||||
if (window.__sseInstalled) return;
|
||||
window.__sseInstalled = true;
|
||||
window.__sseFrames = [];
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function (...args) {
|
||||
const res = await origFetch.apply(this, args);
|
||||
try {
|
||||
const url = typeof args[0] === 'string' ? args[0] : args[0].url;
|
||||
if (url.includes('/api/chat')) {
|
||||
res.clone().text().then((bodyText) => {
|
||||
for (const block of bodyText.split('\\n\\n')) {
|
||||
const line = block.trim();
|
||||
if (line.startsWith('data: ')) {
|
||||
window.__sseFrames.push(line.slice(6));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (e) { /* non-clonable responses: ignored */ }
|
||||
return res;
|
||||
};
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def _install_page_hooks(page: Page) -> None:
|
||||
"""Install all hooks on the loaded page (post-goto, pre-submit)."""
|
||||
page.evaluate(SSE_HOOK)
|
||||
page.evaluate(LABEL_RECORDER)
|
||||
page.evaluate(STATUS_RECORDER)
|
||||
|
||||
|
||||
def _frames(page: Page) -> list[dict]:
|
||||
"""The captured SSE frames, once the hook's background read settles.
|
||||
|
||||
The hook reads ``res.clone().text()`` in a background promise that
|
||||
resolves right after the stream closes — poll briefly until the
|
||||
final ``done`` frame lands (fail loud if the hook captured nothing).
|
||||
"""
|
||||
deadline = time.monotonic() + 10.0
|
||||
while True:
|
||||
raw = page.evaluate("() => window.__sseFrames || []")
|
||||
parsed = [json.loads(line) for line in raw if line]
|
||||
if any(f.get("type") == "done" for f in parsed):
|
||||
return parsed
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError(
|
||||
f"SSE hook captured no `done` frame (frames so far: "
|
||||
f"{len(parsed)}) — hook install failed?"
|
||||
)
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def _tool_frames(frames: list[dict]) -> list[dict]:
|
||||
return [f for f in frames if f.get("type") == "tool"]
|
||||
|
||||
|
||||
def _submit(page: Page, question: str) -> None:
|
||||
page.fill("#message-input", question)
|
||||
page.click("#send-btn")
|
||||
# The user bubble lands synchronously with the submit handler.
|
||||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||||
|
||||
|
||||
def _wait_settled(page: Page) -> None:
|
||||
"""The turn is complete: answer text in the bubble, button recovered.
|
||||
|
||||
Phase 48: the label assertion carries the settle wait with an
|
||||
explicit timeout — the in-flight button is the enabled Stop control
|
||||
(never disabled), so ``to_be_enabled`` no longer blocks until the
|
||||
turn settles, and Playwright expect's default (5s) does not inherit
|
||||
the page default."""
|
||||
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
|
||||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||||
expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
|
||||
|
||||
|
||||
def _assert_no_error_banner(page: Page) -> None:
|
||||
"""The turn must settle WITHOUT the terminal error banner (the red
|
||||
role=alert error banner — the KB-offline banner is a separate,
|
||||
non-error state)."""
|
||||
banner = page.locator("#kb-banner")
|
||||
expect(banner).to_be_hidden()
|
||||
expect(banner).not_to_have_attribute("role", "alert")
|
||||
expect(banner).not_to_have_class(re.compile(r"is-error"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 1. The live search flow: search → matched line → "Found …" answer
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_search_flow_searches_and_answers_from_match(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_pin_fixture()
|
||||
page.set_default_timeout(30_000)
|
||||
_reset_db(mock_llm)
|
||||
page.goto(app_url)
|
||||
_install_page_hooks(page)
|
||||
|
||||
_submit(page, SEARCH_QUESTION)
|
||||
# The "calling tool" STATUS window is transient: the first `tool`
|
||||
# frame sets #send-status and it holds until the FIRST answer delta
|
||||
# (the agent loop completes before the answer stream) — a polling
|
||||
# expect can stride straight over that window (the phase-44 flake),
|
||||
# so the pre-submit MutationObserver records below are the
|
||||
# deterministic source of truth for the status transitions.
|
||||
_wait_settled(page)
|
||||
|
||||
# Phase 48 (owner-locked): the in-flight button is the Stop control
|
||||
# for the whole turn; #send-status walked "… is thinking" →
|
||||
# "… is searching for <sentinel>", in order.
|
||||
labels = page.evaluate("() => window.__labels")
|
||||
assert "Stop" in labels, labels
|
||||
statuses = page.evaluate("() => window.__statuses")
|
||||
i_search = next(
|
||||
(
|
||||
i
|
||||
for i, s in enumerate(statuses)
|
||||
if f"is searching for {SEARCH_PATTERN}" in s
|
||||
),
|
||||
None,
|
||||
)
|
||||
assert i_search is not None, statuses
|
||||
i_think = next(
|
||||
(i for i, s in enumerate(statuses) if "is thinking" in s), None
|
||||
)
|
||||
assert i_think is not None and i_think < i_search, statuses
|
||||
|
||||
# Wire level: exactly ONE `tool` frame — search_documents carrying
|
||||
# the PATTERN as its argument (phase 68 task 02) — ahead of the
|
||||
# first `delta` frame.
|
||||
frames = _frames(page)
|
||||
assert _tool_frames(frames) == [
|
||||
{"type": "tool", "name": "search_documents", "argument": SEARCH_PATTERN}
|
||||
]
|
||||
first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta")
|
||||
assert all(
|
||||
i < first_delta for i, f in enumerate(frames) if f.get("type") == "tool"
|
||||
)
|
||||
done = next(f for f in frames if f.get("type") == "done")
|
||||
assert done["deflected"] is False
|
||||
|
||||
# ONE tool line above the answer: "🔎 Searching for " + the sentinel
|
||||
# in a <code> element (the pattern is data, never markup).
|
||||
lines = page.locator(".msg.brain .tool-call")
|
||||
expect(lines).to_have_count(1)
|
||||
expect(lines.nth(0)).to_contain_text("Searching for")
|
||||
expect(lines.nth(0).locator("code")).to_have_text(SEARCH_PATTERN)
|
||||
|
||||
# The answer quotes the MATCHED LINE — the search result reached the
|
||||
# model and landed in the answer (the mock's deterministic echo).
|
||||
bubble = page.locator(".msg.brain .bubble").last
|
||||
expect(bubble).to_contain_text(FOUND_ANSWER)
|
||||
|
||||
# The turn settled to idle WITHOUT the error banner, and the durable
|
||||
# record is grounded with the retrieval doc only (see test 2 for the
|
||||
# sources contract).
|
||||
_assert_no_error_banner(page)
|
||||
row = _last_query_log()
|
||||
assert row.question == SEARCH_QUESTION
|
||||
assert row.deflected is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 2. Context accounting (locked A5): a search adds no source by itself
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_search_adds_no_source_by_itself(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_pin_fixture()
|
||||
page.set_default_timeout(30_000)
|
||||
_reset_db(mock_llm)
|
||||
page.goto(app_url)
|
||||
_install_page_hooks(page)
|
||||
|
||||
_submit(page, SEARCH_QUESTION)
|
||||
_wait_settled(page)
|
||||
|
||||
# The search really ran (its wire frame is present) — yet the
|
||||
# search-only flow (no read) leaves done.sources at the RETRIEVAL
|
||||
# baseline: the one fixture doc, nothing added by the search.
|
||||
frames = _frames(page)
|
||||
assert _tool_frames(frames) == [
|
||||
{"type": "tool", "name": "search_documents", "argument": SEARCH_PATTERN}
|
||||
]
|
||||
done = next(f for f in frames if f.get("type") == "done")
|
||||
assert done["deflected"] is False
|
||||
assert [(s["source"], s["path"]) for s in done["sources"]] == [
|
||||
(SEED_SOURCE, SEED_PATH)
|
||||
]
|
||||
|
||||
# UI: exactly one source chip — the retrieval doc (the search
|
||||
# renders no chip of its own).
|
||||
chips = page.locator(".msg.brain .source-chip")
|
||||
expect(chips).to_have_count(1)
|
||||
expect(chips.nth(0)).to_contain_text(SEED_SP)
|
||||
|
||||
# Durable record: the sources row is unchanged by the search alone.
|
||||
row = _last_query_log()
|
||||
assert row.deflected is False
|
||||
assert row.sources == SEED_SP
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 3. Persistence: the search tool line re-renders after a reload
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_search_tool_line_re_renders_after_reload(
|
||||
page: Page, app_url: str, mock_llm: int, db_ready: None
|
||||
) -> None:
|
||||
_pin_fixture()
|
||||
page.set_default_timeout(30_000)
|
||||
_reset_db(mock_llm)
|
||||
page.goto(app_url)
|
||||
|
||||
_submit(page, SEARCH_QUESTION)
|
||||
_wait_settled(page)
|
||||
expect(page.locator(".msg.brain .tool-call")).to_have_count(1)
|
||||
|
||||
page.reload()
|
||||
expect(page.locator("#empty-state")).to_be_hidden()
|
||||
|
||||
# The persisted record (the generic {name, argument} toolAcc —
|
||||
# phase 14 convention) re-renders the search line through the same
|
||||
# append helper as the live frames: sentinel in a <code> element.
|
||||
restored = page.locator(".msg.brain .tool-call")
|
||||
expect(restored).to_have_count(1)
|
||||
expect(restored.nth(0)).to_contain_text("Searching for")
|
||||
expect(restored.nth(0).locator("code")).to_have_text(SEARCH_PATTERN)
|
||||
|
||||
# The answer (the matched-line echo) is intact after the restore.
|
||||
bubble = page.locator(".msg.brain .bubble").last
|
||||
expect(bubble).to_contain_text(FOUND_ANSWER)
|
||||
Reference in New Issue
Block a user