feat(agent): search_documents tool — the model can grep the indexed documents for an exact string

tests/unit/test_agent.py

@@ -103,7 +103,8 @@ async def _run(
 
 def test_agent_tools_names_and_parameters() -> None:
     by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
-    assert set(by_name) == {"list_documents", "read_document"}
+    assert len(AGENT_TOOLS) == 3  # list / read / search (phase 68)
+    assert set(by_name) == {"list_documents", "read_document", "search_documents"}
     assert all(t["type"] == "function" for t in AGENT_TOOLS)
     list_params = by_name["list_documents"]["function"]["parameters"]
     assert list_params["type"] == "object"
@@ -128,6 +129,34 @@ def test_agent_tools_names_and_parameters() -> None:
         "list_documents output (e.g. 'homelab/aws-route53.md' from "
         "'source: Homelab | path: homelab/aws-route53.md')."
     )
+    # Phase 68: search_documents — the third tool, a locator (locked A5).
+    search = by_name["search_documents"]["function"]
+    assert search["description"] == (
+        "Search every indexed document for an exact string "
+        "(case-insensitive) and return up to 20 matching lines as "
+        "'source/path:line: text' — use this to locate content, "
+        "then read_document the winner. Optionally pass 'source' "
+        "and 'path' (as shown in list_documents) to search one "
+        "document only."
+    )
+    search_params = search["parameters"]
+    assert search_params["type"] == "object"
+    assert search_params["required"] == ["pattern"]
+    assert set(search_params["properties"]) == {"pattern", "source", "path"}
+    assert search_params["properties"]["pattern"]["description"] == (
+        "The exact text to search for (a plain substring, not a regex)"
+    )
+    # Phase 63 labeled-field wording, same as read_document's parameters.
+    assert search_params["properties"]["source"]["description"] == (
+        "The document's source, as shown after 'source: ' in the "
+        "list_documents output (e.g. 'Homelab' from "
+        "'source: Homelab | path: homelab/aws-route53.md')."
+    )
+    assert search_params["properties"]["path"]["description"] == (
+        "The document's path, as shown after 'path: ' in the "
+        "list_documents output (e.g. 'homelab/aws-route53.md' from "
+        "'source: Homelab | path: homelab/aws-route53.md')."
+    )
 
 
 # ---------- happy path: list → read → answer ----------
@@ -547,6 +576,294 @@ def test_read_document_missing_arguments_refused(
     assert llm.requests[1][1] == AGENT_TOOLS
 
 
+# ---------- search_documents (phase 68, locked A5/A6) ----------
+
+
+def test_grep_document_case_insensitive_line_numbers() -> None:
+    """Case-insensitive fixed substring, 1-based line numbers, file order,
+    repeated matches within a line collapse to one match (grep semantics)."""
+    content = "The NEEDLE is here\nno hit\nneedle again\nNEEDLE NEEDLE\n"
+    assert agent.grep_document(content, "NEEDLE") == [
+        (1, "The NEEDLE is here"),
+        (3, "needle again"),
+        (4, "NEEDLE NEEDLE"),
+    ]
+
+
+def test_grep_document_rstrips_lines_and_empty_content() -> None:
+    assert agent.grep_document("hello \t\nworld ", "WORLD") == [(2, "world")]
+    assert agent.grep_document("", "x") == []
+    assert agent.grep_document("no newlines", "NO") == [(1, "no newlines")]
+    assert agent.grep_document("a\nb\n", "MISSING") == []
+
+
+def test_search_whole_kb_grep_style_output(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Whole-KB search: catalog order, `source/path:line: text` lines,
+    case-insensitive; the call counts in ``tool_calls`` and never touches
+    ``read_docs``; the tools stay offered on the answer request."""
+    d1 = _doc("Alpha", "a/one.md", "One", "first\nNEEDLE in one\nlast")
+    d2 = _doc("Beta", "b/two.md", "Two", "no hit\nneedle in two\n")
+    monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2])
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "needle"}
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == (
+        "Alpha/a/one.md:2: NEEDLE in one\n"
+        "Beta/b/two.md:2: needle in two"
+    )
+    assert holder.tool_calls == 1
+    assert holder.read_docs == []  # locked A5: a search adds no context
+    assert llm.requests[1][1] == AGENT_TOOLS  # tools stay offered
+
+
+def test_search_capped_at_20_matches_in_catalog_order(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The 20-match cap is GLOBAL across documents in catalog order, and
+    the scan stops once it is hit (a 35-match corpus yields exactly 20)."""
+    d1 = _doc("S", "a.md", "A", "\n".join(f"hit-{i}" for i in range(15)))
+    d2 = _doc("S", "b.md", "B", "\n".join(f"hit-{i}" for i in range(20)))
+    monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2])
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "hit-"}
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    lines = llm.requests[1][0][3]["content"].split("\n")
+    assert len(lines) == agent.SEARCH_MAX_MATCHES
+    assert lines[0] == "S/a.md:1: hit-0"
+    assert lines[14] == "S/a.md:15: hit-14"  # all of a.md
+    assert lines[15] == "S/b.md:1: hit-0"  # then b.md, in order
+    assert lines[19] == "S/b.md:5: hit-4"  # cut at the global cap
+    assert holder.tool_calls == 1
+
+
+def test_search_truncates_match_lines_at_200_chars(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A 300-char match line yields exactly 200 chars of it (no crash)."""
+    d1 = _doc("S", "a.md", "A", "top\n" + "x" * 300 + " NEEDLE tail")
+    monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "needle"}
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert (
+        llm.requests[1][0][3]["content"] == f"S/a.md:2: {'x' * agent.SEARCH_LINE_LIMIT}"
+    )
+    assert holder.tool_calls == 1
+
+
+def test_search_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Scoped search: only the named document is loaded (find_document),
+    ``all_documents`` never runs, and the match line carries its path."""
+    d1 = _doc("S", "a.md", "A", "needle here")
+
+    def _find(db: Any, source: str, path: str) -> Document | None:
+        if (source, path) == ("S", "a.md"):
+            return d1
+        raise AssertionError(
+            f"find_document({source}, {path}) — the scoped "
+            "search must not load any other document"
+        )
+
+    def _boom(*_a: Any, **_k: Any) -> None:
+        raise AssertionError("all_documents must not run for a scoped search")
+
+    monkeypatch.setattr(agent, "find_document", _find)
+    monkeypatch.setattr(agent, "all_documents", _boom)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1",
+                name="search_documents",
+                arguments={"pattern": "needle", "source": "S", "path": "a.md"},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here"
+    assert holder.tool_calls == 1
+    assert holder.read_docs == []  # searched doc did not enter the context
+
+
+def test_search_scoped_missing_document_refused(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1",
+                name="search_documents",
+                arguments={"pattern": "x", "source": "S", "path": "ghost.md"},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert (
+        llm.requests[1][0][3]["content"]
+        == "No document at S/ghost.md — check the list_documents output."
+    )
+    assert holder.tool_calls == 0 and holder.read_docs == []  # a refusal
+
+
+@pytest.mark.parametrize(
+    ("arguments", "label"),
+    [
+        ({}, "no arguments"),
+        ({"pattern": ""}, "empty pattern"),
+        ({"pattern": "   "}, "whitespace pattern"),
+        ({"pattern": 42}, "non-string pattern"),
+        ({"pattern": None}, "null pattern"),
+        ({"pattern": "x", "source": "S"}, "source without path"),
+        ({"pattern": "x", "path": "a.md"}, "path without source"),
+    ],
+)
+def test_search_missing_arguments_refused(
+    monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str
+) -> None:
+    """Unusable pattern OR a half-specified source/path pair → the
+    missing-args refusal, with no DB access at all."""
+
+    def _boom(*_a: Any, **_k: Any) -> None:
+        raise AssertionError(f"no DB access for a refused search ({label})")
+
+    monkeypatch.setattr(agent, "all_documents", _boom)
+    monkeypatch.setattr(agent, "find_document", _boom)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == agent.MISSING_SEARCH_ARGS
+    assert holder.tool_calls == 0 and holder.read_docs == []
+    assert llm.requests[1][1] == AGENT_TOOLS  # rejected → tools stay offered
+
+
+def test_search_no_matches_whole_kb(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Zero hits across the KB → the no-match line (pattern quoted); the
+    search still executed, so it counts — and never adds context."""
+    monkeypatch.setattr(
+        agent, "all_documents", lambda db: [_doc("S", "a.md", "A", "nothing here")]
+    )
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "zebra"}
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == (
+        "No matches for 'zebra' in the knowledge base."
+    )
+    assert holder.tool_calls == 1
+    assert holder.read_docs == []
+
+
+def test_search_no_matches_scoped(monkeypatch: pytest.MonkeyPatch) -> None:
+    doc = _doc("S", "a.md", "A", "nothing here")
+    monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1",
+                name="search_documents",
+                arguments={"pattern": "zebra", "source": "S", "path": "a.md"},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == "No matches for 'zebra' in S/a.md."
+    assert holder.tool_calls == 1
+    assert holder.read_docs == []
+
+
+def test_search_no_match_truncates_long_pattern(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A pattern longer than 100 chars is truncated in the no-match line
+    (kept short); the search itself still runs on the full pattern."""
+    monkeypatch.setattr(agent, "all_documents", lambda db: [])
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1",
+                name="search_documents",
+                arguments={"pattern": "p" * 150},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == (
+        f"No matches for '{'p' * 100}' in the knowledge base."
+    )
+    assert holder.tool_calls == 1
+
+
+def test_search_counts_but_never_adds_context(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The locate-then-read workflow: a search finds the document but does
+    NOT add it — the subsequent read_document does (and is not rejected as
+    already-in-context, because the search touched nothing)."""
+    doc = _doc("S", "a.md", "A", "needle here")
+    monkeypatch.setattr(agent, "all_documents", lambda db: [doc])
+    monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "needle"}
+            )
+        ],
+        [
+            ToolCallPiece(
+                id="call_2",
+                name="read_document",
+                arguments={"source": "S", "path": "a.md"},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert holder.tool_calls == 2  # search + read, both executed
+    assert holder.read_docs == [doc]  # only the read added context (A5)
+    assert llm.requests[2][0][5]["content"] == "Document S/a.md:\nneedle here"
+
+
 # ---------- retries inside the agent loop (phase 67, locked A2) ----------
 
 

tests/unit/test_frontend_tool_states.py

@@ -5,7 +5,8 @@ No new Python app logic exists for this task — the behavior lives in
 suite (task 06). Like the other frontend-adjacent unit files, this module
 pins the JS/CSS markers the story depends on, so a silent regression in
 the tool branch, the persistence shape, or the tool-line styling is
-caught without a browser.
+catched without a browser. Phase 68 extends the pins with the
+``search_documents`` status/line contract.
 """
 from __future__ import annotations
 
@@ -39,7 +40,11 @@ def test_tool_branch_is_a_first_class_turn_branch() -> None:
         "the turn handler must branch on tool frames"
     )
     branch = js[tool_idx:delta_idx]
-    assert "toolAcc.push" in branch, "every tool frame is recorded for persistence"
+    assert "toolAcc.push({ name, argument })" in branch, (
+        "every tool frame is recorded for persistence — the record stays"
+        " {name, argument}-generic, no per-tool shape (phase 68: the"
+        " search tool rides the same accumulator)"
+    )
     assert "clearTurnTimeout()" in branch, "a tool frame proves the stream is alive"
     assert 'addMessage("brain", "")' in branch, "first frame creates the brain wrap"
     assert "uiState === UI_STATE.thinking" in branch, (
@@ -69,6 +74,16 @@ def test_calling_tool_label_strings() -> None:
     assert "sendLabel" not in branch, "phase 48: the button keeps its Stop label"
     assert "`${brand()} is listing documents`" in branch
     assert "`${brand()} is reading ${argument}`" in branch
+    # Phase 68: the search status — locked name+argument gate, sitting
+    # BETWEEN the read branch and the listing fallback in the ternary.
+    assert "name === \"search_documents\" && argument" in branch, (
+        "the search status requires the name AND a string argument"
+    )
+    assert "`${brand()} is searching for ${argument}`" in branch
+    read = branch.find("is reading")
+    search = branch.find("is searching for")
+    listing = branch.find("is listing documents")
+    assert -1 < read < search < listing, "ternary order: read → search → listing"
     assert "sendStatus.textContent = toolStatus" in branch, (
         "the #send-status live region announces what Brain is doing"
     )
@@ -106,6 +121,21 @@ def test_tool_lines_render_into_the_bubble_wrap() -> None:
         "the path is data — textContent, never innerHTML"
     )
     assert "name === \"read_document\" && argument" in body
+    # Phase 68: the search branch mirrors the read branch — the same
+    # name+argument gate, a <code> element, and the pattern through
+    # textContent (never markup); the listing stays the final else.
+    assert "name === \"search_documents\" && argument" in body
+    assert 'line.textContent = "🔎 Searching for "' in body
+    search_part = body.split('name === "search_documents"', 1)[1]
+    assert 'document.createElement("code")' in search_part, (
+        "the pattern gets the same <code> treatment as the read path"
+    )
+    assert "code.textContent = argument" in search_part, (
+        "the pattern is data — textContent, never innerHTML"
+    )
+    assert 'line.textContent = "🔎 Listing documents"' in search_part, (
+        "the listing fallback remains the final else"
+    )
 
 
 def test_tool_branch_is_append_only_and_interleaving_safe() -> None:

tests/unit/test_mock_tool_flow.py

@@ -17,7 +17,10 @@ from typing import Any
 
 from tests.e2e.mock_llm import (
     MULTI_READ_TRIGGER,
+    SEARCH_PATTERN,
+    SEARCH_TRIGGER,
     TOOLS_TRIGGER,
+    _search_flow,
     _tool_flow,
 )
 
@@ -256,3 +259,95 @@ def test_multi_trigger_without_tools_trigger_is_none() -> None:
 
 def test_multi_flow_requires_tools_section() -> None:
     assert _tool_flow(_body(MULTI_USER, system=SYSTEM_LOW)) is None
+
+
+# --------------------------------------------------------------------------
+# Phase-68 search flow (task 03)
+# --------------------------------------------------------------------------
+
+#: Carries ONLY the search trigger (never ``use your tools`` — the
+#: phase-68 suite's live question shape, regression-safe by assertion).
+SEARCH_USER = (
+    "Search your documents for the vault passphrase marker in my homelab "
+    "kubernetes backup notes?"
+)
+assert SEARCH_TRIGGER in SEARCH_USER.lower()
+assert TOOLS_TRIGGER not in SEARCH_USER.lower()
+
+#: The agent's ``search_documents`` result for the e2e fixture
+#: (``app/rag/agent.py`` ``_execute_tool``): one ``source/path:LINE: text``
+#: match line (the sentinel line, 200-char-capped server-side).
+SEARCH_RESULT = (
+    f"search_docs/reese-notes.md:6: The offsite vault passphrase marker "
+    f"is {SEARCH_PATTERN}."
+)
+#: The agent's no-match line quotes the pattern — the sentinel-only
+#: shape ``_search_result_line`` also recognizes (degenerate path).
+SEARCH_NO_MATCH = f"No matches for '{SEARCH_PATTERN}' in the knowledge base."
+
+
+def test_search_flow_search_step() -> None:
+    # tools offered, no search result yet: the model greps.
+    assert _search_flow(_body(SEARCH_USER)) == ("search",)
+
+
+def test_search_flow_search_step_requires_tools_offered() -> None:
+    # agent_max_rounds=0 path: trigger + <tools> prompt, but no tools
+    # and no search result — regular answer, not a flow.
+    assert _search_flow(_body(SEARCH_USER, tools=None)) is None
+
+
+def test_search_flow_found_step_quotes_first_match_line() -> None:
+    flow = _search_flow(_body(SEARCH_USER, (SEARCH_RESULT,)))
+    assert flow == ("found", f"The offsite vault passphrase marker is {SEARCH_PATTERN}.")
+
+
+def test_search_flow_found_step_with_nested_path() -> None:
+    # A nested path (``/`` in it) stays intact in the match-line parse.
+    result = f"search_docs/deep/nested-note.md:12: line with {SEARCH_PATTERN} inside"
+    flow = _search_flow(_body(SEARCH_USER, (result,)))
+    assert flow == ("found", f"line with {SEARCH_PATTERN} inside")
+
+
+def test_search_flow_found_step_without_tools_offered() -> None:
+    # The answer is content, not a tool call — it must not be gated on
+    # the ``tools`` parameter (phase 45 keeps the tools offered until
+    # the round cap, but the no-tools final request must still answer).
+    flow = _search_flow(_body(SEARCH_USER, (SEARCH_RESULT,), tools=None))
+    assert flow == ("found", f"The offsite vault passphrase marker is {SEARCH_PATTERN}.")
+
+
+def test_search_flow_ignores_catalog_and_read_results() -> None:
+    # A catalog (labeled lines) and a read result ("Document …" prefix)
+    # are NOT search results — the flow stays at the search step.
+    flow = _search_flow(_body(SEARCH_USER, (CATALOG_2, _read_result(DOC1_SP, DOC1_CONTENT))))
+    assert flow == ("search",)
+
+
+def test_search_flow_sentinel_only_result_is_a_search_result() -> None:
+    # The no-match line quotes the pattern — sentinel-only recognition
+    # (degenerate path; the e2e fixture always matches).
+    flow = _search_flow(_body(SEARCH_USER, (SEARCH_NO_MATCH,)))
+    assert flow == ("found", SEARCH_NO_MATCH)
+
+
+def test_search_flow_requires_tools_section() -> None:
+    # Deflected turns never carry the <tools> section.
+    assert _search_flow(_body(SEARCH_USER, system=SYSTEM_LOW)) is None
+
+
+def test_search_flow_plain_question_is_none() -> None:
+    assert _search_flow(_body(PLAIN_USER)) is None
+
+
+def test_search_trigger_does_not_shadow_the_tool_flow() -> None:
+    # The search question carries no ``use your tools`` — the phase-37
+    # classifier must stay inert on it (regression-safe marker).
+    assert _tool_flow(_body(SEARCH_USER)) is None
+
+
+def test_tool_trigger_does_not_shadow_the_search_flow() -> None:
+    # The phase-37/45 questions carry no ``search your documents`` —
+    # the search classifier must stay inert on them.
+    assert _search_flow(_body(SINGLE_USER)) is None
+    assert _search_flow(_body(MULTI_USER)) is None