feat(agent): search_documents tool — the model can grep the indexed documents for an exact string

tests/unit/test_agent.py

@@ -103,7 +103,8 @@ async def _run(
 
 def test_agent_tools_names_and_parameters() -> None:
     by_name = {t["function"]["name"]: t for t in AGENT_TOOLS}
-    assert set(by_name) == {"list_documents", "read_document"}
+    assert len(AGENT_TOOLS) == 3  # list / read / search (phase 68)
+    assert set(by_name) == {"list_documents", "read_document", "search_documents"}
     assert all(t["type"] == "function" for t in AGENT_TOOLS)
     list_params = by_name["list_documents"]["function"]["parameters"]
     assert list_params["type"] == "object"
@@ -128,6 +129,34 @@ def test_agent_tools_names_and_parameters() -> None:
         "list_documents output (e.g. 'homelab/aws-route53.md' from "
         "'source: Homelab | path: homelab/aws-route53.md')."
     )
+    # Phase 68: search_documents — the third tool, a locator (locked A5).
+    search = by_name["search_documents"]["function"]
+    assert search["description"] == (
+        "Search every indexed document for an exact string "
+        "(case-insensitive) and return up to 20 matching lines as "
+        "'source/path:line: text' — use this to locate content, "
+        "then read_document the winner. Optionally pass 'source' "
+        "and 'path' (as shown in list_documents) to search one "
+        "document only."
+    )
+    search_params = search["parameters"]
+    assert search_params["type"] == "object"
+    assert search_params["required"] == ["pattern"]
+    assert set(search_params["properties"]) == {"pattern", "source", "path"}
+    assert search_params["properties"]["pattern"]["description"] == (
+        "The exact text to search for (a plain substring, not a regex)"
+    )
+    # Phase 63 labeled-field wording, same as read_document's parameters.
+    assert search_params["properties"]["source"]["description"] == (
+        "The document's source, as shown after 'source: ' in the "
+        "list_documents output (e.g. 'Homelab' from "
+        "'source: Homelab | path: homelab/aws-route53.md')."
+    )
+    assert search_params["properties"]["path"]["description"] == (
+        "The document's path, as shown after 'path: ' in the "
+        "list_documents output (e.g. 'homelab/aws-route53.md' from "
+        "'source: Homelab | path: homelab/aws-route53.md')."
+    )
 
 
 # ---------- happy path: list → read → answer ----------
@@ -547,6 +576,294 @@ def test_read_document_missing_arguments_refused(
     assert llm.requests[1][1] == AGENT_TOOLS
 
 
+# ---------- search_documents (phase 68, locked A5/A6) ----------
+
+
+def test_grep_document_case_insensitive_line_numbers() -> None:
+    """Case-insensitive fixed substring, 1-based line numbers, file order,
+    repeated matches within a line collapse to one match (grep semantics)."""
+    content = "The NEEDLE is here\nno hit\nneedle again\nNEEDLE NEEDLE\n"
+    assert agent.grep_document(content, "NEEDLE") == [
+        (1, "The NEEDLE is here"),
+        (3, "needle again"),
+        (4, "NEEDLE NEEDLE"),
+    ]
+
+
+def test_grep_document_rstrips_lines_and_empty_content() -> None:
+    assert agent.grep_document("hello \t\nworld ", "WORLD") == [(2, "world")]
+    assert agent.grep_document("", "x") == []
+    assert agent.grep_document("no newlines", "NO") == [(1, "no newlines")]
+    assert agent.grep_document("a\nb\n", "MISSING") == []
+
+
+def test_search_whole_kb_grep_style_output(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Whole-KB search: catalog order, `source/path:line: text` lines,
+    case-insensitive; the call counts in ``tool_calls`` and never touches
+    ``read_docs``; the tools stay offered on the answer request."""
+    d1 = _doc("Alpha", "a/one.md", "One", "first\nNEEDLE in one\nlast")
+    d2 = _doc("Beta", "b/two.md", "Two", "no hit\nneedle in two\n")
+    monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2])
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "needle"}
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == (
+        "Alpha/a/one.md:2: NEEDLE in one\n"
+        "Beta/b/two.md:2: needle in two"
+    )
+    assert holder.tool_calls == 1
+    assert holder.read_docs == []  # locked A5: a search adds no context
+    assert llm.requests[1][1] == AGENT_TOOLS  # tools stay offered
+
+
+def test_search_capped_at_20_matches_in_catalog_order(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The 20-match cap is GLOBAL across documents in catalog order, and
+    the scan stops once it is hit (a 35-match corpus yields exactly 20)."""
+    d1 = _doc("S", "a.md", "A", "\n".join(f"hit-{i}" for i in range(15)))
+    d2 = _doc("S", "b.md", "B", "\n".join(f"hit-{i}" for i in range(20)))
+    monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2])
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "hit-"}
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    lines = llm.requests[1][0][3]["content"].split("\n")
+    assert len(lines) == agent.SEARCH_MAX_MATCHES
+    assert lines[0] == "S/a.md:1: hit-0"
+    assert lines[14] == "S/a.md:15: hit-14"  # all of a.md
+    assert lines[15] == "S/b.md:1: hit-0"  # then b.md, in order
+    assert lines[19] == "S/b.md:5: hit-4"  # cut at the global cap
+    assert holder.tool_calls == 1
+
+
+def test_search_truncates_match_lines_at_200_chars(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A 300-char match line yields exactly 200 chars of it (no crash)."""
+    d1 = _doc("S", "a.md", "A", "top\n" + "x" * 300 + " NEEDLE tail")
+    monkeypatch.setattr(agent, "all_documents", lambda db: [d1])
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "needle"}
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert (
+        llm.requests[1][0][3]["content"] == f"S/a.md:2: {'x' * agent.SEARCH_LINE_LIMIT}"
+    )
+    assert holder.tool_calls == 1
+
+
+def test_search_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Scoped search: only the named document is loaded (find_document),
+    ``all_documents`` never runs, and the match line carries its path."""
+    d1 = _doc("S", "a.md", "A", "needle here")
+
+    def _find(db: Any, source: str, path: str) -> Document | None:
+        if (source, path) == ("S", "a.md"):
+            return d1
+        raise AssertionError(
+            f"find_document({source}, {path}) — the scoped "
+            "search must not load any other document"
+        )
+
+    def _boom(*_a: Any, **_k: Any) -> None:
+        raise AssertionError("all_documents must not run for a scoped search")
+
+    monkeypatch.setattr(agent, "find_document", _find)
+    monkeypatch.setattr(agent, "all_documents", _boom)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1",
+                name="search_documents",
+                arguments={"pattern": "needle", "source": "S", "path": "a.md"},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here"
+    assert holder.tool_calls == 1
+    assert holder.read_docs == []  # searched doc did not enter the context
+
+
+def test_search_scoped_missing_document_refused(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1",
+                name="search_documents",
+                arguments={"pattern": "x", "source": "S", "path": "ghost.md"},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert (
+        llm.requests[1][0][3]["content"]
+        == "No document at S/ghost.md — check the list_documents output."
+    )
+    assert holder.tool_calls == 0 and holder.read_docs == []  # a refusal
+
+
+@pytest.mark.parametrize(
+    ("arguments", "label"),
+    [
+        ({}, "no arguments"),
+        ({"pattern": ""}, "empty pattern"),
+        ({"pattern": "   "}, "whitespace pattern"),
+        ({"pattern": 42}, "non-string pattern"),
+        ({"pattern": None}, "null pattern"),
+        ({"pattern": "x", "source": "S"}, "source without path"),
+        ({"pattern": "x", "path": "a.md"}, "path without source"),
+    ],
+)
+def test_search_missing_arguments_refused(
+    monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str
+) -> None:
+    """Unusable pattern OR a half-specified source/path pair → the
+    missing-args refusal, with no DB access at all."""
+
+    def _boom(*_a: Any, **_k: Any) -> None:
+        raise AssertionError(f"no DB access for a refused search ({label})")
+
+    monkeypatch.setattr(agent, "all_documents", _boom)
+    monkeypatch.setattr(agent, "find_document", _boom)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == agent.MISSING_SEARCH_ARGS
+    assert holder.tool_calls == 0 and holder.read_docs == []
+    assert llm.requests[1][1] == AGENT_TOOLS  # rejected → tools stay offered
+
+
+def test_search_no_matches_whole_kb(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Zero hits across the KB → the no-match line (pattern quoted); the
+    search still executed, so it counts — and never adds context."""
+    monkeypatch.setattr(
+        agent, "all_documents", lambda db: [_doc("S", "a.md", "A", "nothing here")]
+    )
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "zebra"}
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == (
+        "No matches for 'zebra' in the knowledge base."
+    )
+    assert holder.tool_calls == 1
+    assert holder.read_docs == []
+
+
+def test_search_no_matches_scoped(monkeypatch: pytest.MonkeyPatch) -> None:
+    doc = _doc("S", "a.md", "A", "nothing here")
+    monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1",
+                name="search_documents",
+                arguments={"pattern": "zebra", "source": "S", "path": "a.md"},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == "No matches for 'zebra' in S/a.md."
+    assert holder.tool_calls == 1
+    assert holder.read_docs == []
+
+
+def test_search_no_match_truncates_long_pattern(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A pattern longer than 100 chars is truncated in the no-match line
+    (kept short); the search itself still runs on the full pattern."""
+    monkeypatch.setattr(agent, "all_documents", lambda db: [])
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1",
+                name="search_documents",
+                arguments={"pattern": "p" * 150},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert llm.requests[1][0][3]["content"] == (
+        f"No matches for '{'p' * 100}' in the knowledge base."
+    )
+    assert holder.tool_calls == 1
+
+
+def test_search_counts_but_never_adds_context(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The locate-then-read workflow: a search finds the document but does
+    NOT add it — the subsequent read_document does (and is not rejected as
+    already-in-context, because the search touched nothing)."""
+    doc = _doc("S", "a.md", "A", "needle here")
+    monkeypatch.setattr(agent, "all_documents", lambda db: [doc])
+    monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
+    holder = AgentHolder()
+    llm = ScriptedLLM(
+        [
+            ToolCallPiece(
+                id="call_1", name="search_documents", arguments={"pattern": "needle"}
+            )
+        ],
+        [
+            ToolCallPiece(
+                id="call_2",
+                name="read_document",
+                arguments={"source": "S", "path": "a.md"},
+            )
+        ],
+        [StreamPiece("content", "ans")],
+    )
+    asyncio.run(_run(llm, holder, _settings()))
+    assert holder.tool_calls == 2  # search + read, both executed
+    assert holder.read_docs == [doc]  # only the read added context (A5)
+    assert llm.requests[2][0][5]["content"] == "Document S/a.md:\nneedle here"
+
+
 # ---------- retries inside the agent loop (phase 67, locked A2) ----------