From 801639efccaf13b98a862ba0033207cb5387c4f5 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Thu, 3 Sep 2026 11:17:47 -0400 Subject: [PATCH] =?UTF-8?q?feat(agent):=20align=20the=20document=20tools?= =?UTF-8?q?=20with=20the=20harness-trained=20shape=20=E2=80=94=20ls,=20rea?= =?UTF-8?q?d(path),=20grep(pattern,=20path=3F)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../01_tool_schemas.md | 96 + .../02_prompt_section.md | 42 + .../70_harness_aligned_tools/03_api_sse.md | 47 + .../70_harness_aligned_tools/04_frontend.md | 52 + .../todo/70_harness_aligned_tools/00_phase.md | 143 ++ .../05_mock_e2e_commit.md | 78 + .../71_scaffolding_guardrails/00_phase.md | 152 ++ .../71_scaffolding_guardrails/01_filter.md | 77 + .../02_llm_integration.md | 61 + .../03_recovery_policy.md | 104 ++ .../04_deflect_prompt.md | 35 + .../05_e2e_commit.md | 77 + ...ness_aligned_tools__01_tool_schemas.a1.err | 0 ...rness_aligned_tools__01_tool_schemas.a1.md | 14 + ...aligned_tools__01_tool_schemas.a1.validate | 294 +++ ...ness_aligned_tools__01_tool_schemas.a2.err | 0 ...rness_aligned_tools__01_tool_schemas.a2.md | 11 + ...aligned_tools__01_tool_schemas.a2.validate | 75 + ...ss_aligned_tools__02_prompt_section.a1.err | 0 ...ess_aligned_tools__02_prompt_section.a1.md | 19 + ...igned_tools__02_prompt_section.a1.validate | 75 + ...0_harness_aligned_tools__03_api_sse.a1.err | 0 ...70_harness_aligned_tools__03_api_sse.a1.md | 16 + ...ness_aligned_tools__03_api_sse.a1.validate | 75 + ..._harness_aligned_tools__04_frontend.a1.err | 0 ...0_harness_aligned_tools__04_frontend.a1.md | 11 + ...ess_aligned_tools__04_frontend.a1.validate | 75 + ...s_aligned_tools__05_mock_e2e_commit.a1.err | 0 .env.example | 2 +- README.md | 42 +- app/api/chat.py | 64 +- app/api/docs.py | 2 +- app/rag/agent.py | 398 ++-- app/rag/llm.py | 6 +- app/rag/prompts.py | 49 +- app/schemas.py | 41 +- frontend/assets/app.js | 76 +- frontend/assets/shared.js | 40 +- tests/e2e/mock_llm.py | 118 +- tests/e2e/test_agent_document_tools.py | 38 +- tests/e2e/test_agent_unlimited_tools.py | 42 +- tests/e2e/test_harness_aligned_tools.py | 524 ++++++ tests/e2e/test_search_tool.py | 32 +- tests/integration/test_agent_tools.py | 317 ++-- tests/integration/test_api.py | 12 +- tests/integration/test_chat_api.py | 125 +- tests/integration/test_chats_api.py | 8 +- tests/unit/test_agent.py | 1651 +++++++++-------- tests/unit/test_chat_gate.py | 11 +- tests/unit/test_frontend_tool_states.py | 113 +- tests/unit/test_llm_client.py | 62 +- tests/unit/test_llm_stream_teardown.py | 4 +- tests/unit/test_mock_tool_flow.py | 12 +- tests/unit/test_prompts.py | 62 + tests/unit/test_sse_events.py | 17 +- 55 files changed, 4031 insertions(+), 1466 deletions(-) create mode 100644 .agent/phases/complete/70_harness_aligned_tools/01_tool_schemas.md create mode 100644 .agent/phases/complete/70_harness_aligned_tools/02_prompt_section.md create mode 100644 .agent/phases/complete/70_harness_aligned_tools/03_api_sse.md create mode 100644 .agent/phases/complete/70_harness_aligned_tools/04_frontend.md create mode 100644 .agent/phases/todo/70_harness_aligned_tools/00_phase.md create mode 100644 .agent/phases/todo/70_harness_aligned_tools/05_mock_e2e_commit.md create mode 100644 .agent/phases/todo/71_scaffolding_guardrails/00_phase.md create mode 100644 .agent/phases/todo/71_scaffolding_guardrails/01_filter.md create mode 100644 .agent/phases/todo/71_scaffolding_guardrails/02_llm_integration.md create mode 100644 .agent/phases/todo/71_scaffolding_guardrails/03_recovery_policy.md create mode 100644 .agent/phases/todo/71_scaffolding_guardrails/04_deflect_prompt.md create mode 100644 .agent/phases/todo/71_scaffolding_guardrails/05_e2e_commit.md create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.err create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.md create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.validate create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.err create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.md create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.validate create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.err create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.md create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.validate create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.err create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.md create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.validate create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.err create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.md create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.validate create mode 100644 .agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__05_mock_e2e_commit.a1.err create mode 100644 tests/e2e/test_harness_aligned_tools.py diff --git a/.agent/phases/complete/70_harness_aligned_tools/01_tool_schemas.md b/.agent/phases/complete/70_harness_aligned_tools/01_tool_schemas.md new file mode 100644 index 0000000..eef465d --- /dev/null +++ b/.agent/phases/complete/70_harness_aligned_tools/01_tool_schemas.md @@ -0,0 +1,96 @@ +# Task 01 — Tool Schemas + Execution: `ls` / `read(path)` / `grep(pattern, path?)` + +**Phase:** `70_harness_aligned_tools` · **Story:** `.agent/user_stories/agent-document-tools.md` + +## Objective +Reshape `AGENT_TOOLS` and `_execute_tool` in `app/rag/agent.py` to the pi.dev harness +surface — `ls` (optional `path`), `read` (required `path`), `grep` (required +`pattern`, optional `path`) — with the combined `source/path` form as the canonical +document identity, keeping every locked capability intact. + +## Work +1. `app/rag/agent.py` — `AGENT_TOOLS` (three OpenAI function definitions; keep the + module docstring's loop-contract points accurate, update the names/args references): + - **`ls`** — parameters: optional `path` (string): "Source name to list one + source's documents (e.g. 'homelab'); omit to list every document." Description: + "List the indexed documents as `source: X | path: Y | title: Z` lines." + - **`read`** — parameters: required `path` (string): "The document to add to your + context, as the combined `source/path` string exactly as shown in the `ls` + output (e.g. 'homelab/active/container_caddy/caddy.md')." Description: "Add the + full content of one indexed document to your context." + - **`grep`** — parameters: required `pattern` (string): "The exact text to search + for (a plain substring, not a regex)"; optional `path` (string): "Limit the + search to one document, as a combined `source/path` string from the `ls` output + (omit to search every document)." Description: "Search the indexed documents + for an exact string (case-insensitive) and return up to 20 matching lines as + `source/path:line: text` — a locator, not a context-adder: read the winner with + `read`." +2. `app/rag/agent.py` — `_execute_tool` (branch on the new names; `AgentHolder` + semantics, round counting, and the DB accessors are unchanged): + - `ls`: no `path` → the full `list_catalog` listing (`"N documents:\n"` + the + phase-63 lines, unchanged format). With `path`: strip it; if it equals no + source name → refusal `No source named '{path}' — check the ls output.` (a + rejected call, counts in nothing); else the same listing filtered to that + source (`0 documents:` is a valid, counted result). + - `read`: require a non-blank string `path` (else + `read requires a string argument 'path'.`). Split at the FIRST `/` → + `(source, path)` (source names can never contain `/` — importer contract). + No `/` in the argument → refusal `No document at '{arg}' — check the ls + output.` Already-in-context check (seed + holder) via the resolved pair → + `ALREADY_IN_CONTEXT`. Unknown pair → the same `No document at '{arg}' — check + the ls output.` refusal (echo the argument as passed, so the model sees its + own form). Success → `holder.read_docs.append(doc)`, `holder.tool_calls += 1`, + result `f"Document {doc.source}/{doc.path}:\n{doc.content}"` (full content — + A7-revised, never truncated). + - `grep`: require a non-blank string `pattern` (else + `grep requires a string argument 'pattern'`). Optional `path`: if present it + must resolve to exactly one document (same first-slash split; unknown → + `No document at '{arg}' — check the ls output.`); absent → `all_documents`. + Everything downstream of target selection is the **phase-68 contract verbatim**: + case-insensitive fixed substring via `grep_document`, catalog order, global cap + `SEARCH_MAX_MATCHES=20`, per-line `SEARCH_LINE_LIMIT=200`, no-match lines + (`NO_MATCHES` / the scoped variant keyed on the resolved `source/path`), + counted as a successful call, `holder.read_docs` untouched (locator only). + - `_resolve_document`: replace with the single-input resolver the three tools + share — `_resolve_path(db, combined: str) -> tuple[Document | None, str, str]` + (split at the first slash, exact pair lookup; return the split pair so refusals + can echo/teach). Delete the old two-argument self-correction + "teach the split" + refusal branches (the combined form is now the correct input). + - Update the module-level refusal constants: `MISSING_READ_ARGS = "read requires + a string argument 'path'."`, `MISSING_SEARCH_ARGS = "grep requires a string + argument 'pattern'."`; `ALREADY_IN_CONTEXT` / `UNKNOWN_TOOL` unchanged. + - Update the module docstring: point 2 (the three tools by new name/shape), + point 3 (refusals — no more split-teaching; unknown source for `ls`), and the + phase-70 note (owner permission 2026-09-03: harness-aligned surface; phase-68 + A5 match/output contract preserved; the combined form is canonical). +3. `tests/unit/test_agent.py` — update every existing test to the new names/args and + add: + - schema pins: exactly `ls`/`read`/`grep`; required/optional arg sets; old names + absent from `AGENT_TOOLS`. + - `read`: combined form resolves + full content returned; bare source name + (`read(path='homelab')`) → the no-document refusal; missing/blank/ + non-string `path` → the missing-args refusal; already-in-context (seed and + previously-read) → `ALREADY_IN_CONTEXT`; re-read counts nothing. + - `ls`: no-arg full catalog (format pinned: `source: X | path: Y | title: Z`); + scoped known source (incl. a source with 0 docs → `0 documents:` counted); + unknown source → refusal, not counted. + - `grep`: whole-KB vs `path`-scoped; the A5 pins unchanged (fixed substring, + case-insensitive, 20-cap in catalog order, 200-char truncation, no-match lines + counted, read_docs untouched); unknown `path` target → refusal. + - loop mechanics unchanged: round cap forcing the final no-tools answer, + `agent_max_rounds=0` → exactly one request with `tools=None`, rejected calls + consume a round but count nothing in `holder.tool_calls`. + +## Testing & Quality +- Unit: `tests/unit/test_agent.py` (above) — the DB accessors stay monkeypatched + module-level functions (no database in unit scope). +- Coverage: **>90%** on this task's modified code (`app/rag/agent.py`). + +## Completion Criteria +- [ ] `rg "list_documents|read_document|search_documents" app/rag/agent.py` → no + matches; `AGENT_TOOLS` names are exactly `ls`, `read`, `grep`. +- [ ] `uv run pytest tests/unit/test_agent.py -v --no-cov` green (old + new pins). +- [ ] `uv run pytest --cov=app -k agent` green; `uv run ruff check . && uv run + pyright` clean. +- [ ] No behavior change to the deflected path or the kill switch (pinned by the + updated suite). diff --git a/.agent/phases/complete/70_harness_aligned_tools/02_prompt_section.md b/.agent/phases/complete/70_harness_aligned_tools/02_prompt_section.md new file mode 100644 index 0000000..3216e7f --- /dev/null +++ b/.agent/phases/complete/70_harness_aligned_tools/02_prompt_section.md @@ -0,0 +1,42 @@ +# Task 02 — `` Prompt Section for the New Surface + +**Phase:** `70_harness_aligned_tools` · **Story:** `.agent/user_stories/agent-document-tools.md` + +## Objective +Rewrite `TOOLS_SECTION` in `app/rag/prompts.py` so the HIGH prompt teaches the new +`ls`/`read`/`grep` shapes in harness language — the last thing the model reads on a +grounded turn — while the E2E mock's ``-marker keying and the LOW prompt stay +untouched. + +## Work +1. `app/rag/prompts.py` — replace the `TOOLS_SECTION` constant (keep the name, keep + it HIGH-only, keep it appended after the `` body, keep the `` / + `` markers the mock keys on): + - New copy (harness-aligned, names/args exactly as task 01): instruct that the + context may be extended with three tools — `ls` to list the indexed documents + (`source: X | path: Y | title: Z` lines), `grep` to locate an exact string + (case-insensitive; a locator, not a context-adder), and `read` to pull in one + document by its combined `source/path` (full content) — and that the model + should answer as soon as it has what it needs (the round cap is the bound; the + prompt does not re-state budgets — phase 45). + - Keep the module docstring's statement that the LOW/deflection prompt never + carries the section (byte-identical LOW path — this task must not change + `build_deflect_prompt` or `build_high_prompt`'s section order). +2. `tests/unit/test_prompts.py` (and `test_chat_gate.py` where the section is pinned) + — update the `TOOLS_SECTION` copy pins: markers present; the new tool names + present and the old names (`list_documents`, `read_document`, + `search_documents`) absent from the HIGH prompt; HIGH/LOW section order and the + byte-identical-when-empty steering/overview behavior unchanged; the LOW prompt is + byte-identical to the pre-phase text (no ``, no new copy). + +## Testing & Quality +- Unit: `tests/unit/test_prompts.py` / `test_chat_gate.py` pins above. +- Coverage: **>90%** on this task's modified code (`app/rag/prompts.py`). + +## Completion Criteria +- [ ] `uv run pytest tests/unit/test_prompts.py tests/unit/test_chat_gate.py -v + --no-cov` green. +- [ ] HIGH prompt still ends with the `` section (mock keying intact); + `rg "list_documents|read_document|search_documents" app/rag/prompts.py` → no + matches. +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/complete/70_harness_aligned_tools/03_api_sse.md b/.agent/phases/complete/70_harness_aligned_tools/03_api_sse.md new file mode 100644 index 0000000..2064406 --- /dev/null +++ b/.agent/phases/complete/70_harness_aligned_tools/03_api_sse.md @@ -0,0 +1,47 @@ +# Task 03 — SSE `tool` Frames + API Contract Copy + +**Phase:** `70_harness_aligned_tools` · **Story:** `.agent/user_stories/agent-document-tools.md` + +## Objective +Point the `ToolCallPiece` → SSE `tool`-frame derivation in `app/api/chat.py` at the +new tool shapes (new `name` values, `argument` = the single string the model passed) +and update the module's contract docstrings; `delta`/`thinking`/`retry`/`done`/ +`error` frames and the `done.sources`/`query_log` semantics stay exactly as they are. + +## Work +1. `app/api/chat.py` — the `ToolCallPiece` branch of the piece loop: + - `read` → `argument = piece.arguments.get("path")` (the combined `source/path` + as the model passed it; non-string → null). + - `grep` → `argument = piece.arguments.get("pattern")` (non-string → null, the + existing refusal case). + - `ls` → `argument = piece.arguments.get("path")` (the scope, if the model gave + one; otherwise null). + - One-line rule in the code comment: **`argument` is the single string argument + the model passed, or null** — symmetric across the three tools. + - `ChatToolEvent` schema (`app/schemas.py`) needs no field change (name/argument + are already `str`/`str | None`) — update its docstring's name examples only. +2. `app/api/chat.py` module docstring — the "Agent document tools" paragraph: new + tool names/shapes, the `argument` rule above, phase-70 note (owner permission + 2026-09-03); keep the SSE shape statement (`{"type":"tool","name":…,"argument":…}`) + and the `done.sources`/`query_log`/`tool_calls=N` contract wording accurate. +3. `tests/integration/test_agent_tools.py`, `test_chat_api.py` (+ `test_api.py` / + `test_chats_api.py` where tool frames or names are pinned) — update to the new + `name` values and the `argument` rule; add pins: a `read` frame carries the + combined path as passed; a `grep` frame carries the pattern; an `ls` frame is + null-unscoped / the scope when scoped; a rejected call (unknown `read` path) + still emits its frame with the model's argument (frame emission is + execution-independent — the existing behavior, now pinned). + +## Testing & Quality +- Integration: the updated suites above (mock LLM emitting the new tool names — + `tests/e2e/mock_llm.py` is updated in task 05; until then these integration tests + build their own scripted pieces, so they do not depend on the mock). +- Coverage: **>90%** on this task's modified code (`app/api/chat.py`, + `app/schemas.py`). + +## Completion Criteria +- [ ] `uv run pytest tests/integration/test_agent_tools.py tests/integration/test_chat_api.py -v + --no-cov` green. +- [ ] `done.sources`, `query_log.sources`, and the per-turn `tool_calls=N` field + report exactly what they did before for the same executed calls (pins green). +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/complete/70_harness_aligned_tools/04_frontend.md b/.agent/phases/complete/70_harness_aligned_tools/04_frontend.md new file mode 100644 index 0000000..5d45a7b --- /dev/null +++ b/.agent/phases/complete/70_harness_aligned_tools/04_frontend.md @@ -0,0 +1,52 @@ +# Task 04 — Frontend Tool-Line Rendering for the New Names + +**Phase:** `70_harness_aligned_tools` · **Story:** `.agent/user_stories/agent-document-tools.md` + +## Objective +Point the tool-line rendering at the new `ls`/`read`/`grep` names while keeping +**legacy saved chats** (persisted with the old `list_documents` / `read_document` / +`search_documents` names, phase 14 persistence) rendering exactly as before — no +migration. + +## Work +1. `frontend/assets/app.js` — `appendToolLine(wrap, name, argument)`: + - New branches: `read` + argument → `📄 Reading argument`; + `grep` + argument → `🔎 Searching for argument`; `ls` + argument + (scoped) → `🔎 Listing documents in argument`; `ls` unscoped → + `🔎 Listing documents`. + - **Legacy branches stay**: `read_document` (Reading), `search_documents` + (Searching for), everything else (Listing documents) — persisted turns from + before this phase must render unchanged. (Implementation: a name→kind map + covering both generations, or explicit if-chains — executor's choice; the pins + below define the contract.) + - All arguments through `textContent` only (the existing "path/pattern is data, + never markup" discipline). + - Update the section docstring (phase 37 reference + a phase-70 note: names + remapped to the harness surface; legacy names still render). +2. `frontend/assets/shared.js` — the shared-chat tool-line helper (currently + `read_document` + argument → `📄 Reading …`, else `🔎 Listing + documents`): add the new names (`read` → Reading, `grep` → Searching for with + `` argument, `ls` → Listing documents, `ls` + argument → Listing documents + in `…`), keep the legacy `read_document` branch, keep + `textContent`-only population. Update its docstring comment. +3. `tests/unit/test_frontend_tool_states.py` — update/add pins (the + source-string house pattern): + - `app.js` carries branches for `read`, `grep`, `ls` (with/without argument) and + still carries `read_document` / `search_documents` (legacy render). + - `shared.js` carries the `read` / `grep` / `ls` handling and the legacy + `read_document` branch. + - Every argument population goes through `textContent` (no `innerHTML` on tool + lines). + - The old-only copy pins (e.g. exact "Listing documents" string for the default + case) stay green. + +## Testing & Quality +- Unit: `tests/unit/test_frontend_tool_states.py` pins above. +- Coverage: no `app/` code in this task — the **>90%** gate stays green as part of + the full suite run. + +## Completion Criteria +- [ ] `uv run pytest tests/unit/test_frontend_tool_states.py -v --no-cov` green. +- [ ] A persisted chat containing old tool names renders Reading/Searching/Listing + lines exactly as before (legacy branches pinned). +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/todo/70_harness_aligned_tools/00_phase.md b/.agent/phases/todo/70_harness_aligned_tools/00_phase.md new file mode 100644 index 0000000..a4f60bb --- /dev/null +++ b/.agent/phases/todo/70_harness_aligned_tools/00_phase.md @@ -0,0 +1,143 @@ +# Phase 70 — Harness-Aligned Agent Tools: `ls` / `read(path)` / `grep(pattern, path?)` + +**Source:** owner request (chat, 2026-09-03) — live incident: the question "What are the +correct llama.cpp arguments for Qwen 3.8?" was answered with the raw model text +`<|tool_call_start|>[read(path='/homelab/backup-notes.md')]<|tool_call_end|>` +(`query_log`: top_score=0.040, fts_hits=0, deflected=true — the turn was DEFLECTED, so +no tools were even offered). Diagnosis: the chat model (`lite` per `.env`; owner keeps +`lite` for everything — it's faster, owner decision 2026-09-03) reaches for the tool +shapes it was trained on — a `read` tool taking a **single** `path` argument — and our +`read_document(source, path)` fights that prior (the `_resolve_document` combined-form +self-correction and "teach the split" refusals are the scar tissue). Owner direction: +"match existing harnesses as much as possible" — the mapping below mirrors the +**pi.dev** tool surface (`dist/core/tools/`: `read{path}`, `ls{path?}`, +`grep{pattern, path?}`). +**Story:** `.agent/user_stories/agent-document-tools.md` (this phase reshapes the tools +that story delivered; phase 68's search contract rides along renamed) +**Context:** +- `app/rag/agent.py` — `AGENT_TOOLS` (OpenAI function definitions: + `list_documents` / `read_document(source, path)` / `search_documents(pattern, + source?, path?)`), `_execute_tool` (execution + refusals: `ALREADY_IN_CONTEXT`, + `UNKNOWN_TOOL`, `MISSING_READ_ARGS`, `MISSING_SEARCH_ARGS`; combined-form + self-correction in `_resolve_document`), `AgentHolder`, `run_agent` (round cap + `BOR_AGENT_MAX_ROUNDS`, default 10; `0` = no-tools kill switch). +- `app/rag/prompts.py` — `TOOLS_SECTION` (HIGH prompt only, ends the prompt; the E2E + mock keys off the `` marker's *presence*, not the wording). +- `app/api/chat.py` — the `ToolCallPiece` → SSE `{"type":"tool","name":…,"argument":…}` + derivation (read → `source/path`, search → pattern, list → null), `done.sources`, + per-turn log line (`tool_calls=N`). +- `tests/e2e/mock_llm.py` — the deterministic mock flows keyed on `TOOLS_TRIGGER` + ("use your tools"), `MULTI_READ_TRIGGER`, `SEARCH_TRIGGER` + the `` marker; + the flows emit `tool_calls` deltas with the current names/args and parse the + `source: X | path: Y | title: Z` catalog lines. +- `frontend/assets/app.js` — `.tool-call` lines (one row per `tool` frame, generic + render of `name` + `argument`), docstring name references; `frontend/assets/shared.js` + L159 special-cases `t.name === "read_document"` (shared-chat view). +- `tests/` — unit: `test_agent.py`, `test_prompts.py`, `test_chat_gate.py`, + `test_mock_tool_flow.py`, `test_sse_events.py`; integration: `test_agent_tools.py`, + `test_chat_api.py`, `test_api.py`, `test_chats_api.py`; E2E: + `test_agent_document_tools.py`, `test_agent_unlimited_tools.py`, + `test_search_tool.py`. `README.md` L173–176 documents the current tool names/args. + +## Objective +Rename and reshape the three server-side agent tools to the harness-trained surface — +**`ls`**, **`read(path)`** (single combined `source/path` argument), **`grep(pattern, +path?)`** — so the model's trained priors emit valid calls instead of fighting the +schema. Capabilities and all owner-locked semantics (full-document reads, fixed- +substring locator searches, round cap, kill switch, SSE contract shape) are unchanged; +only the tool surface, its descriptions, and the ripples (prompt copy, SSE names, +mock, tests, docs) change. + +## Dependencies +- `68_search_tool` (complete) — the search semantics this phase renames (A5 locked + match/output contract preserved). +- `45_agent_unlimited_tools` (complete) — the round-cap design the loop keeps. +- No todo dependencies; runs on the current head. Phase `71_scaffolding_guardrails` + (todo) follows this one (its mock changes build on the new tool names). + +## Tasks +1. `01_tool_schemas.md` — `app/rag/agent.py`: `AGENT_TOOLS` → `ls` / `read` / `grep` + schemas + descriptions; `_execute_tool` argument handling (single combined `path`), + updated refusals; unit tests. +2. `02_prompt_section.md` — `app/rag/prompts.py`: `TOOLS_SECTION` rewritten for + `ls`/`read`/`grep`; prompt unit pins. +3. `03_api_sse.md` — `app/api/chat.py`: SSE `tool`-frame `argument` derivation for the + new shapes + module/docstring contract copy; integration tests. +4. `04_frontend.md` — `app.js` + `shared.js` name references and the + `read_document` special case; frontend unit pins; persisted-chat note. +5. `05_mock_e2e_commit.md` — `mock_llm.py` flows on the new names/args, the three + existing E2E suites updated, the new dedicated E2E suite, README, full gates, + commit. + +## Testing & Quality +- Unit: `tests/unit/test_agent.py` — the new schemas (names, required/optional args, + descriptions), `read` path resolution (combined form split at the FIRST slash, bare + source name refusal, already-in-context, full content), `ls` scoping (no-arg full + catalog, `path`=source scope, unknown-source refusal, empty-source "0 documents"), + `grep` (required `pattern`, optional `path` single-doc scope, the locked A5 output + contract: fixed substring, case-insensitive, 20 matches, 200-char lines, locator + only), the round cap + kill switch (`agent_max_rounds=0` → `tools=None`, + byte-identical request) unchanged. +- Unit: `tests/unit/test_prompts.py` / `test_chat_gate.py` — `TOOLS_SECTION` copy + (marker present, HIGH-only), byte-identical LOW prompt (this phase does not touch + the deflection path). +- Integration: `tests/integration/test_agent_tools.py`, `test_chat_api.py` — SSE + `tool` frames with the new `name`/`argument` values end-to-end (mock LLM), + `done.sources` / `query_log.sources` / `tool_calls=N` log line unchanged in meaning. +- E2E (mandatory, house rule): NEW dedicated suite + `tests/e2e/test_harness_aligned_tools.py`, run in isolation — the `ls` → `read` + (combined path) → answer flow and the `grep` → `read` flow over the real UI; plus + the three existing suites (`test_agent_document_tools.py`, + `test_agent_unlimited_tools.py`, `test_search_tool.py`) updated to the new names and + green in isolation. +- Coverage: **>90%** on `app/` (validate.sh gate). + +## Completion Criteria +- [ ] `AGENT_TOOLS` defines exactly `ls` (optional `path`), `read` (required `path`), + `grep` (required `pattern`, optional `path`); the old names exist nowhere in + `app/` (`rg "list_documents|read_document|search_documents" app/` → no matches). +- [ ] `read` accepts the combined `source/path` form (the model's natural shape), + splits at the first slash, appends the **full** document (A7-revised: never + truncated); `grep` keeps the locked A5 match/output contract and is a locator + only; `ls` prints the phase-63 `source: X | path: Y | title: Z` lines. +- [ ] SSE `tool` frames carry the new `name` values; `argument` is the single string + the model passed (`read`'s `path`, `grep`'s `pattern`, `ls`'s `path`) or null. +- [ ] `BOR_AGENT_MAX_ROUNDS=0` still disables the tools entirely (request + byte-identical to the no-tools path); the deflected path is byte-identical + (LOW prompt and `tools=None` untouched by this phase). +- [ ] `uv run pytest` green; `uv run pytest --cov=app` TOTAL **>90%**; + `uv run ruff check . && uv run pyright` clean. +- [ ] `uv run pytest tests/e2e/test_harness_aligned_tools.py -v --no-cov` green in + isolation; regression suites green in isolation: `test_agent_document_tools.py`, + `test_agent_unlimited_tools.py`, `test_search_tool.py`, `test_chat_rag.py`. +- [ ] README tool documentation updated to the new surface. +- [ ] One `--no-gpg-sign` commit (message in the Commit block); phase dir moved to + `.agent/phases/complete/`. + +## Locked decisions +- **Owner (chat, 2026-09-03):** the tool surface is remapped to the pi.dev harness + shape — `ls` / `read(path)` / `grep(pattern, path?)` — "to match existing harnesses + as much as possible"; `lite` stays the chat model for everything (no model swap); + the chat model is NOT flipped to `turbo`. This supersedes the phase-37 tool + names/args and the phase-68 tool *name* (`search_documents` → `grep`); the phase-68 + match/output contract (fixed substring, case-insensitive, 20×200, locator-only) is + preserved verbatim. +- **`read` is `path`-only.** No `offset`/`limit` (pi has them, but the A7-revised + contract is "never truncated" — implementing paging would violate it; `path`-only + still carries the trained shape, which is the point). +- **The combined `source/path` string is the canonical document identity** in every + tool argument, refusal, and result header (it already is in search result lines and + `done.sources`). The old two-argument split and its self-correction/teaching + refusals are deleted — the model's combined form is now *correct*, not a mistake to + fix. +- **SSE contract shape unchanged** (A15 extension honoured): `{"type":"tool", + "name":…,"argument":…}` — only the `name` values and the `argument` derivation + change. `delta`/`thinking`/`retry`/`done`/`error` frames are untouched. +- **No env changes, no schema change.** `BOR_AGENT_MAX_ROUNDS` keeps its meaning + (round cap; `0` = no-tools kill switch). Saved chats persisting old tool names + render fine (the UI renders whatever `name`/`argument` arrive — no migration). + +## Commit +```bash +git add -A .agent/ app/ tests/ frontend/ README.md && git commit --no-gpg-sign -m "feat(agent): align the document tools with the harness-trained shape — ls, read(path), grep(pattern, path?)" +``` diff --git a/.agent/phases/todo/70_harness_aligned_tools/05_mock_e2e_commit.md b/.agent/phases/todo/70_harness_aligned_tools/05_mock_e2e_commit.md new file mode 100644 index 0000000..f945cad --- /dev/null +++ b/.agent/phases/todo/70_harness_aligned_tools/05_mock_e2e_commit.md @@ -0,0 +1,78 @@ +# Task 05 — Mock LLM, E2E Suites, README, Gates, Commit + +**Phase:** `70_harness_aligned_tools` · **Story:** `.agent/user_stories/agent-document-tools.md` + +## Objective +Bring the E2E world onto the new surface: the mock LLM's deterministic tool flows +emit `ls`/`read`/`grep` calls with the new argument shapes, the three existing tool +E2E suites assert the new names/frames, and a NEW dedicated suite pins the remap +end-to-end. Then the full quality gates and the phase commit. + +## Work +1. `tests/e2e/mock_llm.py` — update the deterministic flows (trigger strings + `TOOLS_TRIGGER` / `MULTI_READ_TRIGGER` / `SEARCH_TRIGGER` and the `` + marker keying stay unchanged): + - READ flow: request 1 streams `ls` (no arguments, synthetic id `call_0`, + `finish_reason: "tool_calls"`); request 2 (a `tool`-role catalog result in the + messages) streams `read` with the **combined** `source/path` parsed from the + first `source: X | path: Y` catalog line (the mock joins them — the catalog + format is unchanged, so this is the only parse change); request 3 (a + `tool`-role read result — content starting with the `Document source/path:` + header) streams the plain answer. + - MULTI-READ flow: same, `read` on the first then second combined path. + - SEARCH flow: request 1 streams `grep` with the pattern argument (the existing + SEARCH_TRIGGER pattern value); request 2 (a `tool`-role search result — + `source/path:line: text` lines) streams the plain answer. Update the + `tool`-role result detection strings where they key on the old headers. + - Update the module docstring's flow descriptions (names/args) + the phase-70 + note. +2. Existing E2E suites — update assertions to the new names/frames (behavior + otherwise unchanged): `tests/e2e/test_agent_document_tools.py` (tool frame + `name` values: `ls` then `read`; the `.tool-call` line copy "📄 Reading …"), + `tests/e2e/test_agent_unlimited_tools.py` (multi-read frames), + `tests/e2e/test_search_tool.py` (the `grep` frame + pattern argument). +3. NEW `tests/e2e/test_harness_aligned_tools.py` (the phase's dedicated suite, + house pattern, run in isolation): + - Grounded turn with the READ trigger: the UI shows the `ls` line then the + `📄 Reading ` line, the answer streams, and the done-state + sources include the read document; no raw tool markup anywhere in the DOM. + - Grounded turn with the SEARCH trigger: the `🔎 Searching for ` line + then the answer. + - The SSE wire itself (in-browser `fetch` capture or the existing SSE-capture + house pattern): `tool` frames carry `name` ∈ {`ls`, `read`, `grep`} and the + `argument` rule (read → combined path as passed; grep → pattern; ls → null + unscoped). +4. `README.md` — the agent-tools section (L173–176): document `ls` / `read(path)` / + `grep(pattern, path?)` with the combined-path identity and the A5 locator + semantics; the `.env.example` comment near `BOR_AGENT_MAX_ROUNDS` stays accurate + (no env changes this phase). +5. Gates + commit: + - `uv run pytest` green; `uv run pytest --cov=app --cov-report=term-missing` + TOTAL **>90%**; `uv run ruff check . && uv run pyright` clean. + - E2E in isolation (DB up): `test_harness_aligned_tools.py`, then the regression + suites `test_agent_document_tools.py`, `test_agent_unlimited_tools.py`, + `test_search_tool.py`, `test_chat_rag.py`. + - One atomic commit (message below); move + `.agent/phases/todo/70_harness_aligned_tools/` → + `.agent/phases/complete/70_harness_aligned_tools/`. + +## Testing & Quality +- E2E: the new dedicated suite + the four regression suites (isolation runs). +- Coverage: **>90%** on `app/` (phase-level gate). + +## Completion Criteria +- [ ] `rg "list_documents|read_document|search_documents" app/ frontend/ tests/ + README.md` → matches only in legacy-render pins/comments explicitly marked + legacy (the frontend legacy branches and their unit pins). +- [ ] `uv run pytest` green; `uv run pytest --cov=app` TOTAL **>90%**; + `uv run ruff check . && uv run pyright` clean. +- [ ] `uv run pytest tests/e2e/test_harness_aligned_tools.py -v --no-cov` green in + isolation; regression suites green in isolation. +- [ ] README documents the new surface; no stale tool-name copy in `README.md` / + `.env.example` / `frontend/`. +- [ ] One `--no-gpg-sign` commit; phase dir moved to `.agent/phases/complete/`. + +## Commit +```bash +git add -A .agent/ app/ tests/ frontend/ README.md && git commit --no-gpg-sign -m "feat(agent): align the document tools with the harness-trained shape — ls, read(path), grep(pattern, path?)" +``` diff --git a/.agent/phases/todo/71_scaffolding_guardrails/00_phase.md b/.agent/phases/todo/71_scaffolding_guardrails/00_phase.md new file mode 100644 index 0000000..0e8d7d5 --- /dev/null +++ b/.agent/phases/todo/71_scaffolding_guardrails/00_phase.md @@ -0,0 +1,152 @@ +# Phase 71 — Tool-Scaffolding Guardrails: Deterministic Strip + One Bounded Recovery + +**Source:** owner request (chat, 2026-09-03) — same incident as phase 70: the +deflected answer streamed the raw model text +`<|tool_call_start|>[read(path='/homelab/backup-notes.md')]<|tool_call_end|>` into +the UI (the model's own `<|…|>` chat-template tool syntax, emitted as plain +`delta.content` even though no tools were offered). Owner direction: "We also need +guardrails for situations like this… **deterministic guardrails only** right now, +forget using a model for that" — no model of any kind (no `lite` classifier, no +model-authored repair) in the guardrail path; `lite` stays the chat model for +everything (it's faster). +**Story:** n/a (owner request from chat — tool-scaffolding guardrails, 2026-09-03) +**Context:** +- `app/rag/llm.py` — `chat_stream` yields `StreamPiece("content", delta.content)` + verbatim (L~`content = delta.content; if content: yield`); thinking pieces + (`delta.reasoning_content`) pass through raw by design; tool-call materialization + happens at stream end (after the `async for`); phase-48 teardown closes the + endpoint stream on every exit (must stay intact). `chat_stream_retried` (phase 67) + wraps it with the retry-before-first-piece rule. +- `app/rag/agent.py` — `run_agent`'s round loop: a round with no `ToolCallPiece`s + ends the turn (`if not calls: return`); the round cap forces one final + `tools=None` request (the "forced final answer" pattern this phase reuses for + recovery); `AgentHolder` (read_docs / tool_calls) is unchanged. +- `app/api/chat.py` — the piece loop (thinking/tool/retry/delta handling + + `thinking_chars` counter), the deflected path (`chat_stream_retried(..., + tools=None)` directly), the `LLMError` → terminal error-frame handler, the + per-turn log line (ends `…total_ms=N retries=N`). +- `app/rag/prompts.py` — `build_deflect_prompt`'s `DEFLECT_MODE` body (the E2E mock + keys on the marker's *presence*, not the wording — an appended line is safe). +- `tests/e2e/mock_llm.py` — the trigger-string flow table (task 05 adds the + scaffolding triggers; phase-70 names apply). +- Tests to extend: `tests/unit/test_llm_client.py`, `test_agent.py`, + `tests/integration/test_chat_api.py` (log-line + error-frame pins), + `tests/unit/test_prompts.py`. + +## Objective +Raw tool-scaffolding tokens can never reach the user as answer text: a deterministic +streaming filter strips known scaffolding from `delta.content` as it flows, and a +round/turn whose visible content ends up empty (scaffolding was the whole "answer") +gets **one** bounded, deterministic recovery (same turn, `tools=None`, a fixed +harness-owned correction line in the system prompt); if the recovery also comes back +empty, the turn settles with a dedicated structured error frame. No model is used to +detect or repair anything. + +## Dependencies +- `70_harness_aligned_tools` (todo) — runs first: the mock/prompt/code state this + phase builds on (new tool names in the mock flows and prompt). +- `67_llm_retry` (complete) — `chat_stream_retried`, the primitive every request + (including recoveries) goes through. +- `48_stop_generation` (complete) — the phase-48 stream-teardown contract the + filter integration must not break. + +## Tasks +1. `01_filter.md` — `app/rag/scaffolding.py` (new): the pattern registry + + `ScaffoldingFilter` streaming state machine, with the boundary test matrix. +2. `02_llm_integration.md` — `app/rag/llm.py`: `chat_stream`/`chat_stream_retried` + accept the filter; content deltas are filtered, the tail flushed before tool + materialization; `None` = byte-identical raw path. +3. `03_recovery_policy.md` — `app/rag/agent.py` + `app/api/chat.py`: the empty-round + recovery (one per turn, `tools=None`, correction constant), + `MalformedReplyError`, the dedicated error copy, and the per-turn log line's + `scaffold_stripped=N` field. +4. `04_deflect_prompt.md` — the plain-text line in the `DEFLECT_MODE` body + (prevention; owner-permitted LOW-prompt change). +5. `05_e2e_commit.md` — the mock scaffolding triggers, the dedicated E2E suite, + full gates, commit. + +## Testing & Quality +- Unit (new `tests/unit/test_scaffolding_filter.py`): the filter matrix — span in + one chunk; span split across chunks at **every** boundary offset of the start + token; multiple spans in one chunk; standalone `<|tool_calls|>` / `<|tool_call|>` + tokens stripped; look-alikes **not** stripped (prose containing the words + "tool_call" or `tool_call_start` without the `<|…|>` delimiters, an unknown + `<|some_other_token|>`, a lone `<|tool_call_end|>` without a start); a partial + start token at stream end → `flush()` emits it as-is (no false-positive strip); + `stripped_chars` accounting; empty chunks. +- Unit: `tests/unit/test_llm_client.py` — `chat_stream` with a filter (content + filtered, thinking raw, `None` = raw pass-through pinned byte-identical, flush + order: flushed tail content precedes tool-call pieces, phase-48 teardown intact). +- Unit: `tests/unit/test_agent.py` — grounded recovery matrix (scaffolding-only + round → exactly one recovery request: `tools=None` + correction line in the + system prompt + fresh filter → clean answer ends the turn; scaffolding twice → + `MalformedReplyError`; scaffolding + real content → clean answer, **no** + recovery; round cap and kill switch unchanged). +- Integration: `tests/integration/test_chat_api.py` — deflected-path recovery matrix + (same shapes over the SSE endpoint: clean recovery → `done` frame; terminal → the + dedicated error frame, no `done`, no `query_log` row — same terminal semantics as + today's `LLMError`); log line carries `scaffold_stripped=N` (0 when nothing was + stripped — the field is uniform, the phase-67 `retries=N` pattern). +- E2E (mandatory, house rule): NEW dedicated suite + `tests/e2e/test_tool_scaffolding_guardrails.py`, run in isolation — recovery case + (raw tokens never in the DOM, clean answer shown) and terminal case (error state, + no raw tokens, the app stays usable). +- Coverage: **>90%** on `app/` (validate.sh gate). + +## Completion Criteria +- [ ] `rg "tool_call_start" frontend/` → no matches (no scaffolding rendering + path); the filter lives in `app/rag/scaffolding.py` as a pure module (no I/O, + no model calls). +- [ ] A content stream of pure scaffolding yields zero `delta` frames; a mixed + stream yields the clean remainder; thinking frames are never filtered. +- [ ] Exactly one recovery per turn (grounded and deflected paths); the recovery + request is `tools=None` with the fixed correction line in the system prompt; + a second empty reply settles with the error frame + "The model returned a malformed reply — please try again." +- [ ] The per-turn log line ends `…retries=N scaffold_stripped=N`; `scaffold_stripped=0` + on clean turns (uniform field). +- [ ] `uv run pytest` green; `uv run pytest --cov=app` TOTAL **>90%**; + `uv run ruff check . && uv run pyright` clean. +- [ ] `uv run pytest tests/e2e/test_tool_scaffolding_guardrails.py -v --no-cov` + green in isolation; regression suites green in isolation: + `test_harness_aligned_tools.py`, `test_chat_rag.py`, `test_agent_document_tools.py`. +- [ ] One `--no-gpg-sign` commit (message in the Commit block); phase dir moved to + `.agent/phases/complete/`. + +## Locked decisions +- **Owner (chat, 2026-09-03): deterministic only.** No model — `lite` or any other — + participates in detection or repair. The guardrail is a fixed pattern registry + a + fixed retry policy. (A model-based classifier/repair was proposed and explicitly + rejected for now — if it is ever wanted, it is a later phase with its own + permission.) +- **`lite` stays the chat model** (owner: "I want to use lite for everything since + it's way faster") — the guardrail is what protects the UX while `lite` is the + model; no `.env` change in this phase. +- **The pattern registry is the extension point.** Initial entries: the observed + span form `<|tool_call_start|>…<|tool_call_end|>` (non-greedy, any text between) + plus the standalone sibling tokens `<|tool_calls|>` and `<|tool_call|>` from the + same tokenizer family. Every strip logs a warning with the stripped span + (truncated to 200 chars) — that log line is how a new format gets captured and + added (pattern + unit fixture), keeping the registry honest (every entry traces + to an observed capture or the initial incident). +- **Content only, thinking never filtered.** The Thinking block is the model's raw + reasoning by design (phase 17) and stays raw (collapsible); the guardrail protects + the answer, not the scratchpad. +- **Recovery is a fixed policy, not a conversation.** One extra request per turn, + same messages with the harness-owned constant folded into the system prompt + (single system message — provider-safe), `tools=None`, a fresh filter, the same + phase-67 retry budget. At most one recovery; the second empty reply is terminal. + A round with real visible content plus scaffolding needs no recovery (the clean + content stands). +- **Terminal semantics follow the existing error pattern.** A terminal malformed + turn settles with a structured `error` frame (dedicated copy), writes no + `query_log` row, and the UI shows the existing error state — byte-for-byte the + same shape as today's `LLMError` terminal path. +- **`MalformedReplyError` subclasses `LLMError`** and is raised only by the recovery + policy (never from inside a stream, so `chat_stream_retried`'s retry rule never + sees it); `chat.py` catches it before the generic `LLMError` handler. + +## Commit +```bash +git add -A .agent/ app/ tests/ frontend/ && git commit --no-gpg-sign -m "feat(agent): strip raw tool-scaffolding from streamed answers — deterministic filter with one bounded recovery" +``` diff --git a/.agent/phases/todo/71_scaffolding_guardrails/01_filter.md b/.agent/phases/todo/71_scaffolding_guardrails/01_filter.md new file mode 100644 index 0000000..12164fb --- /dev/null +++ b/.agent/phases/todo/71_scaffolding_guardrails/01_filter.md @@ -0,0 +1,77 @@ +# Task 01 — `app/rag/scaffolding.py`: Pattern Registry + Streaming Filter + +**Phase:** `71_scaffolding_guardrails` · **Story:** n/a (owner request from chat, 2026-09-03) + +## Objective +A pure-Python streaming filter that removes known tool-scaffolding from answer text +as it flows — no I/O, no model, fully unit-testable — plus the pattern registry that +defines "known" (every entry traces to an observed capture). + +## Work +1. `app/rag/scaffolding.py` (new module): + - `SCAFFOLD_PATTERNS: tuple[re.Pattern, ...]` — module-level, compiled: + 1. `re.compile(r"<\|tool_call_start\|>[\s\S]*?<\|tool_call_end\|>")` — the + observed span (incident 2026-09-03), non-greedy so multiple spans each + strip to their own end token. + 2. `re.compile(r"<\|tool_calls\|>")` — standalone sibling token (same + tokenizer family). + 3. `re.compile(r"<\|tool_call\|>")` — standalone sibling token. + Module docstring: the registry is the extension point — a new entry needs an + observed capture (the strip warning log, task 03) + a unit fixture here; no + speculative entries. + - `class ScaffoldingFilter` — streaming state machine over a **content** stream: + - `feed(chunk: str) -> str` — appends to an internal pending buffer and + returns the clean text safe to emit now. Algorithm: repeatedly (a) take the + leftmost complete match among `SCAFFOLD_PATTERNS` in the pending buffer — + drop it (count into `stripped_chars`) and continue; then (b) check the + buffer tail for a **live prefix**: the longest suffix that is either a + proper prefix of any pattern's literal opening (`<|tool_call_start|>`, + `<|tool_calls|>`, `<|tool_call|>`) or an **open span** (a start token with + no end token yet in the buffer — everything from that start token onward is + held, since the span may continue in future chunks). Emit everything before + the held tail; keep the held tail as the new pending state. (Bounded: the + held tail in NORMAL state is ≤ the longest opening token minus one char; + in an open span it is unbounded but is being *stripped*, never emitted.) + - `flush() -> str` — end of stream: the pending tail is emitted **as-is** + (a partial marker at EOF is content, not scaffolding — a documented + choice, pinned). + - `stripped_chars: int` — total characters removed (property or attribute; + read by the caller after the round/turn, task 03). + - One filter instance per model request (callers create fresh instances — + the phase-67 retry-after-dead-attempt case is safe: a dead attempt never + fed the filter). + - No logging, no settings, no imports beyond `re` (pure module; the strip + *warning* log is emitted by the integration layer, task 03, so the module + stays I/O-free). +2. `tests/unit/test_scaffolding_filter.py` (new) — the matrix from the phase + overview: + - span in one chunk → stripped, `stripped_chars` exact. + - span split across chunks: feed the incident text + `<|tool_call_start|>[read(path='/homelab/backup-notes.md')]<|tool_call_end|>` + at **every** split offset of the start token (0..len) plus a mid-span split + and an end-token split → nothing ever emits until the span completes; total + output empty; `stripped_chars` = full span length. + - two spans in one chunk (with text between) → between-text emitted, spans + stripped. + - standalone `<|tool_calls|>` / `<|tool_call|>` stripped (alone and embedded in + a line). + - look-alikes **not** stripped, emitted verbatim: `tool_call_start` as prose; + `<|tool_call_start|` (missing delimiter close) as prose; an unknown + `<|some_other_token|>`; a lone `<|tool_call_end|>` without a start. + - partial start token at EOF (`feed("<|tool_call_st")` then `flush()`) → + flush emits it as-is, `stripped_chars == 0`. + - surrounding text preserved exactly: `hello <|…span…> world` → `hello world` + (no reflow beyond the removal); empty chunk no-op; `flush()` on a clean + stream emits nothing new. + +## Testing & Quality +- Unit: `tests/unit/test_scaffolding_filter.py` (above) — exhaustive boundary + offsets, parametrized. +- Coverage: **>90%** on `app/rag/scaffolding.py` (every branch: match, open-span + hold, live-prefix hold, flush-emits-as-is). + +## Completion Criteria +- [ ] `uv run pytest tests/unit/test_scaffolding_filter.py -v --no-cov` green. +- [ ] `rg "import (os|sys|logging|app)" app/rag/scaffolding.py` → no matches (pure + module). +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/todo/71_scaffolding_guardrails/02_llm_integration.md b/.agent/phases/todo/71_scaffolding_guardrails/02_llm_integration.md new file mode 100644 index 0000000..811e2b8 --- /dev/null +++ b/.agent/phases/todo/71_scaffolding_guardrails/02_llm_integration.md @@ -0,0 +1,61 @@ +# Task 02 — Filter Integration in `chat_stream` / `chat_stream_retried` + +**Phase:** `71_scaffolding_guardrails` · **Story:** n/a (owner request from chat, 2026-09-03) + +## Objective +Route `delta.content` through the caller-supplied `ScaffoldingFilter` in +`app/rag/llm.py` — content filtered, thinking raw, tail flushed before tool-call +materialization — with `None` keeping today's byte-identical raw path. + +## Work +1. `app/rag/llm.py` — `chat_stream(..., scaffolding: "ScaffoldingFilter | None = + None)"` (type imported under `TYPE_CHECKING` or as a string annotation to keep + the module's import graph clean — the filter type is only needed for typing): + - Content path: `content = delta.content` → when a filter is present, + `cleaned = scaffolding.feed(content)`; yield `StreamPiece("content", + cleaned)` only when `cleaned` is non-empty (an empty clean result yields + **nothing** — no empty `delta` frames). Without a filter the existing + `if content: yield` stands untouched (byte-identical). + - Thinking path: unchanged — `reasoning_content` pieces are never filtered + (locked: the scratchpad stays raw). + - End of stream: after the `async for` exhausts and **before** the tool-call + materialization block (`if calls and not emitted: …` and the + finish-reason emission), `tail = scaffolding.flush()` when a filter is + present; yield a content piece for the tail when non-empty. Order pinned: + flushed-tail content precedes `ToolCallPiece`s (content-before-tools wire + convention). + - Teardown (phase 48) untouched: the `finally: await stream.close()` behavior is + independent of the filter. + - Docstring: a short "Scaffolding guardrail (phase 71)" paragraph — the filter + is caller-owned (one per request), content-only, `None` = raw path. +2. `app/rag/llm.py` — `chat_stream_retried(..., scaffolding=None)` — pass-through + to `llm.chat_stream`. The same filter object is used across retry attempts of + one logical request: safe by construction (a restarted attempt only happens + when no piece was emitted, i.e. the filter was never fed — note this in the + docstring). +3. `tests/unit/test_llm_client.py` — scripted-chunk tests (the existing + fake-client pattern): + - with a filter: a chunk carrying a span mid-stream → the `delta` pieces carry + only the clean text; a span split across two chunks → no partial emit; + `filter.stripped_chars` correct after consumption. + - thinking pieces pass through raw even when they contain a span (pinned). + - `scaffolding=None` → the yielded pieces are byte-identical to the pre-phase + raw path (a span in content is yielded verbatim — the raw contract for + callers that opt out). + - flush ordering: a trailing partial-then-complete tail yields its content + piece **before** the materialized `ToolCallPiece` (a chunk stream that ends + with `…clean tail` + tool_calls deltas). + - teardown interaction: the phase-48 `aclose()`/abandon tests + (`tests/unit/test_llm_stream_teardown.py`) stay green with and without a + filter (run, don't rewrite). + +## Testing & Quality +- Unit: `tests/unit/test_llm_client.py` (+ the untouched teardown suite green). +- Coverage: **>90%** on this task's modified code (`app/rag/llm.py`). + +## Completion Criteria +- [ ] `uv run pytest tests/unit/test_llm_client.py tests/unit/test_llm_stream_teardown.py + -v --no-cov` green. +- [ ] `scaffolding=None` callers (including the kill-switch no-tools requests) are + byte-identical (pinned). +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/todo/71_scaffolding_guardrails/03_recovery_policy.md b/.agent/phases/todo/71_scaffolding_guardrails/03_recovery_policy.md new file mode 100644 index 0000000..732813a --- /dev/null +++ b/.agent/phases/todo/71_scaffolding_guardrails/03_recovery_policy.md @@ -0,0 +1,104 @@ +# Task 03 — Recovery Policy: One Bounded `tools=None` Retry + Terminal Error + +**Phase:** `71_scaffolding_guardrails` · **Story:** n/a (owner request from chat, 2026-09-03) + +## Objective +When a round/turn's visible content ends up empty **because scaffolding was the +whole answer**, run exactly one deterministic recovery (same turn, `tools=None`, +fixed correction line in the system prompt, fresh filter); a second empty reply +settles with a dedicated error frame. Both the grounded agent loop and the +deflected path get the policy; the per-turn log line gains `scaffold_stripped=N`. + +## Work +1. `app/rag/agent.py`: + - `CORRECTION_INSTRUCTION: str` — the harness-owned constant (verbatim, single + line): "Your previous reply contained raw tool-call markup, which is not + interpreted here. Answer the user's question directly in plain text — no + tool syntax." (The E2E mock in task 05 keys on a stable substring of it — + pick the exact constant now; the mock copies it.) + - `class MalformedReplyError(LLMError)` — raised only by the recovery policy + (never from inside a stream, so `chat_stream_retried`'s retry rule never + sees it). Module docstring: the phase-71 note (deterministic-only, owner + permission 2026-09-03). + - `run_agent` — per round: create a fresh `ScaffoldingFilter`, pass it to + `chat_stream_retried(..., scaffolding=round_filter)`, and count the + round's visible content (sum of the lengths of the yielded + `StreamPiece("content", …)` texts — the filtered ones). After the round, + when `not calls` (today's "the answer was streamed" exit): + - if round content > 0 → return (as today). + - if round content == 0 and `round_filter.stripped_chars > 0` → **one + recovery**: `messages_recovered = [*messages[:-1], {"role": "system", + "content": system_prompt + "\n" + CORRECTION_INSTRUCTION}, messages[-1]]` + (the correction folds into the ORIGINAL single system message — + provider-safe; the user message stays last) — one + `chat_stream_retried` request with `tools=None`, a fresh filter, the same + `retries`/`delay` budget; yield its pieces through the normal piece flow. + If the recovery's visible content > 0 → return. Otherwise → + `logger.warning` + `raise MalformedReplyError(…)`. + - if round content == 0 and nothing was stripped → return (today's + empty/thinking-only answer behavior — the UI handles it; unchanged). + - Log one warning per strip event here: `logger.warning("agent: stripped + N chars of tool-scaffolding in round %d: %r", …)` with the stripped span + truncated to 200 chars (the capture mechanism for new registry entries — + the filter exposes the stripped spans for this; add a + `stripped_spans: list[str]` to the filter if needed). + - A scaffolding-only round that also **carried tool calls** needs no + recovery (the clean content stands / the tool ran) — the policy keys on + the `not calls` exit only (pinned). +2. `app/api/chat.py` — the **deflected** path (the grounded path is covered by + `run_agent`): + - Create one `ScaffoldingFilter` for the turn's request, pass it to + `chat_stream_retried(..., scaffolding=filter)`; count visible content across + the piece loop (a `content_chars` counter next to `thinking_chars`). + - After the piece loop (deflected branch only): content == 0 and + `filter.stripped_chars > 0` → one recovery request: the same + `messages` with the system prompt extended by + `CORRECTION_INSTRUCTION` (import from `app.rag.agent`), `tools=None`, a + **fresh** filter, the same retry budget; stream its pieces through the + SAME piece-handling code (extract the piece loop into a small inner + helper/coroutine to avoid duplicating the thinking/tool/retry/delta + handling — the extraction must be behavior-preserving for the first pass, + pinned by the existing integration suite). If the recovery content > 0 → + continue to the normal `done` flow; else → the terminal path below. + - Catch `MalformedReplyError` **before** the generic `LLMError` handler: + `settled = True`, yield `ChatErrorEvent(detail="The model returned a + malformed reply — please try again.")`, return (no `query_log` row, no + `done` — the existing terminal-error semantics; the generic + "dropped the connection" copy stays for transport failures). + - Per-turn log line: append `scaffold_stripped=N` after `retries=N` — the sum + across the turn's requests (rounds + any recovery; 0 on clean turns — + uniform field, the phase-67 `retries=N` pattern). The recovery does not + bump `retries=N` (it is not a phase-67 endpoint-retry). + - Module docstring: the phase-71 paragraph (deterministic guardrail + + recovery + log field). +3. `tests/unit/test_agent.py` — the grounded matrix (scripted fake LLM, + monkeypatched DB): scaffolding-only round → exactly **two** model requests, + the second `tools=None` with `CORRECTION_INSTRUCTION` in its system prompt → + clean answer ends the turn, `done`-side state normal (holder untouched by the + recovery); scaffolding twice → `MalformedReplyError` (assert it subclasses + `LLMError`); scaffolding + real content → one request only, clean content + yielded, no recovery; clean turn → one request, no correction in any system + prompt; the round cap + kill-switch tests stay green. +4. `tests/integration/test_chat_api.py` — the deflected matrix over + `POST /api/chat` (mock LLM): scaffolding-only deflected turn → clean recovery + answer + `done` frame; scaffolding twice → the error frame with the dedicated + copy, no `done`, no `query_log` row (assert against the table); mixed + scaffolding+content → clean answer, `scaffold_stripped>0` in the log line; + the log-line pin gains `scaffold_stripped=0` on clean turns and the summed + value on stripped turns. + +## Testing & Quality +- Unit: `tests/unit/test_agent.py` (grounded matrix). +- Integration: `tests/integration/test_chat_api.py` (deflected matrix + log line). +- Coverage: **>90%** on this task's modified code (`app/rag/agent.py`, + `app/api/chat.py`). + +## Completion Criteria +- [ ] `uv run pytest tests/unit/test_agent.py tests/integration/test_chat_api.py -v + --no-cov` green. +- [ ] Exactly one recovery per turn, on both paths; the recovery request is + `tools=None` + correction line; a second empty reply → the dedicated error + frame, no `query_log` row. +- [ ] Clean turns: no correction in any system prompt, `scaffold_stripped=0`, + request counts unchanged (kill-switch/deflection byte-identical pins green). +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/todo/71_scaffolding_guardrails/04_deflect_prompt.md b/.agent/phases/todo/71_scaffolding_guardrails/04_deflect_prompt.md new file mode 100644 index 0000000..8838390 --- /dev/null +++ b/.agent/phases/todo/71_scaffolding_guardrails/04_deflect_prompt.md @@ -0,0 +1,35 @@ +# Task 04 — Deflect Prompt: Plain-Text-Only Line (Prevention) + +**Phase:** `71_scaffolding_guardrails` · **Story:** n/a (owner request from chat, 2026-09-03) + +## Objective +Close the door at the prompt: the deflected (LOW) turn offers no tools, so any tool +markup there is always wrong — add one plain-text-only instruction line to the +`DEFLECT_MODE` body. This is the owner-permitted change to the otherwise-locked LOW +prompt; the `DEFLECT_MODE` marker and everything else in the prompt stay put (the +E2E mock keys on the marker's presence, not the wording). + +## Work +1. `app/rag/prompts.py` — `build_deflect_prompt`: append one line to the + `DEFLECT_MODE` body (after "…propose 2-3 alternative questions."): + "Reply in plain text only — you have no tools in this mode." The weak-hit + title list follows exactly as today; `build_high_prompt`, `PERSONA`, + `TOOLS_SECTION`, and the section order are untouched. Update the module + docstring: the phase-71 note (owner-permitted 2026-09-03 LOW-prompt line; + the marker-keying contract unchanged). +2. `tests/unit/test_prompts.py` (and `test_chat_gate.py` where the deflection body + is pinned) — pins: the new line present in the LOW prompt; the `DEFLECT_MODE` + marker still present (mock keying); the HIGH prompt unchanged (byte-identical + pins stay green — the line must not leak into HIGH); the byte-identical-when- + empty steering/overview behavior unchanged. + +## Testing & Quality +- Unit: the prompt pins above. +- Coverage: **>90%** on this task's modified code (`app/rag/prompts.py`). + +## Completion Criteria +- [ ] `uv run pytest tests/unit/test_prompts.py tests/unit/test_chat_gate.py -v + --no-cov` green. +- [ ] LOW prompt = pre-phase text + exactly the one new line (a diff pin, or a + reconstruction pin in the test). +- [ ] `uv run ruff check . && uv run pyright` clean. diff --git a/.agent/phases/todo/71_scaffolding_guardrails/05_e2e_commit.md b/.agent/phases/todo/71_scaffolding_guardrails/05_e2e_commit.md new file mode 100644 index 0000000..91c8757 --- /dev/null +++ b/.agent/phases/todo/71_scaffolding_guardrails/05_e2e_commit.md @@ -0,0 +1,77 @@ +# Task 05 — Mock Triggers, Dedicated E2E Suite, Gates, Commit + +**Phase:** `71_scaffolding_guardrails` · **Story:** n/a (owner request from chat, 2026-09-03) + +## Objective +Prove the guardrail end-to-end through the real UI: a mock-LLM scaffolding flow +(recovery case + terminal case), a dedicated Playwright suite pinning that raw +tokens never reach the DOM, then the full quality gates and the phase commit. + +## Work +1. `tests/e2e/mock_llm.py` — two new deterministic flows (checked in the flow + table **before** the plain `TOOLS_TRIGGER` flow, after `SEARCH_TRIGGER` + ordering rules as they fit — the triggers are independent of the `` + marker, so both grounded and deflected turns hit them): + - `SCAFFOLD_TRIGGER = "emit raw tool markup"` — request 1 (no correction in + the system prompt): stream ONLY `delta.content` chunks carrying the incident + text `<|tool_call_start|>[read(path='search_docs/reese-notes.md')]<|tool_call_end|>` + (split across ≥2 chunks to exercise the boundary path), `finish_reason: + "stop"`, no structured `tool_calls`, no reasoning. Request 2 (system prompt + contains the stable substring of `CORRECTION_INSTRUCTION` — import it from + `app.rag.agent` so the mock can never drift from the constant): stream a + clean plain answer ("Here is the plain-text answer the recovery produced.") + + `finish_reason: "stop"`. + - `SCAFFOLD_ALWAYS_TRIGGER = "always emit raw tool markup"` — every request + (recovery included): the same scaffolding-only stream, forever. + - Update the module docstring's flow table + the phase-71 note. +2. `tests/e2e/test_tool_scaffolding_guardrails.py` (NEW — the phase's dedicated + suite, house pattern, run in isolation; DB up, mock LLM): + - **Recovery case** — ask a question containing `SCAFFOLD_TRIGGER`: the turn + settles (the composer re-enables, `done` observed); the final answer bubble + contains the recovery's clean text; `document.body.innerText` contains + **neither** `tool_call_start` nor `tool_call_end` (nor the raw + `[read(path=…]` fragment); no error banner. + - **Terminal case** — ask a question containing + `SCAFFOLD_ALWAYS_TRIGGER`: the existing error status renders with the + dedicated copy ("The model returned a malformed reply — please try + again."); no raw tokens in the DOM; no answer bubble with the scaffolding; + the app stays usable — a follow-up plain question (no trigger) gets a + normal streamed answer in the same session. + - **No false positive** — a plain question (existing `CHAT_TRIGGER`-style + flow, no tools needed): the answer streams byte-clean, no error state, no + recovery request visible (the turn settles on the first request). + - The SSE wire itself: in the recovery case, no `delta` frame ever carries a + `tool_call_start`/`tool_call_end` fragment (the existing SSE-capture house + pattern) — the strip happens server-side, not in the UI. +3. Gates + commit: + - `uv run pytest` green; `uv run pytest --cov=app --cov-report=term-missing` + TOTAL **>90%**; `uv run ruff check . && uv run pyright` clean. + - E2E in isolation (DB up): `test_tool_scaffolding_guardrails.py`, then the + regression suites `test_harness_aligned_tools.py`, `test_chat_rag.py`, + `test_agent_document_tools.py`. + - One atomic commit (message below); move + `.agent/phases/todo/71_scaffolding_guardrails/` → + `.agent/phases/complete/71_scaffolding_guardrails/`. + +## Testing & Quality +- E2E: the new dedicated suite (recovery / terminal / no-false-positive / wire + pins) + the three regression suites (isolation runs). +- Coverage: **>90%** on `app/` (phase-level gate). + +## Completion Criteria +- [ ] `rg "tool_call_start|tool_call_end" frontend/` → no matches. +- [ ] Raw scaffolding is never visible in the DOM in any case (recovery, + terminal, mixed) — pinned by the dedicated suite. +- [ ] Exactly one recovery per malformed turn (mock request counts implied by the + flows); clean turns never carry the correction line (pinned by the unit + suites from task 03). +- [ ] `uv run pytest` green; `uv run pytest --cov=app` TOTAL **>90%**; + `uv run ruff check . && uv run pyright` clean. +- [ ] `uv run pytest tests/e2e/test_tool_scaffolding_guardrails.py -v --no-cov` + green in isolation; regression suites green in isolation. +- [ ] One `--no-gpg-sign` commit; phase dir moved to `.agent/phases/complete/`. + +## Commit +```bash +git add -A .agent/ app/ tests/ frontend/ && git commit --no-gpg-sign -m "feat(agent): strip raw tool-scaffolding from streamed answers — deterministic filter with one bounded recovery" +``` diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.err b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.md b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.md new file mode 100644 index 0000000..dca1373 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.md @@ -0,0 +1,14 @@ +All task-01 gates green. Final report: + +**Task 01 complete — `ls` / `read(path)` / `grep(pattern, path?)` tool surface in `app/rag/agent.py`** + +- `AGENT_TOOLS` → exactly `ls` (optional `path`), `read` (required `path`), `grep` (required `pattern`, optional `path`) with the task's pinned descriptions; old names gone from `agent.py` (verified `rg` → no matches) +- `_execute_tool`: `read`/`grep` take the canonical combined `source/path` (first-slash split via new `_resolve_path`; bare source name → no-document refusal, no DB); `ls` scoping by registered source name (`No source named '…'` refusal; `0 documents:` counted); phase-68 A5 grep contract + refusals/round-cap/kill-switch unchanged; split-teaching branches deleted +- New module-level accessor `list_source_names` (registry via `effective_sources` + phase-69 `resolve_source_name`) — the catalog can't distinguish "known source, 0 docs" from "unknown source", the registry can +- Tests: `tests/unit/test_agent.py` rewritten (57 tests, new schema/ls-scope/read/grep pins); `tests/integration/test_agent_tools.py` updated to the new surface incl. real-DB scoped-`ls` registry tests + +**Gates:** `pytest tests/unit/test_agent.py --no-cov` → 57 passed · `pytest --cov=app -k agent` → 76 passed, `app/rag/agent.py` **100%** · `ruff check .` clean · `pyright` 0 errors · full suite 1379 passed, **2 failed** — both in `tests/integration/test_chat_api.py` (they script the *old* tool names; fixing them requires `app/api/chat.py`'s SSE `argument` re-derivation, which is task 03's explicit work item — left for it, per no-next-task rule) + +**Decisions:** blank/non-string `ls` `path` = omitted (full catalog, not refusal); already-in-context dedupe runs pre-DB (seed docs never hit the DB); no commit (task 05 owns it). + +**Next pending task:** `.agent/phases/todo/70_harness_aligned_tools/02_prompt_section.md` diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.validate b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.validate new file mode 100644 index 0000000..cb66709 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a1.validate @@ -0,0 +1,294 @@ +........................................................................ [ 5%] +........................................................................ [ 10%] +........................................................................ [ 15%] +........................................................................ [ 20%] +........................................................................ [ 26%] +........................................................................ [ 31%] +........................................................................ [ 36%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 52%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 67%] +........................................................................ [ 72%] +........................................................................ [ 78%] +.......................F...F............................................ [ 83%] +........................................................................ [ 88%] +........................................................................ [ 93%] +........................................................................ [ 99%] +............. [100%] +=================================== FAILURES =================================== +__________ test_grounded_turn_streams_tool_frames_and_cites_read_doc ___________ + +client = +db = +seeded_kb = +caplog = <_pytest.logging.LogCaptureFixture object at 0x7fc9b01fcad0> + + def test_grounded_turn_streams_tool_frames_and_cites_read_doc( + client, db, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture + ) -> None: + """(a) Grounded turn with tool calls: the event sequence is + ``thinking?/tool/tool/delta…/done``; ``done.sources`` and the + ``query_log`` row include the read document (deduped, order + preserved); the per-turn log line carries ``tool_calls=2``. + Phase 45: the agent loop keeps offering the tools for the whole + turn — the round cap (not per-tool budgets) is the bound.""" + scripted = FakeRagLLM( + tool_script=[ + [ + StreamPiece("thinking", "Let me list what is indexed…"), + ToolCallPiece(id="call_1", name="list_documents", arguments={}), + ], + [ + ToolCallPiece( + id="call_2", + name="read_document", + arguments={"source": "docs", "path": "homelab/backups.md"}, + ) + ], + # the answer request still carries the tools (2 rounds < the + # default cap of 10); the fake's tool_script is exhausted, so + # it falls back to the thinking + answer stream + ] + ) + fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted + try: + caplog.set_level(logging.INFO, logger="app.chat") + _, _, frames = _stream_chat(client, QUESTION) + finally: + fastapi_app.dependency_overrides.clear() + + types = [f["type"] for f in frames] + assert types[0] == "thinking" + assert types[1] == "tool" and types[2] == "tool" # the two executed calls + assert "error" not in types + assert types[3:-1] == ["delta"] * (len(types) - 4) # deltas, then done last + assert frames[-1]["type"] == "done" + + list_frame, read_frame = frames[1], frames[2] + assert set(list_frame) == {"type", "name", "argument"} + assert list_frame["name"] == "list_documents" + assert list_frame["argument"] is None # the tool takes no parameters + assert set(read_frame) == {"type", "name", "argument"} + assert read_frame["name"] == "read_document" + assert read_frame["argument"] == "docs/homelab/backups.md" + + deltas = [f for f in frames if f["type"] == "delta"] + assert len(deltas) >= 2 # genuinely streamed + assert "".join(d["text"] for d in deltas) == scripted.answer + + done = frames[-1] + assert done["deflected"] is False + # done.sources = the retrieval docs + the read doc, deduped, order kept. + sources = [(s["source"], s["path"]) for s in done["sources"]] +> assert sources[-1] == ("docs", "homelab/backups.md") # the read doc is cited + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +E AssertionError: assert ('docs', 'hom...es/deploy.j2') == ('docs', 'homelab/backups.md') +E +E At index 1 diff: 'homelab/templates/deploy.j2' != 'homelab/backups.md' +E Use -v to get more diff + +tests/integration/test_chat_api.py:597: AssertionError +---------------------------- Captured stdout setup ----------------------------- +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=deployments/new-service.md chunks=4 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/backups.md chunks=5 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/container_gitlab/gitlab-compose.yaml chunks=1 +2026-09-03 09:53:07 INFO app.importer :: import: summary source=docs path=homelab/container_gitlab/gitlab-compose.yaml chars=70 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/container_gitlab/gitlab.md chunks=4 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/kubernetes.md chunks=5 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/networking/static-dns.json chunks=1 +2026-09-03 09:53:07 INFO app.importer :: import: summary source=docs path=homelab/networking/static-dns.json chars=60 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/quadlet/cache.volume chunks=1 +2026-09-03 09:53:07 INFO app.importer :: import: summary source=docs path=homelab/quadlet/cache.volume chars=54 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/quadlet/compose.container chunks=2 +2026-09-03 09:53:07 INFO app.importer :: import: summary source=docs path=homelab/quadlet/compose.container chars=59 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/quadlet/lan.network chunks=1 +2026-09-03 09:53:07 INFO app.importer :: import: summary source=docs path=homelab/quadlet/lan.network chars=53 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/scripts/uptime_probe.py chunks=2 +2026-09-03 09:53:07 INFO app.importer :: import: summary source=docs path=homelab/scripts/uptime_probe.py chars=65 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/ssh/ssh_aliases.txt chunks=1 +2026-09-03 09:53:07 INFO app.importer :: import: summary source=docs path=homelab/ssh/ssh_aliases.txt chars=55 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/tables.md chunks=2 +2026-09-03 09:53:07 INFO app.importer :: import: added source=docs path=homelab/templates/deploy.j2 chunks=1 +2026-09-03 09:53:07 INFO app.importer :: import: summary source=docs path=homelab/templates/deploy.j2 chars=54 +2026-09-03 09:53:07 INFO app.importer :: import: summary files=13 added=13 updated=0 unchanged=0 pruned=0 errors=0 chunks=30 embed_batches=21 summaries=8 summary_errors=0 formats=md:5,container:1,j2:1,json:1,network:1,py:1,txt:1,volume:1,yaml:1 +------------------------------ Captured log setup ------------------------------ +INFO app.importer:importer.py:322 import: added source=docs path=deployments/new-service.md chunks=4 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/backups.md chunks=5 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/container_gitlab/gitlab-compose.yaml chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/container_gitlab/gitlab-compose.yaml chars=70 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/container_gitlab/gitlab.md chunks=4 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/kubernetes.md chunks=5 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/networking/static-dns.json chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/networking/static-dns.json chars=60 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/quadlet/cache.volume chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/quadlet/cache.volume chars=54 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/quadlet/compose.container chunks=2 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/quadlet/compose.container chars=59 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/quadlet/lan.network chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/quadlet/lan.network chars=53 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/scripts/uptime_probe.py chunks=2 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/scripts/uptime_probe.py chars=65 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/ssh/ssh_aliases.txt chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/ssh/ssh_aliases.txt chars=55 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/tables.md chunks=2 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/templates/deploy.j2 chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/templates/deploy.j2 chars=54 +INFO app.importer:importer.py:103 import: summary files=13 added=13 updated=0 unchanged=0 pruned=0 errors=0 chunks=30 embed_batches=21 summaries=8 summary_errors=0 formats=md:5,container:1,j2:1,json:1,network:1,py:1,txt:1,volume:1,yaml:1 +----------------------------- Captured stdout call ----------------------------- +2026-09-03 09:53:07 INFO app.agent :: agent tool=list_documents args={} round=1/10 +2026-09-03 09:53:07 INFO app.agent :: agent tool=read_document args={"source": "docs", "path": "homelab/backups.md"} round=2/10 +2026-09-03 09:53:07 INFO app.chat :: question='How is my Kubernetes cluster set up?' embed_ms=0 top_score=0.436 fts_hits=4 summary_hits=1 tuning=0 kb_chars=0 threshold=0.30 deflected=False sources=['docs/homelab/kubernetes.md', 'docs/homelab/templates/deploy.j2'] thinking_chars=28 tool_calls=0 total_ms=11 retries=0 +------------------------------ Captured log call ------------------------------- +INFO app.agent:agent.py:527 agent tool=list_documents args={} round=1/10 +INFO app.agent:agent.py:527 agent tool=read_document args={"source": "docs", "path": "homelab/backups.md"} round=2/10 +INFO app.chat:chat.py:506 question='How is my Kubernetes cluster set up?' embed_ms=0 top_score=0.436 fts_hits=4 summary_hits=1 tuning=0 kb_chars=0 threshold=0.30 deflected=False sources=['docs/homelab/kubernetes.md', 'docs/homelab/templates/deploy.j2'] thinking_chars=28 tool_calls=0 total_ms=11 retries=0 +______________ test_tool_execution_db_failure_yields_error_event _______________ + +client = +db = +seeded_kb = +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7fc9c35c3d20> + + def test_tool_execution_db_failure_yields_error_event( + client, db, seeded_kb: FakeRagLLM, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A tool call that hits a dead DB mid-stream gets the same structured + ``error`` event as the pre-stream retrieval path — never a severed + stream (the "never stale" contract, PLAN §7.4).""" + scripted = FakeRagLLM( + tool_script=[[ToolCallPiece(id="call_1", name="list_documents", arguments={})]] + ) + + def boom(*_a: Any, **_k: Any) -> Any: + raise RuntimeError("db exploded mid tool call") + + monkeypatch.setattr(agent, "list_catalog", boom) + fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted + try: + _, _, frames = _stream_chat(client, QUESTION) + finally: + fastapi_app.dependency_overrides.clear() + + # The ``tool`` frame went out first (the model requested the call); + # the failed execution ends the turn with the structured error event. +> assert [f["type"] for f in frames] == ["tool", "error"] +E AssertionError: assert ['tool', 'del... 'delta', ...] == ['tool', 'error'] +E +E At index 1 diff: 'delta' != 'error' +E Left contains 5 more items, first extra item: 'delta' +E Use -v to get more diff + +tests/integration/test_chat_api.py:817: AssertionError +---------------------------- Captured stdout setup ----------------------------- +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=deployments/new-service.md chunks=4 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/backups.md chunks=5 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/container_gitlab/gitlab-compose.yaml chunks=1 +2026-09-03 09:53:08 INFO app.importer :: import: summary source=docs path=homelab/container_gitlab/gitlab-compose.yaml chars=70 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/container_gitlab/gitlab.md chunks=4 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/kubernetes.md chunks=5 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/networking/static-dns.json chunks=1 +2026-09-03 09:53:08 INFO app.importer :: import: summary source=docs path=homelab/networking/static-dns.json chars=60 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/quadlet/cache.volume chunks=1 +2026-09-03 09:53:08 INFO app.importer :: import: summary source=docs path=homelab/quadlet/cache.volume chars=54 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/quadlet/compose.container chunks=2 +2026-09-03 09:53:08 INFO app.importer :: import: summary source=docs path=homelab/quadlet/compose.container chars=59 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/quadlet/lan.network chunks=1 +2026-09-03 09:53:08 INFO app.importer :: import: summary source=docs path=homelab/quadlet/lan.network chars=53 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/scripts/uptime_probe.py chunks=2 +2026-09-03 09:53:08 INFO app.importer :: import: summary source=docs path=homelab/scripts/uptime_probe.py chars=65 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/ssh/ssh_aliases.txt chunks=1 +2026-09-03 09:53:08 INFO app.importer :: import: summary source=docs path=homelab/ssh/ssh_aliases.txt chars=55 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/tables.md chunks=2 +2026-09-03 09:53:08 INFO app.importer :: import: added source=docs path=homelab/templates/deploy.j2 chunks=1 +2026-09-03 09:53:08 INFO app.importer :: import: summary source=docs path=homelab/templates/deploy.j2 chars=54 +2026-09-03 09:53:08 INFO app.importer :: import: summary files=13 added=13 updated=0 unchanged=0 pruned=0 errors=0 chunks=30 embed_batches=21 summaries=8 summary_errors=0 formats=md:5,container:1,j2:1,json:1,network:1,py:1,txt:1,volume:1,yaml:1 +------------------------------ Captured log setup ------------------------------ +INFO app.importer:importer.py:322 import: added source=docs path=deployments/new-service.md chunks=4 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/backups.md chunks=5 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/container_gitlab/gitlab-compose.yaml chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/container_gitlab/gitlab-compose.yaml chars=70 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/container_gitlab/gitlab.md chunks=4 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/kubernetes.md chunks=5 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/networking/static-dns.json chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/networking/static-dns.json chars=60 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/quadlet/cache.volume chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/quadlet/cache.volume chars=54 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/quadlet/compose.container chunks=2 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/quadlet/compose.container chars=59 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/quadlet/lan.network chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/quadlet/lan.network chars=53 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/scripts/uptime_probe.py chunks=2 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/scripts/uptime_probe.py chars=65 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/ssh/ssh_aliases.txt chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/ssh/ssh_aliases.txt chars=55 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/tables.md chunks=2 +INFO app.importer:importer.py:322 import: added source=docs path=homelab/templates/deploy.j2 chunks=1 +INFO app.importer:importer.py:377 import: summary source=docs path=homelab/templates/deploy.j2 chars=54 +INFO app.importer:importer.py:103 import: summary files=13 added=13 updated=0 unchanged=0 pruned=0 errors=0 chunks=30 embed_batches=21 summaries=8 summary_errors=0 formats=md:5,container:1,j2:1,json:1,network:1,py:1,txt:1,volume:1,yaml:1 +----------------------------- Captured stdout call ----------------------------- +2026-09-03 09:53:08 INFO app.agent :: agent tool=list_documents args={} round=1/10 +2026-09-03 09:53:08 INFO app.chat :: question='How is my Kubernetes cluster set up?' embed_ms=0 top_score=0.436 fts_hits=4 summary_hits=1 tuning=0 kb_chars=0 threshold=0.30 deflected=False sources=['docs/homelab/kubernetes.md', 'docs/homelab/templates/deploy.j2'] thinking_chars=0 tool_calls=0 total_ms=10 retries=0 +------------------------------ Captured log call ------------------------------- +INFO app.agent:agent.py:527 agent tool=list_documents args={} round=1/10 +INFO app.chat:chat.py:506 question='How is my Kubernetes cluster set up?' embed_ms=0 top_score=0.436 fts_hits=4 summary_hits=1 tuning=0 kb_chars=0 threshold=0.30 deflected=False sources=['docs/homelab/kubernetes.md', 'docs/homelab/templates/deploy.j2'] thinking_chars=0 tool_calls=0 total_ms=10 retries=0 +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 153 4 97% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 212 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 126 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 139 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 213 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 205 0 100% +----------------------------------------------- +TOTAL 2726 16 99% +=========================== short test summary info ============================ +FAILED tests/integration/test_chat_api.py::test_grounded_turn_streams_tool_frames_and_cites_read_doc +FAILED tests/integration/test_chat_api.py::test_tool_execution_db_failure_yields_error_event +tests FAILED: unit + integration suite did not pass +All checks passed! +0 errors, 0 warnings, 0 informations +validation FAILED (see output above) diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.err b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.md b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.md new file mode 100644 index 0000000..4c34b35 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.md @@ -0,0 +1,11 @@ +All gates green — full suite included. Final report: + +**Fixed:** the harness gate runs the full unit+integration suite, so I resolved the two `test_chat_api.py` failures that the phase plans queue for task 03 — the minimal coupled fix, no further task-03 work: +- `app/api/chat.py`: SSE `tool`-frame `argument` re-derived for the new surface — `read`'s `path` (combined `source/path`), `grep`'s `pattern`, `ls`'s `path`, else/null for non-strings or unknown tools; module docstring + inline comments updated (old names now gone from `chat.py` too) +- `tests/integration/test_chat_api.py`: 5 tests moved to the new surface — `ls`+`read(path=…)` frame test (`argument` == the single string the model passed), search-frame test renamed to `grep` (raw pattern / non-string → null), DB-failure test scripted as `ls` (asserts `name == "ls"`, `["tool","error"]`), deflected + kill-switch scripts renamed + +**Results:** `uv run pytest` → **1381 passed, 0 failed** · `--cov=app` TOTAL **99%** (`app/rag/agent.py` 100%, `app/api/chat.py` 100%) · `uv run pytest tests/unit/test_agent.py --no-cov` → 57 passed · `uv run pytest --cov=app -k agent` → 76 passed · `uv run ruff check .` clean · `uv run pyright` 0 errors · `rg` old names in `app/rag/agent.py` → no matches + +**Decisions:** remaining old-name test references (chats_api, sse_events, mock_tool_flow, frontend_tool_states, llm_*) all pass — they pin opaque data or files owned by tasks 02/04/05, left untouched; deflected path + kill switch byte-identical (pinned). + +**Next pending task:** `.agent/phases/todo/70_harness_aligned_tools/02_prompt_section.md` diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.validate b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.validate new file mode 100644 index 0000000..e411b71 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__01_tool_schemas.a2.validate @@ -0,0 +1,75 @@ +........................................................................ [ 5%] +........................................................................ [ 10%] +........................................................................ [ 15%] +........................................................................ [ 20%] +........................................................................ [ 26%] +........................................................................ [ 31%] +........................................................................ [ 36%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 52%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 67%] +........................................................................ [ 72%] +........................................................................ [ 78%] +........................................................................ [ 83%] +........................................................................ [ 88%] +........................................................................ [ 93%] +........................................................................ [ 99%] +............. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 149 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 212 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 126 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 139 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 213 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 205 0 100% +----------------------------------------------- +TOTAL 2722 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.err b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.md b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.md new file mode 100644 index 0000000..d6c2a85 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.md @@ -0,0 +1,19 @@ +Task 02 complete — all gates green. + +**Implemented** +- `app/rag/prompts.py`: rewrote `TOOLS_SECTION` copy for the harness-aligned `ls` / `read(path)` / `grep(pattern, path?)` shapes (catalog-line format, case-insensitive locator contract, combined `source/path` + full content, answer-when-ready close — no budget re-statement, phase 45); updated the module docstring and section comment (HIGH-only, LOW byte-identical, mock keys on `` marker presence); `build_high_prompt`/`build_deflect_prompt` and section order untouched +- `tests/unit/test_prompts.py`: new phase-70 pin block (markers, shape copy pins, old names + budget copy absent, HIGH still ends with ``), explicit no-``/no-new-copy LOW assertions +- `tests/unit/test_chat_gate.py`: endpoint pins updated (new names in HIGH system prompt, old names out; new copy absent from the deflected path) +- `tests/unit/test_agent.py`: re-pinned the old `TOOLS_SECTION` copy pins (the in-file NOTE delegated this re-pin to task 02) + +**Results** +- `uv run pytest tests/unit/test_prompts.py tests/unit/test_chat_gate.py -v --no-cov` → 58 passed +- `rg "list_documents|read_document|search_documents" app/rag/prompts.py` → no matches +- `uv run pytest --cov=app --cov-report=term-missing` → 1385 passed; `app/rag/prompts.py` 100% (>90% gate), TOTAL 99% +- `uv run ruff check . && uv run pyright` → All checks passed / 0 errors + +**Decisions** +- E2E/mock still on old names — expected mid-phase; task 05 owns the mock + E2E updates (default `testpaths` excludes E2E; mock keys on the `` marker only, so keying is intact) +- No PLAN.md exists in `.agent/` (referenced by AGENTS.md); the phase overview was treated as binding context + +**Next pending task:** `.agent/phases/todo/70_harness_aligned_tools/03_api_sse.md` diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.validate b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.validate new file mode 100644 index 0000000..87bb208 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__02_prompt_section.a1.validate @@ -0,0 +1,75 @@ +........................................................................ [ 5%] +........................................................................ [ 10%] +........................................................................ [ 15%] +........................................................................ [ 20%] +........................................................................ [ 25%] +........................................................................ [ 31%] +........................................................................ [ 36%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 51%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 67%] +........................................................................ [ 72%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 88%] +........................................................................ [ 93%] +........................................................................ [ 98%] +................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 149 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 212 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 126 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 139 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 213 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 205 0 100% +----------------------------------------------- +TOTAL 2722 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.err b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.md b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.md new file mode 100644 index 0000000..6b4f169 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.md @@ -0,0 +1,16 @@ +All gates green. Task 03 is complete. + +**Task 03 — SSE `tool` frames + API contract copy: done** + +- `app/api/chat.py`: `ToolCallPiece` → SSE derivation now symmetric — `argument` = the single string the model passed (`read`/`ls` → `path`, `grep` → `pattern`), non-string/omitted → null; module docstring carries the phase-70 note, the `{"type":"tool",…}` shape, and the `done.sources`/`query_log`/`tool_calls=N` contract (verified in place; branch was already rewritten by the earlier interrupted run of this task). +- `app/schemas.py`: `ChatToolEvent` docstring re-pointed at `ls`/`read`/`grep` + the argument rule (no field change); `ToolCall` docstring updated incl. the locked no-migration note for legacy saved names. +- `tests/integration/test_chat_api.py`: new pin `test_tool_frames_carry_the_model_arguments_regardless_of_execution` — scoped `ls` frame carries `"docs"`; a rejected `read` (unknown path) still emits its frame with the model's argument as passed; `done.sources` stays retrieval-only; log line `tool_calls=1` (rejected calls count nothing). Existing pins: `read` frame = combined path as passed, `grep` frame = pattern, unscoped `ls` = null. +- `tests/integration/test_chats_api.py`: saved-chat fixture on new names/args; keeps one `read_document` entry pinning that pre-phase-70 chats still validate (locked, no migration). + +**Results** +- `uv run pytest tests/integration/test_agent_tools.py tests/integration/test_chat_api.py -v --no-cov` → 40 passed +- `uv run pytest tests/integration/test_chats_api.py --no-cov` → 54 passed; full `uv run pytest` → green (exit 0) +- `uv run pytest --cov=app --cov-report=term-missing` → `app/api/chat.py` 100%, `app/schemas.py` 100%, TOTAL 99% (>90%) +- `uv run ruff check . && uv run pyright` → clean (0 errors) + +**Notes**: `tests/integration/test_api.py` old-name mentions are app.js asset pins — left for task 04 (frontend). Legacy docstring names in `app/rag/llm.py` and the `docs.py` route function name fall to the phase-level `rg` gate in task 05. Next pending task: `04_frontend.md`. diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.validate b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.validate new file mode 100644 index 0000000..0a1c375 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__03_api_sse.a1.validate @@ -0,0 +1,75 @@ +........................................................................ [ 5%] +........................................................................ [ 10%] +........................................................................ [ 15%] +........................................................................ [ 20%] +........................................................................ [ 25%] +........................................................................ [ 31%] +........................................................................ [ 36%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 51%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 67%] +........................................................................ [ 72%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 88%] +........................................................................ [ 93%] +........................................................................ [ 98%] +.................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 149 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 212 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 126 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 139 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 213 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 205 0 100% +----------------------------------------------- +TOTAL 2722 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.err b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.md b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.md new file mode 100644 index 0000000..cc4f576 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.md @@ -0,0 +1,11 @@ +All green. The other modified files are uncommitted work from completed tasks 01–03 (the commit belongs to task 05 per the phase plan). + +**Task 04 report** + +- `frontend/assets/app.js`: `appendToolLine` now keys on `read`/`grep`/`ls` (new copy: "📄 Reading", "🔎 Searching for", "🔎 Listing documents in ", unscoped "🔎 Listing documents") with legacy `read_document`/`search_documents`/`list_documents` branches preserved byte-identical for persisted turns; status-label ternary in the turn handler extended to the new names (incl. scoped `ls`); section + module docstrings updated with the phase-70 note. +- `frontend/assets/shared.js`: `addToolLines` local copy got the same new-name branches (read/grep/ls + scoped ls) with the legacy branches kept; all arguments `textContent`-only; docstring updated. +- `tests/unit/test_frontend_tool_states.py`: pins updated/added — new+legacy branch gates in app.js, new shared.js pin test (`test_shared_page_tool_lines_cover_new_and_legacy_names`), no-`innerHTML` textContent discipline; old copy pins kept. +- Ripple fix: `tests/integration/test_api.py` emoji guard now strips the fourth tool-line literal `"🔎 Listing documents in "`. +- Tests: `uv run pytest tests/unit/test_frontend_tool_states.py -v --no-cov` → 10 passed; `uv run pytest --cov=app --cov-report=term-missing` → 1387 passed, TOTAL 99% (>90%); `uv run ruff check . && uv run pyright` → clean; both JS files pass `node --check`. +- Decisions: explicit if-chains (not a name map) for the two-generation gates; scoped-`ls` status label added so the live region mirrors the new line copy (PLAN §7.4). E2E suites remain keyed on the old mock names — task 05's scope. +- Next pending task: `.agent/phases/todo/70_harness_aligned_tools/05_mock_e2e_commit.md`. diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.validate b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.validate new file mode 100644 index 0000000..d4dc250 --- /dev/null +++ b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__04_frontend.a1.validate @@ -0,0 +1,75 @@ +........................................................................ [ 5%] +........................................................................ [ 10%] +........................................................................ [ 15%] +........................................................................ [ 20%] +........................................................................ [ 25%] +........................................................................ [ 31%] +........................................................................ [ 36%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 51%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 67%] +........................................................................ [ 72%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 88%] +........................................................................ [ 93%] +........................................................................ [ 98%] +................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +----------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 22 0 100% +app/api/chat.py 149 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 93 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 212 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 8 0 100% +app/api/sync.py 101 0 100% +app/config.py 126 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 20 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/logging.py 13 0 100% +app/db.py 21 0 100% +app/main.py 52 0 100% +app/models.py 86 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 139 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 213 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 65 0 100% +app/rag/retriever.py 94 3 97% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 205 0 100% +----------------------------------------------- +TOTAL 2722 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__05_mock_e2e_commit.a1.err b/.agent/reports/70_harness_aligned_tools/70_harness_aligned_tools__05_mock_e2e_commit.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.env.example b/.env.example index 8bcd900..780ba6d 100644 --- a/.env.example +++ b/.env.example @@ -42,7 +42,7 @@ BOR_HYBRID_VECTOR_CANDIDATES=100 # cosine list width for the fusion BOR_HYBRID_LEXICAL_CANDIDATES=30 # FTS list width for the fusion BOR_RRF_K=60 # Reciprocal Rank Fusion damping constant -# --- Agent document tools (phase 37: grounded turns may list + read) --- +# --- Agent document tools (grounded turns may extend context: ls / read / grep) --- # BOR_AGENT_MAX_ROUNDS=10 # hard cap on agent tool rounds per turn (0 = no tools) # --- Import scope (A9 default; ANY well-formed extension is allowed) --- diff --git a/README.md b/README.md index 8739468..82ccfc9 100644 --- a/README.md +++ b/README.md @@ -163,28 +163,40 @@ exactly as before. To hide it, set `BOR_STREAM_THINKING=0` — the `thinking` events stop (the per-turn log line still counts `thinking_chars`). -## Agent document tools (list + read) +## Agent document tools (ls + read + grep) Retrieval only puts the top documents in context. When an answer depends on a file a note *references* ("the exact JSON shape is in -example-record-file.json"), the model can extend its own context with two -server-side tools — on **grounded** (high-relevance) turns only: +example-record-file.json"), the model can extend its own context with +three server-side tools — on **grounded** (high-relevance) turns only. +The surface mirrors the shape the chat model was trained on (the pi.dev +harness tools, owner decision 2026-09-03), and the canonical document +identity everywhere is the combined `source/path` string: -* **`list_documents`** — lists every indexed document, one - `source: X | path: Y | title: Z` line each (the same order as the - Sources page); -* **`read_document(source, path)`** — appends the **full** text of - one more indexed document to the context (never truncated). +* **`ls`** — lists every indexed document, one `source: X | path: Y | + title: Z` line each (the same order as the Sources page); pass a + source name as `path` to list one source's documents; +* **`read(path)`** — appends the **full** text of one more indexed + document to the context (never truncated); `path` is the combined + `source/path` string exactly as shown in the `ls` output; +* **`grep(pattern, path?)`** — searches the indexed documents for an + exact string (case-insensitive fixed substring) and returns up to 20 + matching `source/path:line: text` lines; an optional `path` limits the + search to one document. A locator, not a context-adder: it never adds + to the answer context — the model `read`s the winner. Each call the model requests is executed against Postgres only (no extra LLM round trip) and streamed as an SSE `tool` frame ahead of the answer — -`{"type": "tool", "name": …, "argument": "source/path" | null}`. In the -chat, each call shows a **"calling tool" state** in addition to -"thinking": the send button keeps its busy state ("Calling tool…") and a -visible tool line (`🔎 Listing documents` / `📄 Reading source/path`) lands -above the answer, one per call, in order. The tool lines persist with the -message, so a reloaded conversation re-renders them. The read document is -reflected in the answer's **source chips** and in the `query_log` row. +`{"type": "tool", "name": "ls" | "read" | "grep", "argument": … | null}` +(`argument` is the single string the model passed — `read`'s `path`, +`grep`'s `pattern`, `ls`'s scope — or null). In the chat, each call shows +a transient **calling-tool status** in addition to "thinking" (the send +button keeps its busy state — "Stop" — for the whole turn) and a visible +tool line (`🔎 Listing documents` / `📄 Reading source/path` / +`🔎 Searching for pattern`) lands above the answer, one per call, in +order. The tool lines persist with the message, so a reloaded +conversation re-renders them. The read document is reflected in the +answer's **source chips** and in the `query_log` row. The tools stay offered for the whole turn — the model may call them as many times as it needs (re-lists included), bounded only by a round cap diff --git a/app/api/chat.py b/app/api/chat.py index 69147a8..6f8b444 100644 --- a/app/api/chat.py +++ b/app/api/chat.py @@ -49,27 +49,28 @@ outline (0 when absent) and the per-turn log line records Agent document tools (phase 37, PLAN §4 extension, owner permission 2026-08-26; phase 45 removed the per-tool budgets — owner permission -2026-08-27; phase 68 added the ``search_documents`` grep): a +2026-08-27; phase 70 aligned the surface to the harness-trained +``ls`` / ``read`` / ``grep`` — owner permission 2026-09-03): a **grounded** turn (``not plan.deflected``) no longer streams a bare ``chat_stream`` — it runs the agent loop (``app.rag.agent.run_agent``), which offers the model the three server-side tools -``list_documents`` / ``read_document`` / ``search_documents`` for the -whole turn (as many calls as the model wants, re-lists and re-searches -included) until it answers or the round cap (``BOR_AGENT_MAX_ROUNDS``, -default 10) forces one final no-tools answer. Each model-requested call -streams as an SSE ``tool`` event — ``{"type": "tool", "name": …, -"argument": "source/path" | pattern | null}`` — ahead of the answer's -``delta`` frames: ``argument`` is the read document's path for -``read_document``, the raw search pattern for ``search_documents`` -(a non-string pattern — a model error the backend refuses — yields -null), and null for ``list_documents``. ``done.sources``, -``query_log.sources`` and the per-turn log line all report the same -combined source list (retrieval docs + the agent's read docs, deduped -by ``(source, path)``, order preserved — a search adds no source; it is -a locator, locked A5), and the log line records ``tool_calls=N`` after -``thinking_chars=N`` (PLAN §9 line extension — ``N`` counts executed -tool calls; rejected calls do not count). **Deflected turns keep the -direct ``chat_stream`` — byte-identical to the pre-phase path (A8):** +``ls`` / ``read`` / ``grep`` for the whole turn (as many calls as the +model wants, re-lists and re-greps included) until it answers or the +round cap (``BOR_AGENT_MAX_ROUNDS``, default 10) forces one final +no-tools answer. Each model-requested call streams as an SSE ``tool`` +event — ``{"type": "tool", "name": …, "argument": … | null}`` — ahead +of the answer's ``delta`` frames: ``argument`` is the single string the +model passed — ``read``'s ``path`` (the combined ``source/path``), +``grep``'s ``pattern``, ``ls``'s ``path`` — or null (a non-string +value — a model error the backend refuses — and an omitted argument +both yield null). ``done.sources``, ``query_log.sources`` and the +per-turn log line all report the same combined source list (retrieval +docs + the agent's read docs, deduped by ``(source, path)``, order +preserved — a grep adds no source; it is a locator, locked A5), and the +log line records ``tool_calls=N`` after ``thinking_chars=N`` (PLAN §9 +line extension — ``N`` counts executed tool calls; rejected calls do not +count). **Deflected turns keep the direct ``chat_stream`` — +byte-identical to the pre-phase path (A8):** the LOW prompt never carries tools, and with ``agent_max_rounds`` at **0** ``run_agent`` makes exactly one ``tools=None`` request, reproducing the pre-phase behavior (the kill switch). @@ -400,21 +401,18 @@ async def chat( try: async for piece in answer_stream: # StreamPiece | ToolCallPiece | RetryPiece if isinstance(piece, ToolCallPiece): - # Phase 37 (PLAN §4 extension): one SSE ``tool`` - # frame per model-requested call. ``argument`` is - # the read_document "source/path"; phase 68 - # extends it with the search_documents pattern - # (a non-string pattern — a model error the - # backend refuses — is null); null otherwise. - if piece.name == "read_document": - argument = ( - f"{piece.arguments.get('source')}/{piece.arguments.get('path')}" - ) - elif piece.name == "search_documents": - pattern = piece.arguments.get("pattern") - argument = pattern if isinstance(pattern, str) else None - else: - argument = None + # Phase 37 (PLAN §4 extension; phase 70): one SSE + # ``tool`` frame per model-requested call. + # ``argument`` is the single string the model + # passed — ``read``'s ``path`` (the combined + # ``source/path``), ``grep``'s ``pattern``, + # ``ls``'s ``path`` — or null (a non-string value + # is a model error the backend refuses, as is an + # omitted argument). + argument = piece.arguments.get( + "pattern" if piece.name == "grep" else "path" + ) + argument = argument if isinstance(argument, str) else None yield sse_event( ChatToolEvent(name=piece.name, argument=argument).model_dump() ) diff --git a/app/api/docs.py b/app/api/docs.py index 3d12b75..61d8eb6 100644 --- a/app/api/docs.py +++ b/app/api/docs.py @@ -36,7 +36,7 @@ def doc_format(path: str) -> str: @router.get("/docs", response_model=DocList) -def list_documents( +def list_indexed_documents( db: Session = Depends(get_db), # noqa: B008 _admin: None = Depends(require_admin), # noqa: B008 ) -> DocList: diff --git a/app/rag/agent.py b/app/rag/agent.py index 0642ce7..1558f4b 100644 --- a/app/rag/agent.py +++ b/app/rag/agent.py @@ -1,4 +1,5 @@ -"""Agent loop: the grounded-turn document tools (phase 37, task 03). +"""Agent loop: the grounded-turn document tools (phase 37, task 03; the +harness-aligned ``ls``/``read``/``grep`` surface, phase 70). Probe verdict (task 01 — ``uv run python -m scripts.llm_probe --tools`` run live against aipi): **``probe: turbo tool_calls=supported 2026-08-26``** @@ -18,44 +19,56 @@ task 04): 1. The model is offered the three OpenAI functions in :data:`AGENT_TOOLS` for the whole turn — phase 45 removed the phase-37 per-tool budgets (owner permission 2026-08-27, ``TODO.md`` L8: "allow the LLM to make - as many tool calls as it wants"): ``list_documents``, - ``read_document`` and ``search_documents`` can each be called as many - times as the model needs, re-lists and re-searches included. With - ``settings.agent_max_rounds`` (``BOR_AGENT_MAX_ROUNDS``, default 10) - at 0 the loop makes exactly one request with ``tools=None`` — - byte-identical to the pre-phase-37 chat path (the kill switch). + as many tool calls as it wants"): ``ls``, ``read`` and ``grep`` can + each be called as many times as the model needs, re-lists and + re-greps included. With ``settings.agent_max_rounds`` + (``BOR_AGENT_MAX_ROUNDS``, default 10) at 0 the loop makes exactly + one request with ``tools=None`` — byte-identical to the + pre-phase-37 chat path (the kill switch). Phase 70 (owner permission + 2026-09-03: "match existing harnesses as much as possible") renamed + and reshaped the tools to the harness-trained surface — + ``ls(path?)`` / ``read(path)`` / ``grep(pattern, path?)``, the + pi.dev tool shapes the model was trained on: the combined + ``source/path`` string is the canonical document identity in every + tool argument, refusal, and result header, and the old two-argument + split (with its self-correction and "teach the split" refusals) is + gone — the model's combined form is now simply correct. The + phase-68 A5 match/output contract rides along under the new name. 2. Each tool call the model emits is executed server-side against - Postgres only (no LLM, no network): ``list_documents`` returns the - indexed catalog — one ``source: X | path: Y | title: Z`` line per - document (phase 63: labeled fields — unambiguous for LLM parsing), + Postgres only (no LLM, no network): ``ls`` returns the indexed + catalog — one ``source: X | path: Y | title: Z`` line per document + (phase 63: labeled fields — unambiguous for LLM parsing), ``GET /api/docs`` order (uncapped in v1; the UI never shows it, only - the model does) — ``read_document`` returns the document's **full** - content (A7-revised contract: never truncated) — and - ``search_documents`` greps the indexed documents (or one named - document) for a case-insensitive fixed substring and returns up to 20 + the model does) — optionally scoped to one source name (a ``path`` + argument matching no source name is a refusal; a registered source + with no indexed documents lists as ``0 documents:`` and counts) — + ``read`` takes the combined ``source/path`` string, splits it at the + FIRST ``'/'`` (source names are directory basenames — they can never + contain ``'/'``), and returns the document's **full** content + (A7-revised contract: never truncated) — and ``grep`` greps the + indexed documents (or the one document a combined ``source/path`` + names) for a case-insensitive fixed substring and returns up to 20 ``source/path:line: text`` match lines (owner-locked A5, phase 68), - each line truncated to 200 chars. A search is a **locator**, not a + each line truncated to 200 chars. A grep is a **locator**, not a context-adder: it never appends to the answer context (only - ``read_document`` does — ``holder.read_docs`` is untouched by a - search). + ``read`` does — ``holder.read_docs`` is untouched by a grep). 3. Rejected calls get a one-line refusal and count in nothing (``holder.tool_calls`` tracks executed calls only): unknown tool name - → ``"Unknown tool."``; missing ``source``/``path`` arguments; a search - without a usable ``pattern`` (missing, blank or non-string) or with a - half-specified ``source``/``path`` target; a document already in - context (seed or previously read) → ``"Already in your - context."``; an unknown ``source/path`` (read or scoped search) → - ``"No document at …"``. A ``source`` argument containing a ``'/'`` - (the model passed the combined ``source/path`` form) is first - self-corrected by splitting at the first slash (see - :func:`_resolve_document` — source names are directory basenames and - can never contain ``'/'``); if the split still matches nothing, the - refusal teaches the split instead of repeating the combined form. - A search that ran but found nothing is NOT a - rejection — its ``"No matches for …"`` line is a (counted) result. - A rejected call still consumes a *round* in the loop, so a - pathological stream that keeps emitting rejected calls is bounded by - the cap (point 4). + → ``"Unknown tool."``; a ``read`` without a usable ``path`` (missing, + blank or non-string) → ``"read requires a string argument + 'path'."``; a ``grep`` without a usable ``pattern`` (missing, blank + or non-string) → ``"grep requires a string argument + 'pattern'."``; a scoped ``ls`` whose ``path`` matches no source name + → ``"No source named '…' — check the ls output."``; a document + already in context (seed or previously read) → ``"Already in your + context."``; an unknown document (a ``read`` or scoped ``grep`` whose + combined ``source/path`` matches nothing — a bare source name, which + can never be a document, included) → ``"No document at '…' — check + the ls output."`` with the argument echoed as passed (the model sees + its own form). A grep that ran but found nothing is NOT a rejection + — its ``"No matches for …"`` line is a (counted) result. A rejected + call still consumes a *round* in the loop, so a pathological stream + that keeps emitting rejected calls is bounded by the cap (point 4). 4. Every call the model emits is appended back to the message history as the assistant tool-call message + the tool result (refusals included), consumes one round, and the model is called again. At the round cap — @@ -82,10 +95,10 @@ task 04): exactly one round. With ``settings.llm_retries=0`` every request is a single plain attempt (the pre-phase-67 path). -The DB accessors (:func:`list_catalog`, :func:`find_document`, -:func:`all_documents`) and the :func:`grep_document` line matcher are -module-level functions so unit tests can monkeypatch them without a -database. +The DB accessors (:func:`list_catalog`, :func:`list_source_names`, +:func:`find_document`, :func:`all_documents`) and the +:func:`grep_document` line matcher are module-level functions so unit +tests can monkeypatch them without a database. """ from __future__ import annotations @@ -100,6 +113,7 @@ from sqlalchemy.orm import Session from app.config import Settings from app.models import Document +from app.rag.git_sources import effective_sources from app.rag.llm import ( LLMClient, RetryPiece, @@ -107,91 +121,78 @@ from app.rag.llm import ( ToolCallPiece, chat_stream_retried, ) +from app.rag.source_removal import resolve_source_name logger = logging.getLogger("app.agent") -#: Parameter descriptions shared by ``read_document`` and -#: ``search_documents``. The model repeatedly conflated the two fields — -#: passing the combined ``source/path`` string (as printed in search -#: result lines, read-result headers and refusals) as ``source`` — so -#: the descriptions define the split explicitly: ``source`` is the part -#: BEFORE the first ``'/'``, ``path`` the part after it, with a worked -#: example in the ``read_document`` description itself. -_SOURCE_PARAM: dict[str, Any] = { - "type": "string", - "description": ( - "Top-level source name only (e.g. 'homelab') — the part BEFORE " - "the first '/' of a combined 'source/path' string, exactly as " - "shown after 'source: ' in the list_documents output. Must not " - "contain '/' itself — do not pass the full source/path here." - ), -} -_PATH_PARAM: dict[str, Any] = { - "type": "string", - "description": ( - "File path relative to the source directory (e.g. " - "'active/container_caddy/caddy.md') — the part AFTER the first " - "'/' of a combined 'source/path' string, exactly as shown after " - "'path: ' in the list_documents output. Must not start with the " - "source name." - ), -} - -#: The three agent tools (phase 37; ``search_documents`` added in phase -#: 68): OpenAI function definitions passed as ``tools=AGENT_TOOLS`` to -#: ``chat_stream`` for the whole grounded turn — phase 45 removed the -#: per-tool budgets; the round cap (``BOR_AGENT_MAX_ROUNDS``) is the only -#: bound. +#: The three agent tools (phase 70: the harness-aligned surface — +#: ``ls`` / ``read`` / ``grep``, the pi.dev tool shapes the model was +#: trained on, replacing the phase-37 list/read and phase-68 search +#: names): OpenAI function +#: definitions passed as ``tools=AGENT_TOOLS`` to ``chat_stream`` for +#: the whole grounded turn — phase 45 removed the per-tool budgets; the +#: round cap (``BOR_AGENT_MAX_ROUNDS``) is the only bound. The combined +#: ``source/path`` string is the canonical document identity in every +#: argument (phase 70, owner permission 2026-09-03). AGENT_TOOLS: list[dict[str, Any]] = [ { "type": "function", "function": { - "name": "list_documents", + "name": "ls", "description": ( - "List every document indexed in the knowledge base, one " - "`source: X | path: Y | title: Z` line each" - ), - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "read_document", - "description": ( - "Add the full content of one more indexed document " - "to your context. A document is identified by the " - "(source, path) pair exactly as shown in the " - "list_documents output: 'source' is the top-level " - "source name only (e.g. 'homelab'), 'path' is the file " - "path inside that source (e.g. " - "'active/container_caddy/caddy.md'). If you only have a " - "combined 'source/path' string (as in search_documents " - "results), split it at the FIRST '/': the part before " - "is the source, the part after is the path. Example: " - "read_document(source='homelab', " - "path='active/container_caddy/caddy.md')." + "List the indexed documents as `source: X | path: Y | " + "title: Z` lines." ), "parameters": { "type": "object", - "properties": {"source": _SOURCE_PARAM, "path": _PATH_PARAM}, - "required": ["source", "path"], + "properties": { + "path": { + "type": "string", + "description": ( + "Source name to list one source's documents " + "(e.g. 'homelab'); omit to list every " + "document." + ), + } + }, + "required": [], }, }, }, { "type": "function", "function": { - "name": "search_documents", + "name": "read", "description": ( - "Search every indexed document for an exact string " - "(case-insensitive) and return up to 20 matching lines as " - "'source/path:line: text' — use this to locate content, " - "then read_document the winner (each result line's " - "'source/path' splits at the first '/': the part before " - "is the source, the part after is the path). Optionally " - "pass 'source' and 'path' (as shown in list_documents) " - "to search one document only." + "Add the full content of one indexed document to your " + "context." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": ( + "The document to add to your context, as the " + "combined `source/path` string exactly as " + "shown in the `ls` output (e.g. " + "'homelab/active/container_caddy/caddy.md')." + ), + } + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "grep", + "description": ( + "Search the indexed documents for an exact string " + "(case-insensitive) and return up to 20 matching lines " + "as `source/path:line: text` — a locator, not a " + "context-adder: read the winner with `read`." ), "parameters": { "type": "object", @@ -203,8 +204,15 @@ AGENT_TOOLS: list[dict[str, Any]] = [ "substring, not a regex)" ), }, - "source": _SOURCE_PARAM, - "path": _PATH_PARAM, + "path": { + "type": "string", + "description": ( + "Limit the search to one document, as a " + "combined `source/path` string from the " + "`ls` output (omit to search every " + "document)." + ), + }, }, "required": ["pattern"], }, @@ -217,8 +225,8 @@ AGENT_TOOLS: list[dict[str, Any]] = [ #: their pathological repetition (phase 45). ALREADY_IN_CONTEXT = "Already in your context." UNKNOWN_TOOL = "Unknown tool." -MISSING_READ_ARGS = "read_document requires string arguments 'source' and 'path'." -MISSING_SEARCH_ARGS = "search_documents requires a string argument 'pattern'." +MISSING_READ_ARGS = "read requires a string argument 'path'." +MISSING_SEARCH_ARGS = "grep requires a string argument 'pattern'." #: Search caps (owner-locked A5, phase 68): a global per-call match cap #: (across documents, in catalog order) and a per-match-line char limit. @@ -227,7 +235,7 @@ SEARCH_LINE_LIMIT = 200 #: No-match result lines (templates — the pattern is truncated to 100 #: chars before formatting, to keep a long pattern from bloating the -#: tool result). A no-match line is a *result* of an executed search, +#: tool result). A no-match line is a *result* of an executed grep, #: not a refusal (see the module docstring, point 3). NO_MATCHES = "No matches for '{pattern}' in the knowledge base." NO_MATCHES_SCOPED = "No matches for '{pattern}' in {source}/{path}." @@ -247,6 +255,31 @@ def list_catalog(db: Session) -> list[tuple[str, str, str]]: return [(source, path, title) for source, path, title in rows] +def list_source_names(db: Session) -> list[str]: + """Every registered source name, deduped, in registry order. + + The source registry (the ``git_sources`` rows — the + ``BOR_GIT_SOURCES`` env fallback while the table is empty) is the + source of truth for *source* names independent of document count: + a registered source with no indexed documents still lists (as + ``0 documents:`` — the scoped ``ls`` must not refuse it as unknown). + Names resolve exactly as the import pipeline indexes them + (:func:`app.rag.source_removal.resolve_source_name` — reuse, or a + scoped ``ls`` would judge the wrong names unknown, the phase-69 + "RAG consistent with the registry" invariant); two rows resolving + to the same name (the phase-69 sibling case) share documents, so + the name is listed once. Module-level (not a method) so unit tests + can monkeypatch it. + """ + rows, _origin = effective_sources(db) + names: list[str] = [] + for row in rows: + name = resolve_source_name(row) + if name not in names: + names.append(name) + return names + + def find_document(db: Session, source: str, path: str) -> Document | None: """The indexed document at ``(source, path)``, or ``None``. @@ -257,42 +290,33 @@ def find_document(db: Session, source: str, path: str) -> Document | None: ) -def _resolve_document( - db: Session, source: str, path: str -) -> tuple[Document | None, str, str]: - """``(source, path)`` → document, with combined-form self-correction. +def _resolve_path(db: Session, combined: str) -> tuple[Document | None, str, str]: + """The combined ``source/path`` identity → document (phase 70). - The exact pair is tried first. If it misses and *source* contains a - ``'/'``, the model passed the combined ``source/path`` form — search - result lines, read-result headers and the generic refusal all print - that form, so the model treats it as the document's identity. Source - names are directory basenames (``app.rag.importer``: ``source = - root.name``) and can never contain a ``'/'``, so the pair is retried - at the FIRST slash: the part before is the source name, the part - after is the path. A second candidate covers a split at a LATER - slash (``source`` carried source + leading directories, ``path`` the - remainder). - - Returns ``(doc, src, p)`` where ``(src, p)`` is the first-slash - split when one was attempted (so a refusal can teach it), else the - original pair. + The canonical document identity in every tool argument, refusal and + result header is the combined string exactly as printed in the + ``ls`` output, the ``Document …`` result headers, and the grep + result lines. Source names are directory basenames (``app.rag.importer``: + ``source = root.name``) and can never contain a ``'/'``, so the + split at the FIRST slash is exact: the part before is the source + name, the part after is the path. Returns ``(doc, source, path)`` + with the split pair (so callers can echo the canonical form, e.g. + the scoped no-match line); no ``'/'`` in the argument → + ``(None, combined, "")`` — a bare source name is never a document + (no DB lookup; the refusal echoes the argument as passed). """ - doc = find_document(db, source, path) - if doc is not None or "/" not in source: - return doc, source, path - split_source, _, split_path = source.partition("/") - doc = find_document(db, split_source, split_path) - if doc is None and path and path != split_path: - doc = find_document(db, split_source, f"{split_path}/{path}") - return doc, split_source, split_path + if "/" not in combined: + return None, combined, "" + source, _, path = combined.partition("/") + return find_document(db, source, path), source, path def all_documents(db: Session) -> list[Document]: """Every indexed document (full rows), ordered by ``(source, path)`` — catalog order. - The whole-KB ``search_documents`` path loads all contents in this one - bulk query (catalog order is the locked match order, owner-locked A5). + The whole-KB ``grep`` path loads all contents in this one bulk query + (catalog order is the locked match order, owner-locked A5). Module-level (not a method) so unit tests can monkeypatch it. """ return list( @@ -322,8 +346,8 @@ def grep_document(content: str, pattern: str) -> list[tuple[int, str]]: class AgentHolder: """Per-turn agent state the API layer reads after the stream (task 04). - ``read_docs``: the documents ``read_document`` added to the context, - in read order (deduped — re-reading a document appends nothing). + ``read_docs``: the documents ``read`` added to the context, in read + order (deduped — re-reading a document appends nothing). ``tool_calls``: how many tool calls executed (re-lists included); rejected calls (unknown tool, unknown/missing arguments or document, already-in-context) do not count. Drives the per-turn log line's @@ -343,78 +367,64 @@ def _execute_tool( """Execute one tool call server-side (DB only). Returns the tool result text. A successful call bumps - ``holder.tool_calls`` (a successful read also appends the - :class:`Document` to ``holder.read_docs``; a search never does — it - is a locator, locked A5); rejected calls return their refusal line - and count in nothing. A search that ran but found nothing is still a - successful (counted) call — its no-match line is a result, not a - refusal. A combined-form ``source`` (containing a ``'/'``) is - self-corrected through :func:`_resolve_document` before any refusal. + ``holder.tool_calls`` (a successful ``read`` also appends the + :class:`Document` to ``holder.read_docs``; a ``grep`` never does — + it is a locator, locked A5); rejected calls return their refusal + line and count in nothing. A grep that ran but found nothing is + still a successful (counted) call — its no-match line is a result, + not a refusal. Document targets are combined ``source/path`` + strings, resolved by :func:`_resolve_path` (the canonical identity, + phase 70). """ - if call.name == "list_documents": + if call.name == "ls": + raw_path = call.arguments.get("path") + scope = raw_path.strip() if isinstance(raw_path, str) else "" rows = list_catalog(db) + if scope: + if scope not in list_source_names(db): + return f"No source named '{scope}' — check the ls output." + rows = [row for row in rows if row[0] == scope] listing = f"{len(rows)} documents:\n" + "\n".join( f"source: {source} | path: {path} | title: {title}" for source, path, title in rows ) holder.tool_calls += 1 return listing - if call.name == "read_document": - raw_source = call.arguments.get("source") + if call.name == "read": raw_path = call.arguments.get("path") - source = raw_source.strip() if isinstance(raw_source, str) else "" - path = raw_path.strip() if isinstance(raw_path, str) else "" - if not source or not path: + arg = raw_path.strip() if isinstance(raw_path, str) else "" + if not arg: return MISSING_READ_ARGS known = {(doc.source, doc.path) for doc in (*seed_docs, *holder.read_docs)} - if (source, path) in known: - return ALREADY_IN_CONTEXT - doc, split_source, split_path = _resolve_document(db, source, path) + # The dedupe check needs no DB: the split pair of a combined + # identity that is in context is in `known` as-is (the resolve + # below would find the same document). + if "/" in arg: + src, _, p = arg.partition("/") + if (src, p) in known: + return ALREADY_IN_CONTEXT + doc, _source, _path = _resolve_path(db, arg) if doc is None: - if "/" in source: - # Educational refusal: the combined form is the model's - # mistake — teach the split instead of repeating it. - return ( - f"source must not contain '/': for '{source}' call " - f"read_document(source='{split_source}', " - f"path='{split_path}')." - ) - return ( - f"No document at {source}/{path} — check the list_documents output." - ) - if (doc.source, doc.path) in known: - # A self-corrected combined form for a document already in - # context (the raw pair above cannot have matched it). - return ALREADY_IN_CONTEXT + # Echo the argument as passed — the model sees its own form + # (a bare source name can never be a document, no DB lookup). + return f"No document at '{arg}' — check the ls output." holder.read_docs.append(doc) holder.tool_calls += 1 return f"Document {doc.source}/{doc.path}:\n{doc.content}" - if call.name == "search_documents": + if call.name == "grep": raw_pattern = call.arguments.get("pattern") pattern = raw_pattern.strip() if isinstance(raw_pattern, str) else "" if not pattern: return MISSING_SEARCH_ARGS - raw_source = call.arguments.get("source") raw_path = call.arguments.get("path") - source = raw_source.strip() if isinstance(raw_source, str) else "" - path = raw_path.strip() if isinstance(raw_path, str) else "" - if (source == "") != (path == ""): - # A half-specified target is a model error — fail loud with - # the missing-args refusal instead of silently widening to a - # whole-KB search (house style). - return MISSING_SEARCH_ARGS - if source: - target, split_source, split_path = _resolve_document(db, source, path) + scope = raw_path.strip() if isinstance(raw_path, str) else "" + scoped_to: tuple[str, str] | None = None + if scope: + target, src, p = _resolve_path(db, scope) if target is None: - if "/" in source: - return ( - f"source must not contain '/': for '{source}' use " - f"source='{split_source}', path='{split_path}'." - ) - return ( - f"No document at {source}/{path} — check the list_documents output." - ) + return f"No document at '{scope}' — check the ls output." docs: list[Document] = [target] + scoped_to = (src, p) # the resolved (canonical) identity else: docs = all_documents(db) matches: list[str] = [] @@ -427,13 +437,15 @@ def _execute_tool( break if len(matches) >= SEARCH_MAX_MATCHES: break # the global cap is hit — stop scanning - holder.tool_calls += 1 # the search executed (no-match counts too) - # Locked A5: a search never adds context — read_docs untouched. + holder.tool_calls += 1 # the grep executed (no-match counts too) + # Locked A5: a grep never adds context — read_docs untouched. if not matches: shown = pattern[:100] # keep a long pattern short in the line - if source: + if scoped_to is not None: + # The scoped no-match line is keyed on the resolved + # source/path (== the argument, stripped). return NO_MATCHES_SCOPED.format( - pattern=shown, source=source, path=path + pattern=shown, source=scoped_to[0], path=scoped_to[1] ) return NO_MATCHES.format(pattern=shown) return "\n".join(matches) diff --git a/app/rag/llm.py b/app/rag/llm.py index 4f21476..af39427 100644 --- a/app/rag/llm.py +++ b/app/rag/llm.py @@ -74,12 +74,12 @@ class ToolCallPiece: ``id`` is the model's tool_call id (synthesized as ``call_`` when the wire never carried one), ``name`` is the function name (whatever the caller's ``tools`` list names — for the agent loop, - ``list_documents`` / ``read_document``), and ``arguments`` is the + ``ls`` / ``read`` / ``grep``, phase 70), and ``arguments`` is the parsed JSON object (``{}`` when the model sent none). """ id: str # the model's tool_call id; synthesized "call_" when absent - name: str # "list_documents" | "read_document" (whatever AGENT_TOOLS names) + name: str # "ls" | "read" | "grep" (whatever AGENT_TOOLS names, phase 70) arguments: dict[str, Any] @@ -124,7 +124,7 @@ def _materialize_tool_calls( Malformed ``arguments`` JSON raises :class:`LLMError` — a silently dropped tool call would corrupt the agent loop (fail-loud house style). Empty/``null`` arguments become ``{}`` (a no-parameter call - such as ``list_documents``). + such as an unscoped ``ls``). """ pieces: list[ToolCallPiece] = [] for index in sorted(slots): diff --git a/app/rag/prompts.py b/app/rag/prompts.py index be113d8..53416dc 100644 --- a/app/rag/prompts.py +++ b/app/rag/prompts.py @@ -24,11 +24,13 @@ the ```` section (order: ```` → roughly what the KB contains before retrieval. With an empty row the prompt is byte-identical to the pre-phase text. -Agent tools (phase 37): the **HIGH** prompt only carries a ```` -section after the ```` body — the grounded turn may call the -server-side ``list_documents`` / ``read_document`` tools (round-capped, -see :mod:`app.rag.agent`). The LOW/deflection prompt never carries it -and stays byte-identical to the pre-phase text. +Agent tools (phase 37; phase 70: the copy teaches the harness-aligned +``ls`` / ``read`` / ``grep`` shapes): the **HIGH** prompt only carries a +```` section after the ```` body — the grounded turn +may extend its context through the three server-side tools (round- +capped, see :mod:`app.rag.agent`; the cap is the bound and this section +does not re-state it, phase 45). The LOW/deflection prompt never +carries it and stays byte-identical to the pre-phase text. """ from __future__ import annotations @@ -73,21 +75,29 @@ _KB_INTRO = ( ) #: The ```` instructions section — **HIGH prompt only** (phase 37, -#: task 03): a grounded turn may extend its context through the two -#: server-side tools (round cap: ``BOR_AGENT_MAX_ROUNDS``, see -#: :mod:`app.rag.agent`). Appended after -#: the mode body (````), so the instructions are the last -#: thing the model reads. The LOW/deflection prompt never carries it — -#: a deflection has no grounded context to extend — and stays -#: byte-identical to the pre-phase text. The E2E mock keys off the -#: ```` marker's *presence*, not this wording. +#: task 03; phase 70: the copy is rewritten for the harness-aligned +#: ``ls`` / ``read`` / ``grep`` shapes, names/args exactly as the +#: ``AGENT_TOOLS`` schemas in :mod:`app.rag.agent`): a grounded turn may +#: extend its context through the three server-side tools (round cap: +#: ``BOR_AGENT_MAX_ROUNDS`` — the cap is the bound and this section does +#: not re-state it, phase 45). Appended after the mode body +#: (````), so the instructions are the last thing the model +#: reads. The LOW/deflection prompt never carries it — a deflection has +#: no grounded context to extend — and stays byte-identical to the +#: pre-phase text. The E2E mock keys off the ```` marker's +#: *presence*, not this wording. TOOLS_SECTION: str = ( "\n" - "If the documents in your context reference other files, or you need " - "content that is not included above, call `list_documents` to see what " - "is indexed, then `read_document` to pull in exactly one more document. " - "Answer as soon as you have what you need — do not read more than one " - "extra document.\n" + "You may extend your context with three tools. `ls` lists the " + "indexed documents as `source: X | path: Y | title: Z` lines " + "(pass a source name as `path` to list one source's documents; " + "omit it to list every document). `grep` locates an exact string " + "(case-insensitive) in the indexed documents and returns up to 20 " + "matching `source/path:line: text` lines — a locator, not a " + "context-adder: read the winner with `read`. `read` pulls in one " + "document by its combined `source/path` string, exactly as shown in " + "the `ls` output, adding its full content to your context. Answer " + "as soon as you have what you need.\n" "" ) @@ -180,7 +190,8 @@ def build_high_prompt( kb_overview: str | None = None, ) -> str: """Grounded turn: locked persona (+ steering, + KB overview) + full -texts of the top documents + the ```` instructions (phase 37). +texts of the top documents + the ```` instructions (phase 37; +the phase-70 copy teaches the ``ls`` / ``read`` / ``grep`` shapes). Section order: ```` → ```` → ```` → ```` → ````; empty steering/overview omit their diff --git a/app/schemas.py b/app/schemas.py index e5194f0..5b62290 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -71,24 +71,26 @@ class ChatThinkingEvent(BaseModel): class ChatToolEvent(BaseModel): """SSE frame for one agent tool call (phase 37, PLAN §4 extension). - A15 extension (owner permission 2026-08-26; ``search_documents`` - added in phase 68): a grounded turn may call the server-side - document tools (``list_documents`` / ``read_document`` / - ``search_documents``, see :mod:`app.rag.agent`); each model-requested + A15 extension (owner permission 2026-08-26; the grep added in phase + 68; phase 70 aligned the surface to the harness-trained + ``ls`` / ``read`` / ``grep`` — owner permission 2026-09-03): a + grounded turn may call the server-side document tools (``ls`` / + ``read`` / ``grep``, see :mod:`app.rag.agent`); each model-requested call streams as ``{type: "tool", name: str, argument: str | null}`` - ahead of the answer's ``delta`` frames. ``argument`` is the read - document's ``"source/path"`` for ``read_document``, the search - pattern for ``search_documents``, and null otherwise (a non-string - pattern — a model error the backend refuses — is null). The client - renders each frame as a "calling tool" line/state (phase 37 task 05); - the ``delta`` / ``done`` shapes are unchanged — the read document is - reflected in ``done.sources`` instead (a search adds no source: it is - a locator, locked A5). + ahead of the answer's ``delta`` frames. ``argument`` is the single + string argument the model passed — ``read``'s ``path`` (the combined + ``source/path``), ``grep``'s ``pattern``, ``ls``'s ``path`` — or + null (a non-string value, a model error the backend refuses, and an + omitted argument both yield null). The client renders each frame as + a "calling tool" line/state (phase 37 task 05); the ``delta`` / + ``done`` shapes are unchanged — the read document is reflected in + ``done.sources`` instead (a grep adds no source: it is a locator, + locked A5). """ type: Literal["tool"] = "tool" - name: str # "list_documents" | "read_document" | "search_documents" - argument: str | None = None # "source/path" for read_document, pattern for search_documents + name: str # "ls" | "read" | "grep" (whatever AGENT_TOOLS names) + argument: str | None = None # the single string argument passed, or null class ChatDoneEvent(BaseModel): @@ -375,10 +377,13 @@ class ToolCall(BaseModel): """One agent tool-call record (the phase-37 ``tools`` record shape). Mirrors the ``{name, argument}`` pair the SSE ``tool`` frames carry - (PLAN §4 extension): ``argument`` is the read document's - ``"source/path"`` for ``read_document`` and null otherwise. Stored - inside :class:`ChatMessage.tools` so a saved chat restores the - "calling tool" lines pixel-identical (phase 50). + (PLAN §4 extension; phase 70): ``argument`` is the single string + argument the model passed (``read``'s combined ``source/path``, + ``grep``'s pattern, ``ls``'s scope) or null. Stored inside + :class:`ChatMessage.tools` so a saved chat restores the "calling + tool" lines pixel-identical (phase 50). Saved chats persisting the + pre-phase-70 tool names still validate — ``name`` is opaque + (no migration, locked). """ name: str diff --git a/frontend/assets/app.js b/frontend/assets/app.js index b16bd6d..5112be1 100644 --- a/frontend/assets/app.js +++ b/frontend/assets/app.js @@ -56,21 +56,24 @@ * phase 34 task 02) clears the key + the list back to the empty state. * * Agent tool calls (phase 37, PLAN §4 extension; phase 68 added - * search_documents): a grounded turn may call the three server-side - * document tools (list_documents / read_document / search_documents, - * bounded only by the round cap — phases 45/68). Each call streams a - * `tool` SSE frame, and the UI shows the "calling tool" state IN - * ADDITION to "thinking": the UI state itself stays "thinking" (the - * button stays the enabled "Stop" control — phase 48 — never stale, - * PLAN §7.4) while the STATUS LABELS change — the #send-status + - * typing-indicator labels say what Brain is doing ("…is listing - * documents" / "…is reading source/path" / "…is searching for - * pattern" — the name prefix resolves from window.BOR_BRAND at call - * time, phase 39) — the button no longer relabels to "Calling tool…" - * (phase 48, owner-locked: it stays "Stop" for the whole turn) — and a - * visible `.tool-call` line (own icon + accent color, distinct from - * the brand-ink Thinking block) is appended above the answer, one per - * call, in order. + * search_documents; phase 70 remapped the surface to the harness + * names ls / read(path) / grep(pattern, path?)): a grounded turn may + * call the three server-side document tools, bounded only by the + * round cap (phases 45/68). Each call streams a `tool` SSE frame, and + * the UI shows the "calling tool" state IN ADDITION to "thinking": the + * UI state itself stays "thinking" (the button stays the enabled + * "Stop" control — phase 48 — never stale, PLAN §7.4) while the STATUS + * LABELS change — the #send-status + typing-indicator labels say what + * Brain is doing ("…is listing documents" / "…is reading source/path" + * / "…is searching for pattern" — the name prefix resolves from + * window.BOR_BRAND at call time, phase 39) — the button no longer + * relabels to "Calling tool…" (phase 48, owner-locked: it stays + * "Stop" for the whole turn) — and a visible `.tool-call` line (own + * icon + accent color, distinct from the brand-ink Thinking block) is + * appended above the answer, one per call, in order. Phase 70: the + * line/label branches key off the NEW names and still carry the legacy + * ones (list_documents / read_document / search_documents) — persisted + * turns from before the remap render exactly as before (no migration). * Append-only like thinking: frames are tolerated in any interleaving * (a frame after the first delta just appends — the agent loop never * emits one, but it must not crash). The turn record persists an @@ -810,9 +813,22 @@ function closeThinkingBlock(wrap) { * interleaving with thinking frames, even after the first delta (the * agent loop never emits one, but a late frame must not crash) — just * append another line, in order. The SAME helper re-renders the - * persisted lines on restore (phase 14 convention): the path/pattern - * argument goes through textContent, so nothing HTML-shaped can come - * from storage. Lines are not interactive (no focus targets). */ + * persisted lines on restore (phase 14 convention): every argument + * (path / pattern / source scope) goes through textContent, so nothing + * HTML-shaped can come from storage. Lines are not interactive (no + * focus targets). + * + * Phase 70 (owner permission 2026-09-03): the server tools were remapped + * to the harness-aligned surface — ls / read(path) / grep(pattern, + * path?) — so the NEW names get their own lines (read → the Reading + * line, grep → the Searching-for line, ls → the Listing-documents line, + * a scoped ls → the Listing-documents-in- line), and the + * pre-phase-70 names (list_documents / read_document / + * search_documents) still render EXACTLY as before: persisted turns + * (phase 14) carry the old names, so both generations render — no + * migration. The content marks (the read/search/list glyphs) stay the + * exact tool-line template literals — the frontend emoji guard strips + * precisely those in this file. */ function appendToolLine(wrap, name, argument) { const body = wrap?.querySelector?.(".msg-body"); if (!body) return; @@ -829,16 +845,24 @@ function appendToolLine(wrap, name, argument) { const line = document.createElement("span"); line.className = "tool-call"; line.setAttribute("role", "listitem"); - if (name === "read_document" && argument) { + // Phase 70: new names first, legacy names kept — a restored turn saved + // before the remap (read_document / search_documents / list_documents) + // renders byte-identical to before (no migration). + if ((name === "read" || name === "read_document") && argument) { line.textContent = "📄 Reading "; const code = document.createElement("code"); code.textContent = argument; // the path is data, never markup line.appendChild(code); - } else if (name === "search_documents" && argument) { + } else if ((name === "grep" || name === "search_documents") && argument) { line.textContent = "🔎 Searching for "; const code = document.createElement("code"); code.textContent = argument; // the pattern is data, never markup line.appendChild(code); + } else if (name === "ls" && argument) { + line.textContent = "🔎 Listing documents in "; + const code = document.createElement("code"); + code.textContent = argument; // the source scope is data, never markup + line.appendChild(code); } else { line.textContent = "🔎 Listing documents"; } @@ -1940,12 +1964,18 @@ async function runTurn(text, { reask = false } = {}) { toolAcc.push({ name, argument }); clearTurnTimeout(); // the stream is alive — a frame arrived if (!wrap) wrap = addMessage("brain", ""); + // Phase 70: the harness-aligned names (read/grep/ls) map to the + // same status copy as their legacy counterparts (read_document / + // search_documents) — a pre-remap frame keeps its label; the + // scoped ls mirrors the scoped tool line. const toolStatus = - name === "read_document" && argument + (name === "read" || name === "read_document") && argument ? `${brand()} is reading ${argument}` - : name === "search_documents" && argument + : (name === "grep" || name === "search_documents") && argument ? `${brand()} is searching for ${argument}` - : `${brand()} is listing documents`; + : name === "ls" && argument + ? `${brand()} is listing documents in ${argument}` + : `${brand()} is listing documents`; if (uiState === UI_STATE.thinking) { sendStatus.textContent = toolStatus; document diff --git a/frontend/assets/shared.js b/frontend/assets/shared.js index 2e49740..791ed5c 100644 --- a/frontend/assets/shared.js +++ b/frontend/assets/shared.js @@ -133,14 +133,22 @@ function addThinkingBlock(wrap, thinking) { body.insertBefore(block, body.querySelector(".bubble")); } -/* Tool-call lines (phase 37) — the local copy of the chat page's - * appendToolLine: one visible "calling tool" row per saved - * {name, argument} record, in saved order, above the answer. The - * path argument goes through textContent, so nothing HTML-shaped can - * come from storage. Lines are not interactive (no focus targets). - * The two content marks (the read glyph / the list glyph) are the - * exact app.js template strings — the frontend emoji guard strips - * precisely those two literals in this file, as in app.js. */ +/* Tool-call lines (phase 37; phase 70 remapped the tool names to the + * harness surface ls / read(path) / grep(pattern, path?)) — the local + * copy of the chat page's appendToolLine: one visible "calling tool" + * row per saved {name, argument} record, in saved order, above the + * answer. Every argument (path / pattern / source scope) goes through + * textContent, so nothing HTML-shaped can come from storage. Lines + * are not interactive (no focus targets). Phase 70: the NEW names + * render (read → the Reading line, grep → the Searching-for line, + * ls → the Listing-documents line, scoped ls → the + * Listing-documents-in- line), and the pre-phase-70 names + * (read_document / search_documents / list_documents) still render + * exactly as before — a row saved before the remap keeps its exact + * line (no migration). The content marks are the exact app.js + * template strings — the frontend emoji guard (tests/integration/ + * test_api.py) strips precisely those literals in this file, as in + * app.js. */ function addToolLines(wrap, tools) { if (!Array.isArray(tools) || !tools.length) return; const body = wrap.querySelector(".msg-body"); @@ -156,11 +164,25 @@ function addToolLines(wrap, tools) { line.setAttribute("role", "listitem"); const argument = typeof t.argument === "string" && t.argument ? t.argument : null; - if (t.name === "read_document" && argument) { + // Phase 70: new names first, legacy names kept — a conversation + // saved before the remap renders byte-identical (no migration). + if ((t.name === "read" || t.name === "read_document") && argument) { line.textContent = "📄 Reading "; const code = document.createElement("code"); code.textContent = argument; // the path is data, never markup line.appendChild(code); + } else if ( + (t.name === "grep" || t.name === "search_documents") && argument + ) { + line.textContent = "🔎 Searching for "; + const code = document.createElement("code"); + code.textContent = argument; // the pattern is data, never markup + line.appendChild(code); + } else if (t.name === "ls" && argument) { + line.textContent = "🔎 Listing documents in "; + const code = document.createElement("code"); + code.textContent = argument; // the source scope is data, never markup + line.appendChild(code); } else { line.textContent = "🔎 Listing documents"; } diff --git a/tests/e2e/mock_llm.py b/tests/e2e/mock_llm.py index 4136cc2..40f6e56 100644 --- a/tests/e2e/mock_llm.py +++ b/tests/e2e/mock_llm.py @@ -56,25 +56,30 @@ Implements just enough of the aipi surface: the echo targets the block itself; its tail still includes the closing tag — same sentinel semantics.) - user message containing ``use your tools`` (phase 37, agent document - tools) **and** the system prompt carries the ```` section -> - the deterministic SINGLE-READ tool flow, discriminated statelessly - from the messages (the ``tools`` parameter gates the list/read - steps — a no-tools request with no tool results is not the flow): + tools; phase 70: the flow emits the harness-aligned names — ``ls`` + / ``read`` with the combined ``source/path`` identity) **and** the + system prompt carries the ```` section -> the deterministic + SINGLE-READ tool flow, discriminated statelessly from the messages + (the ``tools`` parameter gates the list/read steps — a no-tools + request with no tool results is not the flow): * request 1 (``tools`` offered, no tool results yet): stream ONLY - ``tool_calls`` deltas — ``list_documents`` (synthetic id - ``call_0``, no arguments), ``finish_reason: "tool_calls"``, no - content; + ``tool_calls`` deltas — ``ls`` (synthetic id ``call_0``, no + arguments), ``finish_reason: "tool_calls"``, no content; * request 2 (a ``tool``-role catalog result in the messages): parse the FIRST catalog line (``source: X | path: Y | title: Z`` — the labeled ``source:`` / ``path:`` fields, phase 63) and - stream a ``tool_calls`` delta calling ``read_document`` on it - (id ``call_1``); - * request 3 (a ``tool``-role read result in the messages): a - content answer, deterministic: ``Read . `` — so a suite can assert - the read document reached the model and landed in the answer. - Reached regardless of the ``tools`` parameter (phase 45 keeps - the tools offered until the round cap). + stream a ``tool_calls`` delta calling ``read`` on the JOINED + combined ``source/path`` (the mock joins the two labeled fields + — the catalog format is unchanged, so this join is the only + parse change, phase 70) (id ``call_1``); + * request 3 (a ``tool``-role read result in the messages — + content starting with the agent's ``"Document :"`` + header): a content answer, deterministic: ``Read + . `` — so a suite can assert the read document reached + the model and landed in the answer. Reached regardless of the + ``tools`` parameter (phase 45 keeps the tools offered until the + round cap). The single-read flow stops at ONE read result; the MULTI-READ variant below reads two. - user message containing BOTH ``use your tools`` AND ``read two @@ -82,15 +87,18 @@ Implements just enough of the aipi surface: system prompt carries the ```` section -> the deterministic MULTI-READ flow (list → read #1 → read #2 → answer), classified by the COUNT of ``tool``-role read results (content starting with the - agent's ``"Document :"`` prefix): - * 0 read results, no catalog yet: ``list_documents`` (id - ``call_0``); - * 0 read results, catalog present: ``read_document`` on the FIRST - catalog line (id ``call_1``); - * 1 read result: ``read_document`` on the SECOND catalog line — - the first listing line whose ``source/path`` differs from the - one already read (id ``call_2``); a one-document catalog - degenerates to the single-read answer (nothing second to read); + agent's ``"Document :"`` prefix); phase 70: the same + flow on the harness-aligned names — ``ls``, then ``read`` on the + JOINED combined ``source/path`` of each catalog line: + * 0 read results, no catalog yet: ``ls`` (id ``call_0``); + * 0 read results, catalog present: ``read`` on the JOINED + combined ``source/path`` of the FIRST catalog line + (id ``call_1``); + * 1 read result: ``read`` on the JOINED combined ``source/path`` + of the SECOND catalog line — the first listing line whose + ``source/path`` differs from the one already read (id + ``call_2``); a one-document catalog degenerates to the + single-read answer (nothing second to read); * 2 read results: the forced answer, byte-stable: the single-read shape quoting the FIRST read result, plus the line ``I read and .`` naming both read paths in read order — so a @@ -101,12 +109,13 @@ Implements just enough of the aipi surface: ``E2E_REAL_LLM=1`` ignores the mock entirely (the real model does what it does). - user message containing ``search your documents`` - (``SEARCH_TRIGGER``, phase 68, search tool) **and** the system - prompt carries the ```` section -> the deterministic SEARCH - tool flow, discriminated statelessly from the messages (streaming - only): + (``SEARCH_TRIGGER``, phase 68 search tool — renamed to the + harness-aligned ``grep`` in phase 70, same match/output contract) + **and** the system prompt carries the ```` section -> the + deterministic SEARCH tool flow, discriminated statelessly from the + messages (streaming only): * request 1 (``tools`` offered, no search result yet): stream - ONLY ``tool_calls`` deltas — ``search_documents`` with + ONLY ``tool_calls`` deltas — ``grep`` with ``{"pattern": SEARCH_PATTERN}`` (id ``call_0``); * request 2 (a ``tool``-role search result in the messages — recognizable by its ``source/path:line: text`` match lines or @@ -265,11 +274,12 @@ END_OF_NOTES_TRIGGER = "show the end of your notes" #: phase-24 tail echo targets the block, not the raw message tail). _DOCUMENTS_BLOCK_RE = re.compile(r".*?", re.S) -#: Phase 37 (agent-document-tools story): a user message containing this -#: substring (case-insensitive) — combined with the ```` section -#: in the system prompt — drives the deterministic tool flow documented -#: in the module docstring (list_documents → read_document on the first -#: catalog line → the quoted answer). Existing E2E questions do not +#: Phase 37 (agent-document-tools story; phase 70: the flow emits the +#: harness-aligned names): a user message containing this substring +#: (case-insensitive) — combined with the ```` section in the +#: system prompt — drives the deterministic tool flow documented in the +#: module docstring (ls → read on the first catalog line's combined +#: ``source/path`` → the quoted answer). Existing E2E questions do not #: contain the phrase, so every other suite is unaffected. TOOLS_TRIGGER = "use your tools" @@ -282,11 +292,12 @@ TOOLS_TRIGGER = "use your tools" #: so the 3-step flow is untouched. MULTI_READ_TRIGGER = "read two documents" -#: Phase 68 (search tool, TODO.md L4): a user message containing this -#: substring (case-insensitive) — combined with the ```` section -#: in the system prompt — drives the deterministic SEARCH tool flow -#: (search_documents for ``SEARCH_PATTERN`` → the "Found …" answer), -#: documented in the module docstring. Checked BEFORE ``TOOLS_TRIGGER`` +#: Phase 68 (search tool, TODO.md L4; phase 70: renamed to the +#: harness-aligned ``grep``): a user message containing this substring +#: (case-insensitive) — combined with the ```` section in the +#: system prompt — drives the deterministic SEARCH tool flow (grep for +#: ``SEARCH_PATTERN`` → the "Found …" answer), documented in the module +#: docstring. Checked BEFORE ``TOOLS_TRIGGER`` #: (the more specific phrase wins — the same convention as #: ``THINK_PARAS_TRIGGER``); verified 2026-09-01: no existing E2E #: question or fixture file contains the phrase, so every other suite @@ -402,11 +413,11 @@ def _chat_dead(key: str, dead_attempts: int) -> bool: return _bump_fail(key) <= dead_attempts * _HTTPS_PER_DEAD_ATTEMPT -#: The agent's ``read_document`` tool-result prefix (app.rag.agent +#: The agent's ``read`` tool-result prefix (app.rag.agent #: ``_execute_tool``): ``"Document :\n"``. _READ_RESULT_PREFIX = "Document " -#: One line of the agent's ``list_documents`` output (app.rag.agent +#: One line of the agent's ``ls`` output (app.rag.agent #: ``_execute_tool``, phase 63): labeled, pipe-delimited fields — #: ``source: X | path: Y | title: Z`` — unambiguous for LLM parsing even #: when the path contains ``/`` characters. @@ -439,7 +450,7 @@ def _catalog_docs(body: dict[str, Any]) -> list[tuple[str, str]]: """Every ``(source, path)`` in the catalog tool result, in listing order. Catalog lines are ``source: X | path: Y | title: Z`` (the agent's - ``list_documents`` output — phase 63: labeled, pipe-delimited + ``ls`` output — phase 63: labeled, pipe-delimited fields, unambiguous even for paths full of ``/``): the line-level regex recovers the ``source`` and ``path`` fields directly. The ``"N documents:"`` header line matches no line and is skipped; @@ -460,8 +471,9 @@ def _catalog_docs(body: dict[str, Any]) -> list[tuple[str, str]]: return docs -#: One line of the agent's ``search_documents`` output (app.rag.agent -#: ``_execute_tool``, phase 68): ``source/path:LINE: text``. The +#: One line of the agent's ``grep`` output (app.rag.agent +#: ``_execute_tool``, phase 68 — phase 70 renamed the tool, the line +#: format is unchanged): ``source/path:LINE: text``. The #: non-greedy prefix keeps nested paths (``/`` in the path) intact. _SEARCH_LINE_RE = re.compile(r"^(?P.+?):(?P\d+): (?P.*)$") @@ -472,7 +484,7 @@ def _search_result_line(body: dict[str, Any]) -> str | None: A search result is a ``tool``-role message — never a read result (those start with the agent's ``"Document "`` prefix) — that either carries ``source/path:LINE: text`` match lines (the agent's - ``search_documents`` output, phase 68) or the sentinel pattern + ``grep`` output, phase 68) or the sentinel pattern itself (its no-match line quotes the pattern). Returns the first match line's ``text`` part (already 200-char-capped server-side), or the message's first line in the sentinel-only shape, or ``None`` @@ -534,7 +546,9 @@ def _tool_flow(body: dict[str, Any]) -> tuple[str, ...] | None: * ``("read", source, path, "call_1")`` — a ``tool``-role catalog result is in the messages: the model reads its FIRST ``source: X | path: Y | title: Z`` line (the labeled - ``source:`` / ``path:`` fields, phase 63). + ``source:`` / ``path:`` fields, phase 63), emitted as ``read`` on + the JOINED combined ``source/path`` (phase 70: the mock joins + the two fields — the canonical document identity). * ``("answer", "source/path", content)`` — a ``tool``-role read result (``"Document :\n"``) is in the messages: the model answers, quoting the read document. Reached @@ -1054,7 +1068,7 @@ def chat_completions(body: dict[str, Any]) -> Any: if search_flow is not None: if search_flow[0] == "search": stream = _tool_call_stream( - "search_documents", {"pattern": SEARCH_PATTERN}, "call_0" + "grep", {"pattern": SEARCH_PATTERN}, "call_0" ) else: # "found" — quote the first matched line (80 chars) answer = _apply_max_tokens( @@ -1069,16 +1083,16 @@ def chat_completions(body: dict[str, Any]) -> Any: flow = _tool_flow(body) if flow is not None: if flow[0] == "list": - stream = _tool_call_stream("list_documents", {}, "call_0") + stream = _tool_call_stream("ls", {}, "call_0") elif flow[0] == "read": # flow[3] is the synthetic call id — "call_1" for the # single-read flow and the multi-read first read, # "call_2" for the multi-read second read (phase 45, - # task 02). + # task 02). Phase 70: the harness-aligned shape — one + # combined ``source/path`` argument (the mock joins the + # two catalog fields; the catalog format is unchanged). stream = _tool_call_stream( - "read_document", - {"source": flow[1], "path": flow[2]}, - flow[3], + "read", {"path": f"{flow[1]}/{flow[2]}"}, flow[3] ) elif flow[0] == "multi_answer": # Phase 45 (task 02): the multi-read forced answer — diff --git a/tests/e2e/test_agent_document_tools.py b/tests/e2e/test_agent_document_tools.py index 2b0de75..7d32d68 100644 --- a/tests/e2e/test_agent_document_tools.py +++ b/tests/e2e/test_agent_document_tools.py @@ -9,18 +9,20 @@ MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the real ``turbo`` does whatever it does with the tools, while this story's gate is the deterministic marker flow in ``tests/e2e/mock_llm.py`` (user message contains ``use your tools`` **and** the system prompt carries the -```` section of the HIGH prompt): +```` section of the HIGH prompt; phase 70: the flow emits the +harness-aligned names — ``ls`` / ``read`` with the combined +``source/path`` identity): 1. request 1 (``tools`` offered, no tool results yet) → streams ONLY - ``tool_calls`` deltas calling ``list_documents`` (id ``call_0``, no - arguments, ``finish_reason: "tool_calls"``); + ``tool_calls`` deltas calling ``ls`` (id ``call_0``, no arguments, + ``finish_reason: "tool_calls"``); 2. request 2 (a ``tool``-role catalog result in the messages) → streams a - ``tool_calls`` delta calling ``read_document`` on the FIRST catalog - line (id ``call_1``); -3. request 3 (no ``tools`` parameter, the read result in the messages) → - the content answer ``Read . `` — so the suite can assert the read document - reached the model and landed in the answer. + ``tool_calls`` delta calling ``read`` on the JOINED combined + ``source/path`` of the FIRST catalog line (id ``call_1``); +3. request 3 (a ``tool``-role read result in the messages) → the content + answer ``Read . `` — so the suite can assert the read document reached the + model and landed in the answer. KB fixture — reproduces the TODO failure (``aws-route53.md`` references ``example-record-file.json`` "for the exact JSON shape of @@ -43,10 +45,11 @@ reseelink.json" but does not include it): Test → story mapping (Playwright Mapping Rule): 1. ``test_marker_question_lists_reads_and_quotes`` — the SSE carries - ``tool`` frames (list, then read, ahead of any delta), the UI shows - the "calling tool" label while a tool runs, the bubble shows both - tool lines, the final answer quotes the read document, and the - source chips include the read document (viewer link). + ``tool`` frames (``ls``, then ``read`` with the combined path, ahead + of any delta), the UI shows the transient calling-tool status while a + tool runs, the bubble shows both tool lines, the final answer quotes + the read document, and the source chips include the read document + (viewer link). 2. ``test_tool_lines_re_render_after_reload`` — the persisted record (phase 14) re-renders the tool lines. 3. ``test_plain_grounded_question_has_no_tool_frames`` — no marker → no @@ -385,12 +388,13 @@ def test_marker_question_lists_reads_and_quotes( assert i_list is not None and i_read is not None, statuses assert i_list < i_read, statuses - # Wire level: exactly two `tool` frames — list then read — and both - # ahead of the first `delta` frame. + # Wire level: exactly two `tool` frames — ``ls`` then ``read`` (the + # combined source/path as the model passed it) — and both ahead of + # the first `delta` frame. frames = _frames(page) assert _tool_frames(frames) == [ - {"type": "tool", "name": "list_documents", "argument": None}, - {"type": "tool", "name": "read_document", "argument": READ_SP}, + {"type": "tool", "name": "ls", "argument": None}, + {"type": "tool", "name": "read", "argument": READ_SP}, ] first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta") assert all( diff --git a/tests/e2e/test_agent_unlimited_tools.py b/tests/e2e/test_agent_unlimited_tools.py index fa6fc3e..7f833b3 100644 --- a/tests/e2e/test_agent_unlimited_tools.py +++ b/tests/e2e/test_agent_unlimited_tools.py @@ -9,16 +9,20 @@ MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the gate is the deterministic MULTI-READ marker flow in ``tests/e2e/mock_llm.py`` (user message contains BOTH ``use your tools`` (``TOOLS_TRIGGER``) and ``read two documents`` (``MULTI_READ_TRIGGER``) **and** the system prompt -carries the ```` section of the HIGH prompt): +carries the ```` section of the HIGH prompt; phase 70: the flow +emits the harness-aligned names — ``ls``, then ``read`` on the JOINED +combined ``source/path`` of each catalog line): 1. request 1 (``tools`` offered, no tool results yet) → streams ONLY - ``tool_calls`` deltas calling ``list_documents`` (id ``call_0``); -2. request 2 (the ``tool``-role catalog result) → ``read_document`` on - the FIRST catalog line (id ``call_1``); -3. request 3 (one ``tool``-role read result) → ``read_document`` on the - SECOND catalog line (id ``call_2``) — the pre-phase-45 per-tool - budgets would have refused exactly this second read (``No reading - budget left — answer with what you have.``); + ``tool_calls`` deltas calling ``ls`` (id ``call_0``); +2. request 2 (the ``tool``-role catalog result) → ``read`` on the + JOINED combined ``source/path`` of the FIRST catalog line + (id ``call_1``); +3. request 3 (one ``tool``-role read result) → ``read`` on the JOINED + combined ``source/path`` of the SECOND catalog line (id ``call_2``) + — the pre-phase-45 per-tool budgets would have refused exactly this + second read (``No reading budget left — answer with what you + have.``); 4. request 4 (two read results) → the forced answer, byte-stable: the single-read shape quoting the FIRST read result, plus the line ``I read and .`` naming both read paths in read order. @@ -393,14 +397,15 @@ def test_multi_read_turn( _submit(page, MULTI_QUESTION) _wait_settled(page) - # Wire level: exactly THREE `tool` frames — list, read #1, read #2, - # in order — and all ahead of the first `delta` frame. This third + # Wire level: exactly THREE `tool` frames — ls, read #1, read #2 + # (each read's argument is the JOINED combined source/path), in + # order — and all ahead of the first `delta` frame. This third # frame is the one the pre-phase-45 read budget refused. frames = _frames(page) assert _tool_frames(frames) == [ - {"type": "tool", "name": "list_documents", "argument": None}, - {"type": "tool", "name": "read_document", "argument": READ1_SP}, - {"type": "tool", "name": "read_document", "argument": READ2_SP}, + {"type": "tool", "name": "ls", "argument": None}, + {"type": "tool", "name": "read", "argument": READ1_SP}, + {"type": "tool", "name": "read", "argument": READ2_SP}, ] first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta") assert all( @@ -518,7 +523,7 @@ def test_relist_allowed( # per-tool budgets would have refused (list budget 1, read budget # 1 — this turn makes one list and TWO reads). frames = _frames(page) - assert {"type": "tool", "name": "list_documents", "argument": None} in _tool_frames( + assert {"type": "tool", "name": "ls", "argument": None} in _tool_frames( frames ) line0 = page.locator(".msg.brain .tool-call").nth(0) @@ -556,12 +561,13 @@ def test_single_tool_flow_regression( _submit(page, SINGLE_QUESTION) _wait_settled(page) - # Exactly TWO tool frames — list then ONE read of the first catalog - # line — no second read (the marker carries no multi-read trigger). + # Exactly TWO tool frames — ls then ONE read of the first catalog + # line (the JOINED combined source/path) — no second read (the + # marker carries no multi-read trigger). frames = _frames(page) assert _tool_frames(frames) == [ - {"type": "tool", "name": "list_documents", "argument": None}, - {"type": "tool", "name": "read_document", "argument": READ1_SP}, + {"type": "tool", "name": "ls", "argument": None}, + {"type": "tool", "name": "read", "argument": READ1_SP}, ] lines = page.locator(".msg.brain .tool-call") expect(lines).to_have_count(2) diff --git a/tests/e2e/test_harness_aligned_tools.py b/tests/e2e/test_harness_aligned_tools.py new file mode 100644 index 0000000..70f996c --- /dev/null +++ b/tests/e2e/test_harness_aligned_tools.py @@ -0,0 +1,524 @@ +"""Phase 70 E2E (Playwright, mock-only): the harness-aligned tool surface +(``ls`` / ``read(path)`` / ``grep(pattern, path?)``). + +Story: ``.agent/user_stories/agent-document-tools.md`` (phase 70 reshapes +the tools that story delivered — owner decision 2026-09-03: "match +existing harnesses as much as possible", the pi.dev tool shapes). +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_harness_aligned_tools.py -v --no-cov + +MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the gate is the +deterministic marker flows in ``tests/e2e/mock_llm.py`` (phase 70: the +flows emit the NEW names with the NEW argument shapes): + +* the READ flow (``use your tools`` (``TOOLS_TRIGGER``) + the HIGH + prompt's ```` section): ``ls`` (id ``call_0``, no arguments) → + ``read`` on the JOINED combined ``source/path`` of the first catalog + line (id ``call_1``) → the ``Read . `` answer; +* the SEARCH flow (``search your documents`` (``SEARCH_TRIGGER``) + the + ```` section): ``grep`` with ``{"pattern": SEARCH_PATTERN}`` + (id ``call_0``) → the ``Found `` answer. + +The combined ``source/path`` string is the canonical document identity: +the mock joins the two labeled catalog fields itself (the catalog +format is unchanged), and the SSE ``tool`` frames carry exactly what the +model "passed" — ``read``'s combined path, ``grep``'s pattern, ``ls``'s +scope or null when unscoped (the phase-70 argument rule). + +KB fixtures: + +* READ flow — the ``test_agent_document_tools.py`` two-document pair + (TRUNCATE-then-seed): ``Homelab/aws-route53.md`` seeded with one + chunk whose embedding is the mock's own bag-of-words vector (the + marker question cosines ≈0.69 against it, well past the E2E 0.30 + threshold, and it FTS-matches too → grounded) and + ``Deployments/example-record-file.json`` indexed WITHOUT chunks (the + catalog-first line the mock reads; never in the retrieval context). +* SEARCH flow — the phase-68 fixture (``tests/fixtures/search_docs/``) + imported through the real importer, its line 6 carrying the sentinel + ``reese-sentinel-42`` exactly once (``test_search_tool.py`` pattern). + +Test → phase mapping (Playwright Mapping Rule): +1. ``test_read_flow_lines_answer_sources_no_raw_markup`` — the + grounded READ turn: the UI shows the ``ls`` line (unscoped "🔎 + Listing documents", no argument) then the "📄 Reading " + line with the combined path in a ```` element, the answer + streams and quotes the read document, the done-state sources + include the read document, and NO raw tool markup (``<|…|>``, + ``tool_call``) appears anywhere in the DOM — the live incident this + phase fixes. +2. ``test_grep_flow_line_then_answer`` — the grounded SEARCH turn: the + "🔎 Searching for " line (sentinel in ````) then the + matched-line answer. +3. ``test_wire_argument_rule_across_both_flows`` — the SSE wire across + BOTH flows in one session: every ``tool`` frame's name is in + {``ls``, ``read``, ``grep``} (no pre-phase-70 name ever reaches the + client) and the argument rule holds — ``read`` → the combined path + as passed, ``grep`` → the pattern, ``ls`` → null when unscoped. +""" +from __future__ import annotations + +import asyncio +import hashlib +import json +import time +from datetime import UTC, datetime +from pathlib import Path +from threading import Thread +from typing import Any + +from playwright.sync_api import Page, expect +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.config import Settings +from app.db import SessionLocal +from app.models import Chunk, Document +from app.rag.importer import ImportSummary, import_sources +from app.rag.llm import LLMClient +from tests.e2e.mock_llm import SEARCH_PATTERN, embed_text + +REPO = Path(__file__).resolve().parents[2] +FIXTURES = REPO / "tests" / "fixtures" / "search_docs" + +# -------------------------------------------------------------------------- +# READ flow — the two-document pair (cf. test_agent_document_tools.py) +# -------------------------------------------------------------------------- + +SEED_SOURCE = "Homelab" +SEED_PATH = "aws-route53.md" +SEED_SP = f"{SEED_SOURCE}/{SEED_PATH}" + +READ_SOURCE = "Deployments" +READ_PATH = "example-record-file.json" +READ_SP = f"{READ_SOURCE}/{READ_PATH}" + +#: The retrievable document (the grounded seed context): the repeated +#: record-file lines carry the marker question's key tokens — verified +#: ≈0.69 cosine against the mock's embeddings (E2E threshold 0.30) plus +#: FTS hits. +ROUTE53_CONTENT = ( + "# AWS Route 53 Notes\n\n" + "## Record file\n\n" + + ( + "The aws route53 hosted zone for reeselink keeps every record in " + "reseelink.json — the exact JSON shape of reeselink.json is " + "documented in example-record-file.json.\n" + ) + * 10 + + "\n## Sync job\n\n" + "A cron job pushes reeselink.json to the aws route53 hosted zone " + "every fifteen minutes; the diff is applied through the route53 api.\n" +) + +#: The read document (the catalog-first line the mock reads; no chunks, +#: so retrieval never puts it in context). Its FIRST line is longer than +#: 80 chars, so the mock's first-80-chars quote is newline-free. +RECORD_FILE_CONTENT = ( + '{ "version": 3, "comment": "ReeseLink hosted zone records — the exact ' + 'JSON shape of reeselink.json",\n' + ' "hosted_zone_id": "Z0RESEELINK01",\n' + ' "record_sets": [\n' + ' { "name": "www.reeselink.example", "type": "A", "ttl": 300,\n' + ' "resource_records": [ { "value": "10.0.0.20" } ] },\n' + ' { "name": "api.reeselink.example", "type": "CNAME", "ttl": 300,\n' + ' "resource_records": [ { "value": "www.reeselink.example" } ] }\n' + " ]\n" + "}\n" +) +assert "\n" not in RECORD_FILE_CONTENT[:80] # the quote must stay one line + +#: Carries ``TOOLS_TRIGGER`` (and nothing else — no multi-read, no +#: search, no other mock marker). +READ_QUESTION = ( + "Use your tools: what is the exact JSON shape of reeselink.json " + "for my aws route53 hosted zone?" +) +for _other in ( + "read two documents", + "search your documents", + "write a long answer", + "think in paragraphs", + "think out loud", + "show the end of your notes", + "show me a table", + "fail then answer", + "always fail", + "embed fail once", + "pretend to think slowly", +): + assert _other not in READ_QUESTION.lower(), _other + +READ_ANSWER_PREFIX = f"Read {READ_SP}." +READ_ANSWER_QUOTE = RECORD_FILE_CONTENT[:80] + + +def _seed_read_pair(db: Session) -> None: + """The two-document READ-flow KB (see the module docstring).""" + md = Document( + source=SEED_SOURCE, + path=SEED_PATH, + full_path=f"/tmp/{SEED_PATH}", + title="AWS Route 53 Notes", + content=ROUTE53_CONTENT, + content_hash=hashlib.sha256(ROUTE53_CONTENT.encode()).hexdigest(), + indexed_at=datetime.now(UTC), + ) + db.add(md) + db.flush() + # One chunk carrying the mock's own embedding → genuine token + # overlap between the marker question and this document (the only + # retrievable document). + db.add( + Chunk( + document_id=md.id, + position=0, + content=ROUTE53_CONTENT, + embedding=embed_text(ROUTE53_CONTENT), + ) + ) + db.add( + Document( + source=READ_SOURCE, + path=READ_PATH, + full_path=f"/tmp/{READ_PATH}", + title="Example Record File", + content=RECORD_FILE_CONTENT, + content_hash=hashlib.sha256(RECORD_FILE_CONTENT.encode()).hexdigest(), + indexed_at=datetime.now(UTC), + ) + ) + + +# -------------------------------------------------------------------------- +# SEARCH flow — the phase-68 fixture (cf. test_search_tool.py) +# -------------------------------------------------------------------------- + +SEED_SOURCE_S = "search_docs" +SEED_PATH_S = "reese-notes.md" +SEED_SP_S = f"{SEED_SOURCE_S}/{SEED_PATH_S}" + +#: The fixture's sentinel line (line 6) — the mock's grep matches it +#: exactly once; its ``text`` part is what the "Found …" answer quotes. +SENTINEL_LINE = f"The offsite vault passphrase marker is {SEARCH_PATTERN}." +FOUND_ANSWER = f"Found {SENTINEL_LINE[:80]}" + +#: Carries ``SEARCH_TRIGGER`` and is on-topic (cosine ≈0.51 against the +#: fixture + FTS hits → HIGH gate, the ```` section rides along). +SEARCH_QUESTION = ( + "Search your documents for the vault passphrase marker in my homelab " + "kubernetes backup notes?" +) +assert SEARCH_PATTERN.lower() not in SEARCH_QUESTION.lower() + + +def _pin_fixture() -> None: + """The fixture carries the sentinel on line 6, exactly once.""" + content = (FIXTURES / SEED_PATH_S).read_text(encoding="utf-8") + lines = content.split("\n") + assert lines[5] == SENTINEL_LINE, lines[5] + assert sum(SEARCH_PATTERN in line for line in lines) == 1 + + +# -------------------------------------------------------------------------- +# DB seeding (TRUNCATE-then-seed / TRUNCATE-then-import) +# -------------------------------------------------------------------------- + + +async def _import_search_fixtures(mock_port: int) -> ImportSummary: + kwargs: dict[str, Any] = { + "_env_file": None, + "llm_base_url": f"http://127.0.0.1:{mock_port}/v1", + } + settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] + return await import_sources([FIXTURES], LLMClient(settings)) + + +def _run_in_thread(coro: Any) -> Any: + """Run a coroutine on a worker thread. + + Playwright's sync API keeps an asyncio loop running on the test + thread, so ``asyncio.run`` cannot be called directly from a test + body (the established house helper). + """ + box: dict[str, Any] = {} + + def runner() -> None: + try: + box["value"] = asyncio.run(coro) + except BaseException as e: # noqa: BLE001 — re-raised on the test thread + box["error"] = e + + t = Thread(target=runner) + t.start() + t.join() + if "error" in box: + raise box["error"] + return box["value"] + + +def _reset_db_read_pair() -> None: + """Truncate the KB (plus the prompt-shaping tables), then seed the + two-document READ-flow pair. ``steering_notes`` / ``kb_overview`` + are truncated too, so the HIGH prompt is exactly ```` + + ```` + ```` — byte-stable prompts, byte-stable + answers.""" + with SessionLocal() as db: + db.execute( + text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview") + ) + db.commit() + _seed_read_pair(db) + db.commit() + + +def _reset_db_search_fixture(mock_port: int) -> None: + """Truncate the KB (plus the prompt-shaping tables), then import the + phase-68 search fixture through the real importer.""" + with SessionLocal() as db: + db.execute( + text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview") + ) + db.commit() + summary = _run_in_thread(_import_search_fixtures(mock_port)) + assert summary is not None and summary.added == 1, summary + + +# -------------------------------------------------------------------------- +# Page helpers (the test_agent_document_tools.py pattern) +# -------------------------------------------------------------------------- + +#: Captures the raw SSE ``data:`` payloads of the /api/chat stream +#: (a response clone read in the background) — wire-level assertions +#: for the ``tool`` frames, independent of the UI rendering. +SSE_HOOK = """ +() => { + if (window.__sseInstalled) return; + window.__sseInstalled = true; + window.__sseFrames = []; + const origFetch = window.fetch; + window.fetch = async function (...args) { + const res = await origFetch.apply(this, args); + try { + const url = typeof args[0] === 'string' ? args[0] : args[0].url; + if (url.includes('/api/chat')) { + res.clone().text().then((bodyText) => { + for (const block of bodyText.split('\\n\\n')) { + const line = block.trim(); + if (line.startsWith('data: ')) { + window.__sseFrames.push(line.slice(6)); + } + } + }); + } + } catch (e) { /* non-clonable responses: ignored */ } + return res; + }; +} +""" + + +def _install_sse_hook(page: Page) -> None: + page.evaluate(SSE_HOOK) + + +def _drain_frames(page: Page) -> list[dict]: + """One turn's SSE frames: wait for that turn's ``done`` frame, then + return EVERY frame captured since the last drain (the hook's + background read appends the whole stream at once after it closes, so + clearing-and-reading is race-free per turn).""" + deadline = time.monotonic() + 10.0 + while True: + raw = page.evaluate( + "() => { const f = window.__sseFrames || []; " + "window.__sseFrames = []; return f; }" + ) + parsed = [json.loads(line) for line in raw if line] + if any(f.get("type") == "done" for f in parsed): + return parsed + if time.monotonic() > deadline: + raise AssertionError( + f"SSE hook captured no `done` frame (frames so far: " + f"{len(parsed)}) — hook install failed?" + ) + time.sleep(0.05) + + +def _tool_frames(frames: list[dict]) -> list[dict]: + return [f for f in frames if f.get("type") == "tool"] + + +def _submit(page: Page, question: str) -> None: + page.fill("#message-input", question) + page.click("#send-btn") + # The user bubble lands synchronously with the submit handler. + expect(page.locator(".msg.user .bubble").last).to_contain_text(question) + + +def _wait_settled(page: Page) -> None: + """The turn is complete: answer text in the bubble, button recovered. + + Phase 48: the label assertion carries the settle wait with an + explicit timeout — the in-flight button is the enabled Stop control + (never disabled), so ``to_be_enabled`` no longer blocks until the + turn settles.""" + expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000) + expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000) + expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000) + + +# -------------------------------------------------------------------------- +# 1. The grounded READ turn: ls line → Reading line → quoted answer, +# sources include the read doc, no raw tool markup anywhere in the DOM +# -------------------------------------------------------------------------- + + +def test_read_flow_lines_answer_sources_no_raw_markup( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + page.set_default_timeout(30_000) + _reset_db_read_pair() + page.goto(app_url) + _install_sse_hook(page) + + _submit(page, READ_QUESTION) + _wait_settled(page) + + # The UI shows the ls line (UNSCOPED — no argument, no ) then + # the "📄 Reading " line with the COMBINED path in a + # element (the path is data, never markup). + lines = page.locator(".msg.brain .tool-call") + expect(lines).to_have_count(2) + expect(lines.nth(0)).to_contain_text("Listing documents") + expect(lines.nth(0).locator("code")).to_have_count(0) + expect(lines.nth(1)).to_contain_text("Reading ") + expect(lines.nth(1).locator("code")).to_have_text(READ_SP) + + # The answer streamed and quotes the read document (the mock's + # deterministic echo: "Read . "). + bubble = page.locator(".msg.brain .bubble").last + expect(bubble).to_contain_text(READ_ANSWER_PREFIX) + expect(bubble).to_contain_text(READ_ANSWER_QUOTE) + + # Wire level: ls then read — the phase-70 argument rule (ls + # unscoped → null; read → the combined path as passed) — ahead of + # the first delta. + frames = _drain_frames(page) + assert _tool_frames(frames) == [ + {"type": "tool", "name": "ls", "argument": None}, + {"type": "tool", "name": "read", "argument": READ_SP}, + ] + first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta") + assert all( + i < first_delta for i, f in enumerate(frames) if f.get("type") == "tool" + ) + done = next(f for f in frames if f.get("type") == "done") + assert done["deflected"] is False + # Done-state sources include the read document (retrieval doc first, + # the agent's read doc after — the phase-37 extension contract). + assert [(s["source"], s["path"]) for s in done["sources"]] == [ + (SEED_SOURCE, SEED_PATH), + (READ_SOURCE, READ_PATH), + ] + + # The live incident this phase fixes: NO raw tool markup anywhere in + # the DOM — the model's trained wire shapes (<|tool_call_…|>, + # "tool_calls", finish_reason) must never leak into the rendered + # conversation. + dom = page.locator("#messages").inner_html() + for raw in ("<|", "tool_call", "tool_calls", "finish_reason"): + assert raw not in dom, f"raw tool markup {raw!r} leaked into the DOM" + + +# -------------------------------------------------------------------------- +# 2. The grounded SEARCH turn: the "🔎 Searching for " line, +# then the matched-line answer +# -------------------------------------------------------------------------- + + +def test_grep_flow_line_then_answer( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + _pin_fixture() + page.set_default_timeout(30_000) + _reset_db_search_fixture(mock_llm) + page.goto(app_url) + _install_sse_hook(page) + + _submit(page, SEARCH_QUESTION) + _wait_settled(page) + + # ONE tool line above the answer: "🔎 Searching for " + the sentinel + # in a element (the pattern is data, never markup). + lines = page.locator(".msg.brain .tool-call") + expect(lines).to_have_count(1) + expect(lines.nth(0)).to_contain_text("Searching for") + expect(lines.nth(0).locator("code")).to_have_text(SEARCH_PATTERN) + + # The answer quotes the MATCHED LINE — the grep result reached the + # model and landed in the answer (the mock's deterministic echo). + bubble = page.locator(".msg.brain .bubble").last + expect(bubble).to_contain_text(FOUND_ANSWER) + + # Wire level: exactly ONE tool frame — grep carrying the PATTERN as + # its argument (the phase-70 argument rule) — ahead of the first + # delta; the turn is grounded. + frames = _drain_frames(page) + assert _tool_frames(frames) == [ + {"type": "tool", "name": "grep", "argument": SEARCH_PATTERN} + ] + first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta") + assert all( + i < first_delta for i, f in enumerate(frames) if f.get("type") == "tool" + ) + done = next(f for f in frames if f.get("type") == "done") + assert done["deflected"] is False + + +# -------------------------------------------------------------------------- +# 3. The SSE wire across BOTH flows: every tool frame carries a +# phase-70 name and the single-string argument rule +# -------------------------------------------------------------------------- + + +def test_wire_argument_rule_across_both_flows( + page: Page, app_url: str, mock_llm: int, db_ready: None +) -> None: + _pin_fixture() + page.set_default_timeout(30_000) + _reset_db_read_pair() + page.goto(app_url) + _install_sse_hook(page) + + # Turn 1 — the READ flow (ls → read on the combined path). + _submit(page, READ_QUESTION) + _wait_settled(page) + read_frames = _drain_frames(page) + + # Turn 2 — re-seed the search fixture, then the SEARCH flow (grep + # for the sentinel). The app's chat path is single-turn (system + + # user message), so the first turn cannot influence this one. + _reset_db_search_fixture(mock_llm) + _submit(page, SEARCH_QUESTION) + _wait_settled(page) + search_frames = _drain_frames(page) + + read_tools = _tool_frames(read_frames) + search_tools = _tool_frames(search_frames) + # The ordered, combined tool-frame sequence across both flows: the + # argument rule end-to-end — read → the combined path as passed, + # grep → the pattern, ls → null when unscoped. + assert read_tools + search_tools == [ + {"type": "tool", "name": "ls", "argument": None}, + {"type": "tool", "name": "read", "argument": READ_SP}, + {"type": "tool", "name": "grep", "argument": SEARCH_PATTERN}, + ] + # No pre-phase-70 name ever reaches the client. + for frame in read_tools + search_tools: + assert frame["name"] in {"ls", "read", "grep"}, frame + assert frame["argument"] is None or isinstance(frame["argument"], str) + + # And both turns answered (neither flow stalled at a tool round). + assert next(f for f in read_frames if f["type"] == "done")["deflected"] is False + assert next(f for f in search_frames if f["type"] == "done")["deflected"] is False diff --git a/tests/e2e/test_search_tool.py b/tests/e2e/test_search_tool.py index e87d2d3..444e92f 100644 --- a/tests/e2e/test_search_tool.py +++ b/tests/e2e/test_search_tool.py @@ -1,4 +1,6 @@ -"""Phase 68 E2E (Playwright, mock-only): the ``search_documents`` tool. +"""Phase 68 E2E (Playwright, mock-only): the ``grep`` tool (the +phase-68 search tool, renamed to the harness-aligned ``grep`` in +phase 70; the A5 match/output contract is unchanged). Story: n/a (TODO-derived — the owner roadmap confirmation 2026-09-01, TODO.md L4: "Add a search tool that allows the LLM to grep through the @@ -14,8 +16,8 @@ message contains ``search your documents`` (``SEARCH_TRIGGER``) prompt): 1. request 1 (``tools`` offered, no search result yet) → streams ONLY - ``tool_calls`` deltas calling ``search_documents`` with - ``{"pattern": SEARCH_PATTERN}`` (id ``call_0``); + ``tool_calls`` deltas calling ``grep`` with ``{"pattern": + SEARCH_PATTERN}`` (id ``call_0``); 2. request 2 (a ``tool``-role search result — the ``source/path:line: text`` match line) → the content answer ``Found `` — so this @@ -40,13 +42,13 @@ shadow the phase-37/45 flows and vice versa). Test → phase mapping: 1. ``test_search_flow_searches_and_answers_from_match`` — the live - search flow: the SSE carries the ``tool`` frame - (``search_documents`` with ``argument = ``, ahead of any - delta), #send-status recorded the transient "… is searching for - " state, the bubble shows ONE ``🔎 Searching for`` - tool line with the sentinel in a ```` element, the answer - quotes the matched line (``Found …`` — the match reached the - model), and the turn settles to idle with no error banner. + search flow: the SSE carries the ``tool`` frame (``grep`` with + ``argument = ``, ahead of any delta), #send-status + recorded the transient "… is searching for " state, the + bubble shows ONE ``🔎 Searching for`` tool line with the sentinel in + a ```` element, the answer quotes the matched line + (``Found …`` — the match reached the model), and the turn settles to + idle with no error banner. 2. ``test_search_adds_no_source_by_itself`` — context accounting (locked A5): the search-only flow (no read) leaves ``done.sources`` / the source chips / ``query_log.sources`` at the @@ -367,12 +369,12 @@ def test_search_flow_searches_and_answers_from_match( ) assert i_think is not None and i_think < i_search, statuses - # Wire level: exactly ONE `tool` frame — search_documents carrying - # the PATTERN as its argument (phase 68 task 02) — ahead of the - # first `delta` frame. + # Wire level: exactly ONE `tool` frame — grep carrying the PATTERN + # as its argument (phase 68 task 02; phase 70 renamed the tool) — + # ahead of the first `delta` frame. frames = _frames(page) assert _tool_frames(frames) == [ - {"type": "tool", "name": "search_documents", "argument": SEARCH_PATTERN} + {"type": "tool", "name": "grep", "argument": SEARCH_PATTERN} ] first_delta = next(i for i, f in enumerate(frames) if f.get("type") == "delta") assert all( @@ -424,7 +426,7 @@ def test_search_adds_no_source_by_itself( # baseline: the one fixture doc, nothing added by the search. frames = _frames(page) assert _tool_frames(frames) == [ - {"type": "tool", "name": "search_documents", "argument": SEARCH_PATTERN} + {"type": "tool", "name": "grep", "argument": SEARCH_PATTERN} ] done = next(f for f in frames if f.get("type") == "done") assert done["deflected"] is False diff --git a/tests/integration/test_agent_tools.py b/tests/integration/test_agent_tools.py index ec8d1b2..02a6734 100644 --- a/tests/integration/test_agent_tools.py +++ b/tests/integration/test_agent_tools.py @@ -1,17 +1,19 @@ -"""Integration: the agent DB accessors against real Postgres (phase 37). +"""Integration: the agent DB accessors against real Postgres (phase 37; +the harness-aligned ``ls``/``read``/``grep`` surface, phase 70). -``list_catalog`` must order rows by ``(source, path)`` — the same order as -``GET /api/docs`` — and ``find_document`` must resolve a hit to the full -document row (content included, for the never-truncated read) and return -``None`` for unknown ``source``/``path`` pairs. Phase 68: the -``search_documents`` tool is pinned here too — its locked parameter -shape in ``AGENT_TOOLS``, and a scripted ``ToolCallPiece`` executed -through ``run_agent`` against the real DB (``all_documents`` for a -whole-KB search, ``find_document`` for a scoped one). The -combined-form self-correction (a ``source`` argument carrying -``source/path``) is pinned here as well, through ``run_agent``: -the split read executes against the real table, and a still-unknown -split gets the educational refusal. +``list_catalog`` must order rows by ``(source, path)`` — the same order +as ``GET /api/docs`` — ``list_source_names`` must resolve the +registered source names (the scoped ``ls`` join), and ``find_document`` +must resolve a hit to the full document row (content included, for the +never-truncated read) and return ``None`` for unknown pairs. Phase 70: +the ``ls``/``read``/``grep`` tools are pinned here too — the locked +parameter shape in ``AGENT_TOOLS``, and scripted ``ToolCallPiece``s +executed through ``run_agent`` against the real DB: ``ls`` scoped to a +registered source name (unknown name → refusal), ``read`` on the +canonical combined ``source/path`` form (first-slash split; a bare +source name and an unknown identity get the no-document refusal), and +``grep`` (``all_documents`` for a whole-KB search, ``find_document`` for +a scoped one). Requires: podman compose up -d db """ @@ -24,11 +26,11 @@ from copy import deepcopy from typing import Any, cast import pytest -from sqlalchemy import text +from sqlalchemy import delete, text from sqlalchemy.orm import Session from app.config import Settings -from app.models import Document +from app.models import Document, GitSource from app.rag import agent from app.rag.agent import AGENT_TOOLS, AgentHolder, run_agent from app.rag.llm import LLMClient, RetryPiece, StreamPiece, ToolCallPiece @@ -58,6 +60,19 @@ def kb(db) -> Iterator[None]: db.commit() +@pytest.fixture() +def src(db) -> Iterator[GitSource]: + """One registered git source — the scoped ``ls`` source-name check + reads the real registry, so the row is inserted and deleted around + the tests (``repo_name`` resolves the URL to ``Homelab``).""" + row = GitSource(url="https://github.com/reese/Homelab.git", kind="git") + db.add(row) + db.commit() + yield row + db.execute(delete(GitSource).where(GitSource.id == row.id)) + db.commit() + + def test_list_catalog_orders_by_source_then_path(kb, db) -> None: _doc(db, "Zeta", "b/second.md", "Zeta B", "ZB") _doc(db, "Zeta", "a/first.md", "Zeta A", "ZA") @@ -75,6 +90,23 @@ def test_list_catalog_is_empty_without_rows(kb, db) -> None: assert agent.list_catalog(db) == [] +def test_list_source_names_resolves_registry_rows(db) -> None: + """The real registry: git names resolve through the import pipeline's + ``repo_name`` (trailing ``.git`` stripped); a second row resolving to + the same name (the phase-69 sibling case) is listed once.""" + a = GitSource(url="https://github.com/reese/Homelab.git", kind="git") + b = GitSource(url="https://github.com/reese/Homelab", kind="git") # sibling + c = GitSource(url="https://e.com/deployments", kind="git") + db.add_all([a, b, c]) + db.commit() + try: + assert agent.list_source_names(db).count("Homelab") == 1 # deduped + assert "deployments" in agent.list_source_names(db) + finally: + db.execute(delete(GitSource).where(GitSource.id.in_([a.id, b.id, c.id]))) + db.commit() + + def test_find_document_hit_returns_full_row(kb, db) -> None: created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT") db.commit() @@ -97,36 +129,29 @@ def test_find_document_none_for_unknown_pairs(kb, db) -> None: assert agent.find_document(db, "nope", "nope.md") is None # nothing at all -# ---------- search_documents (phase 68) ---------- +# ---------- AGENT_TOOLS surface (phase 70: ls / read / grep) ---------- -def test_all_documents_orders_by_source_then_path(kb, db) -> None: - _doc(db, "Zeta", "b/second.md", "Zeta B", "ZB") - _doc(db, "Zeta", "a/first.md", "Zeta A", "ZA") - _doc(db, "Alpha", "c/third.md", "Alpha C", "AC") - db.commit() - - docs = agent.all_documents(db) - assert [(d.source, d.path) for d in docs] == [ - ("Alpha", "c/third.md"), - ("Zeta", "a/first.md"), - ("Zeta", "b/second.md"), - ] - assert [d.content for d in docs] == ["AC", "ZA", "ZB"] # full rows - - -def test_agent_tools_offers_search_documents_with_locked_shape() -> None: +def test_agent_tools_offers_the_harness_aligned_surface() -> None: by_name = {t["function"]["name"]: t for t in AGENT_TOOLS} - assert list(by_name) == [ # the third tool, in order - "list_documents", - "read_document", - "search_documents", + assert list(by_name) == [ # the harness order, phase 70 + "ls", + "read", + "grep", ] - search = by_name["search_documents"]["function"]["parameters"] - assert search["type"] == "object" - assert search["required"] == ["pattern"] - assert set(search["properties"]) == {"pattern", "source", "path"} - assert all(p["type"] == "string" for p in search["properties"].values()) + ls = by_name["ls"]["function"]["parameters"] + assert ls["type"] == "object" + assert ls["required"] == [] # path is optional + assert set(ls["properties"]) == {"path"} + read = by_name["read"]["function"]["parameters"] + assert read["type"] == "object" + assert read["required"] == ["path"] + assert set(read["properties"]) == {"path"} + grep = by_name["grep"]["function"]["parameters"] + assert grep["type"] == "object" + assert grep["required"] == ["pattern"] + assert set(grep["properties"]) == {"pattern", "path"} + assert all(p["type"] == "string" for p in grep["properties"].values()) class ScriptedToolLLM: @@ -156,26 +181,12 @@ def _settings(**kwargs: Any) -> Settings: return Settings(**kwargs) # pyright: ignore[reportCallIssue] -def _run_search( - db: Session, arguments: dict[str, Any] +def _run_call( + db: Session, name: str, arguments: dict[str, Any] ) -> tuple[AgentHolder, ScriptedToolLLM]: - """Drive one scripted ``search_documents`` call through ``run_agent``.""" + """Drive one scripted tool call through ``run_agent``.""" holder = AgentHolder() - llm = ScriptedToolLLM( - ToolCallPiece(id="call_1", name="search_documents", arguments=arguments) - ) - asyncio.run(_consume(llm, db, holder)) - return holder, llm - - -def _run_read( - db: Session, arguments: dict[str, Any] -) -> tuple[AgentHolder, ScriptedToolLLM]: - """Drive one scripted ``read_document`` call through ``run_agent``.""" - holder = AgentHolder() - llm = ScriptedToolLLM( - ToolCallPiece(id="call_1", name="read_document", arguments=arguments) - ) + llm = ScriptedToolLLM(ToolCallPiece(id="call_1", name=name, arguments=arguments)) asyncio.run(_consume(llm, db, holder)) return holder, llm @@ -197,12 +208,113 @@ async def _consume( return out -def test_search_whole_kb_through_run_agent(kb, db) -> None: +# ---------- ls (scoped through the real registry) ---------- + + +def test_ls_scoped_to_registered_source_through_run_agent(kb, src, db) -> None: + _doc(db, "Homelab", "a.md", "A", "A-CONTENT") + _doc(db, "Other", "b.md", "B", "B-CONTENT") + db.commit() + + holder, llm = _run_call(db, "ls", {"path": "Homelab"}) + + # Offered: the first request carries AGENT_TOOLS (the 3-tool list). + assert llm.requests[0][1] == AGENT_TOOLS + # Executed against the real DB: the listing filtered to the source. + assert llm.requests[1][0][3]["content"] == ( + "1 documents:\nsource: Homelab | path: a.md | title: A" + ) + assert holder.tool_calls == 1 + assert holder.read_docs == [] + + +def test_ls_scoped_unknown_source_refused_through_run_agent(kb, src, db) -> None: + _doc(db, "Homelab", "a.md", "A", "A-CONTENT") + db.commit() + + holder, llm = _run_call(db, "ls", {"path": "Ghost"}) + + assert ( + llm.requests[1][0][3]["content"] == "No source named 'Ghost' — check the ls output." + ) + assert holder.tool_calls == 0 and holder.read_docs == [] + + +# ---------- read (the canonical combined source/path form) ---------- + + +def test_read_combined_path_through_run_agent(kb, db) -> None: + """The combined ``source/path`` identity resolves at the FIRST slash + against the REAL table (a path with further slashes included): the + read executes, the holder records the row, the result header carries + the true source/path.""" + created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT") + db.commit() + + holder, llm = _run_call(db, "read", {"path": "Alpha/deep/nested/doc.md"}) + + assert llm.requests[1][0][3]["content"] == ( + "Document Alpha/deep/nested/doc.md:\nFULL-TEXT" + ) + assert holder.tool_calls == 1 + assert holder.read_docs == [created] + + +def test_read_bare_source_name_refused_through_run_agent(kb, db) -> None: + """A bare source name (no '/') can never be a document — the + no-document refusal echoing the argument as passed; the old + split-teaching refusal is gone (phase 70).""" + _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT") + db.commit() + + holder, llm = _run_call(db, "read", {"path": "Alpha"}) + + assert ( + llm.requests[1][0][3]["content"] == "No document at 'Alpha' — check the ls output." + ) + assert holder.tool_calls == 0 and holder.read_docs == [] + + +def test_read_unknown_combined_path_refused_through_run_agent(kb, db) -> None: + """A combined identity that matches nothing gets the no-document + refusal (the argument echoed as passed — the model sees its own + form).""" + _doc(db, "Alpha", "x.md", "X", "X-CONTENT") + db.commit() + + holder, llm = _run_call(db, "read", {"path": "Alpha/nope/deep.md"}) + + assert ( + llm.requests[1][0][3]["content"] + == "No document at 'Alpha/nope/deep.md' — check the ls output." + ) + assert holder.tool_calls == 0 and holder.read_docs == [] + + +# ---------- grep (the phase-68 A5 contract under the new name) ---------- + + +def test_all_documents_orders_by_source_then_path(kb, db) -> None: + _doc(db, "Zeta", "b/second.md", "Zeta B", "ZB") + _doc(db, "Zeta", "a/first.md", "Zeta A", "ZA") + _doc(db, "Alpha", "c/third.md", "Alpha C", "AC") + db.commit() + + docs = agent.all_documents(db) + assert [(d.source, d.path) for d in docs] == [ + ("Alpha", "c/third.md"), + ("Zeta", "a/first.md"), + ("Zeta", "b/second.md"), + ] + assert [d.content for d in docs] == ["AC", "ZA", "ZB"] # full rows + + +def test_grep_whole_kb_through_run_agent(kb, db) -> None: _doc(db, "Beta", "b/two.md", "Two", "no hit\nNEEDLE in two\nlast") _doc(db, "Alpha", "a/one.md", "One", "first\nneedle in one\nthird") db.commit() - holder, llm = _run_search(db, {"pattern": "needle"}) + holder, llm = _run_call(db, "grep", {"pattern": "needle"}) # Offered: the first request carries AGENT_TOOLS (the 3-tool list). assert llm.requests[0][1] == AGENT_TOOLS @@ -212,17 +324,15 @@ def test_search_whole_kb_through_run_agent(kb, db) -> None: "Beta/b/two.md:2: NEEDLE in two" ) assert holder.tool_calls == 1 - assert holder.read_docs == [] # locked A5: search adds no context + assert holder.read_docs == [] # locked A5: grep adds no context -def test_search_scoped_through_run_agent(kb, db) -> None: +def test_grep_scoped_through_run_agent(kb, db) -> None: _doc(db, "Alpha", "a/one.md", "One", "first\nNeedle here\nthird") _doc(db, "Beta", "b/two.md", "Two", "NEEDLE too") db.commit() - holder, llm = _run_search( - db, {"pattern": "needle", "source": "Alpha", "path": "a/one.md"} - ) + holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/a/one.md"}) # Only the named document is searched — the other one's hit is absent. assert llm.requests[1][0][3]["content"] == "Alpha/a/one.md:2: Needle here" @@ -230,90 +340,27 @@ def test_search_scoped_through_run_agent(kb, db) -> None: assert holder.read_docs == [] -def test_search_scoped_missing_doc_refused_through_run_agent(kb, db) -> None: +def test_grep_scoped_missing_doc_refused_through_run_agent(kb, db) -> None: _doc(db, "Alpha", "a/one.md", "One", "nothing") db.commit() - holder, llm = _run_search( - db, {"pattern": "needle", "source": "Alpha", "path": "ghost.md"} - ) + holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/ghost.md"}) assert ( llm.requests[1][0][3]["content"] - == "No document at Alpha/ghost.md — check the list_documents output." + == "No document at 'Alpha/ghost.md' — check the ls output." ) assert holder.tool_calls == 0 and holder.read_docs == [] -# ---------- combined 'source/path' self-correction (read_document) ---------- - - -def test_read_combined_source_self_corrects_through_run_agent(kb, db) -> None: - """The model's combined 'source' ('Alpha/deep/nested/doc.md') resolves - through the first-slash split against the REAL table: the read - executes, the holder records the row, the result header carries the - true source/path.""" - created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT") - db.commit() - - holder, llm = _run_read( - db, - { - "source": "Alpha/deep/nested/doc.md", - "path": "deep/nested/doc.md", - }, - ) - - assert llm.requests[1][0][3]["content"] == ( - "Document Alpha/deep/nested/doc.md:\nFULL-TEXT" - ) - assert holder.tool_calls == 1 - assert holder.read_docs == [created] - - -def test_read_combined_source_later_slash_split_through_run_agent(kb, db) -> None: - """source='Alpha/deep' + path='nested/doc.md' (a split at a LATER - slash) resolves via the continuation candidate against the real - table.""" - created = _doc(db, "Alpha", "deep/nested/doc.md", "The Doc", "FULL-TEXT") - db.commit() - - holder, llm = _run_read( - db, {"source": "Alpha/deep", "path": "nested/doc.md"} - ) - - assert llm.requests[1][0][3]["content"] == ( - "Document Alpha/deep/nested/doc.md:\nFULL-TEXT" - ) - assert holder.tool_calls == 1 - assert holder.read_docs == [created] - - -def test_read_combined_source_refusal_teaches_split(kb, db) -> None: - """A combined source that matches nothing (even split) gets the - educational refusal naming the corrected arguments.""" - _doc(db, "Alpha", "x.md", "X", "X-CONTENT") - db.commit() - - holder, llm = _run_read( - db, {"source": "Alpha/nope/deep.md", "path": "nope/deep.md"} - ) - - assert llm.requests[1][0][3]["content"] == ( - "source must not contain '/': for 'Alpha/nope/deep.md' call " - "read_document(source='Alpha', path='nope/deep.md')." - ) - assert holder.tool_calls == 0 and holder.read_docs == [] - - -def test_search_no_matches_through_run_agent(kb, db) -> None: +def test_grep_no_matches_through_run_agent(kb, db) -> None: _doc(db, "Alpha", "a/one.md", "One", "nothing matching") db.commit() - holder, llm = _run_search(db, {"pattern": "zebra"}) + holder, llm = _run_call(db, "grep", {"pattern": "zebra"}) assert llm.requests[1][0][3]["content"] == ( "No matches for 'zebra' in the knowledge base." ) - assert holder.tool_calls == 1 # an executed search with zero hits + assert holder.tool_calls == 1 # an executed grep with zero hits assert holder.read_docs == [] diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 61bc366..7ba3e7a 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -300,11 +300,12 @@ def test_ui_chrome_has_no_emoji(client, path: str) -> None: Phase 37 revision (owner permission 2026-08-26, PLAN §4): the agent's ``.tool-call`` line carries the CONTENT marks — 🔎 (list) and 📄 (read) — the only emoji in the whole frontend, and only as the exact - tool-line template strings in app.js. Phase 68 revision: the - ``search_documents`` tool line adds the third template literal - ("🔎 Searching for "). The guard strips precisely those three - literals; any other emoji, or those marks anywhere else, still - fails.""" + tool-line template strings in app.js. Phase 68 revision: the search + tool line (the ``grep`` tool, phase 70) adds the third template + literal ("🔎 Searching for "). Phase 70 revision: the scoped ``ls`` + tool line adds the fourth ("🔎 Listing documents in "). The guard strips + precisely those four literals; any other emoji, or those marks + anywhere else, still fails.""" r = client.get(path) assert r.status_code == 200 text = r.text @@ -312,6 +313,7 @@ def test_ui_chrome_has_no_emoji(client, path: str) -> None: text = text.replace('"🔎 Listing documents"', "") text = text.replace('"📄 Reading "', "") text = text.replace('"🔎 Searching for "', "") + text = text.replace('"🔎 Listing documents in "', "") assert _find_emoji(text) == [], f"emoji found in {path}: {_find_emoji(text)!r}" diff --git a/tests/integration/test_chat_api.py b/tests/integration/test_chat_api.py index 8bdee06..acf4139 100644 --- a/tests/integration/test_chat_api.py +++ b/tests/integration/test_chat_api.py @@ -22,12 +22,12 @@ from typing import Any import pytest from fastapi.testclient import TestClient -from sqlalchemy import func, select, text +from sqlalchemy import delete, func, select, text from app.api import chat as chat_api from app.config import Settings, get_settings from app.main import app as fastapi_app -from app.models import Chunk, QueryLog +from app.models import Chunk, GitSource, QueryLog from app.rag import agent from app.rag.agent import AGENT_TOOLS from app.rag.importer import import_sources @@ -550,13 +550,13 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc( tool_script=[ [ StreamPiece("thinking", "Let me list what is indexed…"), - ToolCallPiece(id="call_1", name="list_documents", arguments={}), + ToolCallPiece(id="call_1", name="ls", arguments={}), ], [ ToolCallPiece( id="call_2", - name="read_document", - arguments={"source": "docs", "path": "homelab/backups.md"}, + name="read", + arguments={"path": "docs/homelab/backups.md"}, ) ], # the answer request still carries the tools (2 rounds < the @@ -580,10 +580,12 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc( list_frame, read_frame = frames[1], frames[2] assert set(list_frame) == {"type", "name", "argument"} - assert list_frame["name"] == "list_documents" - assert list_frame["argument"] is None # the tool takes no parameters + assert list_frame["name"] == "ls" + assert list_frame["argument"] is None # no ``path`` argument was passed assert set(read_frame) == {"type", "name", "argument"} - assert read_frame["name"] == "read_document" + assert read_frame["name"] == "read" + # Phase 70: the frame's argument is the single string the model + # passed — the combined ``source/path``. assert read_frame["argument"] == "docs/homelab/backups.md" deltas = [f for f in frames if f["type"] == "delta"] @@ -621,28 +623,24 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc( assert "'docs/homelab/backups.md'" in lines[-1] -def test_grounded_turn_streams_search_tool_frames( +def test_grounded_turn_streams_grep_tool_frames( client, db, seeded_kb: FakeRagLLM ) -> None: - """Phase 68: a scripted ``search_documents`` call streams as - ``{type: "tool", name: "search_documents", argument: }`` — + """Phase 68 (renamed ``grep`` in phase 70): a scripted ``grep`` call + streams as ``{type: "tool", name: "grep", argument: }`` — the raw pattern is the frame's ``argument`` (the UI renders the "searching for" line from it). A non-string pattern — a model error - the backend refuses — yields ``argument: null``. A search adds no + the backend refuses — yields ``argument: null``. A grep adds no source: ``done.sources`` stays the retrieval docs (locked A5).""" scripted = FakeRagLLM( tool_script=[ [ - ToolCallPiece( - id="call_1", - name="search_documents", - arguments={"pattern": "Cilium"}, - ), + ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "Cilium"}), ], [ ToolCallPiece( id="call_2", - name="search_documents", + name="grep", arguments={"pattern": 42}, # model error: non-string ), ], @@ -659,25 +657,90 @@ def test_grounded_turn_streams_search_tool_frames( types = [f["type"] for f in frames] assert "error" not in types - assert len(scripted.seen_tools) == 3 # both searches executed (rounds) + assert len(scripted.seen_tools) == 3 # both greps executed (rounds) tool_frames = [f for f in frames if f["type"] == "tool"] assert len(tool_frames) == 2 first, second = tool_frames assert set(first) == {"type", "name", "argument"} - assert first["name"] == "search_documents" + assert first["name"] == "grep" assert first["argument"] == "Cilium" # the raw pattern assert set(second) == {"type", "name", "argument"} - assert second["name"] == "search_documents" + assert second["name"] == "grep" assert second["argument"] is None # the non-string pattern → null - # The searches still answered: deltas, then a grounded done. + # The greps still answered: deltas, then a grounded done. assert [f for f in frames if f["type"] == "delta"] done = frames[-1] assert done["type"] == "done" and done["deflected"] is False paths = [s["path"] for s in done["sources"]] assert "homelab/kubernetes.md" in paths # retrieval docs, unchanged - assert "homelab/backups.md" not in paths # a search adds no source + assert "homelab/backups.md" not in paths # a grep adds no source + + +def test_tool_frames_carry_the_model_arguments_regardless_of_execution( + client, db, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture +) -> None: + """Phase 70 pins: the frame's ``argument`` is the single string + argument the model passed — an ``ls`` frame carries the scope when + the model gave one (null only when it is omitted, pinned above) — + and frame emission is execution-independent: a rejected call (an + unknown ``read`` path) still streams its frame with the model's + argument as-is. The rejected read adds no source (``done.sources`` + stays the retrieval docs), and rejected calls count nothing + (``tool_calls=1`` — only the executed scoped ``ls``).""" + # The scoped ``ls`` source-name check reads the registry — insert a + # row resolving to ``docs`` (the fixture's source name) and delete + # it again afterwards. + src = GitSource(url="https://github.com/reese/docs.git", kind="git") + db.add(src) + db.commit() + try: + scripted = FakeRagLLM( + tool_script=[ + [ToolCallPiece(id="call_1", name="ls", arguments={"path": "docs"})], + [ + ToolCallPiece( + id="call_2", name="read", arguments={"path": "docs/homelab/nope.md"} + ) + ], + ] + ) + fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted + try: + caplog.set_level(logging.INFO, logger="app.chat") + _, _, frames = _stream_chat(client, QUESTION) + finally: + fastapi_app.dependency_overrides.clear() + finally: + db.execute(delete(GitSource).where(GitSource.id == src.id)) + db.commit() + + types = [f["type"] for f in frames] + assert "error" not in types + # Both calls stream a frame — the rejected read included. + tool_frames = [f for f in frames if f["type"] == "tool"] + assert len(tool_frames) == 2 + ls_frame, read_frame = tool_frames + assert set(ls_frame) == {"type", "name", "argument"} + assert ls_frame["name"] == "ls" + assert ls_frame["argument"] == "docs" # the model's scope, as passed + assert set(read_frame) == {"type", "name", "argument"} + assert read_frame["name"] == "read" + # The rejected call's frame still carries the model's argument as + # passed — frame emission is execution-independent. + assert read_frame["argument"] == "docs/homelab/nope.md" + + # The rejected read adds no source — done.sources stays retrieval. + done = frames[-1] + assert done["type"] == "done" and done["deflected"] is False + paths = [s["path"] for s in done["sources"]] + assert "homelab/kubernetes.md" in paths # retrieval docs, unchanged + assert "homelab/nope.md" not in paths # the refused read cites nothing + + # The rejected call counts nothing — only the executed scoped ls. + lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()] + assert lines and "tool_calls=1" in lines[-1] def test_deflected_turn_stays_byte_identical_without_tools( @@ -690,12 +753,12 @@ def test_deflected_turn_stays_byte_identical_without_tools( ``tools`` key.""" scripted = FakeRagLLM( tool_script=[ - [ToolCallPiece(id="call_1", name="list_documents", arguments={})], + [ToolCallPiece(id="call_1", name="ls", arguments={})], [ ToolCallPiece( id="call_2", - name="read_document", - arguments={"source": "docs", "path": "homelab/backups.md"}, + name="read", + arguments={"path": "docs/homelab/backups.md"}, ) ], [StreamPiece("content", "never used — the agent never runs")], @@ -743,12 +806,12 @@ def test_zero_max_rounds_reproduce_pre_phase_single_request( the kill switch survives the phase-45 budget removal.""" scripted = FakeRagLLM( tool_script=[ - [ToolCallPiece(id="call_1", name="list_documents", arguments={})], + [ToolCallPiece(id="call_1", name="ls", arguments={})], [ ToolCallPiece( id="call_2", - name="read_document", - arguments={"source": "docs", "path": "homelab/backups.md"}, + name="read", + arguments={"path": "docs/homelab/backups.md"}, ) ], ] @@ -799,7 +862,7 @@ def test_tool_execution_db_failure_yields_error_event( ``error`` event as the pre-stream retrieval path — never a severed stream (the "never stale" contract, PLAN §7.4).""" scripted = FakeRagLLM( - tool_script=[[ToolCallPiece(id="call_1", name="list_documents", arguments={})]] + tool_script=[[ToolCallPiece(id="call_1", name="ls", arguments={})]] ) def boom(*_a: Any, **_k: Any) -> Any: @@ -815,7 +878,7 @@ def test_tool_execution_db_failure_yields_error_event( # The ``tool`` frame went out first (the model requested the call); # the failed execution ends the turn with the structured error event. assert [f["type"] for f in frames] == ["tool", "error"] - assert frames[0]["name"] == "list_documents" + assert frames[0]["name"] == "ls" assert "offline mid-question" in frames[1]["detail"] assert db.scalars(select(QueryLog)).all() == [] # no row for a failed turn diff --git a/tests/integration/test_chats_api.py b/tests/integration/test_chats_api.py index 93afb12..48ad9a5 100644 --- a/tests/integration/test_chats_api.py +++ b/tests/integration/test_chats_api.py @@ -64,8 +64,12 @@ FULL_BRAIN: dict[str, Any] = { "suggestions": ["What ports does Traefik expose?"], "thinking": "The kubernetes doc covers the cluster layout…", "tools": [ - {"name": "read_document", "argument": "Homelab/kubernetes.md"}, - {"name": "list_documents", "argument": None}, + {"name": "read", "argument": "Homelab/kubernetes.md"}, + {"name": "ls", "argument": None}, + # Saved chats persisting the pre-phase-70 tool names still + # validate — ``name`` is opaque to the API (no migration, + # locked: old chats render fine). + {"name": "read_document", "argument": "Homelab/legacy-notes.md"}, ], "stopped": False, } diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index 4127903..c4850fd 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -1,15 +1,23 @@ -"""Unit: the grounded-turn agent loop (phase 37, ``app.rag.agent``). +"""Unit: the grounded-turn agent loop (phase 37, ``app.rag.agent``; the +harness-aligned ``ls``/``read``/``grep`` surface, phase 70). A scripted fake LLM (canned stream sequences) + monkeypatched -``list_catalog`` / ``find_document`` — no database, no network. Covers -the loop mechanics: the list → read → answer happy path (event order, -holder state, the tools staying offered on every request — phase 45 -removed the per-tool budgets, the assistant/tool message history), the -kill switch (``agent_max_rounds=0`` single-call path), the round cap -forcing a final no-tools answer (an always-calling stream and an -always-rejected stream), re-lists and multi-reads executing without -budgets, dedupe, unknown tool / missing args / unknown path, the -```` prompt section (HIGH only), and the phase-67 per-round +``list_catalog`` / ``list_source_names`` / ``find_document`` / +``all_documents`` — no database, no network. Covers the loop mechanics: +the ls → read (combined ``source/path``) → answer happy path (event +order, holder state, the tools staying offered on every request — +phase 45 removed the per-tool budgets, the assistant/tool message +history), the ``ls`` scoping (no-arg full catalog in the phase-63 +labeled-field format, a one-source scope, a known source with 0 +documents → ``0 documents:`` counted, an unknown-source refusal that +counts nothing), ``read`` on the canonical combined form (split at the +FIRST slash, full content, the bare-source-name refusal, the +already-in-context dedupe, missing-args refusals), the phase-68 ``grep`` +contract under its new name (the locked A5 pins: fixed substring, +case-insensitive, 20-cap in catalog order, 200-char lines, locator-only +— ``read_docs`` untouched, no-match lines counted), the round cap +forcing a final no-tools answer, the kill switch +(``agent_max_rounds=0`` single-call path), and the phase-67 per-round retries (a dead-then-recovered round restarts before its first piece with a ``RetryPiece``; a mid-stream drop stays terminal — locked A2; the forced final no-tools call retries too; ``llm_retries=0`` is one @@ -30,7 +38,7 @@ import pytest from sqlalchemy.orm import Session from app.config import Settings -from app.models import Document +from app.models import Document, GitSource from app.rag import agent from app.rag.agent import ( AGENT_TOOLS, @@ -98,99 +106,137 @@ async def _run( return out -# ---------- AGENT_TOOLS shape ---------- +# ---------- AGENT_TOOLS shape (phase 70: ls / read / grep) ---------- def test_agent_tools_names_and_parameters() -> None: by_name = {t["function"]["name"]: t for t in AGENT_TOOLS} - assert len(AGENT_TOOLS) == 3 # list / read / search (phase 68) - assert set(by_name) == {"list_documents", "read_document", "search_documents"} + assert len(AGENT_TOOLS) == 3 # ls / read / grep (phase 70) + assert set(by_name) == {"ls", "read", "grep"} + # The phase-37/68 names exist nowhere in the tool surface. + assert not set(by_name) & {"list_documents", "read_document", "search_documents"} assert all(t["type"] == "function" for t in AGENT_TOOLS) - list_params = by_name["list_documents"]["function"]["parameters"] - assert list_params["type"] == "object" - assert list_params["properties"] == {} # no parameters - read_params = by_name["read_document"]["function"]["parameters"] - assert read_params["required"] == ["source", "path"] - assert set(read_params["properties"]) == {"source", "path"} - # The model repeatedly conflated the two fields — passing the - # combined 'source/path' string as 'source' — so the read_document - # description pins the split rule with a worked example. - assert by_name["read_document"]["function"]["description"] == ( - "Add the full content of one more indexed document to your " - "context. A document is identified by the (source, path) pair " - "exactly as shown in the list_documents output: 'source' is " - "the top-level source name only (e.g. 'homelab'), 'path' is " - "the file path inside that source (e.g. " - "'active/container_caddy/caddy.md'). If you only have a " - "combined 'source/path' string (as in search_documents " - "results), split it at the FIRST '/': the part before is the " - "source, the part after is the path. Example: " - "read_document(source='homelab', " - "path='active/container_caddy/caddy.md')." + ls = by_name["ls"]["function"] + assert ls["description"] == ( + "List the indexed documents as `source: X | path: Y | title: Z` lines." ) - # The parameter descriptions define the split: source = before the - # first '/', path = after it. - assert read_params["properties"]["source"]["description"] == ( - "Top-level source name only (e.g. 'homelab') — the part BEFORE " - "the first '/' of a combined 'source/path' string, exactly as " - "shown after 'source: ' in the list_documents output. Must not " - "contain '/' itself — do not pass the full source/path here." + ls_params = ls["parameters"] + assert ls_params["type"] == "object" + assert ls_params["required"] == [] # path is optional + assert set(ls_params["properties"]) == {"path"} + assert ls_params["properties"]["path"]["type"] == "string" + assert ls_params["properties"]["path"]["description"] == ( + "Source name to list one source's documents (e.g. 'homelab'); " + "omit to list every document." ) + read = by_name["read"]["function"] + assert read["description"] == ( + "Add the full content of one indexed document to your context." + ) + read_params = read["parameters"] + assert read_params["type"] == "object" + assert read_params["required"] == ["path"] + assert set(read_params["properties"]) == {"path"} + assert read_params["properties"]["path"]["type"] == "string" + # The combined source/path string is the canonical document identity + # (phase 70) — the description pins it with a worked example. assert read_params["properties"]["path"]["description"] == ( - "File path relative to the source directory (e.g. " - "'active/container_caddy/caddy.md') — the part AFTER the first " - "'/' of a combined 'source/path' string, exactly as shown " - "after 'path: ' in the list_documents output. Must not start " - "with the source name." + "The document to add to your context, as the combined " + "`source/path` string exactly as shown in the `ls` output (e.g. " + "'homelab/active/container_caddy/caddy.md')." ) - # Phase 68: search_documents — the third tool, a locator (locked - # A5); its description maps result lines back onto the split. - search = by_name["search_documents"]["function"] - assert search["description"] == ( - "Search every indexed document for an exact string " + grep = by_name["grep"]["function"] + assert grep["description"] == ( + "Search the indexed documents for an exact string " "(case-insensitive) and return up to 20 matching lines as " - "'source/path:line: text' — use this to locate content, " - "then read_document the winner (each result line's " - "'source/path' splits at the first '/': the part before " - "is the source, the part after is the path). Optionally " - "pass 'source' and 'path' (as shown in list_documents) " - "to search one document only." + "`source/path:line: text` — a locator, not a context-adder: " + "read the winner with `read`." ) - search_params = search["parameters"] - assert search_params["type"] == "object" - assert search_params["required"] == ["pattern"] - assert set(search_params["properties"]) == {"pattern", "source", "path"} - assert search_params["properties"]["pattern"]["description"] == ( + grep_params = grep["parameters"] + assert grep_params["type"] == "object" + assert grep_params["required"] == ["pattern"] + assert set(grep_params["properties"]) == {"pattern", "path"} + assert all(p["type"] == "string" for p in grep_params["properties"].values()) + assert grep_params["properties"]["pattern"]["description"] == ( "The exact text to search for (a plain substring, not a regex)" ) - # Shared constants: search's source/path params ARE read_document's - # (one definition, no drift between the two tools). - assert search_params["properties"]["source"] is read_params["properties"]["source"] - assert search_params["properties"]["path"] is read_params["properties"]["path"] + assert grep_params["properties"]["path"]["description"] == ( + "Limit the search to one document, as a combined `source/path` " + "string from the `ls` output (omit to search every document)." + ) -# ---------- happy path: list → read → answer ---------- +def test_agent_tools_order_is_ls_read_grep() -> None: + """The listing → context → locator order the prompt teaches (the API + layer and the mock key off the names).""" + assert [t["function"]["name"] for t in AGENT_TOOLS] == ["ls", "read", "grep"] -def test_list_then_read_then_answer( +def test_refusal_constants_are_harness_aligned() -> None: + """The updated module-level refusal lines (the names moved to the + harness surface; ALREADY_IN_CONTEXT / UNKNOWN_TOOL unchanged).""" + assert agent.ALREADY_IN_CONTEXT == "Already in your context." + assert agent.UNKNOWN_TOOL == "Unknown tool." + assert agent.MISSING_READ_ARGS == "read requires a string argument 'path'." + assert agent.MISSING_SEARCH_ARGS == "grep requires a string argument 'pattern'." + + +# ---------- list_source_names (the scoped ls registry join) ---------- + + +def test_list_source_names_resolves_registry_rows( monkeypatch: pytest.MonkeyPatch, ) -> None: + """Names resolve exactly as the import pipeline indexes them (the + phase-69 ``resolve_source_name`` expressions — reuse, not + re-derivation), deduped (two rows resolving to the same name share + documents), in registry order.""" + rows = [ + GitSource(url="https://github.com/reese/homelab.git", kind="git"), + GitSource( + url="/srv/reese/deployments", kind="local", path="/srv/reese/deployments" + ), + # The phase-69 sibling case: a second row, same resolved name. + GitSource(url="https://github.com/reese/homelab", kind="git"), + ] + monkeypatch.setattr(agent, "effective_sources", lambda db: (rows, "db")) + assert agent.list_source_names(cast("Session", object())) == [ + "homelab", + "deployments", + ] + + +def test_list_source_names_empty_registry(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(agent, "effective_sources", lambda db: ([], "env")) + assert agent.list_source_names(cast("Session", object())) == [] + + +# ---------- happy path: ls → read (combined path) → answer ---------- + + +def test_ls_then_read_then_answer(monkeypatch: pytest.MonkeyPatch) -> None: catalog = [ ("Deployments", "backups.md", "Backup Strategy"), ("Homelab", "aws-route53.md", "AWS Route53 Records"), ] monkeypatch.setattr(agent, "list_catalog", lambda db: catalog) target = _doc("Homelab", "aws-route53.md", "AWS Route53 Records", "R53-CONTENT") - monkeypatch.setattr(agent, "find_document", lambda db, source, path: target) + calls: list[tuple[str, str]] = [] + + def _find(db: Any, source: str, path: str) -> Document | None: + calls.append((source, path)) + return target if (source, path) == ("Homelab", "aws-route53.md") else None + + monkeypatch.setattr(agent, "find_document", _find) seed = [_doc("Homelab", "kubernetes.md", "Kubernetes", "K8S-CONTENT")] holder = AgentHolder() llm = ScriptedLLM( - [ToolCallPiece(id="call_1", name="list_documents", arguments={})], + [ToolCallPiece(id="call_1", name="ls", arguments={})], [ ToolCallPiece( id="call_2", - name="read_document", - arguments={"source": "Homelab", "path": "aws-route53.md"}, + name="read", + arguments={"path": "Homelab/aws-route53.md"}, ) ], [StreamPiece("thinking", "hmm "), StreamPiece("content", "Done! ")], @@ -205,9 +251,10 @@ def test_list_then_read_then_answer( StreamPiece, StreamPiece, ] - assert pieces[0] == ToolCallPiece(id="call_1", name="list_documents", arguments={}) - assert isinstance(pieces[1], ToolCallPiece) - assert pieces[1].name == "read_document" + assert pieces[0] == ToolCallPiece(id="call_1", name="ls", arguments={}) + assert pieces[1] == ToolCallPiece( + id="call_2", name="read", arguments={"path": "Homelab/aws-route53.md"} + ) assert pieces[3] == StreamPiece("content", "Done! ") # The read document is recorded for done.sources / query_log (task 04). assert holder.read_docs == [target] @@ -221,6 +268,10 @@ def test_list_then_read_then_answer( assert llm.requests[2][1] == AGENT_TOOLS assert len(llm.requests) == 3 + # read splits the combined form at the FIRST slash — one exact + # lookup, no self-correction candidates (phase 70). + assert calls == [("Homelab", "aws-route53.md")] + # The follow-up request carries the assistant tool-call + tool result. msgs = llm.requests[1][0] assert msgs[0] == {"role": "system", "content": "SYSTEM_PROMPT"} @@ -232,7 +283,7 @@ def test_list_then_read_then_answer( { "id": "call_1", "type": "function", - "function": {"name": "list_documents", "arguments": "{}"}, + "function": {"name": "ls", "arguments": "{}"}, } ], } @@ -250,8 +301,7 @@ def test_list_then_read_then_answer( assert msgs[4]["role"] == "assistant" assert msgs[4]["tool_calls"][0]["id"] == "call_2" assert json.loads(msgs[4]["tool_calls"][0]["function"]["arguments"]) == { - "source": "Homelab", - "path": "aws-route53.md", + "path": "Homelab/aws-route53.md" } assert msgs[5] == { "role": "tool", @@ -260,18 +310,6 @@ def test_list_then_read_then_answer( } -def test_empty_catalog_listing_says_zero_documents(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(agent, "list_catalog", lambda db: []) - holder = AgentHolder() - llm = ScriptedLLM( - [ToolCallPiece(id="call_1", name="list_documents", arguments={})], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == "0 documents:\n" - assert holder.tool_calls == 1 - - def test_content_and_tool_call_in_one_stream_keeps_both( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -282,7 +320,7 @@ def test_content_and_tool_call_in_one_stream_keeps_both( llm = ScriptedLLM( [ StreamPiece("content", "Let me check "), - ToolCallPiece(id="call_1", name="list_documents", arguments={}), + ToolCallPiece(id="call_1", name="ls", arguments={}), ], [StreamPiece("content", "the answer")], ) @@ -292,20 +330,686 @@ def test_content_and_tool_call_in_one_stream_keeps_both( assert llm.requests[1][0][3]["content"] == "0 documents:\n" +# ---------- ls: full catalog + scoping ---------- + + +def test_ls_full_catalog_format(monkeypatch: pytest.MonkeyPatch) -> None: + """No argument: the full catalog in the phase-63 labeled-field format + (``source: X | path: Y | title: Z``) — counted; no registry lookup.""" + catalog = [ + ("Deployments", "backups.md", "Backup Strategy"), + ("Homelab", "aws-route53.md", "AWS Route53 Records"), + ] + monkeypatch.setattr(agent, "list_catalog", lambda db: catalog) + + def _boom_sources(*_a: Any, **_k: Any) -> None: + raise AssertionError("no registry lookup for an unscoped ls") + + monkeypatch.setattr(agent, "list_source_names", _boom_sources) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="ls", arguments={})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "2 documents:\n" + "source: Deployments | path: backups.md | title: Backup Strategy\n" + "source: Homelab | path: aws-route53.md | title: AWS Route53 Records" + ) + assert holder.tool_calls == 1 + + +def test_ls_empty_catalog_says_zero_documents(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(agent, "list_catalog", lambda db: []) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="ls", arguments={})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == "0 documents:\n" + assert holder.tool_calls == 1 + + +@pytest.mark.parametrize( + ("arguments", "label"), + [ + ({"path": " "}, "blank path"), + ({"path": 7}, "non-string path"), + ], +) +def test_ls_blank_path_lists_full_catalog( + monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str +) -> None: + """A blank (or non-string) ``path`` is treated as omitted — the full + catalog, counted (no refusal for an empty scope).""" + catalog = [("S", "a.md", "A")] + monkeypatch.setattr(agent, "list_catalog", lambda db: catalog) + monkeypatch.setattr(agent, "list_source_names", lambda db: ["S"]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="ls", arguments=arguments)], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert ( + llm.requests[1][0][3]["content"] == "1 documents:\nsource: S | path: a.md | title: A" + ) + assert holder.tool_calls == 1 + + +def test_ls_scoped_to_known_source(monkeypatch: pytest.MonkeyPatch) -> None: + """A known source name: the same listing filtered to that source — + counted.""" + catalog = [ + ("Deployments", "backups.md", "Backup Strategy"), + ("Homelab", "a.md", "A"), + ("Homelab", "b.md", "B"), + ] + monkeypatch.setattr(agent, "list_catalog", lambda db: catalog) + monkeypatch.setattr(agent, "list_source_names", lambda db: ["Deployments", "Homelab"]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="ls", arguments={"path": "Homelab"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "2 documents:\n" + "source: Homelab | path: a.md | title: A\n" + "source: Homelab | path: b.md | title: B" + ) + assert holder.tool_calls == 1 + + +def test_ls_scoped_known_source_with_zero_docs_counts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A registered source with no indexed documents is KNOWN (the + registry is the source of truth, not the catalog): it lists as + ``0 documents:`` — a valid, counted result, not a refusal.""" + monkeypatch.setattr(agent, "list_catalog", lambda db: [("Other", "a.md", "A")]) + monkeypatch.setattr(agent, "list_source_names", lambda db: ["Homelab", "Other"]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="ls", arguments={"path": "Homelab"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == "0 documents:\n" + assert holder.tool_calls == 1 # an executed ls, not a refusal + assert llm.requests[1][1] == AGENT_TOOLS + + +def test_ls_scoped_unknown_source_refused(monkeypatch: pytest.MonkeyPatch) -> None: + """A ``path`` matching no source name is a refusal — not counted, the + round cap bounds its repetition.""" + monkeypatch.setattr(agent, "list_catalog", lambda db: [("S", "a.md", "A")]) + monkeypatch.setattr(agent, "list_source_names", lambda db: ["S"]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="ls", arguments={"path": "Ghost"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.tool_calls == 0 # a refusal counts in nothing + assert ( + llm.requests[1][0][3]["content"] == "No source named 'Ghost' — check the ls output." + ) + assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered + + +# ---------- read: the canonical combined source/path form ---------- + + +def test_read_combined_path_resolves_and_returns_full_content( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The combined ``source/path`` form (the model's trained shape) is + the canonical identity: split at the FIRST '/', one exact lookup, + the full content returned (A7-revised: never truncated) — even when + the path itself carries further slashes.""" + doc = _doc("Homelab", "active/container_caddy/caddy.md", "Caddy", "CADDY-CONTENT") + calls: list[tuple[str, str]] = [] + + def _find(db: Any, source: str, path: str) -> Document | None: + calls.append((source, path)) + return ( + doc + if (source, path) == ("Homelab", "active/container_caddy/caddy.md") + else None + ) + + monkeypatch.setattr(agent, "find_document", _find) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", + name="read", + arguments={"path": "Homelab/active/container_caddy/caddy.md"}, + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + # First-slash split — exactly one lookup, the canonical pair. + assert calls == [("Homelab", "active/container_caddy/caddy.md")] + assert holder.read_docs == [doc] + assert holder.tool_calls == 1 + assert llm.requests[1][0][3]["content"] == ( + "Document Homelab/active/container_caddy/caddy.md:\nCADDY-CONTENT" + ) + + +def test_read_bare_source_name_refused_without_db(monkeypatch: pytest.MonkeyPatch) -> None: + """A bare source name (no '/') can never be a document — the + no-document refusal (the argument echoed as passed), no DB lookup, + nothing counted.""" + monkeypatch.setattr(agent, "list_catalog", lambda db: [("Homelab", "a.md", "A")]) + + def _boom(*_a: Any, **_k: Any) -> None: + raise AssertionError("find_document must not run for a bare source name") + + monkeypatch.setattr(agent, "find_document", _boom) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="read", arguments={"path": "Homelab"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.read_docs == [] and holder.tool_calls == 0 + assert llm.requests[1][0][3]["content"] == ( + "No document at 'Homelab' — check the ls output." + ) + assert llm.requests[1][1] == AGENT_TOOLS + + +def test_read_unknown_path_refused_echoing_argument( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An unknown combined identity → the refusal echoing the argument as + passed (the model sees its own form) — the old split-teaching refusal + is gone (phase 70).""" + monkeypatch.setattr(agent, "find_document", lambda db, source, path: None) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="read", arguments={"path": "S/ghost.md"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.read_docs == [] and holder.tool_calls == 0 + assert llm.requests[1][0][3]["content"] == ( + "No document at 'S/ghost.md' — check the ls output." + ) + assert llm.requests[1][1] == AGENT_TOOLS # tools stay offered (cap bounds) + + +@pytest.mark.parametrize( + ("arguments", "label"), + [ + ({}, "no arguments"), + ({"path": ""}, "empty path"), + ({"path": " "}, "blank path"), + ({"path": 7}, "non-string path"), + ({"path": None}, "null path"), + ], +) +def test_read_missing_arguments_refused( + monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str +) -> None: + def _boom(*_a: Any, **_k: Any) -> None: + raise AssertionError(f"find_document must not be called ({label})") + + monkeypatch.setattr(agent, "find_document", _boom) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="read", arguments=arguments)], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.read_docs == [] and holder.tool_calls == 0 + assert llm.requests[1][0][3]["content"] == agent.MISSING_READ_ARGS + assert llm.requests[1][1] == AGENT_TOOLS + + +def test_reading_a_seed_doc_is_already_in_context(monkeypatch: pytest.MonkeyPatch) -> None: + """The combined identity of a seeded document: its split pair is in + the known set → ALREADY_IN_CONTEXT with no DB lookup (the dedupe + check precedes the resolve).""" + seed = [_doc("Homelab", "kubernetes.md", "Kubernetes", "K8S-CONTENT")] + + def _boom(*_a: Any, **_k: Any) -> None: + raise AssertionError("find_document must not be called for a seeded doc") + + monkeypatch.setattr(agent, "find_document", _boom) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", + name="read", + arguments={"path": "Homelab/kubernetes.md"}, + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings(), seed_docs=seed)) + assert holder.read_docs == [] and holder.tool_calls == 0 + assert llm.requests[1][0][3]["content"] == agent.ALREADY_IN_CONTEXT + # Rejected → the tools are still offered on the next request (the + # round cap is the only bound). + assert llm.requests[1][1] == AGENT_TOOLS + + +def test_reading_an_already_read_doc_is_deduped(monkeypatch: pytest.MonkeyPatch) -> None: + """The second read of the same document (holder.read_docs) → + ALREADY_IN_CONTEXT — appended once, counted once.""" + doc = _doc("S", "a.md", "A", "A-CONTENT") + monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="read", arguments={"path": "S/a.md"})], + [ToolCallPiece(id="call_2", name="read", arguments={"path": "S/a.md"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.read_docs == [doc] # appended exactly once + assert holder.tool_calls == 1 # the re-read counts nothing + assert llm.requests[1][0][3]["content"] == "Document S/a.md:\nA-CONTENT" + assert llm.requests[2][0][5]["content"] == agent.ALREADY_IN_CONTEXT + # Rejected → the tools are still offered on the next request… + assert llm.requests[2][1] == AGENT_TOOLS + + +def test_unknown_tool_name_refused(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(agent, "list_catalog", lambda db: []) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="delete_universe", arguments={"x": 1})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.read_docs == [] and holder.tool_calls == 0 + assert llm.requests[1][0][3]["content"] == agent.UNKNOWN_TOOL + assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered + + +# ---------- grep (the phase-68 A5 contract under the new name) ---------- + + +def test_grep_document_case_insensitive_line_numbers() -> None: + """Case-insensitive fixed substring, 1-based line numbers, file order, + repeated matches within a line collapse to one match (grep semantics).""" + content = "The NEEDLE is here\nno hit\nneedle again\nNEEDLE NEEDLE\n" + assert agent.grep_document(content, "NEEDLE") == [ + (1, "The NEEDLE is here"), + (3, "needle again"), + (4, "NEEDLE NEEDLE"), + ] + + +def test_grep_document_rstrips_lines_and_empty_content() -> None: + assert agent.grep_document("hello \t\nworld ", "WORLD") == [(2, "world")] + assert agent.grep_document("", "x") == [] + assert agent.grep_document("no newlines", "NO") == [(1, "no newlines")] + assert agent.grep_document("a\nb\n", "MISSING") == [] + + +def test_grep_whole_kb_grep_style_output(monkeypatch: pytest.MonkeyPatch) -> None: + """Whole-KB grep: catalog order, `source/path:line: text` lines, + case-insensitive; the call counts in ``tool_calls`` and never touches + ``read_docs``; the tools stay offered on the answer request.""" + d1 = _doc("Alpha", "a/one.md", "One", "first\nNEEDLE in one\nlast") + d2 = _doc("Beta", "b/two.md", "Two", "no hit\nneedle in two\n") + monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "needle"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "Alpha/a/one.md:2: NEEDLE in one\n" + "Beta/b/two.md:2: needle in two" + ) + assert holder.tool_calls == 1 + assert holder.read_docs == [] # locked A5: a grep adds no context + assert llm.requests[1][1] == AGENT_TOOLS # tools stay offered + + +def test_grep_capped_at_20_matches_in_catalog_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The 20-match cap is GLOBAL across documents in catalog order, and + the scan stops once it is hit (a 35-match corpus yields exactly 20).""" + d1 = _doc("S", "a.md", "A", "\n".join(f"hit-{i}" for i in range(15))) + d2 = _doc("S", "b.md", "B", "\n".join(f"hit-{i}" for i in range(20))) + monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "hit-"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + lines = llm.requests[1][0][3]["content"].split("\n") + assert len(lines) == agent.SEARCH_MAX_MATCHES + assert lines[0] == "S/a.md:1: hit-0" + assert lines[14] == "S/a.md:15: hit-14" # all of a.md + assert lines[15] == "S/b.md:1: hit-0" # then b.md, in order + assert lines[19] == "S/b.md:5: hit-4" # cut at the global cap + assert holder.tool_calls == 1 + + +def test_grep_truncates_match_lines_at_200_chars(monkeypatch: pytest.MonkeyPatch) -> None: + """A 300-char match line yields exactly 200 chars of it (no crash).""" + d1 = _doc("S", "a.md", "A", "top\n" + "x" * 300 + " NEEDLE tail") + monkeypatch.setattr(agent, "all_documents", lambda db: [d1]) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "needle"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == f"S/a.md:2: {'x' * agent.SEARCH_LINE_LIMIT}" + assert holder.tool_calls == 1 + + +def test_grep_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None: + """Scoped grep: only the named document is loaded (find_document on + the first-slash split), ``all_documents`` never runs, and the match + line carries its path.""" + d1 = _doc("S", "a.md", "A", "needle here") + + def _find(db: Any, source: str, path: str) -> Document | None: + if (source, path) == ("S", "a.md"): + return d1 + raise AssertionError( + f"find_document({source}, {path}) — the scoped " + "grep must not load any other document" + ) + + def _boom(*_a: Any, **_k: Any) -> None: + raise AssertionError("all_documents must not run for a scoped grep") + + monkeypatch.setattr(agent, "find_document", _find) + monkeypatch.setattr(agent, "all_documents", _boom) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", + name="grep", + arguments={"pattern": "needle", "path": "S/a.md"}, + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here" + assert holder.tool_calls == 1 + assert holder.read_docs == [] # grepped doc did not enter the context + + +def test_grep_scoped_combined_path_with_nested_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A combined target whose path itself contains slashes: the split is + at the FIRST slash — the scoped grep runs on the right document.""" + d1 = _doc("S", "deep/nested/a.md", "A", "needle here") + + def _find(db: Any, source: str, path: str) -> Document | None: + if (source, path) == ("S", "deep/nested/a.md"): + return d1 + raise AssertionError(f"find_document({source}, {path}) — wrong first-slash split") + + monkeypatch.setattr(agent, "find_document", _find) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", + name="grep", + arguments={"pattern": "needle", "path": "S/deep/nested/a.md"}, + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == "S/deep/nested/a.md:1: needle here" + assert holder.tool_calls == 1 + + +def test_grep_scoped_missing_document_refused(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(agent, "find_document", lambda db, source, path: None) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", + name="grep", + arguments={"pattern": "x", "path": "S/ghost.md"}, + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "No document at 'S/ghost.md' — check the ls output." + ) + assert holder.tool_calls == 0 and holder.read_docs == [] # a refusal + assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered + + +def test_grep_scoped_bare_source_name_refused_without_db( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A bare source name as the grep target can never resolve to exactly + one document — the no-document refusal, no DB lookup, not counted.""" + def _boom(*_a: Any, **_k: Any) -> None: + raise AssertionError("find_document must not run for a bare source name") + + monkeypatch.setattr(agent, "find_document", _boom) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", + name="grep", + arguments={"pattern": "x", "path": "Homelab"}, + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "No document at 'Homelab' — check the ls output." + ) + assert holder.tool_calls == 0 and holder.read_docs == [] + + +@pytest.mark.parametrize( + ("arguments", "label"), + [ + ({}, "no arguments"), + ({"pattern": ""}, "empty pattern"), + ({"pattern": " "}, "whitespace pattern"), + ({"pattern": 42}, "non-string pattern"), + ({"pattern": None}, "null pattern"), + ], +) +def test_grep_missing_arguments_refused( + monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str +) -> None: + """A missing/blank/non-string pattern → the missing-args refusal, with + no DB access at all.""" + + def _boom(*_a: Any, **_k: Any) -> None: + raise AssertionError(f"no DB access for a refused grep ({label})") + + monkeypatch.setattr(agent, "all_documents", _boom) + monkeypatch.setattr(agent, "find_document", _boom) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments=arguments)], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == agent.MISSING_SEARCH_ARGS + assert holder.tool_calls == 0 and holder.read_docs == [] + assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered + + +def test_grep_no_matches_whole_kb(monkeypatch: pytest.MonkeyPatch) -> None: + """Zero hits across the KB → the no-match line (pattern quoted); the + grep still executed, so it counts — and never adds context.""" + monkeypatch.setattr( + agent, "all_documents", lambda db: [_doc("S", "a.md", "A", "nothing here")] + ) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "zebra"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + "No matches for 'zebra' in the knowledge base." + ) + assert holder.tool_calls == 1 + assert holder.read_docs == [] + + +def test_grep_no_matches_scoped(monkeypatch: pytest.MonkeyPatch) -> None: + """The scoped no-match line is keyed on the resolved source/path.""" + doc = _doc("S", "a.md", "A", "nothing here") + + def _find(db: Any, source: str, path: str) -> Document | None: + return doc if (source, path) == ("S", "a.md") else None + + monkeypatch.setattr(agent, "find_document", _find) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece( + id="call_1", + name="grep", + arguments={"pattern": "zebra", "path": "S/a.md"}, + ) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == "No matches for 'zebra' in S/a.md." + assert holder.tool_calls == 1 + assert holder.read_docs == [] + + +def test_grep_no_match_truncates_long_pattern(monkeypatch: pytest.MonkeyPatch) -> None: + """A pattern longer than 100 chars is truncated in the no-match line + (kept short); the grep itself still runs on the full pattern.""" + monkeypatch.setattr(agent, "all_documents", lambda db: []) + holder = AgentHolder() + llm = ScriptedLLM( + [ + ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "p" * 150}) + ], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert llm.requests[1][0][3]["content"] == ( + f"No matches for '{'p' * 100}' in the knowledge base." + ) + assert holder.tool_calls == 1 + + +def test_grep_counts_but_never_adds_context(monkeypatch: pytest.MonkeyPatch) -> None: + """The locate-then-read workflow: a grep finds the document but does + NOT add it — the subsequent read does (and is not rejected as + already-in-context, because the grep touched nothing).""" + doc = _doc("S", "a.md", "A", "needle here") + monkeypatch.setattr(agent, "all_documents", lambda db: [doc]) + monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="grep", arguments={"pattern": "needle"})], + [ToolCallPiece(id="call_2", name="read", arguments={"path": "S/a.md"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.tool_calls == 2 # grep + read, both executed + assert holder.read_docs == [doc] # only the read added context (A5) + assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here" + assert llm.requests[2][0][5]["content"] == "Document S/a.md:\nneedle here" + + +# ---------- unlimited calls: re-lists and multi-reads (phase 45) ---------- + + +def test_relist_executes_and_counts(monkeypatch: pytest.MonkeyPatch) -> None: + """Re-lists execute — a second ``ls`` in one turn returns the catalog + again and counts in ``tool_calls`` (no budget to exhaust).""" + catalog = [ + ("Deployments", "backups.md", "Backup Strategy"), + ("Homelab", "aws-route53.md", "AWS Route53 Records"), + ] + monkeypatch.setattr(agent, "list_catalog", lambda db: catalog) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="ls", arguments={})], + [ToolCallPiece(id="call_2", name="ls", arguments={})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.tool_calls == 2 # both re-lists executed and counted + listing = ( + "2 documents:\n" + "source: Deployments | path: backups.md | title: Backup Strategy\n" + "source: Homelab | path: aws-route53.md | title: AWS Route53 Records" + ) + # The answer request carries the catalog a second time as a tool result. + assert llm.requests[2][0][3]["content"] == listing # first listing + assert llm.requests[2][0][5]["content"] == listing # the re-list + assert llm.requests[2][1] == AGENT_TOOLS # still offered (no budgets) + + +def test_multi_read_executes_without_budgets(monkeypatch: pytest.MonkeyPatch) -> None: + """Reads are no longer budgeted either — two different documents can + be read in one turn (re-reading the same one is still deduped via + ALREADY_IN_CONTEXT — see the rejection tests).""" + a = _doc("S", "a.md", "A", "A-CONTENT") + b = _doc("S", "b.md", "B", "B-CONTENT") + monkeypatch.setattr( + agent, "find_document", lambda db, source, path: {"a.md": a, "b.md": b}[path] + ) + holder = AgentHolder() + llm = ScriptedLLM( + [ToolCallPiece(id="call_1", name="read", arguments={"path": "S/a.md"})], + [ToolCallPiece(id="call_2", name="read", arguments={"path": "S/b.md"})], + [StreamPiece("content", "ans")], + ) + asyncio.run(_run(llm, holder, _settings())) + assert holder.read_docs == [a, b] # both reads appended, in order + assert holder.tool_calls == 2 + assert llm.requests[1][0][3]["content"] == "Document S/a.md:\nA-CONTENT" + assert llm.requests[2][0][5]["content"] == "Document S/b.md:\nB-CONTENT" + assert llm.requests[2][1] == AGENT_TOOLS # the second read was still offered + + # ---------- round cap (phase 45: replaces the per-tool budgets) ---------- -def test_always_list_bounded_by_round_cap(monkeypatch: pytest.MonkeyPatch) -> None: - """A model that keeps calling ``list_documents`` gets exactly +def test_always_ls_bounded_by_round_cap(monkeypatch: pytest.MonkeyPatch) -> None: + """A model that keeps calling ``ls`` gets exactly ``agent_max_rounds`` tool rounds, then one forced ``tools=None`` request streams the answer — the cap is the only forced exit.""" monkeypatch.setattr(agent, "list_catalog", lambda db: [("S", "a.md", "A")]) listing = "1 documents:\nsource: S | path: a.md | title: A" holder = AgentHolder() llm = ScriptedLLM( - [ToolCallPiece(id="call_1", name="list_documents", arguments={})], - [ToolCallPiece(id="call_2", name="list_documents", arguments={})], - [ToolCallPiece(id="call_3", name="list_documents", arguments={})], + [ToolCallPiece(id="call_1", name="ls", arguments={})], + [ToolCallPiece(id="call_2", name="ls", arguments={})], + [ToolCallPiece(id="call_3", name="ls", arguments={})], [StreamPiece("content", "forced answer")], ) pieces = asyncio.run(_run(llm, holder, _settings(agent_max_rounds=3))) @@ -349,36 +1053,16 @@ def test_zero_max_rounds_is_one_request_without_tools() -> None: assert holder.read_docs == [] and holder.tool_calls == 0 -def test_rejected_read_spam_runs_to_round_cap( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Every call rejected (unknown path — "No document at …"): rejections - no longer end the loop early via budgets — the round cap bounds them - and forces the final no-tools answer.""" +def test_rejected_read_spam_runs_to_round_cap(monkeypatch: pytest.MonkeyPatch) -> None: + """Every call rejected (unknown document — "No document at …"): + rejections no longer end the loop early via budgets — the round cap + bounds them and forces the final no-tools answer.""" monkeypatch.setattr(agent, "find_document", lambda db, source, path: None) holder = AgentHolder() llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="read_document", - arguments={"source": "S", "path": "ghost.md"}, - ) - ], - [ - ToolCallPiece( - id="call_2", - name="read_document", - arguments={"source": "S", "path": "ghost.md"}, - ) - ], - [ - ToolCallPiece( - id="call_3", - name="read_document", - arguments={"source": "S", "path": "ghost.md"}, - ) - ], + [ToolCallPiece(id="call_1", name="read", arguments={"path": "S/ghost.md"})], + [ToolCallPiece(id="call_2", name="read", arguments={"path": "S/ghost.md"})], + [ToolCallPiece(id="call_3", name="read", arguments={"path": "S/ghost.md"})], [StreamPiece("content", "forced answer")], ) asyncio.run(_run(llm, holder, _settings(agent_max_rounds=3))) @@ -388,675 +1072,12 @@ def test_rejected_read_spam_runs_to_round_cap( assert llm.requests[2][1] == AGENT_TOOLS assert llm.requests[3][1] is None # the forced final request: no tools assert holder.read_docs == [] and holder.tool_calls == 0 # nothing executed - refusal = "No document at S/ghost.md — check the list_documents output." + refusal = "No document at 'S/ghost.md' — check the ls output." assert llm.requests[1][0][3]["content"] == refusal assert llm.requests[2][0][5]["content"] == refusal assert llm.requests[3][0][7]["content"] == refusal -# ---------- unlimited calls: re-lists and multi-reads (phase 45) ---------- - - -def test_relist_executes_and_counts(monkeypatch: pytest.MonkeyPatch) -> None: - """Re-lists execute — a second ``list_documents`` in one turn returns - the catalog again and counts in ``tool_calls`` (no budget to - exhaust).""" - catalog = [ - ("Deployments", "backups.md", "Backup Strategy"), - ("Homelab", "aws-route53.md", "AWS Route53 Records"), - ] - monkeypatch.setattr(agent, "list_catalog", lambda db: catalog) - holder = AgentHolder() - llm = ScriptedLLM( - [ToolCallPiece(id="call_1", name="list_documents", arguments={})], - [ToolCallPiece(id="call_2", name="list_documents", arguments={})], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert holder.tool_calls == 2 # both re-lists executed and counted - listing = ( - "2 documents:\n" - "source: Deployments | path: backups.md | title: Backup Strategy\n" - "source: Homelab | path: aws-route53.md | title: AWS Route53 Records" - ) - # The answer request carries the catalog a second time as a tool result. - assert llm.requests[2][0][3]["content"] == listing # first listing - assert llm.requests[2][0][5]["content"] == listing # the re-list - assert llm.requests[2][1] == AGENT_TOOLS # still offered (no budgets) - - -def test_multi_read_executes_without_budgets( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Reads are no longer budgeted either — two different documents can - be read in one turn (re-reading the same one is still deduped via - ALREADY_IN_CONTEXT — see the rejection tests).""" - a = _doc("S", "a.md", "A", "A-CONTENT") - b = _doc("S", "b.md", "B", "B-CONTENT") - monkeypatch.setattr( - agent, "find_document", lambda db, source, path: {"a.md": a, "b.md": b}[path] - ) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", name="read_document", arguments={"source": "S", "path": "a.md"} - ) - ], - [ - ToolCallPiece( - id="call_2", name="read_document", arguments={"source": "S", "path": "b.md"} - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert holder.read_docs == [a, b] # both reads appended, in order - assert holder.tool_calls == 2 - assert llm.requests[1][0][3]["content"] == "Document S/a.md:\nA-CONTENT" - assert llm.requests[2][0][5]["content"] == "Document S/b.md:\nB-CONTENT" - assert llm.requests[2][1] == AGENT_TOOLS # the second read was still offered - - -# ---------- rejections (non-budget; the round cap bounds their repetition) ---------- - - -def test_reading_a_seed_doc_is_already_in_context( - monkeypatch: pytest.MonkeyPatch, -) -> None: - seed = [_doc("Homelab", "kubernetes.md", "Kubernetes", "K8S-CONTENT")] - - def _boom(*_a: Any, **_k: Any) -> None: - raise AssertionError("find_document must not be called for a seeded doc") - - monkeypatch.setattr(agent, "list_catalog", lambda db: []) - monkeypatch.setattr(agent, "find_document", _boom) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="read_document", - arguments={"source": "Homelab", "path": "kubernetes.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings(), seed_docs=seed)) - assert holder.read_docs == [] and holder.tool_calls == 0 - assert llm.requests[1][0][3]["content"] == agent.ALREADY_IN_CONTEXT - # Rejected → the tools are still offered on the next request (the - # round cap is the only bound). - assert llm.requests[1][1] == AGENT_TOOLS - - -def test_reading_an_already_read_doc_is_deduped( - monkeypatch: pytest.MonkeyPatch, -) -> None: - doc = _doc("S", "a.md", "A", "A-CONTENT") - monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", name="read_document", arguments={"source": "S", "path": "a.md"} - ) - ], - [ - ToolCallPiece( - id="call_2", name="read_document", arguments={"source": "S", "path": "a.md"} - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert holder.read_docs == [doc] # appended exactly once - assert holder.tool_calls == 1 - assert llm.requests[2][0][5]["content"] == agent.ALREADY_IN_CONTEXT - # Rejected → the tools are still offered on the next request… - assert llm.requests[2][1] == AGENT_TOOLS - - -def test_unknown_path_refused( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(agent, "find_document", lambda db, source, path: None) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="read_document", - arguments={"source": "S", "path": "ghost.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert holder.read_docs == [] and holder.tool_calls == 0 - assert ( - llm.requests[1][0][3]["content"] - == "No document at S/ghost.md — check the list_documents output." - ) - assert llm.requests[1][1] == AGENT_TOOLS # tools stay offered (cap bounds) - - -def test_unknown_tool_name_refused( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(agent, "list_catalog", lambda db: []) - holder = AgentHolder() - llm = ScriptedLLM( - [ToolCallPiece(id="call_1", name="delete_universe", arguments={"x": 1})], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert holder.read_docs == [] and holder.tool_calls == 0 - assert llm.requests[1][0][3]["content"] == agent.UNKNOWN_TOOL - assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered - - -@pytest.mark.parametrize( - ("arguments", "label"), - [ - ({}, "no arguments"), - ({"source": "S"}, "path missing"), - ({"path": "p.md"}, "source missing"), - ({"source": "", "path": "p.md"}, "empty source"), - ({"source": "S", "path": " "}, "blank path"), - ({"source": 7, "path": "p.md"}, "non-string source"), - ], -) -def test_read_document_missing_arguments_refused( - monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str -) -> None: - def _boom(*_a: Any, **_k: Any) -> None: - raise AssertionError(f"find_document must not be called ({label})") - - monkeypatch.setattr(agent, "list_catalog", lambda db: []) - monkeypatch.setattr(agent, "find_document", _boom) - holder = AgentHolder() - llm = ScriptedLLM( - [ToolCallPiece(id="call_1", name="read_document", arguments=arguments)], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert holder.read_docs == [] and holder.tool_calls == 0 - assert llm.requests[1][0][3]["content"] == agent.MISSING_READ_ARGS - assert llm.requests[1][1] == AGENT_TOOLS - - -# ---------- combined 'source/path' self-correction ---------- -# The model treats the combined 'source/path' string (search result -# lines, read-result headers, refusals) as the document's identity and -# sometimes passes it as 'source'. _resolve_document splits it at the -# first slash (source names are directory basenames — they can never -# contain '/'); a refusal for a still-unknown split teaches the split. - - -def test_read_combined_source_is_split_and_read( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """source='S/a/b.md' (the combined form) + path='a/b.md': the exact - lookup misses, the first-slash split hits — the read executes, the - holder records the document, and the result header carries the TRUE - source/path (not the model's raw arguments).""" - doc = _doc("S", "a/b.md", "B", "B-CONTENT") - calls: list[tuple[str, str]] = [] - - def _find(db: Any, source: str, path: str) -> Document | None: - calls.append((source, path)) - return doc if (source, path) == ("S", "a/b.md") else None - - monkeypatch.setattr(agent, "find_document", _find) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="read_document", - arguments={"source": "S/a/b.md", "path": "a/b.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - # Exact pair first, then the first-slash split (no third attempt). - assert calls == [("S/a/b.md", "a/b.md"), ("S", "a/b.md")] - assert holder.read_docs == [doc] - assert holder.tool_calls == 1 - assert llm.requests[1][0][3]["content"] == "Document S/a/b.md:\nB-CONTENT" - - -def test_read_combined_source_split_at_later_slash( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """source='S/a' + path='b.md' — a split at a LATER slash (source - carried source + leading directory, path the remainder): the exact - lookup and the first-slash split miss, the continuation candidate - (source, split/path) hits.""" - doc = _doc("S", "a/b.md", "B", "B-CONTENT") - calls: list[tuple[str, str]] = [] - - def _find(db: Any, source: str, path: str) -> Document | None: - calls.append((source, path)) - return doc if (source, path) == ("S", "a/b.md") else None - - monkeypatch.setattr(agent, "find_document", _find) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="read_document", - arguments={"source": "S/a", "path": "b.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert calls == [("S/a", "b.md"), ("S", "a"), ("S", "a/b.md")] - assert holder.read_docs == [doc] - assert holder.tool_calls == 1 - assert llm.requests[1][0][3]["content"] == "Document S/a/b.md:\nB-CONTENT" - - -def test_read_combined_source_unknown_teaches_the_split( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A combined source that matches nothing — even split — gets the - EDUCATIONAL refusal: it names the corrected arguments instead of - repeating the combined form (the old generic line reinforced the - mistake).""" - monkeypatch.setattr(agent, "find_document", lambda db, source, path: None) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="read_document", - arguments={"source": "S/a/b.md", "path": "a/b.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert holder.read_docs == [] and holder.tool_calls == 0 - assert llm.requests[1][0][3]["content"] == ( - "source must not contain '/': for 'S/a/b.md' call " - "read_document(source='S', path='a/b.md')." - ) - assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered - - -def test_read_combined_source_for_seed_doc_is_already_in_context( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The combined form of a document ALREADY in context: the raw pair - cannot match the dedupe set, so the split resolves it — and it is - still rejected as already-in-context (no duplicate read_docs entry, - no re-read into the context).""" - seed = [_doc("S", "a.md", "A", "A-CONTENT")] - - def _find(db: Any, source: str, path: str) -> Document | None: - return seed[0] if (source, path) == ("S", "a.md") else None - - monkeypatch.setattr(agent, "find_document", _find) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="read_document", - arguments={"source": "S/a.md", "path": "a.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings(), seed_docs=seed)) - assert holder.read_docs == [] and holder.tool_calls == 0 - assert llm.requests[1][0][3]["content"] == agent.ALREADY_IN_CONTEXT - - -def test_search_scoped_combined_source_is_split( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A scoped search whose 'source' carries the combined form resolves - through the split — the search runs on the right document.""" - doc = _doc("S", "a.md", "A", "needle here") - - def _find(db: Any, source: str, path: str) -> Document | None: - return doc if (source, path) == ("S", "a.md") else None - - monkeypatch.setattr(agent, "find_document", _find) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="search_documents", - arguments={"pattern": "needle", "source": "S/a.md", "path": "a.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here" - assert holder.tool_calls == 1 - assert holder.read_docs == [] # searched doc did not enter the context - - -def test_search_scoped_combined_source_unknown_teaches_the_split( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(agent, "find_document", lambda db, source, path: None) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="search_documents", - arguments={"pattern": "x", "source": "S/ghost.md", "path": "ghost.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == ( - "source must not contain '/': for 'S/ghost.md' use " - "source='S', path='ghost.md'." - ) - assert holder.tool_calls == 0 and holder.read_docs == [] # a refusal - - -# ---------- search_documents (phase 68, locked A5/A6) ---------- - - -def test_grep_document_case_insensitive_line_numbers() -> None: - """Case-insensitive fixed substring, 1-based line numbers, file order, - repeated matches within a line collapse to one match (grep semantics).""" - content = "The NEEDLE is here\nno hit\nneedle again\nNEEDLE NEEDLE\n" - assert agent.grep_document(content, "NEEDLE") == [ - (1, "The NEEDLE is here"), - (3, "needle again"), - (4, "NEEDLE NEEDLE"), - ] - - -def test_grep_document_rstrips_lines_and_empty_content() -> None: - assert agent.grep_document("hello \t\nworld ", "WORLD") == [(2, "world")] - assert agent.grep_document("", "x") == [] - assert agent.grep_document("no newlines", "NO") == [(1, "no newlines")] - assert agent.grep_document("a\nb\n", "MISSING") == [] - - -def test_search_whole_kb_grep_style_output( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Whole-KB search: catalog order, `source/path:line: text` lines, - case-insensitive; the call counts in ``tool_calls`` and never touches - ``read_docs``; the tools stay offered on the answer request.""" - d1 = _doc("Alpha", "a/one.md", "One", "first\nNEEDLE in one\nlast") - d2 = _doc("Beta", "b/two.md", "Two", "no hit\nneedle in two\n") - monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2]) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", name="search_documents", arguments={"pattern": "needle"} - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == ( - "Alpha/a/one.md:2: NEEDLE in one\n" - "Beta/b/two.md:2: needle in two" - ) - assert holder.tool_calls == 1 - assert holder.read_docs == [] # locked A5: a search adds no context - assert llm.requests[1][1] == AGENT_TOOLS # tools stay offered - - -def test_search_capped_at_20_matches_in_catalog_order( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The 20-match cap is GLOBAL across documents in catalog order, and - the scan stops once it is hit (a 35-match corpus yields exactly 20).""" - d1 = _doc("S", "a.md", "A", "\n".join(f"hit-{i}" for i in range(15))) - d2 = _doc("S", "b.md", "B", "\n".join(f"hit-{i}" for i in range(20))) - monkeypatch.setattr(agent, "all_documents", lambda db: [d1, d2]) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", name="search_documents", arguments={"pattern": "hit-"} - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - lines = llm.requests[1][0][3]["content"].split("\n") - assert len(lines) == agent.SEARCH_MAX_MATCHES - assert lines[0] == "S/a.md:1: hit-0" - assert lines[14] == "S/a.md:15: hit-14" # all of a.md - assert lines[15] == "S/b.md:1: hit-0" # then b.md, in order - assert lines[19] == "S/b.md:5: hit-4" # cut at the global cap - assert holder.tool_calls == 1 - - -def test_search_truncates_match_lines_at_200_chars( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A 300-char match line yields exactly 200 chars of it (no crash).""" - d1 = _doc("S", "a.md", "A", "top\n" + "x" * 300 + " NEEDLE tail") - monkeypatch.setattr(agent, "all_documents", lambda db: [d1]) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", name="search_documents", arguments={"pattern": "needle"} - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert ( - llm.requests[1][0][3]["content"] == f"S/a.md:2: {'x' * agent.SEARCH_LINE_LIMIT}" - ) - assert holder.tool_calls == 1 - - -def test_search_scoped_to_one_document(monkeypatch: pytest.MonkeyPatch) -> None: - """Scoped search: only the named document is loaded (find_document), - ``all_documents`` never runs, and the match line carries its path.""" - d1 = _doc("S", "a.md", "A", "needle here") - - def _find(db: Any, source: str, path: str) -> Document | None: - if (source, path) == ("S", "a.md"): - return d1 - raise AssertionError( - f"find_document({source}, {path}) — the scoped " - "search must not load any other document" - ) - - def _boom(*_a: Any, **_k: Any) -> None: - raise AssertionError("all_documents must not run for a scoped search") - - monkeypatch.setattr(agent, "find_document", _find) - monkeypatch.setattr(agent, "all_documents", _boom) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="search_documents", - arguments={"pattern": "needle", "source": "S", "path": "a.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == "S/a.md:1: needle here" - assert holder.tool_calls == 1 - assert holder.read_docs == [] # searched doc did not enter the context - - -def test_search_scoped_missing_document_refused( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(agent, "find_document", lambda db, source, path: None) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="search_documents", - arguments={"pattern": "x", "source": "S", "path": "ghost.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert ( - llm.requests[1][0][3]["content"] - == "No document at S/ghost.md — check the list_documents output." - ) - assert holder.tool_calls == 0 and holder.read_docs == [] # a refusal - - -@pytest.mark.parametrize( - ("arguments", "label"), - [ - ({}, "no arguments"), - ({"pattern": ""}, "empty pattern"), - ({"pattern": " "}, "whitespace pattern"), - ({"pattern": 42}, "non-string pattern"), - ({"pattern": None}, "null pattern"), - ({"pattern": "x", "source": "S"}, "source without path"), - ({"pattern": "x", "path": "a.md"}, "path without source"), - ], -) -def test_search_missing_arguments_refused( - monkeypatch: pytest.MonkeyPatch, arguments: dict[str, Any], label: str -) -> None: - """Unusable pattern OR a half-specified source/path pair → the - missing-args refusal, with no DB access at all.""" - - def _boom(*_a: Any, **_k: Any) -> None: - raise AssertionError(f"no DB access for a refused search ({label})") - - monkeypatch.setattr(agent, "all_documents", _boom) - monkeypatch.setattr(agent, "find_document", _boom) - holder = AgentHolder() - llm = ScriptedLLM( - [ToolCallPiece(id="call_1", name="search_documents", arguments=arguments)], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == agent.MISSING_SEARCH_ARGS - assert holder.tool_calls == 0 and holder.read_docs == [] - assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered - - -def test_search_no_matches_whole_kb(monkeypatch: pytest.MonkeyPatch) -> None: - """Zero hits across the KB → the no-match line (pattern quoted); the - search still executed, so it counts — and never adds context.""" - monkeypatch.setattr( - agent, "all_documents", lambda db: [_doc("S", "a.md", "A", "nothing here")] - ) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", name="search_documents", arguments={"pattern": "zebra"} - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == ( - "No matches for 'zebra' in the knowledge base." - ) - assert holder.tool_calls == 1 - assert holder.read_docs == [] - - -def test_search_no_matches_scoped(monkeypatch: pytest.MonkeyPatch) -> None: - doc = _doc("S", "a.md", "A", "nothing here") - monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="search_documents", - arguments={"pattern": "zebra", "source": "S", "path": "a.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == "No matches for 'zebra' in S/a.md." - assert holder.tool_calls == 1 - assert holder.read_docs == [] - - -def test_search_no_match_truncates_long_pattern( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A pattern longer than 100 chars is truncated in the no-match line - (kept short); the search itself still runs on the full pattern.""" - monkeypatch.setattr(agent, "all_documents", lambda db: []) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", - name="search_documents", - arguments={"pattern": "p" * 150}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert llm.requests[1][0][3]["content"] == ( - f"No matches for '{'p' * 100}' in the knowledge base." - ) - assert holder.tool_calls == 1 - - -def test_search_counts_but_never_adds_context( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The locate-then-read workflow: a search finds the document but does - NOT add it — the subsequent read_document does (and is not rejected as - already-in-context, because the search touched nothing).""" - doc = _doc("S", "a.md", "A", "needle here") - monkeypatch.setattr(agent, "all_documents", lambda db: [doc]) - monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc) - holder = AgentHolder() - llm = ScriptedLLM( - [ - ToolCallPiece( - id="call_1", name="search_documents", arguments={"pattern": "needle"} - ) - ], - [ - ToolCallPiece( - id="call_2", - name="read_document", - arguments={"source": "S", "path": "a.md"}, - ) - ], - [StreamPiece("content", "ans")], - ) - asyncio.run(_run(llm, holder, _settings())) - assert holder.tool_calls == 2 # search + read, both executed - assert holder.read_docs == [doc] # only the read added context (A5) - assert llm.requests[2][0][5]["content"] == "Document S/a.md:\nneedle here" - - # ---------- retries inside the agent loop (phase 67, locked A2) ---------- @@ -1133,7 +1154,7 @@ def test_round_retried_before_first_piece( llm = FailingLLM( [ ([], LLMError("connection refused")), - ([ToolCallPiece(id="call_1", name="list_documents", arguments={})], None), + ([ToolCallPiece(id="call_1", name="ls", arguments={})], None), ([StreamPiece("content", "Done!")], None), ] ) @@ -1144,7 +1165,7 @@ def test_round_retried_before_first_piece( ) assert pieces == [ RetryPiece(2, 4), # default llm_retries=3 → 4 attempts - ToolCallPiece(id="call_1", name="list_documents", arguments={}), + ToolCallPiece(id="call_1", name="ls", arguments={}), StreamPiece("content", "Done!"), ] assert holder.tool_calls == 1 @@ -1158,12 +1179,10 @@ def test_round_retried_before_first_piece( assert sleeps == [2.5] tool_logs = [r for r in caplog.records if r.getMessage().startswith("agent tool=")] assert len(tool_logs) == 1 # the retry did not re-run the tool or log - assert tool_logs[0].getMessage() == "agent tool=list_documents args={} round=1/2" + assert tool_logs[0].getMessage() == "agent tool=ls args={} round=1/2" -def test_round_failure_after_first_piece_is_terminal( - monkeypatch: pytest.MonkeyPatch, -) -> None: +def test_round_failure_after_first_piece_is_terminal(monkeypatch: pytest.MonkeyPatch) -> None: """Locked A2: a round that already streamed a piece fails the turn — the LLMError propagates out of ``run_agent``, no RetryPiece, no sleep, no second request, and the holder is untouched (the tool @@ -1197,9 +1216,7 @@ def test_round_failure_after_first_piece_is_terminal( assert holder.read_docs == [] and holder.tool_calls == 0 -def test_forced_final_no_tools_call_is_retried( - monkeypatch: pytest.MonkeyPatch, -) -> None: +def test_forced_final_no_tools_call_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: """The forced final request (round cap reached) goes through the same retry rule: a failure before its first piece yields a RetryPiece and restarts with ``tools=None``; the answer from the retry streams.""" @@ -1207,8 +1224,8 @@ def test_forced_final_no_tools_call_is_retried( holder = AgentHolder() llm = FailingLLM( [ - ([ToolCallPiece(id="call_1", name="list_documents", arguments={})], None), - ([ToolCallPiece(id="call_2", name="list_documents", arguments={})], None), + ([ToolCallPiece(id="call_1", name="ls", arguments={})], None), + ([ToolCallPiece(id="call_2", name="ls", arguments={})], None), ([], LLMError("down at the cap")), ([StreamPiece("content", "forced answer")], None), ] @@ -1231,9 +1248,7 @@ def test_forced_final_no_tools_call_is_retried( assert holder.tool_calls == 2 -def test_zero_retries_is_one_plain_attempt( - monkeypatch: pytest.MonkeyPatch, -) -> None: +def test_zero_retries_is_one_plain_attempt(monkeypatch: pytest.MonkeyPatch) -> None: """The kill-switch path (``llm_retries=0``): a dead round raises immediately — one request, no RetryPiece, no sleep (pre-phase-67 behavior).""" @@ -1264,9 +1279,7 @@ def test_zero_retries_is_one_plain_attempt( assert holder.read_docs == [] and holder.tool_calls == 0 -def test_abandon_mid_retry_sleep_leaks_nothing( - monkeypatch: pytest.MonkeyPatch, -) -> None: +def test_abandon_mid_retry_sleep_leaks_nothing(monkeypatch: pytest.MonkeyPatch) -> None: """Consumer abandon while a retried round is parked in the pre-retry sleep (client disconnect): the driving task is cancelled cleanly, the production teardown ``aclose()`` on ``run_agent`` does not raise, the @@ -1329,8 +1342,8 @@ def test_retries_are_invisible_to_the_round_cap( llm = FailingLLM( [ ([], LLMError("down")), - ([ToolCallPiece(id="call_1", name="list_documents", arguments={})], None), - ([ToolCallPiece(id="call_2", name="list_documents", arguments={})], None), + ([ToolCallPiece(id="call_1", name="ls", arguments={})], None), + ([ToolCallPiece(id="call_2", name="ls", arguments={})], None), ([StreamPiece("content", "forced answer")], None), ] ) @@ -1346,20 +1359,34 @@ def test_retries_are_invisible_to_the_round_cap( assert llm.requests[3][1] is None # the forced final, after round 2 assert holder.tool_calls == 2 msgs = [r.getMessage() for r in caplog.records] - assert "agent tool=list_documents args={} round=1/2" in msgs - assert "agent tool=list_documents args={} round=2/2" in msgs + assert "agent tool=ls args={} round=1/2" in msgs + assert "agent tool=ls args={} round=2/2" in msgs assert any("round cap reached (rounds=2)" in m for m in msgs) # ---------- prompts: section (HIGH only) ---------- +# NOTE (phase 70, task 02): these pins cover the phase-70 TOOLS_SECTION +# copy — the harness-aligned ls/read/grep names (the old phase-37/68 +# names and the phase-37 per-tool budget line are gone; the round cap +# is the bound, not re-stated in the prompt, phase 45). The +# LOW/deflection path is untouched by this phase. def test_high_prompt_carries_tools_section_after_documents() -> None: prompt = build_high_prompt([_doc("S", "a.md", "A", "A-CONTENT")]) assert TOOLS_SECTION in prompt - assert "call `list_documents`" in prompt - assert "then `read_document` to pull in exactly one more document" in prompt - assert "do not read more than one extra document" in prompt + # Phase 70: the harness-aligned ls/read/grep copy. + assert "`ls`" in prompt + assert "`grep`" in prompt + assert "`read`" in prompt + assert "source: X | path: Y | title: Z" in prompt + assert "locator, not a context-adder" in prompt + assert "combined `source/path`" in prompt + assert "Answer as soon as you have what you need" in prompt + # The old names and the per-tool budget restatement are gone. + for old in ("list_documents", "read_document", "search_documents"): + assert old not in prompt + assert "more than one extra document" not in prompt # After the mode body: follows . assert prompt.index("") < prompt.index("") assert prompt.rstrip().endswith("") diff --git a/tests/unit/test_chat_gate.py b/tests/unit/test_chat_gate.py index 84893af..85040bf 100644 --- a/tests/unit/test_chat_gate.py +++ b/tests/unit/test_chat_gate.py @@ -578,10 +578,16 @@ def test_endpoint_grounded_turn_runs_agent_loop_with_tools( assert not any(f["type"] == "tool" for f in frames) assert len(llm.seen) == 1 assert llm.seen_tools == [AGENT_TOOLS] # one request, tools offered - # The system prompt is the HIGH prompt with the instructions. + # The system prompt is the HIGH prompt with the instructions + # (phase 70: the harness-aligned ls/read/grep copy — new names in, + # old phase-37/68 names out). (system, _user) = llm.seen[0][0], llm.seen[0][1] assert "HIGH" in system["content"] assert "" in system["content"] + for tool in ("`ls`", "`grep`", "`read`"): + assert tool in system["content"] + for old in ("list_documents", "read_document", "search_documents"): + assert old not in system["content"] def test_endpoint_deflected_turn_never_offers_tools( @@ -605,6 +611,9 @@ def test_endpoint_deflected_turn_never_offers_tools( assert llm.seen_tools == [None] (system, _user) = llm.seen[0][0], llm.seen[0][1] assert "" not in system["content"] # the LOW prompt never carries it + # Phase 70: the rewritten copy stays out of the deflected path + # (the LOW prompt is byte-identical to the pre-phase text). + assert "You may extend your context with three tools" not in system["content"] def test_endpoint_score_at_threshold_answers( diff --git a/tests/unit/test_frontend_tool_states.py b/tests/unit/test_frontend_tool_states.py index 0c923d1..0c762a0 100644 --- a/tests/unit/test_frontend_tool_states.py +++ b/tests/unit/test_frontend_tool_states.py @@ -6,7 +6,12 @@ suite (task 06). Like the other frontend-adjacent unit files, this module pins the JS/CSS markers the story depends on, so a silent regression in the tool branch, the persistence shape, or the tool-line styling is catched without a browser. Phase 68 extends the pins with the -``search_documents`` status/line contract. +``search_documents`` status/line contract. Phase 70 extends the pins to +the harness-aligned names (``ls`` / ``read`` / ``grep``) in both +``app.js`` and the shared page's local copy (``shared.js``) — the legacy +names (``list_documents`` / ``read_document`` / ``search_documents``) +must keep rendering exactly as before for persisted turns (no +migration). """ from __future__ import annotations @@ -15,6 +20,7 @@ from pathlib import Path FRONTEND = Path(__file__).resolve().parents[2] / "frontend" APP_JS = FRONTEND / "assets" / "app.js" +SHARED_JS = FRONTEND / "assets" / "shared.js" STYLES_CSS = FRONTEND / "assets" / "styles.css" @@ -22,6 +28,10 @@ def _js() -> str: return APP_JS.read_text(encoding="utf-8") +def _shared_js() -> str: + return SHARED_JS.read_text(encoding="utf-8") + + def _css() -> str: return STYLES_CSS.read_text(encoding="utf-8") @@ -66,7 +76,10 @@ def test_calling_tool_label_strings() -> None: status lives in #send-status + the typing-indicator aria-label only. Phase 39 centralizes the brand prefix: the name resolves from window.BOR_BRAND at call time via brand() (the default name renders - the same bytes).""" + the same bytes). Phase 70: the ternary keys off the harness-aligned + names (read / grep / ls) and still carries the legacy names + (read_document / search_documents) — a pre-remap label stays + accurate.""" js = _js() tool_idx = js.find('ev.type === "tool"') delta_idx = js.find('ev.type === "delta"') @@ -74,12 +87,19 @@ def test_calling_tool_label_strings() -> None: assert "sendLabel" not in branch, "phase 48: the button keeps its Stop label" assert "`${brand()} is listing documents`" in branch assert "`${brand()} is reading ${argument}`" in branch - # Phase 68: the search status — locked name+argument gate, sitting - # BETWEEN the read branch and the listing fallback in the ternary. - assert "name === \"search_documents\" && argument" in branch, ( - "the search status requires the name AND a string argument" + # Phase 70: the read status — new + legacy name, locked + # name+argument gate, first in the ternary. + assert 'name === "read" || name === "read_document") && argument' in branch, ( + "the read status requires the name (new or legacy) AND a string argument" ) + # The search status — new + legacy name, sitting BETWEEN the read + # branch and the listing fallback in the ternary. + assert 'name === "grep" || name === "search_documents") && argument' in branch assert "`${brand()} is searching for ${argument}`" in branch + # Phase 70: the scoped ls status mirrors the scoped tool line; the + # unscoped listing stays the final fallback. + assert 'name === "ls" && argument' in branch + assert "`${brand()} is listing documents in ${argument}`" in branch read = branch.find("is reading") search = branch.find("is searching for") listing = branch.find("is listing documents") @@ -120,21 +140,41 @@ def test_tool_lines_render_into_the_bubble_wrap() -> None: assert "code.textContent = argument" in body, ( "the path is data — textContent, never innerHTML" ) - assert "name === \"read_document\" && argument" in body - # Phase 68: the search branch mirrors the read branch — the same - # name+argument gate, a element, and the pattern through - # textContent (never markup); the listing stays the final else. - assert "name === \"search_documents\" && argument" in body + # Phase 70: the harness-aligned names key the branches, with the + # legacy names kept — a persisted turn from before the remap + # (read_document / search_documents / list_documents) renders + # unchanged (no migration). + assert '(name === "read" || name === "read_document") && argument' in body, ( + "read (new) and read_document (legacy) both render the Reading line" + ) + assert '(name === "grep" || name === "search_documents") && argument' in body, ( + "grep (new) and search_documents (legacy) both render the Searching line" + ) assert 'line.textContent = "🔎 Searching for "' in body - search_part = body.split('name === "search_documents"', 1)[1] - assert 'document.createElement("code")' in search_part, ( + grep_part = body.split('name === "grep"', 1)[1] + assert 'document.createElement("code")' in grep_part, ( "the pattern gets the same treatment as the read path" ) - assert "code.textContent = argument" in search_part, ( + assert "code.textContent = argument" in grep_part, ( "the pattern is data — textContent, never innerHTML" ) - assert 'line.textContent = "🔎 Listing documents"' in search_part, ( - "the listing fallback remains the final else" + # Phase 70: the scoped ls line — the scope through textContent, and + # the unscoped "Listing documents" stays the final else (legacy + # list_documents, and a nameless/unknown frame, land there too). + assert 'name === "ls" && argument' in body + assert 'line.textContent = "🔎 Listing documents in "' in body + ls_part = body.split('name === "ls" && argument', 1)[1] + assert 'document.createElement("code")' in ls_part, ( + "the scope gets the same treatment as the read path" + ) + assert "code.textContent = argument" in ls_part, ( + "the scope is data — textContent, never innerHTML" + ) + assert 'line.textContent = "🔎 Listing documents"' in ls_part, ( + "the unscoped listing fallback remains the final else" + ) + assert "innerHTML" not in body, ( + "no HTML injection surface on tool lines — textContent only" ) @@ -231,6 +271,47 @@ def test_tool_call_style_is_accent_and_contrast_safe() -> None: ) +def test_shared_page_tool_lines_cover_new_and_legacy_names() -> None: + """Phase 70: the shared page's local copy (``addToolLines``) renders + the harness-aligned names — read → Reading, grep → Searching for, + ls → Listing documents, scoped ls → Listing documents in — + and keeps the legacy branches (read_document / search_documents), so + a conversation saved before the remap renders exactly as before (no + migration). Every argument through textContent; the lines carry no + innerHTML at all.""" + js = _shared_js() + fn = js.find("function addToolLines") + assert fn != -1, "addToolLines must exist in shared.js" + body = js[fn : js.find("\n}\n", fn)] + assert '(t.name === "read" || t.name === "read_document") && argument' in body, ( + "read (new) and read_document (legacy) both render the Reading line" + ) + assert '(t.name === "grep" || t.name === "search_documents") && argument' in body, ( + "grep (new) and search_documents (legacy) both render the Searching line" + ) + assert 'line.textContent = "📄 Reading "' in body + assert 'line.textContent = "🔎 Searching for "' in body + assert 't.name === "ls" && argument' in body + assert 'line.textContent = "🔎 Listing documents in "' in body + ls_part = body.split('t.name === "ls" && argument', 1)[1] + assert 'document.createElement("code")' in ls_part, ( + "the scope gets the same treatment as the read path" + ) + assert "code.textContent = argument" in ls_part, ( + "the scope is data — textContent, never innerHTML" + ) + assert 'line.textContent = "🔎 Listing documents"' in ls_part, ( + "the unscoped listing fallback remains the final else (legacy" + " list_documents renders unchanged)" + ) + assert body.count("code.textContent = argument") == 3, ( + "all three argument-bearing lines (read / grep / ls) are textContent-only" + ) + assert "innerHTML" not in body, ( + "no HTML injection surface on shared tool lines — textContent only" + ) + + def test_no_cdn_added() -> None: """AGENTS.md rule 6: the tool state adds no external script/link.""" index = (FRONTEND / "index.html").read_text(encoding="utf-8") diff --git a/tests/unit/test_llm_client.py b/tests/unit/test_llm_client.py index 218cd77..bf461ae 100644 --- a/tests/unit/test_llm_client.py +++ b/tests/unit/test_llm_client.py @@ -500,29 +500,27 @@ def test_chat_stream_llm_error_passes_through_unwrapped() -> None: # ---------- tool-call streaming (phase 37, task 02) ---------- -#: The agent's tool list (phase 37) — the exact wire shape AGENT_TOOLS will -#: pass through (the names are whatever the caller's tools list names). +#: The agent's tool list (phase 70: the harness-aligned surface) — the +#: exact wire shape AGENT_TOOLS passes through (the names are whatever +#: the caller's tools list names). _AGENT_TOOLS: list[dict[str, Any]] = [ { "type": "function", "function": { - "name": "list_documents", + "name": "ls", "description": "List the indexed documents.", - "parameters": {"type": "object", "properties": {}}, + "parameters": {"type": "object", "properties": {}, "required": []}, }, }, { "type": "function", "function": { - "name": "read_document", + "name": "read", "description": "Add one indexed document's full text to the context.", "parameters": { "type": "object", - "properties": { - "source": {"type": "string"}, - "path": {"type": "string"}, - }, - "required": ["source", "path"], + "properties": {"path": {"type": "string"}}, + "required": ["path"], }, }, }, @@ -557,12 +555,12 @@ def test_chat_stream_accumulates_tool_call_across_chunk_partials() -> None: _tool_call( 0, id="call_abc", - name="read_document", - arguments='{"source": "Homelab", "pa', + name="read", + arguments='{"path": "Homelab/ku', ) ], ), - _chunk(None, tool_calls=[_tool_call(0, arguments='th": "kubernetes.md"}')]), + _chunk(None, tool_calls=[_tool_call(0, arguments='bernetes.md"}')]), _chunk(None, finish_reason="tool_calls"), ] ) @@ -572,8 +570,8 @@ def test_chat_stream_accumulates_tool_call_across_chunk_partials() -> None: assert pieces == [ ToolCallPiece( id="call_abc", - name="read_document", - arguments={"source": "Homelab", "path": "kubernetes.md"}, + name="read", + arguments={"path": "Homelab/kubernetes.md"}, ) ] @@ -586,14 +584,14 @@ def test_chat_stream_two_tool_calls_yielded_in_index_order() -> None: _chunk( None, tool_calls=[ - _tool_call(1, id="call_b", name="read_document", arguments='{"sou') + _tool_call(1, id="call_b", name="read", arguments='{"pa') ], ), _chunk( None, tool_calls=[ - _tool_call(0, id="call_a", name="list_documents"), - _tool_call(1, arguments='rce": "Homelab", "path": "a.md"}') + _tool_call(0, id="call_a", name="ls"), + _tool_call(1, arguments='th": "Homelab/a.md"}') ], ), _chunk(None, finish_reason="tool_calls"), @@ -603,11 +601,11 @@ def test_chat_stream_two_tool_calls_yielded_in_index_order() -> None: llm, [{"role": "user", "content": "q"}], _AGENT_TOOLS ) assert pieces == [ - ToolCallPiece(id="call_a", name="list_documents", arguments={}), + ToolCallPiece(id="call_a", name="ls", arguments={}), ToolCallPiece( id="call_b", - name="read_document", - arguments={"source": "Homelab", "path": "a.md"}, + name="read", + arguments={"path": "Homelab/a.md"}, ), ] @@ -619,21 +617,21 @@ def test_chat_stream_tool_calls_yielded_at_stream_end_without_finish_reason() -> [ _chunk( None, - tool_calls=[_tool_call(0, id="call_z", name="list_documents")], + tool_calls=[_tool_call(0, id="call_z", name="ls")], ) ] ) pieces = _collect_with_tools( llm, [{"role": "user", "content": "q"}], _AGENT_TOOLS ) - assert pieces == [ToolCallPiece(id="call_z", name="list_documents", arguments={})] + assert pieces == [ToolCallPiece(id="call_z", name="ls", arguments={})] def test_chat_stream_synthesizes_call_id_when_absent() -> None: """Wire never carried the call id ⇒ synthesized "call_".""" llm, _ = _make_stream_client( [ - _chunk(None, tool_calls=[_tool_call(2, name="read_document", arguments="{}")]), + _chunk(None, tool_calls=[_tool_call(2, name="read", arguments="{}")]), _chunk(None, finish_reason="tool_calls"), ] ) @@ -643,7 +641,7 @@ def test_chat_stream_synthesizes_call_id_when_absent() -> None: assert pieces == [ ToolCallPiece( id="call_2", - name="read_document", + name="read", arguments={}, ) ] @@ -656,7 +654,7 @@ def test_chat_stream_null_arguments_become_empty_dict() -> None: _chunk( None, tool_calls=[ - _tool_call(0, id="call_n", name="list_documents", arguments="null") + _tool_call(0, id="call_n", name="ls", arguments="null") ], ), _chunk(None, finish_reason="tool_calls"), @@ -665,7 +663,7 @@ def test_chat_stream_null_arguments_become_empty_dict() -> None: pieces = _collect_with_tools( llm, [{"role": "user", "content": "q"}], _AGENT_TOOLS ) - assert pieces == [ToolCallPiece(id="call_n", name="list_documents", arguments={})] + assert pieces == [ToolCallPiece(id="call_n", name="ls", arguments={})] def test_chat_stream_malformed_tool_arguments_raise_llm_error() -> None: @@ -679,8 +677,8 @@ def test_chat_stream_malformed_tool_arguments_raise_llm_error() -> None: _tool_call( 0, id="call_x", - name="read_document", - arguments='{"source": "Homelab",', + name="read", + arguments='{"path": "Homelab",', ) ], ), @@ -706,7 +704,7 @@ def test_chat_stream_non_object_tool_arguments_raise_llm_error() -> None: _chunk( None, tool_calls=[ - _tool_call(0, id="call_y", name="read_document", arguments='[1, 2]') + _tool_call(0, id="call_y", name="grep", arguments='[1, 2]') ], ), _chunk(None, finish_reason="tool_calls"), @@ -987,7 +985,7 @@ def test_retried_healthy_stream_is_untouched( a healthy turn is byte-identical to the plain chat_stream.""" answer = [ StreamPiece("thinking", "hmm"), - ToolCallPiece(id="call_1", name="list_documents", arguments={}), + ToolCallPiece(id="call_1", name="ls", arguments={}), StreamPiece("content", "Talos."), ] client = _ScriptedClient([(answer, None)]) @@ -995,7 +993,7 @@ def test_retried_healthy_stream_is_untouched( tools = [ { "type": "function", - "function": {"name": "list_documents", "parameters": {}}, + "function": {"name": "ls", "parameters": {}}, } ] pieces = _collect_retried( diff --git a/tests/unit/test_llm_stream_teardown.py b/tests/unit/test_llm_stream_teardown.py index 76e8aeb..4808201 100644 --- a/tests/unit/test_llm_stream_teardown.py +++ b/tests/unit/test_llm_stream_teardown.py @@ -39,7 +39,7 @@ def _chunk(content: str) -> SimpleNamespace: def _tool_chunk() -> SimpleNamespace: """One chunk carrying a malformed-arguments tool call (index 0).""" - fn = SimpleNamespace(name="read_document", arguments='{"source": "Homelab",') + fn = SimpleNamespace(name="read", arguments='{"path": "Homelab",') tc = SimpleNamespace(index=0, id="call_x", function=fn) delta = SimpleNamespace(content=None, tool_calls=[tc]) return SimpleNamespace(choices=[SimpleNamespace(delta=delta)]) @@ -194,7 +194,7 @@ def test_llm_error_materialization_passes_through_and_closes() -> None: async def drain() -> None: async for _ in llm.chat_stream( [{"role": "user", "content": "q"}], - tools=[{"type": "function", "function": {"name": "read_document"}}], + tools=[{"type": "function", "function": {"name": "read"}}], ): pass diff --git a/tests/unit/test_mock_tool_flow.py b/tests/unit/test_mock_tool_flow.py index 10608c6..433ef04 100644 --- a/tests/unit/test_mock_tool_flow.py +++ b/tests/unit/test_mock_tool_flow.py @@ -32,10 +32,11 @@ from tests.e2e.mock_llm import ( SYSTEM_HIGH = "HIGH\n\n\n\n…\n" SYSTEM_LOW = "LOW\n" -#: A minimal truthy ``tools`` parameter (the mock only checks presence). -TOOLS = [{"type": "function", "function": {"name": "list_documents"}}] +#: A minimal truthy ``tools`` parameter (the mock only checks presence; +#: the phase-70 harness-aligned names). +TOOLS = [{"type": "function", "function": {"name": "ls"}}] -#: The agent's ``list_documents`` output for a two-document KB +#: The agent's ``ls`` output for a two-document KB #: (``app/rag/agent.py`` ``_execute_tool``): one #: ``source: X | path: Y | title: Z`` line per document (phase 63: labeled, #: unambiguous fields), ``(source, path)`` order. @@ -103,7 +104,7 @@ def _body( { "id": f"call_{i}", "type": "function", - "function": {"name": "list_documents", "arguments": "{}"}, + "function": {"name": "ls", "arguments": "{}"}, } ], } @@ -274,7 +275,8 @@ SEARCH_USER = ( assert SEARCH_TRIGGER in SEARCH_USER.lower() assert TOOLS_TRIGGER not in SEARCH_USER.lower() -#: The agent's ``search_documents`` result for the e2e fixture +#: The agent's ``grep`` result for the e2e fixture (phase 70 renamed +#: the phase-68 tool; the line format is unchanged) #: (``app/rag/agent.py`` ``_execute_tool``): one ``source/path:LINE: text`` #: match line (the sentinel line, 200-char-capped server-side). SEARCH_RESULT = ( diff --git a/tests/unit/test_prompts.py b/tests/unit/test_prompts.py index 4e93384..8535da0 100644 --- a/tests/unit/test_prompts.py +++ b/tests/unit/test_prompts.py @@ -138,6 +138,68 @@ def test_zero_note_prompt_is_byte_identical_to_pre_steering() -> None: ) assert "" not in build_high_prompt([doc]) assert "" not in build_deflect_prompt([]) + # Phase 70: the rewritten copy stays out of the LOW path — + # the byte-identical equality above already proves it; this names + # the contract (no , no new copy) on both empty/non-empty LOW + # builds. + for low in (build_deflect_prompt(["T1"]), build_deflect_prompt([])): + assert "" not in low + assert TOOLS_SECTION not in low + + +# ---------- section copy (phase 70: ls / read / grep) ---------- + + +def test_tools_section_markers_and_new_tool_names() -> None: + """Phase 70: the section keeps the ````/```` markers + the E2E mock keys on and teaches the harness-aligned tool names + (backticked, exactly as the ``AGENT_TOOLS`` schemas name them).""" + assert TOOLS_SECTION.startswith("\n") + assert TOOLS_SECTION.rstrip().endswith("") + for tool in ("`ls`", "`grep`", "`read`"): + assert tool in TOOLS_SECTION + + +def test_tools_section_teaches_the_harness_shapes() -> None: + """Copy pins: ``ls``'s phase-63 catalog-line format (and its + optional one-source scope), ``grep``'s case-insensitive exact-string + locator contract (up to 20 ``source/path:line: text`` lines, a + locator not a context-adder), and ``read``'s combined + ``source/path`` + full content.""" + assert "source: X | path: Y | title: Z" in TOOLS_SECTION + assert "pass a source name as `path`" in TOOLS_SECTION + assert "case-insensitive" in TOOLS_SECTION + assert "up to 20" in TOOLS_SECTION + assert "source/path:line: text" in TOOLS_SECTION + assert "locator, not a context-adder" in TOOLS_SECTION + assert "combined `source/path`" in TOOLS_SECTION + assert "full content" in TOOLS_SECTION + assert "Answer as soon as you have what you need" in TOOLS_SECTION + + +def test_tools_section_old_names_and_budget_copy_gone() -> None: + """The phase-37/68 tool names and the phase-37 per-tool budget line + (phase 45: the round cap is the bound — the prompt does not + re-state it) are out of the copy.""" + for old in ("list_documents", "read_document", "search_documents"): + assert old not in TOOLS_SECTION + assert "more than one" not in TOOLS_SECTION + assert "extra document" not in TOOLS_SECTION + + +def test_high_prompt_still_ends_with_tools_section() -> None: + """Mock keying intact: the HIGH prompt still ends with the + ```` section after ````, now in the phase-70 + copy — new names in, old names out.""" + doc = _doc("kubernetes.md", "Talos Linux on three nodes.", "Kubernetes Homelab Cluster") + prompt = build_high_prompt([doc]) + assert TOOLS_SECTION in prompt + assert prompt.index("") < prompt.index("") + assert prompt.rstrip().endswith("") + for tool in ("`ls`", "`grep`", "`read`"): + assert tool in prompt + for old in ("list_documents", "read_document", "search_documents"): + assert old not in prompt def test_relevance_placeholder_rejected_for_garbage() -> None: diff --git a/tests/unit/test_sse_events.py b/tests/unit/test_sse_events.py index 2acdf65..63ec1f8 100644 --- a/tests/unit/test_sse_events.py +++ b/tests/unit/test_sse_events.py @@ -88,21 +88,22 @@ def test_tool_frame_serializes_exactly() -> None: ``{type: "tool", name: str, argument: str | null}`` — one per model-requested document tool call, streamed ahead of the ``delta`` frames of the answer.""" - frame = sse_event(ChatToolEvent(name="read_document", argument="S/p.md").model_dump()) - assert frame == 'data: {"type": "tool", "name": "read_document", "argument": "S/p.md"}\n\n' - assert _payload(frame) == {"type": "tool", "name": "read_document", "argument": "S/p.md"} + frame = sse_event(ChatToolEvent(name="read", argument="S/p.md").model_dump()) + assert frame == 'data: {"type": "tool", "name": "read", "argument": "S/p.md"}\n\n' + assert _payload(frame) == {"type": "tool", "name": "read", "argument": "S/p.md"} def test_tool_frame_argument_is_null_for_parameterless_tools() -> None: - """``list_documents`` takes no parameters, so its frame's ``argument`` - serializes as JSON null (the client renders the name alone).""" - dumped = ChatToolEvent(name="list_documents").model_dump() - assert dumped == {"type": "tool", "name": "list_documents", "argument": None} + """``ls`` (unscoped) carries no string argument, so its frame's + ``argument`` serializes as JSON null (the client renders the name + alone).""" + dumped = ChatToolEvent(name="ls").model_dump() + assert dumped == {"type": "tool", "name": "ls", "argument": None} assert _payload(sse_event(dumped))["argument"] is None def test_tool_event_shape_is_type_name_argument_only() -> None: - dumped = ChatToolEvent(name="read_document", argument="S/p.md").model_dump() + dumped = ChatToolEvent(name="read", argument="S/p.md").model_dump() assert set(dumped.keys()) == {"type", "name", "argument"} assert dumped["type"] == "tool" # default — call sites never spell it out