feat(rag): unbounded agent tool calls behind a round cap (owner revision)
Phase 45 (owner permission 2026-08-27, TODO.md L8: "allow the LLM
to make as many tool calls as it wants"): the phase-37 per-turn tool
budgets (BOR_AGENT_LIST_CALLS / BOR_AGENT_READ_CALLS, default 1 each)
and their exhaustion refusals are removed — a grounded turn now offers
list_documents / read_document for the whole turn (re-lists included),
bounded only by the round cap:
- app/config.py: agent_max_rounds (BOR_AGENT_MAX_ROUNDS, default 10,
negative rejected) replaces agent_list_calls / agent_read_calls;
.env.example + README document the single knob; app/rag/prompts.py
docstrings follow.
- app/rag/agent.py: the loop runs tools until the model answers or
rounds >= max_rounds, at which point it forces one final no-tools
answer (the cap is the only forced exit); 0 = no tools — exactly one
tools=None request, byte-identical to the pre-phase-37 path (the
kill switch). Rejected calls (unknown tool / missing args /
already-in-context / unknown path) still consume a round, so
pathological rejected-call streams are bounded by the cap. The
per-call log line is now tool/args/round=N/M; the per-turn
tool_calls=N field and the tool SSE event are unchanged.
- tests/e2e/mock_llm.py: MULTI_READ_TRIGGER ("read two documents") —
the deterministic list -> read #1 -> read #2 -> forced-answer flow
(byte-stable "I read <sp1> and <sp2>." line), classified by the
count of tool-role read results; the phase-37 single-read flow stays
byte-identical (unit-pinned in tests/unit/test_mock_tool_flow.py).
- tests/e2e/test_agent_unlimited_tools.py (new, story suite,
mock-only): three tool frames/lines in order (one list, two reads —
the second read is what the old read budget refused) + the
both-named non-deflected answer; done.sources + chips = retrieval
doc + both reads, deduped; no budget refusal rendered; the
single-read marker flow regression (exactly one read, single tool
pair).
- .agent/PLAN.md: the phase-45 SSE revision note (owner-locked, R2) —
the only PLAN edit this phase; the phase-37 note's budget clause is
marked removed.
Unit/integration rewrites (test_agent.py round-cap matrix incl. the
kill switch and rejected-call spam, test_config.py, test_chat_api.py
agent_max_rounds=0 fixtures) landed with the server core so every gate
stays green.
uv run pytest: 756 passed, app/ coverage 99%; ruff + pyright clean;
story E2E 4/4 in isolation (ran twice); regression E2E suites
(agent_document_tools unmodified, chat_rag, smoke) green in isolation.
Also records the 45_agent_unlimited_tools todo/ -> complete/ task-file
moves (00/01/02 pending in the working tree, task 03 moves on success).
This commit is contained in:
+155
-103
@@ -3,10 +3,13 @@
|
||||
A scripted fake LLM (canned stream sequences) + monkeypatched
|
||||
``list_catalog`` / ``find_document`` — no database, no network. Covers
|
||||
the loop mechanics: the list → read → answer happy path (event order,
|
||||
holder state, the ``tools=None`` request after the budgets are spent,
|
||||
the assistant/tool message history), the 0/0 single-call path, budget
|
||||
exhaustion, dedupe, unknown tool / missing args / unknown path, the
|
||||
round cap, and the ``<tools>`` prompt section (HIGH only).
|
||||
holder state, the tools staying offered on every request — phase 45
|
||||
removed the per-tool budgets, the assistant/tool message history), the
|
||||
kill switch (``agent_max_rounds=0`` single-call path), the round cap
|
||||
forcing a final no-tools answer (an always-calling stream and an
|
||||
always-rejected stream), re-lists and multi-reads executing without
|
||||
budgets, dedupe, unknown tool / missing args / unknown path, and the
|
||||
``<tools>`` prompt section (HIGH only).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -102,6 +105,11 @@ def test_agent_tools_names_and_parameters() -> None:
|
||||
read_params = by_name["read_document"]["function"]["parameters"]
|
||||
assert read_params["required"] == ["source", "path"]
|
||||
assert set(read_params["properties"]) == {"source", "path"}
|
||||
# Phase 45: the per-tool budgets are gone — "exactly one more"
|
||||
# dropped out of the read_document description.
|
||||
assert by_name["read_document"]["function"]["description"] == (
|
||||
"Add the full content of one more indexed document to your context"
|
||||
)
|
||||
|
||||
|
||||
# ---------- happy path: list → read → answer ----------
|
||||
@@ -148,11 +156,12 @@ def test_list_then_read_then_answer(
|
||||
assert holder.read_docs == [target]
|
||||
assert holder.tool_calls == 2
|
||||
|
||||
# Default budgets (1/1): tools offered while any budget remains…
|
||||
# Phase 45: no per-tool budgets — the tools stay offered on every
|
||||
# request (the round cap, not spent budgets, bounds the loop), so
|
||||
# the answer request still carries them (2 rounds < default cap 10).
|
||||
assert llm.requests[0][1] == AGENT_TOOLS
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
# …and dropped (tools=None) once both are spent.
|
||||
assert llm.requests[2][1] is None
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
assert len(llm.requests) == 3
|
||||
|
||||
# The follow-up request carries the assistant tool-call + tool result.
|
||||
@@ -226,17 +235,53 @@ def test_content_and_tool_call_in_one_stream_keeps_both(
|
||||
assert llm.requests[1][0][3]["content"] == "0 documents:\n"
|
||||
|
||||
|
||||
# ---------- budgets ----------
|
||||
# ---------- round cap (phase 45: replaces the per-tool budgets) ----------
|
||||
|
||||
|
||||
def test_zero_budgets_is_one_request_without_tools() -> None:
|
||||
"""BOR_AGENT_LIST_CALLS=0 BOR_AGENT_READ_CALLS=0 → byte-identical
|
||||
single-call path: exactly one request, tools=None, no history growth."""
|
||||
def test_always_list_bounded_by_round_cap(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A model that keeps calling ``list_documents`` gets exactly
|
||||
``agent_max_rounds`` tool rounds, then one forced ``tools=None``
|
||||
request streams the answer — the cap is the only forced exit."""
|
||||
monkeypatch.setattr(agent, "list_catalog", lambda db: [("S", "a.md", "A")])
|
||||
listing = "1 documents:\nS/a.md — A"
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_2", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_3", name="list_documents", arguments={})],
|
||||
[StreamPiece("content", "forced answer")],
|
||||
)
|
||||
pieces = asyncio.run(_run(llm, holder, _settings(agent_max_rounds=3)))
|
||||
assert [type(p) for p in pieces] == [
|
||||
ToolCallPiece,
|
||||
ToolCallPiece,
|
||||
ToolCallPiece,
|
||||
StreamPiece,
|
||||
]
|
||||
assert len(llm.requests) == 4 # 3 tool rounds + the forced answer
|
||||
# The three tool rounds were offered the tools…
|
||||
assert llm.requests[0][1] == AGENT_TOOLS
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
# …and the forced final request carries no tools, whatever is left.
|
||||
assert llm.requests[3][1] is None
|
||||
# Every re-list executed and counted.
|
||||
assert holder.tool_calls == 3
|
||||
# The final request carries all three executed listings as history.
|
||||
final_msgs = llm.requests[3][0]
|
||||
assert len(final_msgs) == 8 # 2 + 3 rounds × (assistant + tool)
|
||||
assert final_msgs[3]["content"] == listing
|
||||
assert final_msgs[5]["content"] == listing
|
||||
assert final_msgs[7]["content"] == listing
|
||||
|
||||
|
||||
def test_zero_max_rounds_is_one_request_without_tools() -> None:
|
||||
"""``agent_max_rounds=0`` — the kill switch: exactly one request,
|
||||
``tools=None``, no tool lines, no history growth (byte-identical to
|
||||
the pre-phase-37 path)."""
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM([StreamPiece("thinking", "t "), StreamPiece("content", "direct answer")])
|
||||
pieces = asyncio.run(
|
||||
_run(llm, holder, _settings(agent_list_calls=0, agent_read_calls=0))
|
||||
)
|
||||
pieces = asyncio.run(_run(llm, holder, _settings(agent_max_rounds=0)))
|
||||
assert [type(p) for p in pieces] == [StreamPiece, StreamPiece]
|
||||
assert len(llm.requests) == 1
|
||||
assert llm.requests[0][1] is None
|
||||
@@ -247,12 +292,92 @@ def test_zero_budgets_is_one_request_without_tools() -> None:
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0
|
||||
|
||||
|
||||
def test_read_budget_exhausted_refuses_and_appends_nothing(
|
||||
def test_rejected_read_spam_runs_to_round_cap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Every call rejected (unknown path — "No document at …"): rejections
|
||||
no longer end the loop early via budgets — the round cap bounds them
|
||||
and forces the final no-tools answer."""
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1",
|
||||
name="read_document",
|
||||
arguments={"source": "S", "path": "ghost.md"},
|
||||
)
|
||||
],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_2",
|
||||
name="read_document",
|
||||
arguments={"source": "S", "path": "ghost.md"},
|
||||
)
|
||||
],
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_3",
|
||||
name="read_document",
|
||||
arguments={"source": "S", "path": "ghost.md"},
|
||||
)
|
||||
],
|
||||
[StreamPiece("content", "forced answer")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings(agent_max_rounds=3)))
|
||||
assert len(llm.requests) == 4 # 3 rejected rounds + the forced answer
|
||||
assert llm.requests[0][1] == AGENT_TOOLS
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
assert llm.requests[3][1] is None # the forced final request: no tools
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0 # nothing executed
|
||||
refusal = "No document at S/ghost.md — check the list_documents output."
|
||||
assert llm.requests[1][0][3]["content"] == refusal
|
||||
assert llm.requests[2][0][5]["content"] == refusal
|
||||
assert llm.requests[3][0][7]["content"] == refusal
|
||||
|
||||
|
||||
# ---------- unlimited calls: re-lists and multi-reads (phase 45) ----------
|
||||
|
||||
|
||||
def test_relist_executes_and_counts(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Re-lists execute — a second ``list_documents`` in one turn returns
|
||||
the catalog again and counts in ``tool_calls`` (no budget to
|
||||
exhaust)."""
|
||||
catalog = [
|
||||
("Deployments", "backups.md", "Backup Strategy"),
|
||||
("Homelab", "aws-route53.md", "AWS Route53 Records"),
|
||||
]
|
||||
monkeypatch.setattr(agent, "list_catalog", lambda db: catalog)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_2", name="list_documents", arguments={})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.tool_calls == 2 # both re-lists executed and counted
|
||||
listing = (
|
||||
"2 documents:\n"
|
||||
"Deployments/backups.md — Backup Strategy\n"
|
||||
"Homelab/aws-route53.md — AWS Route53 Records"
|
||||
)
|
||||
# The answer request carries the catalog a second time as a tool result.
|
||||
assert llm.requests[2][0][3]["content"] == listing # first listing
|
||||
assert llm.requests[2][0][5]["content"] == listing # the re-list
|
||||
assert llm.requests[2][1] == AGENT_TOOLS # still offered (no budgets)
|
||||
|
||||
|
||||
def test_multi_read_executes_without_budgets(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Reads are no longer budgeted either — two different documents can
|
||||
be read in one turn (re-reading the same one is still deduped via
|
||||
ALREADY_IN_CONTEXT — see the rejection tests)."""
|
||||
a = _doc("S", "a.md", "A", "A-CONTENT")
|
||||
b = _doc("S", "b.md", "B", "B-CONTENT")
|
||||
monkeypatch.setattr(
|
||||
agent, "find_document", lambda db, source, path: a if path == "a.md" else None
|
||||
agent, "find_document", lambda db, source, path: {"a.md": a, "b.md": b}[path]
|
||||
)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
@@ -268,38 +393,15 @@ def test_read_budget_exhausted_refuses_and_appends_nothing(
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings(agent_list_calls=1, agent_read_calls=1)))
|
||||
|
||||
assert holder.read_docs == [a] # the refused read appended nothing
|
||||
assert holder.tool_calls == 1 # …and consumed no budget
|
||||
refusal = llm.requests[2][0][5]
|
||||
assert refusal == {
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_2",
|
||||
"content": agent.READ_EXHAUSTED,
|
||||
}
|
||||
# The list budget is still open, so tools stay offered after the refusal.
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.read_docs == [a, b] # both reads appended, in order
|
||||
assert holder.tool_calls == 2
|
||||
assert llm.requests[1][0][3]["content"] == "Document S/a.md:\nA-CONTENT"
|
||||
assert llm.requests[2][0][5]["content"] == "Document S/b.md:\nB-CONTENT"
|
||||
assert llm.requests[2][1] == AGENT_TOOLS # the second read was still offered
|
||||
|
||||
|
||||
def test_list_budget_exhausted_refuses_with_its_own_message(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(agent, "list_catalog", lambda db: [])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_2", name="list_documents", arguments={})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings(agent_list_calls=1, agent_read_calls=1)))
|
||||
assert holder.tool_calls == 1
|
||||
assert llm.requests[2][0][5]["content"] == agent.LIST_EXHAUSTED
|
||||
# The read budget is still open, so tools stay offered after the refusal.
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
|
||||
|
||||
# ---------- rejections (no budget consumed) ----------
|
||||
# ---------- rejections (non-budget; the round cap bounds their repetition) ----------
|
||||
|
||||
|
||||
def test_reading_a_seed_doc_is_already_in_context(
|
||||
@@ -326,7 +428,8 @@ def test_reading_a_seed_doc_is_already_in_context(
|
||||
asyncio.run(_run(llm, holder, _settings(), seed_docs=seed))
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0
|
||||
assert llm.requests[1][0][3]["content"] == agent.ALREADY_IN_CONTEXT
|
||||
# No budget consumed → tools are still offered on the next request.
|
||||
# Rejected → the tools are still offered on the next request (the
|
||||
# round cap is the only bound).
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
|
||||
|
||||
@@ -349,15 +452,15 @@ def test_reading_an_already_read_doc_is_deduped(
|
||||
],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings(agent_list_calls=1, agent_read_calls=1)))
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.read_docs == [doc] # appended exactly once
|
||||
assert holder.tool_calls == 1
|
||||
assert llm.requests[2][0][5]["content"] == agent.ALREADY_IN_CONTEXT
|
||||
# The read budget is intact after the deduped refusal…
|
||||
# Rejected → the tools are still offered on the next request…
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
|
||||
|
||||
def test_unknown_path_refused_without_budget(
|
||||
def test_unknown_path_refused(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
@@ -378,7 +481,7 @@ def test_unknown_path_refused_without_budget(
|
||||
llm.requests[1][0][3]["content"]
|
||||
== "No document at S/ghost.md — check the list_documents output."
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # budget intact
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # tools stay offered (cap bounds)
|
||||
|
||||
|
||||
def test_unknown_tool_name_refused(
|
||||
@@ -393,7 +496,7 @@ def test_unknown_tool_name_refused(
|
||||
asyncio.run(_run(llm, holder, _settings()))
|
||||
assert holder.read_docs == [] and holder.tool_calls == 0
|
||||
assert llm.requests[1][0][3]["content"] == agent.UNKNOWN_TOOL
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # nothing was consumed
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -426,57 +529,6 @@ def test_read_document_missing_arguments_refused(
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
|
||||
|
||||
# ---------- round cap (pathological stream) ----------
|
||||
|
||||
|
||||
def test_round_cap_forces_a_final_no_tools_answer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A model that keeps calling a budget-exhausted tool must be forced
|
||||
to answer at ``max_rounds = 2 + list + read`` (= 4 for 1/1)."""
|
||||
monkeypatch.setattr(agent, "list_catalog", lambda db: [])
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_2", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_3", name="list_documents", arguments={})],
|
||||
[ToolCallPiece(id="call_4", name="list_documents", arguments={})],
|
||||
[StreamPiece("content", "forced answer")],
|
||||
)
|
||||
pieces = asyncio.run(_run(llm, holder, _settings(agent_list_calls=1, agent_read_calls=1)))
|
||||
assert [type(p) for p in pieces] == [
|
||||
ToolCallPiece,
|
||||
ToolCallPiece,
|
||||
ToolCallPiece,
|
||||
ToolCallPiece,
|
||||
StreamPiece,
|
||||
]
|
||||
assert len(llm.requests) == 5
|
||||
# The forced final request carries no tools, whatever is left.
|
||||
assert llm.requests[4][1] is None
|
||||
# Only the first call consumed budget; the three rejections did not.
|
||||
assert holder.tool_calls == 1
|
||||
# The 4th rejection sits at messages[2 + 4*2 - 1] of the final request.
|
||||
assert llm.requests[4][0][9]["content"] == agent.LIST_EXHAUSTED
|
||||
|
||||
|
||||
# ---------- settings ----------
|
||||
|
||||
|
||||
def test_agent_budget_settings_default_to_one_each() -> None:
|
||||
s = _settings()
|
||||
assert s.agent_list_calls == 1
|
||||
assert s.agent_read_calls == 1
|
||||
|
||||
|
||||
def test_agent_budget_settings_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("BOR_AGENT_LIST_CALLS", "0")
|
||||
monkeypatch.setenv("BOR_AGENT_READ_CALLS", "2")
|
||||
s = _settings()
|
||||
assert s.agent_list_calls == 0
|
||||
assert s.agent_read_calls == 2
|
||||
|
||||
|
||||
# ---------- prompts: <tools> section (HIGH only) ----------
|
||||
|
||||
|
||||
|
||||
@@ -563,9 +563,10 @@ def test_endpoint_grounded_turn_runs_agent_loop_with_tools(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 37: a grounded endpoint turn runs the agent loop — the
|
||||
single no-tool-call request carries ``AGENT_TOOLS`` (default 1/1
|
||||
budgets), no ``tool`` frames stream, and the ``done`` event is the
|
||||
plain retrieval shape (the tool-free answer is byte-identical)."""
|
||||
single no-tool-call request carries ``AGENT_TOOLS`` (the default
|
||||
round cap keeps the tools offered), no ``tool`` frames stream, and
|
||||
the ``done`` event is the plain retrieval shape (the tool-free
|
||||
answer is byte-identical)."""
|
||||
_session, llm = gate_env
|
||||
doc = _doc("Kubernetes Homelab Cluster", "TALOS_DOC_SENT")
|
||||
monkeypatch.setattr(chat_api, "retrieve", _fake_retriever([_chunk(doc, 0.90)]))
|
||||
|
||||
@@ -78,6 +78,26 @@ def test_max_output_tokens_env_override(monkeypatch) -> None:
|
||||
assert s.max_output_tokens == 1234
|
||||
|
||||
|
||||
def test_agent_max_rounds_default_and_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Phase 45: the per-tool budgets are gone — ``BOR_AGENT_MAX_ROUNDS``
|
||||
(default 10) is the single agent-loop knob; ``0`` is the no-tools
|
||||
kill switch."""
|
||||
monkeypatch.delenv("BOR_AGENT_MAX_ROUNDS", raising=False)
|
||||
assert _settings().agent_max_rounds == 10
|
||||
monkeypatch.setenv("BOR_AGENT_MAX_ROUNDS", "5")
|
||||
assert _settings().agent_max_rounds == 5
|
||||
monkeypatch.setenv("BOR_AGENT_MAX_ROUNDS", "0")
|
||||
assert _settings().agent_max_rounds == 0
|
||||
|
||||
|
||||
def test_agent_max_rounds_rejects_negative(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""``0`` is the kill switch — a negative value is a typo, so the
|
||||
validator fails loudly at startup."""
|
||||
monkeypatch.setenv("BOR_AGENT_MAX_ROUNDS", "-1")
|
||||
with pytest.raises(ValidationError, match="agent_max_rounds"):
|
||||
_settings()
|
||||
|
||||
|
||||
def test_stream_thinking_default_true_and_env_parse(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Phase 17 kill-switch (``BOR_STREAM_THINKING``): on by default,
|
||||
``0``/``false`` turn the ``thinking`` SSE frames off."""
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Unit tests for the E2E mock's tool-flow classifier (phase 45, task 02).
|
||||
|
||||
The mock (``tests/e2e/mock_llm.py``) classifies marker requests
|
||||
statelessly into one step of the agent tool flow. This file pins the
|
||||
classification at unit speed — no Playwright, no LLM process:
|
||||
|
||||
* the phase-37 SINGLE-READ flow (``TOOLS_TRIGGER`` only) stays
|
||||
byte-identical: list → read (first catalog line, ``call_1``) → answer;
|
||||
* the phase-45 MULTI-READ flow (``TOOLS_TRIGGER`` + ``MULTI_READ_TRIGGER``)
|
||||
classifies by the count of ``tool``-role read results: list → read #1
|
||||
(``call_1``) → read #2 (second catalog line, ``call_2``) → the
|
||||
byte-stable ``multi_answer`` naming both read paths.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from tests.e2e.mock_llm import (
|
||||
MULTI_READ_TRIGGER,
|
||||
TOOLS_TRIGGER,
|
||||
_tool_flow,
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Wire fixtures — byte-identical to what app/rag/agent.py produces
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: The ``<tools>`` section marks the HIGH prompt (app/rag/prompts.py).
|
||||
SYSTEM_HIGH = "<relevance>HIGH</relevance>\n<documents>\n</documents>\n<tools>\n…\n</tools>"
|
||||
SYSTEM_LOW = "<relevance>LOW</relevance>\n"
|
||||
|
||||
#: A minimal truthy ``tools`` parameter (the mock only checks presence).
|
||||
TOOLS = [{"type": "function", "function": {"name": "list_documents"}}]
|
||||
|
||||
#: The agent's ``list_documents`` output for a two-document KB
|
||||
# (``app/rag/agent.py`` ``_execute_tool``): one ``source/path — title``
|
||||
#: line per document, ``(source, path)`` order.
|
||||
CATALOG_2 = (
|
||||
"2 documents:\n"
|
||||
"Deployments/example-record-file.json — Example Record File\n"
|
||||
"Homelab/aws-route53.md — AWS Route 53 Notes"
|
||||
)
|
||||
|
||||
CATALOG_1 = "1 documents:\nDeployments/example-record-file.json — Example Record File"
|
||||
|
||||
CATALOG_3 = (
|
||||
"3 documents:\n"
|
||||
"Deployments/aaa.md — AAA\n"
|
||||
"Deployments/bbb.md — BBB\n"
|
||||
"Homelab/ccc.md — CCC"
|
||||
)
|
||||
|
||||
DOC1_SP = "Deployments/example-record-file.json"
|
||||
DOC1_CONTENT = (
|
||||
"The record file keeps every hosted zone record — first line is longer "
|
||||
"than eighty characters so the quote truncation below is observable.\n"
|
||||
"second line of the document content"
|
||||
)
|
||||
assert len(DOC1_CONTENT) > 80
|
||||
|
||||
DOC2_SP = "Homelab/aws-route53.md"
|
||||
DOC2_CONTENT = "Route 53 notes — the second read, short on purpose."
|
||||
|
||||
SINGLE_USER = "Use your tools: what is the exact shape of the record file?"
|
||||
#: Carries BOTH markers — ``use your tools`` then ``read two documents``.
|
||||
MULTI_USER = "Use your tools and read two documents: compare the zone notes with the record file."
|
||||
#: The multi marker alone — no ``use your tools``.
|
||||
MULTI_ONLY_USER = "Please read two documents and compare them."
|
||||
PLAIN_USER = "How does the sync job push records to the zone?"
|
||||
assert TOOLS_TRIGGER in SINGLE_USER.lower() and MULTI_READ_TRIGGER not in SINGLE_USER.lower()
|
||||
assert TOOLS_TRIGGER in MULTI_USER.lower() and MULTI_READ_TRIGGER in MULTI_USER.lower()
|
||||
|
||||
|
||||
def _read_result(sp: str, content: str) -> str:
|
||||
"""The agent's read-result text (``_execute_tool`` prefix)."""
|
||||
return f"Document {sp}:\n{content}"
|
||||
|
||||
|
||||
def _body(
|
||||
user: str,
|
||||
tool_msgs: tuple[str, ...] = (),
|
||||
tools: Any = TOOLS,
|
||||
system: str = SYSTEM_HIGH,
|
||||
) -> dict[str, Any]:
|
||||
"""A chat-completion body: system + user + the tool results in order."""
|
||||
messages: list[dict[str, Any]] = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
for i, content in enumerate(tool_msgs):
|
||||
messages.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": f"call_{i}",
|
||||
"type": "function",
|
||||
"function": {"name": "list_documents", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
messages.append({"role": "tool", "tool_call_id": f"call_{i}", "content": content})
|
||||
return {"messages": messages, "tools": tools}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Phase-37 single-read flow — must stay byte-identical
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_single_flow_list_step() -> None:
|
||||
assert _tool_flow(_body(SINGLE_USER)) == ("list", "", "")
|
||||
|
||||
|
||||
def test_single_flow_read_step_first_catalog_line() -> None:
|
||||
flow = _tool_flow(_body(SINGLE_USER, (CATALOG_3,)))
|
||||
# The FIRST listing line (Deployments/aaa.md), rsplit convention.
|
||||
assert flow == ("read", "Deployments", "aaa.md", "call_1")
|
||||
|
||||
|
||||
def test_single_flow_answer_step_with_tools_offered() -> None:
|
||||
# Phase 45: the round cap keeps the tools offered until it is hit —
|
||||
# the answer step fires regardless of the ``tools`` parameter.
|
||||
flow = _tool_flow(
|
||||
_body(SINGLE_USER, (CATALOG_2, _read_result(DOC1_SP, DOC1_CONTENT)))
|
||||
)
|
||||
assert flow == ("answer", DOC1_SP, DOC1_CONTENT)
|
||||
|
||||
|
||||
def test_single_flow_answer_step_without_tools() -> None:
|
||||
flow = _tool_flow(
|
||||
_body(
|
||||
SINGLE_USER,
|
||||
(CATALOG_2, _read_result(DOC1_SP, DOC1_CONTENT)),
|
||||
tools=None,
|
||||
)
|
||||
)
|
||||
assert flow == ("answer", DOC1_SP, DOC1_CONTENT)
|
||||
|
||||
|
||||
def test_single_flow_no_tools_no_results_is_not_the_flow() -> None:
|
||||
# agent_max_rounds=0 path: marker + <tools> prompt, but the request
|
||||
# carries no tools and no tool results — regular answer, not a flow.
|
||||
assert _tool_flow(_body(SINGLE_USER, tools=None)) is None
|
||||
|
||||
|
||||
def test_single_flow_marker_without_tools_section_is_none() -> None:
|
||||
assert _tool_flow(_body(SINGLE_USER, system=SYSTEM_LOW)) is None
|
||||
|
||||
|
||||
def test_single_flow_plain_question_is_none() -> None:
|
||||
assert _tool_flow(_body(PLAIN_USER)) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Phase-45 multi-read flow (task 02)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_multi_flow_list_step() -> None:
|
||||
assert _tool_flow(_body(MULTI_USER)) == ("list", "", "")
|
||||
|
||||
|
||||
def test_multi_flow_read_first_step() -> None:
|
||||
flow = _tool_flow(_body(MULTI_USER, (CATALOG_2,)))
|
||||
assert flow == ("read", DOC1_SP.split("/", 1)[0], DOC1_SP.rsplit("/", 1)[1], "call_1")
|
||||
|
||||
|
||||
def test_multi_flow_read_second_step_skips_already_read() -> None:
|
||||
flow = _tool_flow(_body(MULTI_USER, (CATALOG_2, _read_result(DOC1_SP, DOC1_CONTENT))))
|
||||
# The second catalog line — the first line differing from DOC1.
|
||||
assert flow == ("read", "Homelab", "aws-route53.md", "call_2")
|
||||
|
||||
|
||||
def test_multi_flow_read_second_is_listing_order_not_last() -> None:
|
||||
# Three-doc catalog, first doc read: read #2 is the SECOND line
|
||||
# (Deployments/bbb.md), not the last one.
|
||||
flow = _tool_flow(_body(MULTI_USER, (CATALOG_3, _read_result("Deployments/aaa.md", "x"))))
|
||||
assert flow == ("read", "Deployments", "bbb.md", "call_2")
|
||||
|
||||
|
||||
def test_multi_flow_answer_step_names_both_paths() -> None:
|
||||
flow = _tool_flow(
|
||||
_body(
|
||||
MULTI_USER,
|
||||
(
|
||||
CATALOG_2,
|
||||
_read_result(DOC1_SP, DOC1_CONTENT),
|
||||
_read_result(DOC2_SP, DOC2_CONTENT),
|
||||
),
|
||||
)
|
||||
)
|
||||
assert flow is not None
|
||||
assert flow[0] == "multi_answer"
|
||||
# Byte-stable: the single-read shape quoting the FIRST read result
|
||||
# (first 80 chars), plus both read paths in read order.
|
||||
assert flow[2] == f"Read {DOC1_SP}. {DOC1_CONTENT[:80]} I read {DOC1_SP} and {DOC2_SP}."
|
||||
|
||||
|
||||
def test_multi_flow_answer_step_without_tools_offered() -> None:
|
||||
# The forced answer is content, not a tool call — it must not be
|
||||
# gated on the ``tools`` parameter.
|
||||
flow = _tool_flow(
|
||||
_body(
|
||||
MULTI_USER,
|
||||
(
|
||||
CATALOG_2,
|
||||
_read_result(DOC1_SP, DOC1_CONTENT),
|
||||
_read_result(DOC2_SP, DOC2_CONTENT),
|
||||
),
|
||||
tools=None,
|
||||
)
|
||||
)
|
||||
assert flow is not None
|
||||
assert flow[0] == "multi_answer"
|
||||
|
||||
|
||||
def test_multi_flow_one_document_catalog_degenerates_to_single_answer() -> None:
|
||||
# Nothing second to read — the single-read answer shape, quoting the
|
||||
# only read result.
|
||||
flow = _tool_flow(
|
||||
_body(MULTI_USER, (CATALOG_1, _read_result(DOC1_SP, DOC1_CONTENT)))
|
||||
)
|
||||
assert flow == ("answer", DOC1_SP, DOC1_CONTENT)
|
||||
|
||||
|
||||
def test_multi_flow_no_tools_no_results_is_not_the_flow() -> None:
|
||||
assert _tool_flow(_body(MULTI_USER, tools=None)) is None
|
||||
|
||||
|
||||
def test_multi_trigger_without_tools_trigger_is_none() -> None:
|
||||
# The multi marker alone (no ``use your tools``) is not the flow.
|
||||
assert MULTI_READ_TRIGGER in MULTI_ONLY_USER
|
||||
assert TOOLS_TRIGGER not in MULTI_ONLY_USER.lower()
|
||||
assert _tool_flow(_body(MULTI_ONLY_USER)) is None
|
||||
|
||||
|
||||
def test_multi_flow_requires_tools_section() -> None:
|
||||
assert _tool_flow(_body(MULTI_USER, system=SYSTEM_LOW)) is None
|
||||
Reference in New Issue
Block a user