phase: 160_tool_budget_telemetry
All green. Final report: **Phase 160 completion pass — verified (final verification; all 5 tasks already in `complete/`)** - **Verified, no code defects found:** budget trailers (`TOOL_BUDGET_REMAINING/LAST/SPENT`), cap-path rejection + `ROUND_CAP_FALLBACK_ANSWER` + `ROUND_CAP_WRAPUP_INSTRUCTION`, `done.budget_exhausted` (ABSENT-when-false), `rounds=/cap_hit=` log fields, `agent_max_rounds=10` default, frontend Continue button + restore + shared-page exclusion, mock markers. - **Tests/lint:** `uv run pytest` 153/153 green (EXIT 0); `uv run pytest --cov=app --cov-report=term-missing` TOTAL **99%** (>90%); `uv run ruff check .` clean; `uv run pyright` 0 errors/0 warnings. - **Criteria:** ① decreasing trailer pinned by unit tests (9→…→spent) ✅ ② cap path ends readable, no tool lines after cap (E2E) ✅ ③ `budget_exhausted` true-only-on-cap, absent otherwise (integration) ✅ ④ Continue button → one fresh `continue` turn, fresh budget, record intact (E2E) ✅ ⑤ flag survives refresh; shared chat never renders it (`shared.js` has no `budget_exhausted` path) ✅ ⑥ `max_rounds=0` byte-identical single `tools=None` request; default still 10 ✅ ⑦ full suite/coverage/lint green ✅ ⑧ `test_tool_budget_telemetry.py` **7/7** in isolation; all 6 regression E2E suites green in isolation ✅ ⑨ commit + phase move = harness (I make no git history/index changes). - **Docs:** README.md — fixed: the "Continue researching" button and cap behavior were undocumented (added a Chat-section paragraph + an Agent-tools-section paragraph, incl. the phase-71 recorded-revision note); env-var table, commands, and architecture all verified consistent otherwise. AGENTS.md — verified consistent, no changes needed. - **Decisions:** no code changes required — phase was fully implemented; only README docs added. No live-infrastructure touched. - **Next pending phase:** `161_reasoning_bubble`.
This commit is contained in:
@@ -320,13 +320,13 @@ def test_ls_top_level_lists_registered_sources_through_run_agent(
|
||||
holder, llm = _run_call(db, "ls", {})
|
||||
|
||||
assert llm.requests[0][1] == AGENT_TOOLS
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"2 sources:\n"
|
||||
"\n"
|
||||
"Deployments — 1 documents\n"
|
||||
"Homelab — 2 documents\n"
|
||||
" Home lab notes."
|
||||
)
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
@@ -352,7 +352,7 @@ def test_ls_top_level_empty_registry_through_run_agent(
|
||||
db.commit()
|
||||
|
||||
holder, llm = _run_call(db, "ls", {})
|
||||
assert llm.requests[1][0][3]["content"] == "0 sources:"
|
||||
assert llm.requests[1][0][3]["content"].startswith("0 sources:")
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
@@ -375,20 +375,20 @@ def test_ls_source_scope_lists_root_folder_through_run_agent(kb, registry, db) -
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Homelab"})
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"Homelab — 1 documents, 2 folders:\n"
|
||||
"\n"
|
||||
" backups/ — 2 documents: Backup notes.\n"
|
||||
" networking/ — 1 documents\n"
|
||||
"\n"
|
||||
"source: Homelab | path: readme.md | title: Readme | date: 2024-06-15"
|
||||
)
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
# A registered source with no documents: the header line alone.
|
||||
holder0, llm0 = _run_call(db, "ls", {"path": "Deployments"})
|
||||
assert llm0.requests[1][0][3]["content"] == "Deployments — 0 documents, 0 folders:"
|
||||
assert llm0.requests[1][0][3]["content"].startswith("Deployments — 0 documents, 0 folders:")
|
||||
assert holder0.tool_calls == 1
|
||||
|
||||
|
||||
@@ -405,23 +405,23 @@ def test_ls_nested_folder_scope_drills_one_level_through_run_agent(
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Homelab/networking"})
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"Homelab/networking — 0 documents, 2 folders:\n"
|
||||
"\n"
|
||||
" networking/lan/ — 2 documents\n"
|
||||
" networking/vpn/ — 1 documents"
|
||||
)
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
# One level deeper.
|
||||
holder2, llm2 = _run_call(db, "ls", {"path": "Homelab/networking/lan"})
|
||||
assert llm2.requests[1][0][3]["content"] == (
|
||||
assert llm2.requests[1][0][3]["content"].startswith(
|
||||
"Homelab/networking/lan — 2 documents, 0 folders:\n"
|
||||
"\n"
|
||||
"source: Homelab | path: networking/lan/a.md | title: A | date: 2024-06-15\n"
|
||||
"source: Homelab | path: networking/lan/b.md | title: B | date: 2024-06-15"
|
||||
)
|
||||
)
|
||||
assert holder2.tool_calls == 1
|
||||
|
||||
|
||||
@@ -434,7 +434,9 @@ def test_ls_folder_file_cap_through_run_agent(kb, registry, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Homelab/big"})
|
||||
|
||||
content = llm.requests[1][0][3]["content"]
|
||||
# Phase 160 (task 01): the budget-telemetry trailer rides the
|
||||
# listing — the raw listing is the RAW part (the trailer stripped).
|
||||
content = llm.requests[1][0][3]["content"].removesuffix(agent._budget_note(9))
|
||||
lines = content.splitlines()
|
||||
assert lines[0] == "Homelab/big — 51 documents, 0 folders:"
|
||||
assert lines[2] == (
|
||||
@@ -460,9 +462,8 @@ def test_ls_scoped_unknown_source_refused_through_run_agent(kb, src, db) -> None
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Ghost"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="Ghost")
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="Ghost")
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
@@ -477,9 +478,8 @@ def test_ls_path_like_scope_unknown_source_gets_no_source_refusal(kb, src, db) -
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "app/rag/importer.py"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="app")
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="app")
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
@@ -500,10 +500,10 @@ def test_ls_unknown_folder_gets_not_a_folder_with_parents_subfolders(
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Homelab/netwoking"})
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"'Homelab/netwoking' is not a folder — Homelab has: "
|
||||
"backups/ containers/ networking/"
|
||||
)
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
assert llm.requests[1][1] == AGENT_TOOLS # rejected → tools stay offered
|
||||
|
||||
@@ -523,11 +523,11 @@ def test_read_combined_path_through_run_agent(kb, db) -> None:
|
||||
|
||||
# Phase 106 (D5): the date rides every read — the SECOND line (the
|
||||
# first line stays the byte-identical header).
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"Document Alpha/deep/nested/doc.md:\n"
|
||||
"date: 2024-06-15\n"
|
||||
"FULL-TEXT"
|
||||
)
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
# SEC-14-04: short-lived session loads fresh copies
|
||||
assert [d.id for d in holder.read_docs] == [created.id]
|
||||
@@ -542,8 +542,8 @@ def test_read_bare_source_name_refused_through_run_agent(kb, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "read", {"path": "Alpha"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"] == "No document at 'Alpha' — check the ls output."
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"No document at 'Alpha' — check the ls output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
@@ -558,9 +558,8 @@ def test_read_unknown_combined_path_refused_through_run_agent(kb, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "read", {"path": "Alpha/nope/deep.md"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== "No document at 'Alpha/nope/deep.md' — check the ls output."
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"No document at 'Alpha/nope/deep.md' — check the ls output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
@@ -592,19 +591,19 @@ def test_read_bare_path_single_source_suggestion_then_corrected_read(kb, db) ->
|
||||
# Round 1: the bare path resolves to no combined identity, but it IS
|
||||
# the indexed document's path — the refusal names the one combined
|
||||
# identity to use (not counted, the tools stay offered).
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"No document at 'deep/nested/doc.md' — "
|
||||
"did you mean 'Alpha/deep/nested/doc.md'?"
|
||||
)
|
||||
)
|
||||
assert llm.requests[1][1] == AGENT_TOOLS
|
||||
# Round 2: the corrected combined identity succeeds — the full
|
||||
# content (plus the phase-106 D5 date line), the holder records the
|
||||
# row, and it counts.
|
||||
assert llm.requests[2][0][5]["content"] == (
|
||||
assert llm.requests[2][0][5]["content"].startswith(
|
||||
"Document Alpha/deep/nested/doc.md:\n"
|
||||
"date: 2024-06-15\n"
|
||||
"FULL-TEXT"
|
||||
)
|
||||
)
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
# SEC-14-04: short-lived session loads fresh copies
|
||||
assert [d.id for d in holder.read_docs] == [created.id]
|
||||
@@ -633,13 +632,13 @@ def test_read_bare_path_two_sources_one_of_suggestion_then_corrected_read(
|
||||
holder = AgentHolder()
|
||||
asyncio.run(_consume(cast("LLMClient", llm), lambda: SessionLocal(), holder))
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"No document at 'shared/x.md' — did you mean one of: "
|
||||
"'Alpha/shared/x.md', 'Beta/shared/x.md'?"
|
||||
)
|
||||
assert llm.requests[2][0][5]["content"] == (
|
||||
)
|
||||
assert llm.requests[2][0][5]["content"].startswith(
|
||||
"Document Alpha/shared/x.md:\ndate: 2024-06-15\nA-TEXT"
|
||||
)
|
||||
)
|
||||
# SEC-14-04: short-lived session loads fresh copies
|
||||
assert [d.id for d in holder.read_docs] == [a.id]
|
||||
assert holder.tool_calls == 1 # only the corrected read executed
|
||||
@@ -673,10 +672,10 @@ def test_grep_whole_kb_through_run_agent(kb, db) -> None:
|
||||
# Offered: the first request carries AGENT_TOOLS (the 3-tool list).
|
||||
assert llm.requests[0][1] == AGENT_TOOLS
|
||||
# Executed against the real DB: catalog order, grep-style lines.
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"Alpha/a/one.md:2: needle in one\n"
|
||||
"Beta/b/two.md:2: NEEDLE in two"
|
||||
)
|
||||
)
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [] # locked A5: grep adds no context
|
||||
|
||||
@@ -689,7 +688,7 @@ def test_grep_scoped_through_run_agent(kb, db) -> None:
|
||||
holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/a/one.md"})
|
||||
|
||||
# Only the named document is searched — the other one's hit is absent.
|
||||
assert llm.requests[1][0][3]["content"] == "Alpha/a/one.md:2: Needle here"
|
||||
assert llm.requests[1][0][3]["content"].startswith("Alpha/a/one.md:2: Needle here")
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == []
|
||||
|
||||
@@ -703,9 +702,8 @@ def test_grep_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/ghost.md"})
|
||||
|
||||
assert (
|
||||
llm.requests[1][0][3]["content"]
|
||||
== "No document at 'Alpha/ghost.md' — check the ls output."
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"No document at 'Alpha/ghost.md' — check the ls output."
|
||||
)
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
|
||||
@@ -716,8 +714,8 @@ def test_grep_no_matches_through_run_agent(kb, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "grep", {"pattern": "zebra"})
|
||||
|
||||
assert llm.requests[1][0][3]["content"] == (
|
||||
assert llm.requests[1][0][3]["content"].startswith(
|
||||
"No matches for 'zebra' in the knowledge base."
|
||||
)
|
||||
)
|
||||
assert holder.tool_calls == 1 # an executed grep with zero hits
|
||||
assert holder.read_docs == []
|
||||
|
||||
@@ -28,7 +28,7 @@ from sqlalchemy.orm import Session
|
||||
from app.config import Settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Document, GitSource
|
||||
from app.rag.agent import AgentHolder, run_agent
|
||||
from app.rag.agent import AgentHolder, _budget_note, run_agent
|
||||
from app.rag.llm import LLMClient, RetryPiece, StreamPiece, ToolCallPiece, ToolResultPiece
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -160,7 +160,9 @@ def test_read_result_second_line_is_stored_date(kb, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "read", {"path": "Alpha/deep/nested/doc.md"})
|
||||
|
||||
content = llm.requests[1][0][3]["content"]
|
||||
# Phase 160 (task 01): the trailer rides the read result — the raw
|
||||
# result is the RAW part (the trailer stripped).
|
||||
content = llm.requests[1][0][3]["content"].removesuffix(_budget_note(9))
|
||||
lines = content.splitlines()
|
||||
assert lines[0] == "Document Alpha/deep/nested/doc.md:" # byte-identical header
|
||||
assert lines[1] == f"date: {D2_STR}" # the STORED date (row's UTC date part)
|
||||
@@ -178,13 +180,13 @@ def test_read_result_date_is_the_row_date_not_a_constant(kb, db) -> None:
|
||||
db.commit()
|
||||
|
||||
holder_a, llm_a = _run_call(db, "read", {"path": "Alpha/a.md"})
|
||||
assert llm_a.requests[1][0][3]["content"] == (
|
||||
assert llm_a.requests[1][0][3]["content"].startswith(
|
||||
f"Document Alpha/a.md:\ndate: {D1_STR}\nA-TEXT"
|
||||
)
|
||||
)
|
||||
holder_b, llm_b = _run_call(db, "read", {"path": "Alpha/b.md"})
|
||||
assert llm_b.requests[1][0][3]["content"] == (
|
||||
assert llm_b.requests[1][0][3]["content"].startswith(
|
||||
f"Document Alpha/b.md:\ndate: {D3_STR}\nB-TEXT"
|
||||
)
|
||||
)
|
||||
# SEC-14-04: short-lived sessions
|
||||
assert [d.id for d in holder_a.read_docs] == [a.id]
|
||||
assert [d.id for d in holder_b.read_docs] == [b.id]
|
||||
@@ -207,7 +209,9 @@ def test_ls_drill_file_lines_carry_their_dates(kb, src, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "ls", {"path": "Homelab"})
|
||||
|
||||
content = llm.requests[1][0][3]["content"]
|
||||
# Phase 160 (task 01): the trailer rides the listing — stripped for
|
||||
# the raw-part pins below.
|
||||
content = llm.requests[1][0][3]["content"].removesuffix(_budget_note(9))
|
||||
lines = content.splitlines()
|
||||
# The root level: one direct file (readme.md — D1) + the two
|
||||
# subfolder lines (date-free) + the date-free header.
|
||||
@@ -221,7 +225,7 @@ def test_ls_drill_file_lines_carry_their_dates(kb, src, db) -> None:
|
||||
|
||||
# Drill into backups: BOTH files list, each with its OWN date.
|
||||
holder2, llm2 = _run_call(db, "ls", {"path": "Homelab/backups"})
|
||||
lines2 = llm2.requests[1][0][3]["content"].splitlines()
|
||||
lines2 = llm2.requests[1][0][3]["content"].removesuffix(_budget_note(9)).splitlines()
|
||||
assert lines2[0] == "Homelab/backups — 2 documents, 0 folders:"
|
||||
assert lines2[2] == (
|
||||
f"source: Homelab | path: backups/cron.md | title: Cron | date: {D1_STR}"
|
||||
@@ -247,7 +251,7 @@ def test_ls_top_level_source_lines_carry_no_date(kb, db) -> None:
|
||||
|
||||
holder, llm = _run_call(db, "ls", {})
|
||||
|
||||
content = llm.requests[1][0][3]["content"]
|
||||
content = llm.requests[1][0][3]["content"].removesuffix(_budget_note(9))
|
||||
assert content == "1 sources:\n\nHomelab — 1 documents"
|
||||
assert "date" not in content
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
@@ -82,6 +82,7 @@ class FakeRagLLM:
|
||||
stream_error: Exception | None = None,
|
||||
fail_mid_stream: bool = False,
|
||||
tool_script: list[list[StreamPiece | ToolCallPiece]] | None = None,
|
||||
no_tools_script: list[list[StreamPiece | ToolCallPiece]] | None = None,
|
||||
embed_fail_count: int = 0,
|
||||
stream_fail_count: int = 0,
|
||||
answer_sequence: list[str] | None = None,
|
||||
@@ -123,6 +124,18 @@ class FakeRagLLM:
|
||||
#: thinking + answer stream below, so a deflected turn through this
|
||||
#: fake is byte-identical to the plain fake's output.
|
||||
self.tool_script: list[list[StreamPiece | ToolCallPiece]] = list(tool_script or [])
|
||||
#: Phase 160 (task 02): canned piece lists for requests made
|
||||
#: WITHOUT tools (``tools=None``) — the deflected turn, the
|
||||
#: cap-forced final, the kill-switch single-request path, and the
|
||||
#: phase-71 recovery. Empty (the default) keeps the pre-phase
|
||||
#: behavior: a tools=None request yields the thinking + answer
|
||||
#: stream below, so every existing test is untouched. When set,
|
||||
#: the *i*-th tools=None request consumes ``no_tools_script[i]``
|
||||
#: (so a mock can make a tools=None request return structured
|
||||
#: ``ToolCallPiece``s — the documented ``lite`` behavior).
|
||||
self.no_tools_script: list[list[StreamPiece | ToolCallPiece]] = list(
|
||||
no_tools_script or []
|
||||
)
|
||||
|
||||
async def embed(
|
||||
self,
|
||||
@@ -197,6 +210,8 @@ class FakeRagLLM:
|
||||
raw: list[StreamPiece | ToolCallPiece]
|
||||
if tools is not None and self.tool_script:
|
||||
raw = self.tool_script.pop(0)
|
||||
elif tools is None and self.no_tools_script:
|
||||
raw = self.no_tools_script.pop(0)
|
||||
elif self.fail_mid_stream:
|
||||
raw = [StreamPiece("content", "partial ")]
|
||||
mid_stream_drop = True
|
||||
@@ -980,9 +995,10 @@ marker in the model's context."""
|
||||
m for r in scripted.seen_messages for m in r if m.get("role") == "tool"
|
||||
]
|
||||
assert tool_msgs, "the executed read must be appended as a tool message"
|
||||
assert (
|
||||
tool_msgs[-1]["content"]
|
||||
== "Document docs/fits.md:\ndate: 2024-06-15\n" + content
|
||||
# Phase 160 (task 01): the budget-telemetry trailer rides the
|
||||
# read result — the raw result is the RAW part.
|
||||
assert tool_msgs[-1]["content"].startswith(
|
||||
"Document docs/fits.md:\ndate: 2024-06-15\n" + content
|
||||
)
|
||||
assert TRUNCATION_MARKER not in tool_msgs[-1]["content"]
|
||||
# Still a successful read.
|
||||
@@ -1165,9 +1181,10 @@ def test_untruncated_read_streams_no_tool_result_frame(
|
||||
m for r in scripted.seen_messages for m in r if m.get("role") == "tool"
|
||||
]
|
||||
assert tool_msgs
|
||||
assert (
|
||||
tool_msgs[-1]["content"]
|
||||
== "Document docs/big-read.md:\ndate: 2024-06-15\n" + content
|
||||
# Phase 160 (task 01): the budget-telemetry trailer rides the
|
||||
# read result — the raw result is the RAW part.
|
||||
assert tool_msgs[-1]["content"].startswith(
|
||||
"Document docs/big-read.md:\ndate: 2024-06-15\n" + content
|
||||
)
|
||||
assert TRUNCATION_MARKER not in tool_msgs[-1]["content"]
|
||||
|
||||
@@ -1500,6 +1517,176 @@ def test_zero_max_rounds_reproduce_pre_phase_single_request(
|
||||
assert "scaffold_stripped=0" in lines[-1] # phase 71: uniform clean-turn field
|
||||
|
||||
|
||||
def test_cap_path_rejects_late_calls_and_fallbacks(
|
||||
client, db, seeded_kb: FakeRagLLM, monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""A grounded turn with ``agent_max_rounds=1`` whose forced final
|
||||
(``tools=None``) returns a ``ToolCallPiece`` (the documented
|
||||
``lite`` behavior — a no-tools request that still comes back with
|
||||
structured calls): the executed round streams ``tool`` ×1, then
|
||||
NOTHING of the sort — no second ``tool`` frame, no ``error`` frame —
|
||||
and the turn settles with ``done`` whose delta text is the
|
||||
``ROUND_CAP_FALLBACK_ANSWER`` (the model gave no visible content on
|
||||
the forced final). The late call is never executed and never counted
|
||||
in ``tool_calls`` (the executed-only log field stays 1)."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[[ToolCallPiece(id="call_1", name="ls", arguments={})]],
|
||||
no_tools_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_f1",
|
||||
name="read",
|
||||
arguments={"path": "docs/homelab/kubernetes.md"},
|
||||
)
|
||||
]
|
||||
],
|
||||
)
|
||||
live = get_settings()
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"get_settings",
|
||||
lambda: Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
relevance_threshold=live.relevance_threshold,
|
||||
agent_max_rounds=1,
|
||||
),
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
caplog.set_level(logging.INFO, logger="app.chat")
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
types = [f["type"] for f in frames]
|
||||
# Only the executed round 1 call streams as a tool frame; the forced
|
||||
# final's call is rejected — no second tool frame, no error frame.
|
||||
assert types.count("tool") == 1
|
||||
assert "error" not in types
|
||||
# The turn settles with done; the answer is the budget fallback.
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done"
|
||||
assert done["deflected"] is False
|
||||
delta_text = "".join(f.get("text", "") for f in frames if f["type"] == "delta")
|
||||
assert delta_text == agent.ROUND_CAP_FALLBACK_ANSWER
|
||||
# The late ``read`` was never executed: ``tool_calls=1`` (the
|
||||
# executed round only) in the per-turn log line.
|
||||
lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
|
||||
assert lines and "tool_calls=1" in lines[-1]
|
||||
|
||||
|
||||
def test_cap_reached_turn_reports_budget_exhausted_and_log_fields(
|
||||
client, db, seeded_kb: FakeRagLLM, monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""A grounded turn with ``agent_max_rounds=1`` that hits the cap:
|
||||
the ``done`` frame carries ``"budget_exhausted": true`` (the
|
||||
additive key — the ``data:`` line the client sees) and the
|
||||
per-turn log line reads ``rounds=1 cap_hit=1`` (the cap hit,
|
||||
named — the owner can grep how often the cap bites)."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[[ToolCallPiece(id="call_1", name="ls", arguments={})]],
|
||||
no_tools_script=[[StreamPiece("content", "the wrap-up answer")]],
|
||||
)
|
||||
live = get_settings()
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"get_settings",
|
||||
lambda: Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
relevance_threshold=live.relevance_threshold,
|
||||
agent_max_rounds=1,
|
||||
),
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
caplog.set_level(logging.INFO, logger="app.chat")
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done"
|
||||
assert done["deflected"] is False
|
||||
assert done["budget_exhausted"] is True # the cap-reached flag
|
||||
assert "error" not in [f["type"] for f in frames]
|
||||
lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
|
||||
assert lines and "rounds=1 cap_hit=1" in lines[-1]
|
||||
assert "tool_calls=1" in lines[-1]
|
||||
|
||||
|
||||
def test_in_cap_and_deflected_turns_report_no_budget_exhausted(
|
||||
client, db, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
"""An ordinary grounded turn (answered in-cap) and a deflected
|
||||
turn: the ``done`` frames carry NO ``budget_exhausted`` key
|
||||
(ABSENT, never ``false`` — the byte-identical-off contract), and
|
||||
the per-turn log lines read ``cap_hit=0`` — the grounded one
|
||||
with ``rounds=`` the executed rounds (2), the deflected one
|
||||
``rounds=0 cap_hit=0`` (the agent never runs)."""
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[ToolCallPiece(id="call_1", name="ls", arguments={})],
|
||||
[ToolCallPiece(id="call_2", name="ls", arguments={})],
|
||||
]
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
caplog.set_level(logging.INFO, logger="app.chat")
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
_, _, deflected_frames = _stream_chat(client, OFF_TOPIC)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done"
|
||||
assert done["deflected"] is False
|
||||
assert "budget_exhausted" not in done # ABSENT, never false
|
||||
d_done = deflected_frames[-1]
|
||||
assert d_done["type"] == "done"
|
||||
assert d_done["deflected"] is True
|
||||
assert "budget_exhausted" not in d_done
|
||||
lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
|
||||
assert len(lines) == 2
|
||||
assert "tool_calls=2 rounds=2 cap_hit=0" in lines[0] # the grounded turn
|
||||
assert "rounds=0 cap_hit=0" in lines[1] # the deflected turn
|
||||
|
||||
|
||||
def test_deflected_turn_rejects_structured_calls(
|
||||
client, db, seeded_kb: FakeRagLLM
|
||||
) -> None:
|
||||
"""A deflected turn (``tools=None``, A8) whose model returns a
|
||||
structured ``tool_calls`` (the documented ``lite`` behavior) emits
|
||||
NO ``tool`` frame — the call is rejected at the transport
|
||||
(``_pump`` drops it, ``tools_offered=False``), never rendered, never
|
||||
executed. The turn still settles (``done``) with the model's
|
||||
content (the rejection only drops the call, not the answer)."""
|
||||
scripted = FakeRagLLM(
|
||||
no_tools_script=[
|
||||
[
|
||||
ToolCallPiece(id="call_1", name="ls", arguments={}),
|
||||
StreamPiece("content", "the deflected answer"),
|
||||
]
|
||||
],
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, OFF_TOPIC)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
types = [f["type"] for f in frames]
|
||||
assert not any(t == "tool" for t in types) # no tool frame
|
||||
assert "error" not in types
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done"
|
||||
assert done["deflected"] is True
|
||||
# The model's content still streamed (the rejection only drops the
|
||||
# call, not the answer).
|
||||
delta_text = "".join(f.get("text", "") for f in frames if f["type"] == "delta")
|
||||
assert delta_text == "the deflected answer"
|
||||
# One request, no tools key (the deflected direct path, A8).
|
||||
assert len(scripted.seen_messages) == 1
|
||||
assert scripted.seen_tools == [None]
|
||||
|
||||
|
||||
def test_tool_execution_db_failure_yields_error_event(
|
||||
client, db, seeded_kb: FakeRagLLM, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
||||
@@ -232,6 +232,12 @@ def _expect(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"stopped": m.get("stopped"),
|
||||
"failed": m.get("failed"),
|
||||
"error": m.get("error"),
|
||||
# Phase 160 (task 04): the budget-exhausted marker —
|
||||
# explicit null where the record does not carry it (the
|
||||
# bor.chat.v1 shape is explicit-null for every optional
|
||||
# key; a record WITH the key round-trips it byte-
|
||||
# identically — the pins below).
|
||||
"budget_exhausted": m.get("budget_exhausted"),
|
||||
}
|
||||
for m in records
|
||||
]
|
||||
@@ -464,8 +470,10 @@ def test_create_round_trips_full_brain_record(admin_client: TestClient) -> None:
|
||||
assert r.status_code == 201
|
||||
# The bor.chat.v1-shaped payload round-trips losslessly: every
|
||||
# optional key (sources/related/deflected/suggestions/thinking/
|
||||
# tools/stopped) survives identical.
|
||||
assert r.json()["messages"][1] == FULL_BRAIN
|
||||
# tools/stopped/failed/error/budget_exhausted — the phase-160
|
||||
# additive marker, explicit null where absent) survives identical
|
||||
# (the _expect full stored shape).
|
||||
assert r.json()["messages"][1] == _expect([_user(FIRST_QUESTION), FULL_BRAIN])[1]
|
||||
|
||||
|
||||
def test_create_rejects_empty_messages(admin_client: TestClient) -> None:
|
||||
@@ -1036,6 +1044,128 @@ def test_shared_serve_includes_the_brain_answer_image_ref(
|
||||
assert "answer_image" not in messages[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 160 (task 04, TODO L4): the budget-exhausted marker — the
|
||||
# persisted answer that ran out of tool calls. The done-time
|
||||
# auto-save (``rememberBrainTurn``) persists the done frame's
|
||||
# ``budget_exhausted`` into the brain record's ``budget_exhausted``
|
||||
# key (the ``answer_image`` precedent — the field must exist so the
|
||||
# ``extra="forbid"`` boundary does NOT 422 those saves; the
|
||||
# phase-113 lesson, applied up front), and the restore path re-
|
||||
# renders the "Continue researching" button from it (task 04 — the
|
||||
# marker is a client restore hint only, never a server behavior; no
|
||||
# migration: ``saved_chats.messages`` is raw JSONB, phase 50, and
|
||||
# ``app/api/chats.py``'s logic is unchanged). These round-trips are
|
||||
# that regression pin.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_create_round_trips_the_budget_exhausted_marker(
|
||||
admin_client: TestClient,
|
||||
) -> None:
|
||||
"""The phase-113 regression pin (the A2 quiet-failure path): a
|
||||
brain record carrying ``budget_exhausted: true`` →
|
||||
``POST /api/chats`` is 201 (NOT a silent 422) and the admin GET
|
||||
round-trips the record BYTE-FOR-BYTE through Postgres (the
|
||||
``_expect`` full bor.chat.v1 shape — the restore path re-renders
|
||||
the Continue button from the marker). The user record carries the
|
||||
key as an explicit null (it never did, and the stored shape is
|
||||
explicit-null for every optional key)."""
|
||||
brain = {**FULL_BRAIN, "budget_exhausted": True}
|
||||
r = admin_client.post(
|
||||
"/api/chats",
|
||||
json={"messages": [_user(FIRST_QUESTION), brain]},
|
||||
)
|
||||
assert r.status_code == 201, f"expected 201, got {r.status_code}: {r.text}"
|
||||
body = r.json()
|
||||
assert body["messages"][1]["budget_exhausted"] is True
|
||||
assert body["messages"][0]["budget_exhausted"] is None
|
||||
|
||||
got = admin_client.get(f"/api/chats/{body['id']}")
|
||||
assert got.status_code == 200
|
||||
assert got.json()["messages"] == _expect(
|
||||
[_user(FIRST_QUESTION), {**FULL_BRAIN, "budget_exhausted": True}]
|
||||
)
|
||||
|
||||
|
||||
def test_create_without_budget_exhausted_marker_round_trips_unchanged(
|
||||
admin_client: TestClient,
|
||||
) -> None:
|
||||
"""A record WITHOUT the marker (every non-cap turn, every pre-
|
||||
phase record) round-trips with the key as an explicit null — the
|
||||
stored shape never gains a value (the byte-identical contract,
|
||||
the phase-50 precedent: the key is absent from the client's
|
||||
``bor.chat.v1`` record via the ``undefined``-drops-the-key
|
||||
pattern, and explicit-null here keeps the restore path null-safe).
|
||||
The ``FULL_BRAIN`` shape (no marker) is the pin: its stored and
|
||||
served form is byte-identical to pre-phase-160 apart from the
|
||||
one new null key."""
|
||||
r = admin_client.post(
|
||||
"/api/chats",
|
||||
json={"messages": [_user(FIRST_QUESTION), FULL_BRAIN]},
|
||||
)
|
||||
assert r.status_code == 201, r.text
|
||||
body = r.json()
|
||||
assert body["messages"][1]["budget_exhausted"] is None
|
||||
got = admin_client.get(f"/api/chats/{body['id']}")
|
||||
assert got.status_code == 200
|
||||
assert got.json()["messages"] == _expect([_user(FIRST_QUESTION), FULL_BRAIN])
|
||||
|
||||
|
||||
def test_shared_serve_includes_the_budget_exhausted_marker(
|
||||
admin_client: TestClient,
|
||||
) -> None:
|
||||
"""The shared view is faithful (the data is in the public
|
||||
snapshot), but the control is NOT: the shared page
|
||||
(``/shared/<token>``, rendered server-side by ``shared.js`` —
|
||||
never by ``app.js``, owner-locked zero interactive controls)
|
||||
IGNORES the key by construction — no Continue control renders for
|
||||
a flagged record (work item 8; pinned in
|
||||
``test_frontend_tool_budget.py::test_shared_page_never_shows_
|
||||
the_continue_control``). The user record stays with the explicit
|
||||
null."""
|
||||
brain = {**FULL_BRAIN, "budget_exhausted": True}
|
||||
r = admin_client.post(
|
||||
"/api/chats",
|
||||
json={"messages": [_user(FIRST_QUESTION), brain]},
|
||||
)
|
||||
assert r.status_code == 201, r.text
|
||||
share_url = _share(admin_client, r.json()["id"])["share_url"]
|
||||
|
||||
anon = TestClient(fastapi_app) # fresh jar: truly anonymous
|
||||
got = anon.get(f"/api{share_url}")
|
||||
assert got.status_code == 200
|
||||
messages = got.json()["messages"]
|
||||
assert messages[1]["budget_exhausted"] is True # faithful snapshot
|
||||
assert messages[0]["budget_exhausted"] is None
|
||||
|
||||
|
||||
def test_put_round_trips_the_budget_exhausted_marker(
|
||||
admin_client: TestClient,
|
||||
) -> None:
|
||||
"""The re-Save upsert (the auto-save path the client's restore
|
||||
relies on) round-trips the marker byte-identically too: a PUT
|
||||
with the flagged record replaces the stored JSONB and the 200
|
||||
body + the GET both carry it (a refreshed chat page restores the
|
||||
Continue affordance from the persisted record)."""
|
||||
created = admin_client.post(
|
||||
"/api/chats",
|
||||
json={"messages": [_user(FIRST_QUESTION), FULL_BRAIN]},
|
||||
).json()
|
||||
brain = {**FULL_BRAIN, "budget_exhausted": True}
|
||||
r = admin_client.put(
|
||||
f"/api/chats/{created['id']}",
|
||||
json={"messages": [_user(FIRST_QUESTION), brain]},
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["messages"][1]["budget_exhausted"] is True
|
||||
got = admin_client.get(f"/api/chats/{created['id']}")
|
||||
assert got.status_code == 200
|
||||
assert got.json()["messages"] == _expect(
|
||||
[_user(FIRST_QUESTION), {**FULL_BRAIN, "budget_exhausted": True}]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 159 (task 02): edit-and-resend — the SAVE shape. The client
|
||||
# edits the last question: the record is replaced in place, EVERY
|
||||
|
||||
@@ -443,7 +443,10 @@ def test_plain_read_truncates_then_range_read_answers_the_find_traps_question(
|
||||
assert isinstance(plain_result["content"], str)
|
||||
notice = READ_TRUNCATION_NOTICE.format(shown=READ_CAP, total=len(doc.content))
|
||||
assert plain_result["content"].startswith(f"Document {IDENTITY}:\n")
|
||||
assert plain_result["content"].endswith(f"\n{TRUNCATION_MARKER}\n{notice}")
|
||||
# Phase 160 (task 01): the budget-telemetry trailer rides the
|
||||
# truncated result — the marker + notice end the RAW part.
|
||||
raw = plain_result["content"].removesuffix(agent._budget_note(9))
|
||||
assert raw.endswith(f"\n{TRUNCATION_MARKER}\n{notice}")
|
||||
# The truncation surfaced to the API layer as a ToolResultPiece
|
||||
# (the SSE ``tool_result`` frame's wire value).
|
||||
truncations = [p for p in pieces if isinstance(p, ToolResultPiece)]
|
||||
|
||||
@@ -411,7 +411,7 @@ def test_two_ranges_of_one_doc_both_reach_the_model(two_range_doc: Document) ->
|
||||
# the byte-identical line is now true).
|
||||
reread_result = llm.requests[3][0][7]
|
||||
assert reread_result["role"] == "tool"
|
||||
assert reread_result["content"] == agent.ALREADY_IN_CONTEXT
|
||||
assert reread_result["content"].startswith(agent.ALREADY_IN_CONTEXT)
|
||||
|
||||
# Round 4: the answer echoes BOTH markers (the bug: pre-phase,
|
||||
# the echo could contain only marker A — range B's slice never
|
||||
@@ -485,7 +485,10 @@ def test_truncated_plain_read_then_range_b_proceeds_and_plain_reread_teaches(
|
||||
assert isinstance(plain_result["content"], str)
|
||||
notice = READ_TRUNCATION_NOTICE.format(shown=cap, total=len(doc.content))
|
||||
assert plain_result["content"].startswith(f"Document {IDENTITY}:\n")
|
||||
assert plain_result["content"].endswith(f"\n{TRUNCATION_MARKER}\n{notice}")
|
||||
# Phase 160 (task 01): the budget-telemetry trailer rides the
|
||||
# truncated result — the marker + notice end the RAW part.
|
||||
raw = plain_result["content"].removesuffix(agent._budget_note(9))
|
||||
assert raw.endswith(f"\n{TRUNCATION_MARKER}\n{notice}")
|
||||
assert MARKER_A in plain_result["content"]
|
||||
truncations = [p for p in pieces if isinstance(p, ToolResultPiece)]
|
||||
assert [(p.argument, p.chars_shown, p.chars_total) for p in truncations] == [
|
||||
@@ -512,12 +515,12 @@ def test_truncated_plain_read_then_range_b_proceeds_and_plain_reread_teaches(
|
||||
# resolved doc.
|
||||
reread_result = llm.requests[3][0][7]
|
||||
assert reread_result["role"] == "tool"
|
||||
assert reread_result["content"] == agent.REREAD_PARTIAL.format(
|
||||
assert reread_result["content"].startswith(agent.REREAD_PARTIAL.format(
|
||||
covered=f"1–{covered}, {marker_b_line - 10}–{marker_b_line - 10 + RANGE_B_LIMIT - 1}",
|
||||
source=SOURCE,
|
||||
path=DOC_PATH,
|
||||
total=total_lines,
|
||||
)
|
||||
))
|
||||
|
||||
# Round 4: the answer echoes both markers.
|
||||
answer = _answer(pieces)
|
||||
|
||||
Reference in New Issue
Block a user