phase: 160_tool_budget_telemetry

All green. Final report: **Phase 160 completion pass — verified (final verification; all 5 tasks already in `complete/`)** - **Verified, no code defects found:** budget trailers (`TOOL_BUDGET_REMAINING/LAST/SPENT`), cap-path rejection + `ROUND_CAP_FALLBACK_ANSWER` + `ROUND_CAP_WRAPUP_INSTRUCTION`, `done.budget_exhausted` (ABSENT-when-false), `rounds=/cap_hit=` log fields, `agent_max_rounds=10` default, frontend Continue button + restore + shared-page exclusion, mock markers. - **Tests/lint:** `uv run pytest` 153/153 green (EXIT 0); `uv run pytest --cov=app --cov-report=term-missing` TOTAL **99%** (>90%); `uv run ruff check .` clean; `uv run pyright` 0 errors/0 warnings. - **Criteria:** ① decreasing trailer pinned by unit tests (9→…→spent) ✅ ② cap path ends readable, no tool lines after cap (E2E) ✅ ③ `budget_exhausted` true-only-on-cap, absent otherwise (integration) ✅ ④ Continue button → one fresh `continue` turn, fresh budget, record intact (E2E) ✅ ⑤ flag survives refresh; shared chat never renders it (`shared.js` has no `budget_exhausted` path) ✅ ⑥ `max_rounds=0` byte-identical single `tools=None` request; default still 10 ✅ ⑦ full suite/coverage/lint green ✅ ⑧ `test_tool_budget_telemetry.py` **7/7** in isolation; all 6 regression E2E suites green in isolation ✅ ⑨ commit + phase move = harness (I make no git history/index changes). - **Docs:** README.md — fixed: the "Continue researching" button and cap behavior were undocumented (added a Chat-section paragraph + an Agent-tools-section paragraph, incl. the phase-71 recorded-revision note); env-var table, commands, and architecture all verified consistent otherwise. AGENTS.md — verified consistent, no changes needed. - **Decisions:** no code changes required — phase was fully implemented; only README docs added. No live-infrastructure touched. - **Next pending phase:** `161_reasoning_bubble`.
2026-10-06 01:42:44 -04:00
parent cb8767512a
commit 8172956c15
55 changed files with 4124 additions and 344 deletions
@@ -320,13 +320,13 @@ def test_ls_top_level_lists_registered_sources_through_run_agent(
    holder, llm = _run_call(db, "ls", {})

    assert llm.requests[0][1] == AGENT_TOOLS
-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "2 sources:\n"
        "\n"
        "Deployments — 1 documents\n"
        "Homelab — 2 documents\n"
        "  Home lab notes."
-    )
+        )
    assert holder.tool_calls == 1
    assert holder.read_docs == []

@@ -352,7 +352,7 @@ def test_ls_top_level_empty_registry_through_run_agent(
    db.commit()

    holder, llm = _run_call(db, "ls", {})
-    assert llm.requests[1][0][3]["content"] == "0 sources:"
+    assert llm.requests[1][0][3]["content"].startswith("0 sources:")
    assert holder.tool_calls == 1


@@ -375,20 +375,20 @@ def test_ls_source_scope_lists_root_folder_through_run_agent(kb, registry, db) -

    holder, llm = _run_call(db, "ls", {"path": "Homelab"})

-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "Homelab — 1 documents, 2 folders:\n"
        "\n"
        "  backups/ — 2 documents: Backup notes.\n"
        "  networking/ — 1 documents\n"
        "\n"
        "source: Homelab | path: readme.md | title: Readme | date: 2024-06-15"
-    )
+        )
    assert holder.tool_calls == 1
    assert holder.read_docs == []

    # A registered source with no documents: the header line alone.
    holder0, llm0 = _run_call(db, "ls", {"path": "Deployments"})
-    assert llm0.requests[1][0][3]["content"] == "Deployments — 0 documents, 0 folders:"
+    assert llm0.requests[1][0][3]["content"].startswith("Deployments — 0 documents, 0 folders:")
    assert holder0.tool_calls == 1


@@ -405,23 +405,23 @@ def test_ls_nested_folder_scope_drills_one_level_through_run_agent(

    holder, llm = _run_call(db, "ls", {"path": "Homelab/networking"})

-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "Homelab/networking — 0 documents, 2 folders:\n"
        "\n"
        "  networking/lan/ — 2 documents\n"
        "  networking/vpn/ — 1 documents"
-    )
+        )
    assert holder.tool_calls == 1
    assert holder.read_docs == []

    # One level deeper.
    holder2, llm2 = _run_call(db, "ls", {"path": "Homelab/networking/lan"})
-    assert llm2.requests[1][0][3]["content"] == (
+    assert llm2.requests[1][0][3]["content"].startswith(
        "Homelab/networking/lan — 2 documents, 0 folders:\n"
        "\n"
        "source: Homelab | path: networking/lan/a.md | title: A | date: 2024-06-15\n"
        "source: Homelab | path: networking/lan/b.md | title: B | date: 2024-06-15"
-    )
+        )
    assert holder2.tool_calls == 1


@@ -434,7 +434,9 @@ def test_ls_folder_file_cap_through_run_agent(kb, registry, db) -> None:

    holder, llm = _run_call(db, "ls", {"path": "Homelab/big"})

-    content = llm.requests[1][0][3]["content"]
+    # Phase 160 (task 01): the budget-telemetry trailer rides the
+    # listing — the raw listing is the RAW part (the trailer stripped).
+    content = llm.requests[1][0][3]["content"].removesuffix(agent._budget_note(9))
    lines = content.splitlines()
    assert lines[0] == "Homelab/big — 51 documents, 0 folders:"
    assert lines[2] == (
@@ -460,9 +462,8 @@ def test_ls_scoped_unknown_source_refused_through_run_agent(kb, src, db) -> None

    holder, llm = _run_call(db, "ls", {"path": "Ghost"})

-    assert (
-        llm.requests[1][0][3]["content"]
-        == agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="Ghost")
+    assert llm.requests[1][0][3]["content"].startswith(
+        agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="Ghost")
    )
    assert holder.tool_calls == 0 and holder.read_docs == []

@@ -477,9 +478,8 @@ def test_ls_path_like_scope_unknown_source_gets_no_source_refusal(kb, src, db) -

    holder, llm = _run_call(db, "ls", {"path": "app/rag/importer.py"})

-    assert (
-        llm.requests[1][0][3]["content"]
-        == agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="app")
+    assert llm.requests[1][0][3]["content"].startswith(
+        agent.NO_SOURCE_NOT_A_DIRECTORY.format(scope="app")
    )
    assert holder.tool_calls == 0 and holder.read_docs == []
    assert llm.requests[1][1] == AGENT_TOOLS  # rejected → tools stay offered
@@ -500,10 +500,10 @@ def test_ls_unknown_folder_gets_not_a_folder_with_parents_subfolders(

    holder, llm = _run_call(db, "ls", {"path": "Homelab/netwoking"})

-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "'Homelab/netwoking' is not a folder — Homelab has: "
        "backups/ containers/ networking/"
-    )
+        )
    assert holder.tool_calls == 0 and holder.read_docs == []
    assert llm.requests[1][1] == AGENT_TOOLS  # rejected → tools stay offered

@@ -523,11 +523,11 @@ def test_read_combined_path_through_run_agent(kb, db) -> None:

    # Phase 106 (D5): the date rides every read — the SECOND line (the
    # first line stays the byte-identical header).
-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "Document Alpha/deep/nested/doc.md:\n"
        "date: 2024-06-15\n"
        "FULL-TEXT"
-    )
+        )
    assert holder.tool_calls == 1
    # SEC-14-04: short-lived session loads fresh copies
    assert [d.id for d in holder.read_docs] == [created.id]
@@ -542,8 +542,8 @@ def test_read_bare_source_name_refused_through_run_agent(kb, db) -> None:

    holder, llm = _run_call(db, "read", {"path": "Alpha"})

-    assert (
-        llm.requests[1][0][3]["content"] == "No document at 'Alpha' — check the ls output."
+    assert llm.requests[1][0][3]["content"].startswith(
+        "No document at 'Alpha' — check the ls output."
    )
    assert holder.tool_calls == 0 and holder.read_docs == []

@@ -558,9 +558,8 @@ def test_read_unknown_combined_path_refused_through_run_agent(kb, db) -> None:

    holder, llm = _run_call(db, "read", {"path": "Alpha/nope/deep.md"})

-    assert (
-        llm.requests[1][0][3]["content"]
-        == "No document at 'Alpha/nope/deep.md' — check the ls output."
+    assert llm.requests[1][0][3]["content"].startswith(
+        "No document at 'Alpha/nope/deep.md' — check the ls output."
    )
    assert holder.tool_calls == 0 and holder.read_docs == []

@@ -592,19 +591,19 @@ def test_read_bare_path_single_source_suggestion_then_corrected_read(kb, db) ->
    # Round 1: the bare path resolves to no combined identity, but it IS
    # the indexed document's path — the refusal names the one combined
    # identity to use (not counted, the tools stay offered).
-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "No document at 'deep/nested/doc.md' — "
        "did you mean 'Alpha/deep/nested/doc.md'?"
-    )
+        )
    assert llm.requests[1][1] == AGENT_TOOLS
    # Round 2: the corrected combined identity succeeds — the full
    # content (plus the phase-106 D5 date line), the holder records the
    # row, and it counts.
-    assert llm.requests[2][0][5]["content"] == (
+    assert llm.requests[2][0][5]["content"].startswith(
        "Document Alpha/deep/nested/doc.md:\n"
        "date: 2024-06-15\n"
        "FULL-TEXT"
-    )
+        )
    assert llm.requests[2][1] == AGENT_TOOLS
    # SEC-14-04: short-lived session loads fresh copies
    assert [d.id for d in holder.read_docs] == [created.id]
@@ -633,13 +632,13 @@ def test_read_bare_path_two_sources_one_of_suggestion_then_corrected_read(
    holder = AgentHolder()
    asyncio.run(_consume(cast("LLMClient", llm), lambda: SessionLocal(), holder))

-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "No document at 'shared/x.md' — did you mean one of: "
        "'Alpha/shared/x.md', 'Beta/shared/x.md'?"
-    )
-    assert llm.requests[2][0][5]["content"] == (
+        )
+    assert llm.requests[2][0][5]["content"].startswith(
        "Document Alpha/shared/x.md:\ndate: 2024-06-15\nA-TEXT"
-    )
+        )
    # SEC-14-04: short-lived session loads fresh copies
    assert [d.id for d in holder.read_docs] == [a.id]
    assert holder.tool_calls == 1  # only the corrected read executed
@@ -673,10 +672,10 @@ def test_grep_whole_kb_through_run_agent(kb, db) -> None:
    # Offered: the first request carries AGENT_TOOLS (the 3-tool list).
    assert llm.requests[0][1] == AGENT_TOOLS
    # Executed against the real DB: catalog order, grep-style lines.
-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "Alpha/a/one.md:2: needle in one\n"
        "Beta/b/two.md:2: NEEDLE in two"
-    )
+        )
    assert holder.tool_calls == 1
    assert holder.read_docs == []  # locked A5: grep adds no context

@@ -689,7 +688,7 @@ def test_grep_scoped_through_run_agent(kb, db) -> None:
    holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/a/one.md"})

    # Only the named document is searched — the other one's hit is absent.
-    assert llm.requests[1][0][3]["content"] == "Alpha/a/one.md:2: Needle here"
+    assert llm.requests[1][0][3]["content"].startswith("Alpha/a/one.md:2: Needle here")
    assert holder.tool_calls == 1
    assert holder.read_docs == []

@@ -703,9 +702,8 @@ def test_grep_scoped_missing_doc_refused_through_run_agent(kb, db) -> None:

    holder, llm = _run_call(db, "grep", {"pattern": "needle", "path": "Alpha/ghost.md"})

-    assert (
-        llm.requests[1][0][3]["content"]
-        == "No document at 'Alpha/ghost.md' — check the ls output."
+    assert llm.requests[1][0][3]["content"].startswith(
+        "No document at 'Alpha/ghost.md' — check the ls output."
    )
    assert holder.tool_calls == 0 and holder.read_docs == []

@@ -716,8 +714,8 @@ def test_grep_no_matches_through_run_agent(kb, db) -> None:

    holder, llm = _run_call(db, "grep", {"pattern": "zebra"})

-    assert llm.requests[1][0][3]["content"] == (
+    assert llm.requests[1][0][3]["content"].startswith(
        "No matches for 'zebra' in the knowledge base."
-    )
+        )
    assert holder.tool_calls == 1  # an executed grep with zero hits
    assert holder.read_docs == []