phase: 95_read_truncation_cap
All gates are green. Final verification is complete — no fixes were needed; the phase is fully implemented and passing. **Phase 95 final verification report** - Verified all 3 tasks already implemented (task files in `complete/`): read cap + notice + `ToolResultPiece` (task 01), `tool_result` SSE + live/saved/shared marker (task 02), E2E suite (task 03). - Code checks passed: `BOR_READ_MAX_CHARS` (default 128 000, validator), pinned `READ_TRUNCATION_NOTICE` + `TRUNCATION_MARKER`, byte-identical at/under-cap read, holder `read_truncations` (no `tool_calls` impact), piece order (after `tool`, before next round), `ChatToolResultEvent`, `ToolCall` back-compat fields, `app.js` live/restore handlers, `shared.js`, `.truncated-note` CSS, `.env.example` entry, mock-LLM echo script. - **Tests:** `uv run pytest` → 1966 passed; `uv run pytest --cov=app --cov-report=term-missing` → all green, TOTAL **99%** (>90% gate). - **E2E:** `uv run pytest tests/e2e/test_read_truncation_cap.py -v --no-cov` → **3 passed** (frame order + live marker + LLM notice via echo; save→shared fidelity; short-read control). - **Regression (isolated):** `test_agent_document_tools` 4 ✓, `test_chat_history` 5 ✓, `test_share_chat` 4 ✓, `test_big_read_progress` 4 ✓, `test_stop_generation` 3 ✓. - **Lint/types:** `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors, 0 warnings. **Completion criteria:** ① over-cap read → first-cap-chars + marker + pinned notice — ✓ (unit-pinned: at-cap/cap+1/notice tests); ② user marker live/saved/shared — ✓ (E2E + frontend tests); ③ at/under cap byte-identical, no frame — ✓ (unit + control E2E); ④ top-2 `<documents>` retrieval untouched — ✓ (`app/rag/retriever.py` unmodified vs HEAD); ⑤ suite green, >90% coverage, ruff+pyright clean — ✓; ⑥ no completed-phase behavior change — ✓ (all gates green; commit left to harness per pass rules). - No defects found; no changes made this pass. Next pending phase: none in `todo/` (96 is the next free number).
This commit is contained in:
+205
-12
@@ -47,12 +47,21 @@ from app.models import Document, GitSource
|
||||
from app.rag import agent
|
||||
from app.rag.agent import (
|
||||
AGENT_TOOLS,
|
||||
READ_TRUNCATION_NOTICE,
|
||||
AgentHolder,
|
||||
MalformedReplyError,
|
||||
run_agent,
|
||||
)
|
||||
from app.rag.llm import LLMClient, LLMError, RetryPiece, StreamPiece, ToolCallPiece
|
||||
from app.rag.llm import (
|
||||
LLMClient,
|
||||
LLMError,
|
||||
RetryPiece,
|
||||
StreamPiece,
|
||||
ToolCallPiece,
|
||||
ToolResultPiece,
|
||||
)
|
||||
from app.rag.prompts import TOOLS_SECTION, _base, build_deflect_prompt, build_high_prompt
|
||||
from app.rag.retriever import TRUNCATION_MARKER
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.rag.scaffolding import ScaffoldingFilter
|
||||
@@ -121,11 +130,12 @@ async def _run(
|
||||
settings: Settings,
|
||||
seed_docs: list[Document] | None = None,
|
||||
history: Sequence[dict[str, Any]] = (),
|
||||
) -> list[StreamPiece | ToolCallPiece | RetryPiece]:
|
||||
) -> list[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece]:
|
||||
"""Consume one ``run_agent`` turn; *history* (phase 74) is the
|
||||
client's prior turns spliced between system and user (default
|
||||
``()`` — the pre-phase-74 two-message request)."""
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece] = []
|
||||
``()`` — the pre-phase-74 two-message request). Phase 95: the loop
|
||||
may also yield a ``ToolResultPiece`` (a truncated ``read``)."""
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece] = []
|
||||
async for piece in run_agent(
|
||||
cast("LLMClient", llm),
|
||||
cast("Session", None),
|
||||
@@ -192,15 +202,25 @@ def test_agent_tools_names_and_parameters() -> None:
|
||||
# section already carries (every refusal of a 12-call run was
|
||||
# ALREADY_IN_CONTEXT); the rule now leads the description instead
|
||||
# of sitting mid-paragraph, and the tool is framed as "only for
|
||||
# documents NOT already in <documents>".
|
||||
# documents NOT already in <documents>". Phase 95 (task 01): the
|
||||
# read-truncation sentence is inserted before the one-call-at-a-
|
||||
# time discipline clause (the discipline rule stays last, as in the
|
||||
# other two tools) — a capped read carries the TRUNCATED notice and
|
||||
# the `grep` follow-up (the pinned copy).
|
||||
assert read["description"] == (
|
||||
"Do not call this tool for a document already shown in "
|
||||
"the <documents> section, even when the user asks you to "
|
||||
"open or read it — its full text is already in your "
|
||||
"prompt; answer directly from it. Use it only to add a "
|
||||
"document NOT already in <documents> to your context, "
|
||||
"by its combined `source/path` string. Call one tool at "
|
||||
"a time — wait for this result before your next call."
|
||||
"by its combined `source/path` string. Very large "
|
||||
"documents are truncated: you receive the first part "
|
||||
"plus a TRUNCATED notice naming how many more characters "
|
||||
"exist — the notice is authoritative, the document did "
|
||||
"NOT end where it stopped. Follow it and use `grep` "
|
||||
"(pattern) to locate the rest — it searches the whole "
|
||||
"document. Call one tool at a time — wait for this "
|
||||
"result before your next call."
|
||||
)
|
||||
read_params = read["parameters"]
|
||||
assert read_params["type"] == "object"
|
||||
@@ -1519,6 +1539,177 @@ def test_reading_an_already_read_doc_is_deduped(monkeypatch: pytest.MonkeyPatch)
|
||||
assert llm.requests[2][1] == AGENT_TOOLS
|
||||
|
||||
|
||||
# ---------- phase 95: the read cap (bounded reads, honest truncation) ----------
|
||||
|
||||
|
||||
def test_read_exactly_at_cap_is_byte_identical_and_untruncated(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 95 boundary: a document whose content length EQUALS the cap
|
||||
fits — read whole, byte-identical to the pre-phase-95 result (no
|
||||
marker, no notice, no holder entry, no ``ToolResultPiece``)."""
|
||||
cap = 20
|
||||
content = "x" * cap
|
||||
doc = _doc("S", "big.md", "Big", content)
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "S/big.md"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
out = asyncio.run(_run(llm, holder, _settings(read_max_chars=cap)))
|
||||
# Byte-identical to today's read result (no marker, no notice).
|
||||
assert llm.requests[1][0][3]["content"] == "Document S/big.md:\n" + content
|
||||
assert TRUNCATION_MARKER not in llm.requests[1][0][3]["content"]
|
||||
# No truncation recorded, none surfaced to the loop.
|
||||
assert holder.read_truncations == []
|
||||
assert not any(isinstance(p, ToolResultPiece) for p in out)
|
||||
# Still a successful read.
|
||||
assert holder.read_docs == [doc]
|
||||
assert holder.tool_calls == 1
|
||||
|
||||
|
||||
def test_read_at_cap_plus_one_truncates_with_marker_and_notice(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 95 boundary: ONE char over the cap truncates — the first
|
||||
``cap`` chars + the shared :data:`TRUNCATION_MARKER` + the pinned
|
||||
grep-pointer notice (``{total}`` = the true length, ``{shown}`` = the
|
||||
cap), and the truncation is recorded on the holder. A truncated read
|
||||
is still a successful call (``tool_calls`` / ``read_docs`` as today)."""
|
||||
cap = 20
|
||||
content = "x" * (cap + 1)
|
||||
doc = _doc("S", "big.md", "Big", content)
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "S/big.md"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings(read_max_chars=cap)))
|
||||
expected = (
|
||||
"Document S/big.md:\n"
|
||||
+ content[:cap]
|
||||
+ "\n"
|
||||
+ TRUNCATION_MARKER
|
||||
+ "\n"
|
||||
+ READ_TRUNCATION_NOTICE.format(shown=cap, total=cap + 1)
|
||||
)
|
||||
assert llm.requests[1][0][3]["content"] == expected
|
||||
# (argument, chars_shown, chars_total) — the raw argument the tool
|
||||
# frame carries, the cap kept, the true length.
|
||||
assert holder.read_truncations == [("S/big.md", cap, cap + 1)]
|
||||
assert holder.read_docs == [doc] # still added to the context
|
||||
assert holder.tool_calls == 1 # still a counted, successful call
|
||||
|
||||
|
||||
def test_read_truncation_notice_is_pinned() -> None:
|
||||
"""Phase 95: the notice copy is pinned — it names the true length
|
||||
(``{total}``), the cap kept (``{shown}``), states the rest is NOT
|
||||
shown (the document did not end where it stopped), and points at
|
||||
``grep`` (which searches the whole document)."""
|
||||
assert READ_TRUNCATION_NOTICE.format(shown=100, total=250) == (
|
||||
"TRUNCATED — this document is 250 characters; only the first "
|
||||
"100 are in your context. The rest is NOT shown. Use grep "
|
||||
"(pattern) to locate what you need — grep searches the whole "
|
||||
"document."
|
||||
)
|
||||
|
||||
|
||||
def test_run_agent_yields_tool_result_after_tool_frame_before_next_round(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 95: ``run_agent`` yields exactly ONE ``ToolResultPiece`` per
|
||||
truncated read — AFTER the round's ``tool`` frame (the matching
|
||||
``ToolCallPiece``) and BEFORE the next model round (the answer
|
||||
pieces). It carries (argument, shown, total); ``argument`` is the
|
||||
same value the matching ``tool`` frame carries (the raw
|
||||
``source/path`` the model passed)."""
|
||||
cap = 20
|
||||
content = "y" * (cap + 5)
|
||||
doc = _doc("S", "big.md", "Big", content)
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "S/big.md"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
out = asyncio.run(_run(llm, holder, _settings(read_max_chars=cap)))
|
||||
# The piece order: the read's ToolCallPiece, then the ToolResultPiece,
|
||||
# then the next round's content (the answer).
|
||||
assert isinstance(out[0], ToolCallPiece) and out[0].name == "read"
|
||||
piece = out[1]
|
||||
assert isinstance(piece, ToolResultPiece)
|
||||
assert isinstance(out[2], StreamPiece)
|
||||
assert piece.name == "read"
|
||||
assert piece.argument == "S/big.md"
|
||||
assert piece.truncated is True
|
||||
assert piece.chars_shown == cap
|
||||
assert piece.chars_total == cap + 5
|
||||
# Exactly one ToolResultPiece for the one truncated read.
|
||||
assert [p for p in out if isinstance(p, ToolResultPiece)] == [piece]
|
||||
|
||||
|
||||
def test_run_agent_short_read_yields_no_tool_result_piece(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 95: a read at or under the cap yields NO ``ToolResultPiece``
|
||||
— the non-truncated stream is byte-identical to the pre-phase-95
|
||||
one (just the ``ToolCallPiece`` + the answer)."""
|
||||
cap = 20
|
||||
content = "z" * cap # exactly at the cap → not truncated
|
||||
doc = _doc("S", "small.md", "Small", content)
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: doc)
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "S/small.md"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
out = asyncio.run(_run(llm, holder, _settings(read_max_chars=cap)))
|
||||
assert not any(isinstance(p, ToolResultPiece) for p in out)
|
||||
assert holder.read_truncations == []
|
||||
# Order: the ToolCallPiece then the answer content (no piece between).
|
||||
assert isinstance(out[0], ToolCallPiece) and out[0].name == "read"
|
||||
assert isinstance(out[1], StreamPiece)
|
||||
|
||||
|
||||
def test_read_truncation_does_not_touch_refusal_paths(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 95: the read refusal paths are untouched by the cap — a SEED
|
||||
document that is over the cap is still refused with
|
||||
``ALREADY_IN_CONTEXT`` (not truncated, nothing recorded, nothing
|
||||
counted), and an unknown path is still the no-document refusal (no
|
||||
content is read, so no truncation either)."""
|
||||
big = "B" * 5000 # far over the tiny cap below
|
||||
seed = _doc("S", "seed.md", "Seed", big)
|
||||
monkeypatch.setattr(agent, "find_document", lambda db, source, path: None)
|
||||
monkeypatch.setattr(agent, "all_documents", lambda db: [])
|
||||
# (a) Reading the (over-cap) seed doc → ALREADY_IN_CONTEXT (refusal).
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "S/seed.md"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm, holder, _settings(read_max_chars=100), seed_docs=[seed]))
|
||||
assert llm.requests[1][0][3]["content"] == agent.ALREADY_IN_CONTEXT
|
||||
assert holder.read_truncations == []
|
||||
assert holder.tool_calls == 0 and holder.read_docs == []
|
||||
# (b) An unknown path → the no-document refusal (argument echoed),
|
||||
# even though a big doc could have truncated — no content is read.
|
||||
holder2 = AgentHolder()
|
||||
llm2 = ScriptedLLM(
|
||||
[ToolCallPiece(id="call_1", name="read", arguments={"path": "S/missing.md"})],
|
||||
[StreamPiece("content", "ans")],
|
||||
)
|
||||
asyncio.run(_run(llm2, holder2, _settings(read_max_chars=100), seed_docs=[seed]))
|
||||
assert llm2.requests[1][0][3]["content"] == (
|
||||
"No document at 'S/missing.md' — check the ls output."
|
||||
)
|
||||
assert holder2.read_truncations == []
|
||||
assert holder2.tool_calls == 0 and holder2.read_docs == []
|
||||
|
||||
|
||||
def test_unknown_tool_name_refused(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
holder = AgentHolder()
|
||||
llm = ScriptedLLM(
|
||||
@@ -2322,8 +2513,8 @@ def test_round_failure_after_first_piece_is_terminal(monkeypatch: pytest.MonkeyP
|
||||
)
|
||||
sleeps = _record_sleeps(monkeypatch)
|
||||
|
||||
async def drain() -> list[StreamPiece | ToolCallPiece | RetryPiece]:
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece] = []
|
||||
async def drain() -> list[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece]:
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece] = []
|
||||
with pytest.raises(LLMError, match="mid-stream drop"):
|
||||
async for piece in run_agent(
|
||||
cast("LLMClient", llm),
|
||||
@@ -2384,8 +2575,8 @@ def test_zero_retries_is_one_plain_attempt(monkeypatch: pytest.MonkeyPatch) -> N
|
||||
llm = FailingLLM([([], LLMError("connection refused"))])
|
||||
sleeps = _record_sleeps(monkeypatch)
|
||||
|
||||
async def drain() -> list[StreamPiece | ToolCallPiece | RetryPiece]:
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece] = []
|
||||
async def drain() -> list[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece]:
|
||||
out: list[StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece] = []
|
||||
with pytest.raises(LLMError, match="connection refused"):
|
||||
async for piece in run_agent(
|
||||
cast("LLMClient", llm),
|
||||
@@ -2434,7 +2625,9 @@ def test_abandon_mid_retry_sleep_leaks_nothing(monkeypatch: pytest.MonkeyPatch)
|
||||
holder=holder,
|
||||
)
|
||||
|
||||
async def consumer() -> list[StreamPiece | ToolCallPiece | RetryPiece]:
|
||||
async def consumer() -> list[
|
||||
StreamPiece | ToolCallPiece | RetryPiece | ToolResultPiece
|
||||
]:
|
||||
return [p async for p in gen]
|
||||
|
||||
task = asyncio.ensure_future(consumer())
|
||||
|
||||
Reference in New Issue
Block a user