phase: 95_read_truncation_cap
All gates are green. Final verification is complete — no fixes were needed; the phase is fully implemented and passing. **Phase 95 final verification report** - Verified all 3 tasks already implemented (task files in `complete/`): read cap + notice + `ToolResultPiece` (task 01), `tool_result` SSE + live/saved/shared marker (task 02), E2E suite (task 03). - Code checks passed: `BOR_READ_MAX_CHARS` (default 128 000, validator), pinned `READ_TRUNCATION_NOTICE` + `TRUNCATION_MARKER`, byte-identical at/under-cap read, holder `read_truncations` (no `tool_calls` impact), piece order (after `tool`, before next round), `ChatToolResultEvent`, `ToolCall` back-compat fields, `app.js` live/restore handlers, `shared.js`, `.truncated-note` CSS, `.env.example` entry, mock-LLM echo script. - **Tests:** `uv run pytest` → 1966 passed; `uv run pytest --cov=app --cov-report=term-missing` → all green, TOTAL **99%** (>90% gate). - **E2E:** `uv run pytest tests/e2e/test_read_truncation_cap.py -v --no-cov` → **3 passed** (frame order + live marker + LLM notice via echo; save→shared fidelity; short-read control). - **Regression (isolated):** `test_agent_document_tools` 4 ✓, `test_chat_history` 5 ✓, `test_share_chat` 4 ✓, `test_big_read_progress` 4 ✓, `test_stop_generation` 3 ✓. - **Lint/types:** `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors, 0 warnings. **Completion criteria:** ① over-cap read → first-cap-chars + marker + pinned notice — ✓ (unit-pinned: at-cap/cap+1/notice tests); ② user marker live/saved/shared — ✓ (E2E + frontend tests); ③ at/under cap byte-identical, no frame — ✓ (unit + control E2E); ④ top-2 `<documents>` retrieval untouched — ✓ (`app/rag/retriever.py` unmodified vs HEAD); ⑤ suite green, >90% coverage, ruff+pyright clean — ✓; ⑥ no completed-phase behavior change — ✓ (all gates green; commit left to harness per pass rules). - No defects found; no changes made this pass. Next pending phase: none in `todo/` (96 is the next free number).
This commit is contained in:
@@ -16,6 +16,7 @@ import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
@@ -24,15 +25,18 @@ import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api import chat as chat_api
|
||||
from app.config import Settings, get_settings
|
||||
from app.main import app as fastapi_app
|
||||
from app.models import Chunk, GitSource, QueryLog
|
||||
from app.models import Chunk, Document, GitSource, QueryLog
|
||||
from app.rag import agent
|
||||
from app.rag.agent import AGENT_TOOLS
|
||||
from app.rag.agent import AGENT_TOOLS, READ_TRUNCATION_NOTICE
|
||||
from app.rag.importer import import_sources
|
||||
from app.rag.llm import EmbeddingError, LLMError, StreamPiece, ToolCallPiece
|
||||
from app.rag.llm import EmbeddingError, LLMError, StreamPiece, ToolCallPiece, ToolResultPiece
|
||||
from app.rag.prompts import build_high_prompt
|
||||
from app.rag.retriever import TRUNCATION_MARKER
|
||||
from app.schemas import ChatDoneEvent, SourceRef
|
||||
from tests.conftest import ADMIN_PASSWORD
|
||||
|
||||
@@ -610,6 +614,352 @@ def test_chat_query_log_failure_still_sends_done(client, db, seeded_kb: FakeRagL
|
||||
# ---------- phase 37: agent document tools on grounded turns ----------
|
||||
|
||||
|
||||
async def _collect_run_agent(
|
||||
llm: FakeRagLLM,
|
||||
db: Session,
|
||||
system_prompt: str,
|
||||
settings: Settings,
|
||||
seed_docs: list[Document],
|
||||
) -> tuple[list[Any], agent.AgentHolder]:
|
||||
"""Consume one ``run_agent`` turn, returning the yielded pieces (in
|
||||
order) and the holder. Phase 95 (task 01): the direct agent-loop
|
||||
drive — the agent-loop yield order on the real prompt path, the
|
||||
complement of the endpoint-level ``tool_result`` SSE tests below
|
||||
(task 02)."""
|
||||
holder = agent.AgentHolder()
|
||||
pieces: list[Any] = []
|
||||
async for piece in agent.run_agent(
|
||||
llm, # pyright: ignore[reportArgumentType] # duck-typed LLMClient
|
||||
db,
|
||||
system_prompt=system_prompt,
|
||||
user_message=QUESTION,
|
||||
seed_docs=seed_docs,
|
||||
settings=settings,
|
||||
holder=holder,
|
||||
):
|
||||
pieces.append(piece)
|
||||
return pieces, holder
|
||||
|
||||
|
||||
def test_read_cap_truncates_and_yields_tool_result_on_real_prompt_path(
|
||||
db,
|
||||
) -> None:
|
||||
"""Phase 95 (task 01): on the REAL prompt path (a real Postgres
|
||||
document + the real ``build_high_prompt``), a ``read`` of a document
|
||||
LONGER than ``settings.read_max_chars`` truncates the result the model
|
||||
sees — first ``cap`` chars + the shared :data:`TRUNCATION_MARKER` + the
|
||||
pinned grep-pointer notice — and ``run_agent`` yields exactly ONE
|
||||
``ToolResultPiece``: AFTER the read's ``tool`` frame (the matching
|
||||
``ToolCallPiece``) and BEFORE the next model round. The endpoint-level
|
||||
``tool_result`` SSE frame is asserted separately below (task 02); this
|
||||
pins the agent-loop yield order on the real prompt path."""
|
||||
cap = 100
|
||||
content = "K" * (cap + 40) # 40 chars over the cap
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source="docs",
|
||||
path="big.md",
|
||||
full_path="/tmp/big.md",
|
||||
title="Big Doc",
|
||||
content=content,
|
||||
content_hash="1" * 64,
|
||||
)
|
||||
db.add(doc)
|
||||
db.commit()
|
||||
try:
|
||||
# The real prompt path: the actual HIGH prompt for the one doc.
|
||||
system_prompt = build_high_prompt([doc])
|
||||
settings = Settings(_env_file=None, read_max_chars=cap) # pyright: ignore[reportCallIssue]
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="read", arguments={"path": "docs/big.md"}
|
||||
)
|
||||
]
|
||||
# the answer request (tools still offered, script
|
||||
# exhausted) falls back to the thinking + answer stream
|
||||
]
|
||||
)
|
||||
pieces, holder = asyncio.run(
|
||||
_collect_run_agent(scripted, db, system_prompt, settings, seed_docs=[])
|
||||
)
|
||||
# The model's context carried the truncated read — the first cap
|
||||
# chars, then the shared marker + the pinned grep-pointer notice
|
||||
# (so a downstream grep is the model's path to the rest). The
|
||||
# fake records the (mutated-in-place) messages list, so the same
|
||||
# tool message is aliased across requests — they all carry the
|
||||
# same content; take the last.
|
||||
tool_msgs = [
|
||||
m for r in scripted.seen_messages for m in r if m.get("role") == "tool"
|
||||
]
|
||||
assert tool_msgs, "the executed read must be appended as a tool message"
|
||||
body = tool_msgs[-1]["content"]
|
||||
assert body.startswith("Document docs/big.md:\n" + content[:cap])
|
||||
assert TRUNCATION_MARKER in body
|
||||
assert (
|
||||
READ_TRUNCATION_NOTICE.format(shown=cap, total=len(content)) in body
|
||||
)
|
||||
# The yield order: the read's ToolCallPiece, then the ONE
|
||||
# ToolResultPiece, then the next round's answer content.
|
||||
kinds: list[str] = []
|
||||
for p in pieces:
|
||||
if isinstance(p, ToolCallPiece):
|
||||
kinds.append("toolcall")
|
||||
elif isinstance(p, ToolResultPiece):
|
||||
kinds.append("toolresult")
|
||||
elif isinstance(p, StreamPiece):
|
||||
kinds.append(p.kind)
|
||||
assert kinds.count("toolresult") == 1
|
||||
assert kinds.index("toolcall") < kinds.index("toolresult")
|
||||
assert kinds.index("toolresult") < kinds.index("content")
|
||||
# The piece carries (argument, shown, total) — the raw
|
||||
# source/path the model passed (what the tool frame carries), the
|
||||
# cap kept, the true length.
|
||||
(result_piece,) = [p for p in pieces if isinstance(p, ToolResultPiece)]
|
||||
assert result_piece.name == "read"
|
||||
assert result_piece.argument == "docs/big.md"
|
||||
assert result_piece.truncated is True
|
||||
assert result_piece.chars_shown == cap
|
||||
assert result_piece.chars_total == len(content)
|
||||
# Holder accounting: a truncated read is still a SUCCESSFUL call
|
||||
# (counted + added to context); the tuple is the signal only.
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [doc]
|
||||
assert holder.read_truncations == [("docs/big.md", cap, len(content))]
|
||||
finally:
|
||||
db.delete(doc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_read_at_or_under_cap_yields_no_tool_result_on_real_prompt_path(
|
||||
db,
|
||||
) -> None:
|
||||
"""Phase 95 (task 01): the complement — a ``read`` of a document at or
|
||||
under the cap on the real prompt path is byte-identical to the
|
||||
pre-phase-95 agent loop: NO ``ToolResultPiece``, no holder entry, no
|
||||
marker in the model's context."""
|
||||
cap = 100
|
||||
content = "K" * cap # exactly at the cap → fits, not truncated
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source="docs",
|
||||
path="fits.md",
|
||||
full_path="/tmp/fits.md",
|
||||
title="Fits Doc",
|
||||
content=content,
|
||||
content_hash="2" * 64,
|
||||
)
|
||||
db.add(doc)
|
||||
db.commit()
|
||||
try:
|
||||
system_prompt = build_high_prompt([doc])
|
||||
settings = Settings(_env_file=None, read_max_chars=cap) # pyright: ignore[reportCallIssue]
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="read", arguments={"path": "docs/fits.md"}
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
pieces, holder = asyncio.run(
|
||||
_collect_run_agent(scripted, db, system_prompt, settings, seed_docs=[])
|
||||
)
|
||||
# No ToolResultPiece, no holder entry.
|
||||
assert not any(isinstance(p, ToolResultPiece) for p in pieces)
|
||||
assert holder.read_truncations == []
|
||||
# The model's context is the whole document, byte-identical to
|
||||
# the pre-phase-95 read result (no marker, no notice). (The fake
|
||||
# aliases the mutated messages list, so take the last tool msg.)
|
||||
tool_msgs = [
|
||||
m for r in scripted.seen_messages for m in r if m.get("role") == "tool"
|
||||
]
|
||||
assert tool_msgs, "the executed read must be appended as a tool message"
|
||||
assert tool_msgs[-1]["content"] == "Document docs/fits.md:\n" + content
|
||||
assert TRUNCATION_MARKER not in tool_msgs[-1]["content"]
|
||||
# Still a successful read.
|
||||
assert holder.tool_calls == 1
|
||||
assert holder.read_docs == [doc]
|
||||
finally:
|
||||
db.delete(doc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _insert_big_doc(db, content: str) -> Document:
|
||||
"""One bare ``documents`` row (no chunks — the ``read`` lookup is a
|
||||
(source, path) identity match, not a retrieval) for the SSE-level
|
||||
read-cap tests: a document the model can only reach through the
|
||||
``read`` tool."""
|
||||
doc = Document(
|
||||
id=uuid.uuid4(),
|
||||
source="docs",
|
||||
path="big-read.md",
|
||||
full_path="/tmp/big-read.md",
|
||||
title="Big Read Doc",
|
||||
content=content,
|
||||
content_hash="3" * 64,
|
||||
)
|
||||
db.add(doc)
|
||||
db.commit()
|
||||
return doc
|
||||
|
||||
|
||||
def test_truncated_read_streams_tool_result_frame_after_tool_frame(
|
||||
client,
|
||||
db,
|
||||
seeded_kb: FakeRagLLM,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 95 (task 02, the A15 extension): a grounded turn whose
|
||||
scripted ``read`` hits a document LONGER than ``read_max_chars``
|
||||
(the cap lowered via the settings override — the task 01
|
||||
``Settings(_env_file=None, read_max_chars=…)`` pattern) streams the
|
||||
``tool`` → ``tool_result`` → ``delta…`` → ``done`` sequence: EXACTLY
|
||||
ONE ``tool_result`` frame, AFTER the matching ``tool`` frame (the
|
||||
line is already on screen) and BEFORE the next round's first frame,
|
||||
with the right shape and counts (``chars_shown`` = the cap,
|
||||
``chars_total`` = the true length). The model's context carried the
|
||||
truncated read (marker + pinned grep-pointer notice); the read is
|
||||
still cited (a truncated read is a successful call)."""
|
||||
cap = 100
|
||||
content = "K" * (cap + 150)
|
||||
doc = _insert_big_doc(db, content)
|
||||
live = get_settings()
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"get_settings",
|
||||
lambda: Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
relevance_threshold=live.relevance_threshold,
|
||||
read_max_chars=cap,
|
||||
),
|
||||
)
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="read", arguments={"path": "docs/big-read.md"}
|
||||
)
|
||||
]
|
||||
# the answer request still carries the tools (1 round < the
|
||||
# default cap of 10); the script is exhausted, so the fake
|
||||
# falls back to the thinking + answer stream
|
||||
]
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
db.delete(doc)
|
||||
db.commit()
|
||||
|
||||
types = [f["type"] for f in frames]
|
||||
assert "error" not in types
|
||||
tool_i = types.index("tool")
|
||||
tool_result_i = types.index("tool_result")
|
||||
# Exactly one tool_result frame…
|
||||
assert types.count("tool_result") == 1
|
||||
# …AFTER the matching tool frame and BEFORE the next model round's
|
||||
# first frame (the answer's deltas): tool → tool_result → delta…
|
||||
assert tool_i + 1 == tool_result_i
|
||||
assert tool_result_i < min(i for i, t in enumerate(types) if t == "delta")
|
||||
# The frame's exact shape: the additive seventh event type carries
|
||||
# the name/argument of the matching tool frame + the counts.
|
||||
frame = frames[tool_result_i]
|
||||
assert set(frame) == {
|
||||
"type",
|
||||
"name",
|
||||
"argument",
|
||||
"truncated",
|
||||
"chars_shown",
|
||||
"chars_total",
|
||||
}
|
||||
assert frame["name"] == frames[tool_i]["name"] == "read"
|
||||
assert frame["argument"] == frames[tool_i]["argument"] == "docs/big-read.md"
|
||||
assert frame["truncated"] is True
|
||||
assert frame["chars_shown"] == cap # the cap kept
|
||||
assert frame["chars_total"] == len(content) # the true length
|
||||
# The LLM's context carried the honest truncation: first cap chars +
|
||||
# the shared marker + the pinned grep-pointer notice (the fake
|
||||
# aliases the mutated messages list — take the last tool msg).
|
||||
tool_msgs = [
|
||||
m for r in scripted.seen_messages for m in r if m.get("role") == "tool"
|
||||
]
|
||||
assert tool_msgs
|
||||
body = tool_msgs[-1]["content"]
|
||||
assert body.startswith(f"Document docs/big-read.md:\n{content[:cap]}")
|
||||
assert TRUNCATION_MARKER in body
|
||||
assert READ_TRUNCATION_NOTICE.format(shown=cap, total=len(content)) in body
|
||||
# The truncated read is still a SUCCESSFUL call — cited in done.
|
||||
done = frames[-1]
|
||||
assert done["type"] == "done" and done["deflected"] is False
|
||||
assert ("docs", "big-read.md") in [(s["source"], s["path"]) for s in done["sources"]]
|
||||
assert ("docs", "homelab/kubernetes.md") in [
|
||||
(s["source"], s["path"]) for s in done["sources"]
|
||||
]
|
||||
|
||||
|
||||
def test_untruncated_read_streams_no_tool_result_frame(
|
||||
client,
|
||||
db,
|
||||
seeded_kb: FakeRagLLM,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Phase 95 (task 02): the complement at the SSE level — a ``read``
|
||||
of a document AT OR UNDER the cap (the same long document, cap
|
||||
raised past its true length) streams NO ``tool_result`` frame (one
|
||||
frame = one noteworthy event; the six pre-existing event types are
|
||||
byte-identical), the ``tool`` frame is unchanged, and the model's
|
||||
context is the WHOLE document (no marker, no notice)."""
|
||||
content = "K" * 250
|
||||
doc = _insert_big_doc(db, content)
|
||||
live = get_settings()
|
||||
monkeypatch.setattr(
|
||||
chat_api,
|
||||
"get_settings",
|
||||
lambda: Settings(
|
||||
_env_file=None, # pyright: ignore[reportCallIssue]
|
||||
relevance_threshold=live.relevance_threshold,
|
||||
read_max_chars=10_000, # far over the doc's true length
|
||||
),
|
||||
)
|
||||
scripted = FakeRagLLM(
|
||||
tool_script=[
|
||||
[
|
||||
ToolCallPiece(
|
||||
id="call_1", name="read", arguments={"path": "docs/big-read.md"}
|
||||
)
|
||||
]
|
||||
]
|
||||
)
|
||||
fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: scripted
|
||||
try:
|
||||
_, _, frames = _stream_chat(client, QUESTION)
|
||||
finally:
|
||||
fastapi_app.dependency_overrides.clear()
|
||||
db.delete(doc)
|
||||
db.commit()
|
||||
|
||||
types = [f["type"] for f in frames]
|
||||
assert "error" not in types
|
||||
assert types.count("tool_result") == 0 # one frame = one noteworthy event
|
||||
assert types.count("tool") == 1
|
||||
(tool_frame,) = [f for f in frames if f["type"] == "tool"]
|
||||
assert set(tool_frame) == {"type", "name", "argument"} # byte-identical shape
|
||||
assert tool_frame["argument"] == "docs/big-read.md"
|
||||
assert frames[-1]["type"] == "done" and frames[-1]["deflected"] is False
|
||||
# The model saw the WHOLE document — no marker, no notice.
|
||||
tool_msgs = [
|
||||
m for r in scripted.seen_messages for m in r if m.get("role") == "tool"
|
||||
]
|
||||
assert tool_msgs
|
||||
assert tool_msgs[-1]["content"] == "Document docs/big-read.md:\n" + content
|
||||
assert TRUNCATION_MARKER not in tool_msgs[-1]["content"]
|
||||
|
||||
|
||||
def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
|
||||
client, db, seeded_kb: FakeRagLLM, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user