feat(agent): strip raw tool-scaffolding from streamed answers — deterministic filter with one bounded recovery

tests/integration/test_agent_tools.py

@@ -23,7 +23,7 @@ import asyncio
 import uuid
 from collections.abc import AsyncIterator, Iterator
 from copy import deepcopy
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
 
 import pytest
 from sqlalchemy import delete, text
@@ -35,6 +35,9 @@ from app.rag import agent
 from app.rag.agent import AGENT_TOOLS, AgentHolder, run_agent
 from app.rag.llm import LLMClient, RetryPiece, StreamPiece, ToolCallPiece
 
+if TYPE_CHECKING:
+    from app.rag.scaffolding import ScaffoldingFilter
+
 
 def _doc(db: Session, source: str, path: str, title: str, content: str) -> Document:
     doc = Document(
@@ -168,6 +171,7 @@ class ScriptedToolLLM:
         self,
         messages: list[dict[str, str]],
         tools: list[dict[str, Any]] | None = None,
+        scaffolding: ScaffoldingFilter | None = None,  # phase 71 pass-through
     ) -> AsyncIterator[StreamPiece | ToolCallPiece]:
         self.requests.append((deepcopy(messages), deepcopy(tools)))
         if len(self.requests) == 1:

tests/integration/test_chat_api.py

@@ -18,7 +18,7 @@ import math
 import re
 from collections.abc import Iterator
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any, cast
 
 import pytest
 from fastapi.testclient import TestClient
@@ -33,6 +33,9 @@ from app.rag.agent import AGENT_TOOLS
 from app.rag.importer import import_sources
 from app.rag.llm import EmbeddingError, LLMError, StreamPiece, ToolCallPiece
 
+if TYPE_CHECKING:
+    from app.rag.scaffolding import ScaffoldingFilter
+
 FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "docs"
 QUESTION = "How is my Kubernetes cluster set up?"
 OFF_TOPIC = "How do I bake sourdough bread?"
@@ -63,6 +66,7 @@ class FakeRagLLM:
         tool_script: list[list[StreamPiece | ToolCallPiece]] | None = None,
         embed_fail_count: int = 0,
         stream_fail_count: int = 0,
+        answer_sequence: list[str] | None = None,
     ) -> None:
         self.settings = Settings(_env_file=None)  # pyright: ignore[reportCallIssue]
         self.embed_batches = 0
@@ -71,6 +75,11 @@ class FakeRagLLM:
         self.embed_error = embed_error
         self.stream_error = stream_error
         self.fail_mid_stream = fail_mid_stream
+        #: Phase 71: per-request canned answers (the recovery matrix):
+        #: request *i* (0-based, in ``seen_messages`` order) yields
+        #: ``answer_sequence[i]``; once exhausted it falls back to
+        #: ``answer``. ``None`` keeps the single-``answer`` behavior.
+        self.answer_sequence = answer_sequence
         #: Phase 67: the first N ``embed_one`` calls raise an
         #: ``EmbeddingError`` (then succeed) — a dead-then-recovered
         #: embeddings endpoint for the retry loop.
@@ -117,17 +126,35 @@ class FakeRagLLM:
         self.question_embeds.append(text)
         return _token_vec(text)
 
+    def _answer_for_request(self) -> str:
+        """The canned answer for the request that was just recorded
+        (phase 71 ``answer_sequence``; ``None`` → the single answer)."""
+        if self.answer_sequence is None:
+            return self.answer
+        index = len(self.seen_messages) - 1
+        if index < len(self.answer_sequence):
+            return self.answer_sequence[index]
+        return self.answer
+
     async def chat_stream(
         self,
         messages: list[dict[str, str]],
         tools: list[dict[str, Any]] | None = None,
+        scaffolding: ScaffoldingFilter | None = None,
     ):
         """Typed stream (phase 17): ``thinking`` slices (same 12-char
         cadence as content) **before** the content pieces. With the
         default ``thinking=""`` this yields content-only pieces — today's
         behavior, new yield type. Phase 37: *tools* is the agent loop's
         ``tools=…`` passthrough (recorded in ``seen_tools``); a request
-        with tools consumes the next ``tool_script`` entry, if any."""
+        with tools consumes the next ``tool_script`` entry, if any.
+        Phase 71: *scaffolding* mirrors ``LLMClient.chat_stream`` — the
+        canned content pieces are fed through the caller's filter (an
+        empty clean result yields nothing) and the held tail is flushed
+        on normal completion, so a scaffolding-only canned answer streams
+        zero content pieces and leaves ``stripped_chars`` behind for the
+        recovery policy to key on. ``None`` (e.g. pre-phase callers) keeps
+        the byte-identical raw path."""
         self.seen_messages.append(messages)
         self.seen_tools.append(tools)
         if self.stream_error is not None:
@@ -135,17 +162,46 @@ class FakeRagLLM:
         if self.stream_fail_count > 0:
             self.stream_fail_count -= 1
             raise LLMError("simulated pre-piece endpoint failure")
+        mid_stream_drop = False
+        raw: list[StreamPiece | ToolCallPiece]
         if tools is not None and self.tool_script:
-            for piece in self.tool_script.pop(0):
+            raw = self.tool_script.pop(0)
+        elif self.fail_mid_stream:
+            raw = [StreamPiece("content", "partial ")]
+            mid_stream_drop = True
+        else:
+            answer = self._answer_for_request()
+            raw = cast(
+                "list[StreamPiece | ToolCallPiece]",
+                [
+                    StreamPiece("thinking", self.thinking[i : i + 12])
+                    for i in range(0, len(self.thinking), 12)
+                ]
+                + [
+                    StreamPiece("content", answer[i : i + 12])
+                    for i in range(0, len(answer), 12)
+                ],
+            )
+        if scaffolding is None:
+            for piece in raw:
                 yield piece
+            if mid_stream_drop:
+                raise LLMError("mid-stream dropout")
             return
-        if self.fail_mid_stream:
-            yield StreamPiece("content", "partial ")
+        for piece in raw:
+            if isinstance(piece, StreamPiece) and piece.kind == "content":
+                cleaned = scaffolding.feed(piece.text)
+                if cleaned:
+                    yield StreamPiece("content", cleaned)
+            else:
+                yield piece
+        if mid_stream_drop:
+            # The tail is NOT flushed on a failed stream — the real
+            # client only flushes a cleanly completed one.
             raise LLMError("mid-stream dropout")
-        for i in range(0, len(self.thinking), 12):
-            yield StreamPiece("thinking", self.thinking[i : i + 12])
-        for i in range(0, len(self.answer), 12):
-            yield StreamPiece("content", self.answer[i : i + 12])
+        tail = scaffolding.flush()
+        if tail:
+            yield StreamPiece("content", tail)
 
 
 @pytest.fixture()
@@ -621,6 +677,7 @@ def test_grounded_turn_streams_tool_frames_and_cites_read_doc(
     assert lines and "tool_calls=2" in lines[-1]
     assert "'docs/homelab/kubernetes.md'" in lines[-1]
     assert "'docs/homelab/backups.md'" in lines[-1]
+    assert "scaffold_stripped=0" in lines[-1]  # phase 71: uniform clean-turn field
 
 
 def test_grounded_turn_streams_grep_tool_frames(
@@ -853,6 +910,7 @@ def test_zero_max_rounds_reproduce_pre_phase_single_request(
     assert "backups.md" not in row.sources
     lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
     assert lines and "tool_calls=0" in lines[-1]
+    assert "scaffold_stripped=0" in lines[-1]  # phase 71: uniform clean-turn field
 
 
 def test_tool_execution_db_failure_yields_error_event(
@@ -929,6 +987,7 @@ def test_embed_failure_retries_then_turn_completes(
     assert frames[-1]["type"] == "done"
     lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
     assert lines and "retries=1" in lines[-1]
+    assert "scaffold_stripped=0" in lines[-1]  # phase 71: uniform clean-turn field
 
 
 def test_embed_failure_exhausts_retries_then_terminal_error(
@@ -988,6 +1047,7 @@ def test_deflected_stream_retries_before_the_first_piece(
     assert flaky.seen_tools == [None, None]  # …byte-identical (no tools key)
     lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
     assert lines and "retries=1" in lines[-1]
+    assert "scaffold_stripped=0" in lines[-1]  # phase 71: uniform clean-turn field
 
 
 def test_deflected_stream_failure_after_first_frame_is_terminal(
@@ -1034,3 +1094,135 @@ def test_zero_retries_keep_the_pre_phase_wire_shape(
     assert frames[0]["type"] == "error"
     assert "embedding" in frames[0]["detail"]
     assert not any(f["type"] == "retry" for f in frames)
+
+
+# ---------- phase 71: the deterministic scaffolding guardrail (deflected path) ----------
+
+
+def _scaffold_span() -> str:
+    """The raw span from the 2026-09-03 incident (the E2E mock's trigger,
+    task 05) — a complete span the filter strips in full."""
+    return "<|tool_call_start|>[read(path='/homelab/backup-notes.md')]<|tool_call_end|>"
+
+
+def test_deflected_scaffolding_only_reply_recovers_once(
+    client,
+    db,
+    seeded_kb: FakeRagLLM,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """(a) A deflected reply that is pure scaffolding streams ZERO delta
+    frames (no raw tokens on the wire); the one bounded recovery —
+    ``tools=None``, the correction folded into the single system prompt,
+    a fresh filter, the same retry budget — streams the clean answer, the
+    turn settles with ``done`` + a query_log row, and the log line counts
+    the stripped chars (the recovery does not bump ``retries=N``)."""
+    span = _scaffold_span()
+    clean = "I don't have that on hand — try one of the chips below?"
+    flaky = FakeRagLLM(answer_sequence=[span, clean])
+    live = get_settings()
+    monkeypatch.setattr(chat_api, "get_settings", lambda: _retry_settings(live))
+    fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: flaky
+    try:
+        caplog.set_level(logging.INFO, logger="app.chat")
+        _, _, frames = _stream_chat(client, OFF_TOPIC)
+    finally:
+        fastapi_app.dependency_overrides.clear()
+
+    # The raw tokens never reach the wire; the deltas reassemble to the
+    # clean recovery answer.
+    assert span not in json.dumps(frames)
+    deltas = [f for f in frames if f["type"] == "delta"]
+    assert "".join(d["text"] for d in deltas) == clean
+    assert not any(f["type"] == "error" for f in frames)
+    done = frames[-1]
+    assert done["type"] == "done" and done["deflected"] is True
+
+    # Exactly two requests: the stripped round + the one recovery, both
+    # without a tools key…
+    assert len(flaky.seen_messages) == 2
+    assert flaky.seen_tools == [None, None]
+    # …and the recovery's system prompt is the ORIGINAL deflected prompt
+    # with the correction folded in (a single system message — the user
+    # message stays last).
+    recovered = flaky.seen_messages[1]
+    assert len(recovered) == 2
+    assert recovered[1] == {"role": "user", "content": OFF_TOPIC}
+    first_system = flaky.seen_messages[0][0]["content"]
+    assert "DEFLECT_MODE" in first_system
+    assert recovered[0] == {
+        "role": "system",
+        "content": first_system + "\n" + agent.CORRECTION_INSTRUCTION,
+    }
+
+    # The turn settled normally: one query_log row…
+    (row,) = db.scalars(select(QueryLog)).all()
+    assert row.deflected is True
+    # …and the log line carries the summed stripped count (the clean
+    # recovery stripped nothing) with retries untouched.
+    lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
+    assert lines and f"scaffold_stripped={len(span)}" in lines[-1]
+    assert "retries=0" in lines[-1]  # the recovery is not an endpoint-retry
+
+
+def test_deflected_scaffolding_twice_settles_malformed(
+    client, db, seeded_kb: FakeRagLLM, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """(b) The recovery answer is scaffolding again — a second empty
+    reply is terminal: the DEDICATED error frame (the exact copy), no
+    ``done``, no query_log row — byte-for-byte today's ``LLMError``
+    terminal shape — and no third request (at most one recovery per
+    turn)."""
+    span = _scaffold_span()
+    dead = FakeRagLLM(answer_sequence=[span, span])
+    live = get_settings()
+    monkeypatch.setattr(chat_api, "get_settings", lambda: _retry_settings(live))
+    fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: dead
+    try:
+        _, _, frames = _stream_chat(client, OFF_TOPIC)
+    finally:
+        fastapi_app.dependency_overrides.clear()
+
+    assert [f["type"] for f in frames] == ["error"]
+    assert frames[0]["detail"] == (
+        "The model returned a malformed reply — please try again."
+    )
+    assert set(frames[0].keys()) == {"type", "detail"}  # the contract shape
+    assert span not in json.dumps(frames)
+    assert not any(f["type"] == "done" for f in frames)
+    assert db.scalars(select(QueryLog)).all() == []
+    assert len(dead.seen_messages) == 2  # round + one recovery — no more
+    assert dead.seen_tools == [None, None]
+
+
+def test_deflected_mixed_scaffolding_and_content_needs_no_recovery(
+    client,
+    db,
+    seeded_kb: FakeRagLLM,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """(c) Real visible content plus scaffolding: the clean remainder
+    streams (no raw tokens on the wire), NO recovery runs, and the log
+    line counts the stripped span (``scaffold_stripped>0``)."""
+    span = _scaffold_span()
+    mixed = f"I don't have that. {span} Try the chips below?"
+    flaky = FakeRagLLM(answer=mixed)
+    live = get_settings()
+    monkeypatch.setattr(chat_api, "get_settings", lambda: _retry_settings(live))
+    fastapi_app.dependency_overrides[chat_api.get_llm] = lambda: flaky
+    try:
+        caplog.set_level(logging.INFO, logger="app.chat")
+        _, _, frames = _stream_chat(client, OFF_TOPIC)
+    finally:
+        fastapi_app.dependency_overrides.clear()
+
+    assert span not in json.dumps(frames)
+    deltas = [f for f in frames if f["type"] == "delta"]
+    assert "".join(d["text"] for d in deltas) == "I don't have that.  Try the chips below?"
+    assert not any(f["type"] == "error" for f in frames)
+    assert frames[-1]["type"] == "done"
+    assert len(flaky.seen_messages) == 1  # the clean content stands — no recovery
+    lines = [r.getMessage() for r in caplog.records if "question=" in r.getMessage()]
+    assert lines and f"scaffold_stripped={len(span)}" in lines[-1]