feat(agent): strip raw tool-scaffolding from streamed answers — deterministic filter with one bounded recovery

app/rag/scaffolding.py

@@ -0,0 +1,170 @@
+"""Deterministic tool-scaffolding guardrail — pattern registry + streaming
+filter (phase 71, task 01).
+
+The ``lite`` chat model occasionally emits its own chat-template
+tool-scaffolding as plain answer text (incident 2026-09-03: a deflected
+round streamed ``<|tool_call_start|>[read(path='…')]<|tool_call_end|>``
+into the UI although no tools were offered). This module is the
+detection half of the guardrail: a fixed registry of *observed*
+scaffolding forms and a streaming state machine that strips them from
+``delta.content`` as it flows. Pure Python — no I/O, no logging, no
+model: the strip *warning* log and the one bounded recovery are emitted
+by the integration layer (phase 71, tasks 02–03), so this module stays
+unit-testable in isolation.
+
+Content only: thinking pieces are the model's raw reasoning by design
+(phase 17) and are never filtered — the guardrail protects the answer,
+not the scratchpad.
+"""
+import re
+
+__all__ = ["SCAFFOLD_PATTERNS", "ScaffoldingFilter"]
+
+#: The known scaffolding forms. The registry is the extension point: a
+#: new entry needs an observed capture (the strip warning log, task 03)
+#: + a unit fixture in ``tests/unit/test_scaffolding_filter.py`` — no
+#: speculative entries. Every entry here traces to the 2026-09-03
+#: incident (the span form is the deflected round's raw text; the
+#: standalone siblings are from the same tokenizer family).
+SCAFFOLD_PATTERNS: tuple[re.Pattern, ...] = (
+    # The observed span (incident 2026-09-03) — non-greedy, so multiple
+    # spans in one buffer each strip to their own end token.
+    re.compile(r"<\|tool_call_start\|>[\s\S]*?<\|tool_call_end\|>"),
+    # Standalone sibling token (same tokenizer family).
+    re.compile(r"<\|tool_calls\|>"),
+    # Standalone sibling token (same tokenizer family).
+    re.compile(r"<\|tool_call\|>"),
+)
+
+_SPAN_START = "<|tool_call_start|>"
+_SPAN_END = "<|tool_call_end|>"
+#: The literal openings a live (possibly incomplete) match can begin with.
+_OPENINGS: tuple[str, ...] = (_SPAN_START, "<|tool_calls|>", "<|tool_call|>")
+#: Boundedness bound: in NORMAL state the held tail is at most one char
+#: short of the longest opening (an open span is unbounded, but it is
+#: being *stripped*, never emitted).
+_MAX_OPENING = max(len(opening) for opening in _OPENINGS)
+
+
+def _leftmost_match(buf: str) -> re.Match | None:
+    """The leftmost complete match among :data:`SCAFFOLD_PATTERNS`, or
+    None when the buffer holds no complete scaffolding form.
+
+    (Two patterns cannot match at the same start position — their
+    literals diverge after ``<|tool_call`` — so the leftmost start is
+    unambiguous.)
+    """
+    best: re.Match | None = None
+    for pattern in SCAFFOLD_PATTERNS:
+        match = pattern.search(buf)
+        if match is not None and (best is None or match.start() < best.start()):
+            best = match
+    return best
+
+
+def _hold_index(buf: str) -> int:
+    """Index where the live hold-tail begins; everything from it stays
+    pending (called only after the strip loop found no complete match).
+
+    The tail is the leftmost of two candidates: an **open span** (a
+    start token with no end token after it — the span may continue in
+    future chunks, so everything from that start token onward is held)
+    or a **live prefix** (the longest suffix that is a proper prefix of
+    any opening literal — the token may complete in future chunks).
+    Everything before the leftmost candidate is safe to emit: no
+    complete match remains, and no match can grow from earlier text,
+    since a complete opening earlier in the buffer would already have
+    matched (span) or be a candidate in its own right (standalone).
+    """
+    if not buf:
+        return 0
+    hold = len(buf)
+    start = buf.find(_SPAN_START)
+    if start != -1 and buf.find(_SPAN_END, start + 1) == -1:
+        hold = start
+    for k in range(min(len(buf), _MAX_OPENING - 1), 0, -1):
+        suffix = buf[-k:]
+        if any(k < len(opening) and opening.startswith(suffix) for opening in _OPENINGS):
+            hold = min(hold, len(buf) - k)
+            break
+    return hold
+
+
+class ScaffoldingFilter:
+    """Streaming state machine over a model **content** stream (phase 71).
+
+    One instance per model request (callers create fresh instances —
+    the phase-67 retry-after-dead-attempt case is safe: a dead attempt
+    never fed the filter). Feed the raw ``delta.content`` chunks; each
+    :meth:`feed` returns the clean text safe to emit *now*; :meth:`flush`
+    emits the tail at stream end.
+
+    ``feed`` appends the chunk to the internal pending buffer, then
+    (a) repeatedly takes the leftmost complete match among
+    :data:`SCAFFOLD_PATTERNS` — drops it (counting it into
+    :attr:`stripped_chars`) and continues — until none remains; then
+    (b) checks the buffer tail for a **live prefix** via
+    :func:`_hold_index`: the longest suffix that is a proper prefix of
+    any opening literal, or an **open span** (a start token with no end
+    token yet — everything from that start token onward is held, since
+    the span may continue in future chunks). Everything before the held
+    tail is emitted; the held tail becomes the new pending state.
+    Bounded: the held tail in NORMAL state is ≤ the longest opening
+    token minus one char; in an open span it is unbounded but is being
+    *stripped*, never emitted.
+
+    ``flush`` (end of stream) emits the pending tail **as-is**: a
+    partial marker at EOF is content, not scaffolding — a documented,
+    pinned choice (a stream that ends mid-``<|tool_call_st`` must not
+    be silently eaten, and a lone ``<|tool_call_end|>`` without a start
+    is prose the user sent or the model produced outside a span).
+
+    The stripped spans themselves are exposed read-only
+    (:attr:`stripped_spans`) so the integration layer can log one
+    warning per strip event (phase 71 task 03) — that log line is how
+    a *new* scaffolding format gets captured and added to the registry.
+    """
+
+    def __init__(self) -> None:
+        self._pending = ""
+        self._stripped_chars = 0
+        self._stripped_spans: list[str] = []
+
+    @property
+    def stripped_chars(self) -> int:
+        """Total characters removed so far (read by the caller after the
+        round/turn — the strip warning log, phase 71 task 03)."""
+        return self._stripped_chars
+
+    @property
+    def stripped_spans(self) -> list[str]:
+        """The raw spans removed so far, in stream order (one entry per
+        strip event — the integration layer's per-span warning log
+        truncates each to 200 chars). Read-only: callers must not
+        mutate the filter's state."""
+        return list(self._stripped_spans)
+
+    def feed(self, chunk: str) -> str:
+        """Append *chunk* to the pending buffer; return the clean text
+        safe to emit now (possibly ``""`` — e.g. while an open span or a
+        partial marker is still held). An empty chunk is a no-op."""
+        if not chunk:
+            return ""
+        buf = self._strip_complete(self._pending + chunk)
+        hold = _hold_index(buf)
+        self._pending = buf[hold:]
+        return buf[:hold]
+
+    def flush(self) -> str:
+        """End of stream: emit the pending tail as-is and reset it."""
+        tail = self._pending
+        self._pending = ""
+        return tail
+
+    def _strip_complete(self, buf: str) -> str:
+        """Drop leftmost-complete matches until none remains (step a)."""
+        while (match := _leftmost_match(buf)) is not None:
+            self._stripped_chars += match.end() - match.start()
+            self._stripped_spans.append(match.group(0))
+            buf = buf[: match.start()] + buf[match.end():]
+        return buf