feat(agent): strip raw tool-scaffolding from streamed answers — deterministic filter with one bounded recovery
This commit is contained in:
+139
-5
@@ -91,6 +91,30 @@ the same budget, and the deflected answer stream goes through
|
||||
The per-turn log line records ``retries=N`` after ``total_ms=N`` (0
|
||||
when nothing was retried — the field is uniform across all turn
|
||||
shapes).
|
||||
|
||||
Tool-scaffolding guardrail (phase 71, deterministic only — owner
|
||||
permission 2026-09-03: "deterministic guardrails only right now, forget
|
||||
using a model for that"): the raw chat-template tokens
|
||||
``<|tool_call_start|>…<|tool_call_end|>`` the ``lite`` model sometimes
|
||||
emits as plain answer text can never reach the user. The DEFLECTED
|
||||
path's request runs ``delta.content`` through a caller-owned
|
||||
``ScaffoldingFilter`` (content only — thinking stays raw); when the
|
||||
filter wipes the whole reply (visible content 0, ``stripped_chars > 0``)
|
||||
the turn gets exactly ONE bounded recovery: the same messages with
|
||||
``agent.CORRECTION_INSTRUCTION`` folded into the single system prompt,
|
||||
``tools=None``, a fresh filter, the same phase-67 retry budget, streamed
|
||||
through the same piece loop (extracted as the inner ``_pump`` helper).
|
||||
A recovery that also comes back empty — and any grounded round where the
|
||||
recovery policy in ``run_agent`` fails — settles with the dedicated
|
||||
structured ``error`` frame (``MalformedReplyError`` caught before the
|
||||
generic ``LLMError`` handler): no ``done``, no ``query_log`` row,
|
||||
byte-for-byte the existing terminal-error shape. The per-turn log line
|
||||
records ``scaffold_stripped=N`` after ``retries=N`` — the sum across the
|
||||
turn's requests (grounded: the agent's rounds + forced final + any
|
||||
recovery, via the holder; deflected: this turn's filters), 0 on clean
|
||||
turns (the field is uniform, the phase-67 ``retries=N`` pattern); the
|
||||
recovery does not bump ``retries=N`` (it is not a phase-67
|
||||
endpoint-retry).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -110,7 +134,12 @@ from app.api.steering import load_steering_notes
|
||||
from app.config import Settings, get_settings
|
||||
from app.db import db_available, get_db
|
||||
from app.models import Document, QueryLog
|
||||
from app.rag.agent import AgentHolder, run_agent
|
||||
from app.rag.agent import (
|
||||
CORRECTION_INSTRUCTION, # phase 71: the harness-owned recovery line
|
||||
AgentHolder,
|
||||
MalformedReplyError, # phase 71: the recovery policy's terminal signal
|
||||
run_agent,
|
||||
)
|
||||
from app.rag.llm import (
|
||||
EmbeddingError,
|
||||
LLMClient,
|
||||
@@ -123,6 +152,7 @@ from app.rag.llm import (
|
||||
from app.rag.overview import load_kb_overview
|
||||
from app.rag.prompts import build_deflect_prompt, build_high_prompt
|
||||
from app.rag.retriever import RetrievedChunk, retrieve, select_documents, weak_hit_titles
|
||||
from app.rag.scaffolding import ScaffoldingFilter # phase 71: the streaming filter
|
||||
from app.rag.suggestions import derive_suggestions
|
||||
from app.schemas import (
|
||||
ChatDoneEvent,
|
||||
@@ -373,19 +403,27 @@ async def chat(
|
||||
# ``tools=None`` request anyway (the kill switch).
|
||||
holder = AgentHolder()
|
||||
answer_stream: AsyncIterator[StreamPiece | ToolCallPiece | RetryPiece]
|
||||
deflected_filter: ScaffoldingFilter | None = None
|
||||
if plan.deflected:
|
||||
# Phase 67: the deflected stream goes through the retry
|
||||
# primitive — a dead endpoint is restarted (SSE ``retry``
|
||||
# frames) only before its first piece (locked A2); the
|
||||
# grounded path stays a plain ``run_agent`` call (task 03
|
||||
# makes IT retry internally) — its ``RetryPiece``s flow
|
||||
# through the shared piece loop below.
|
||||
# through the shared piece loop below. Phase 71: the
|
||||
# request's content also runs through a caller-owned
|
||||
# filter (one per request) — a scaffolding-only reply
|
||||
# streams zero ``delta`` frames instead of raw tokens, and
|
||||
# the filter's ``stripped_chars`` drives the recovery
|
||||
# decision after the piece loop.
|
||||
deflected_filter = ScaffoldingFilter()
|
||||
answer_stream = chat_stream_retried(
|
||||
llm,
|
||||
messages,
|
||||
tools=None,
|
||||
retries=settings.llm_retries,
|
||||
delay=settings.llm_retry_delay,
|
||||
scaffolding=deflected_filter,
|
||||
)
|
||||
else:
|
||||
answer_stream = run_agent(
|
||||
@@ -398,8 +436,19 @@ async def chat(
|
||||
holder=holder,
|
||||
)
|
||||
thinking_chars = 0
|
||||
try:
|
||||
async for piece in answer_stream: # StreamPiece | ToolCallPiece | RetryPiece
|
||||
content_chars = 0 # phase 71: the turn's visible (clean) content
|
||||
scaffold_stripped = 0 # phase 71: sum across the turn's requests
|
||||
|
||||
async def _pump(
|
||||
pieces: AsyncIterator[StreamPiece | ToolCallPiece | RetryPiece],
|
||||
) -> AsyncIterator[str]:
|
||||
"""One request's piece loop (phase 71 extraction): the
|
||||
thinking/tool/retry/delta handling shared by the turn's
|
||||
first pass and — deflected path only — the one bounded
|
||||
recovery. Behavior-preserving for the first pass (pinned
|
||||
by the existing integration suite)."""
|
||||
nonlocal thinking_chars, content_chars, retries_used
|
||||
async for piece in pieces: # StreamPiece | ToolCallPiece | RetryPiece
|
||||
if isinstance(piece, ToolCallPiece):
|
||||
# Phase 37 (PLAN §4 extension; phase 70): one SSE
|
||||
# ``tool`` frame per model-requested call.
|
||||
@@ -435,7 +484,91 @@ async def chat(
|
||||
if settings.stream_thinking:
|
||||
yield sse_event(ChatThinkingEvent(text=piece.text).model_dump())
|
||||
else:
|
||||
content_chars += len(piece.text)
|
||||
yield sse_event({"type": "delta", "text": piece.text})
|
||||
|
||||
try:
|
||||
async for frame in _pump(answer_stream):
|
||||
yield frame
|
||||
if plan.deflected and deflected_filter is not None:
|
||||
scaffold_stripped = deflected_filter.stripped_chars
|
||||
# Phase 71: the deflected reply's visible content was
|
||||
# wiped by the filter (the scaffolding was the whole
|
||||
# "answer") — the ONE bounded recovery: the same
|
||||
# messages with the correction folded into the single
|
||||
# system prompt, ``tools=None``, a FRESH filter, the
|
||||
# same phase-67 retry budget, streamed through the
|
||||
# same piece loop. A round with real visible content
|
||||
# needs no recovery (the clean content stands).
|
||||
if content_chars == 0 and deflected_filter.stripped_chars > 0:
|
||||
logger.warning(
|
||||
"chat: deflected reply was pure tool-scaffolding "
|
||||
"(%d chars stripped) — running the one bounded "
|
||||
"recovery",
|
||||
deflected_filter.stripped_chars,
|
||||
)
|
||||
recovery_filter = ScaffoldingFilter()
|
||||
recovery_stream = chat_stream_retried(
|
||||
llm,
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
plan.system_prompt + "\n"
|
||||
+ CORRECTION_INSTRUCTION
|
||||
),
|
||||
},
|
||||
*messages[1:],
|
||||
],
|
||||
tools=None,
|
||||
retries=settings.llm_retries,
|
||||
delay=settings.llm_retry_delay,
|
||||
scaffolding=recovery_filter,
|
||||
)
|
||||
async for frame in _pump(recovery_stream):
|
||||
yield frame
|
||||
scaffold_stripped += recovery_filter.stripped_chars
|
||||
if content_chars == 0:
|
||||
# The second empty reply is terminal (at most
|
||||
# one recovery per turn) — the dedicated error
|
||||
# frame below (no done, no query_log row).
|
||||
logger.warning(
|
||||
"chat: the recovery reply was still empty "
|
||||
"(scaffold_stripped=%d) — settling with a "
|
||||
"malformed-reply error",
|
||||
scaffold_stripped,
|
||||
)
|
||||
raise MalformedReplyError(
|
||||
"the deflected model answered in raw "
|
||||
"tool-scaffolding twice in a row — no "
|
||||
"clean answer to stream"
|
||||
)
|
||||
else:
|
||||
# Grounded turns: the agent's rounds + forced final +
|
||||
# any recovery already accumulated the turn total on
|
||||
# the holder (the deflected fallback is 0 — the agent
|
||||
# never runs, so this branch is grounded-only).
|
||||
scaffold_stripped = holder.scaffold_stripped
|
||||
except MalformedReplyError as e:
|
||||
# Phase 71: the recovery policy's terminal signal —
|
||||
# caught BEFORE the generic LLMError handler (it
|
||||
# subclasses it), so the dedicated copy reaches the UI;
|
||||
# the generic "dropped the connection" copy stays for
|
||||
# transport failures.
|
||||
logger.error(
|
||||
"chat: malformed reply after the one bounded recovery "
|
||||
"question=%r total_ms=%d — %s",
|
||||
request.message,
|
||||
int((time.monotonic() - started) * 1000),
|
||||
e,
|
||||
)
|
||||
settled = True # terminal: the error frame settles the turn
|
||||
yield sse_event(
|
||||
ChatErrorEvent(
|
||||
detail="The model returned a malformed reply — please try again."
|
||||
).model_dump()
|
||||
)
|
||||
return
|
||||
except LLMError as e:
|
||||
logger.error(
|
||||
"chat: LLM stream failed question=%r total_ms=%d — %s",
|
||||
@@ -504,7 +637,7 @@ async def chat(
|
||||
logger.info(
|
||||
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d summary_hits=%d tuning=%d "
|
||||
"kb_chars=%d threshold=%.2f deflected=%s sources=%r thinking_chars=%d "
|
||||
"tool_calls=%d total_ms=%d retries=%d",
|
||||
"tool_calls=%d total_ms=%d retries=%d scaffold_stripped=%d",
|
||||
request.message,
|
||||
embed_ms,
|
||||
plan.top_score,
|
||||
@@ -519,6 +652,7 @@ async def chat(
|
||||
holder.tool_calls,
|
||||
total_ms,
|
||||
retries_used,
|
||||
scaffold_stripped,
|
||||
)
|
||||
settled = True # terminal: the done frame settles the turn
|
||||
yield sse_event(
|
||||
|
||||
+192
-2
@@ -95,6 +95,27 @@ task 04):
|
||||
exactly one round. With ``settings.llm_retries=0`` every request is a
|
||||
single plain attempt (the pre-phase-67 path).
|
||||
|
||||
Scaffolding guardrail (phase 71, deterministic only — owner permission
|
||||
2026-09-03: "deterministic guardrails only right now, forget using a
|
||||
model for that"): every model request (each round, the forced final,
|
||||
and any recovery) runs its ``delta.content`` through a fresh caller-
|
||||
owned :class:`app.rag.scaffolding.ScaffoldingFilter`, so raw
|
||||
``<|tool_call_start|>…<|tool_call_end|>`` tokens can never reach the
|
||||
user as answer text. A round that ends with NO visible content AND a
|
||||
non-empty strip (the scaffolding was the whole "answer") gets exactly
|
||||
ONE bounded recovery: one extra request with ``tools=None``, the same
|
||||
messages with :data:`CORRECTION_INSTRUCTION` folded into the original
|
||||
single system message, a fresh filter, and the same phase-67 retry
|
||||
budget. A recovery that also comes back empty — or a round with no
|
||||
strip and no content (today's empty/thinking-only answer) — settles as
|
||||
before; a second empty reply raises :class:`MalformedReplyError` (the
|
||||
API layer turns it into the dedicated error frame). A round with real
|
||||
visible content plus scaffolding needs no recovery (the clean content
|
||||
stands), and a scaffolding-only round that also carried tool calls
|
||||
needs none either (the tool ran) — the policy keys on the no-calls
|
||||
exit only. No model participates in detection or repair: the
|
||||
registry + the fixed retry policy are the whole guardrail.
|
||||
|
||||
The DB accessors (:func:`list_catalog`, :func:`list_source_names`,
|
||||
:func:`find_document`, :func:`all_documents`) and the
|
||||
:func:`grep_document` line matcher are module-level functions so unit
|
||||
@@ -116,11 +137,13 @@ from app.models import Document
|
||||
from app.rag.git_sources import effective_sources
|
||||
from app.rag.llm import (
|
||||
LLMClient,
|
||||
LLMError,
|
||||
RetryPiece,
|
||||
StreamPiece,
|
||||
ToolCallPiece,
|
||||
chat_stream_retried,
|
||||
)
|
||||
from app.rag.scaffolding import ScaffoldingFilter
|
||||
from app.rag.source_removal import resolve_source_name
|
||||
|
||||
logger = logging.getLogger("app.agent")
|
||||
@@ -228,6 +251,33 @@ UNKNOWN_TOOL = "Unknown tool."
|
||||
MISSING_READ_ARGS = "read requires a string argument 'path'."
|
||||
MISSING_SEARCH_ARGS = "grep requires a string argument 'pattern'."
|
||||
|
||||
#: The harness-owned recovery line (phase 71, task 03) — folded into the
|
||||
#: ORIGINAL single system message of the one bounded recovery request
|
||||
#: (``system_prompt + "\n" + CORRECTION_INSTRUCTION``; provider-safe,
|
||||
#: the user message stays last). Verbatim constant: the E2E mock
|
||||
#: (task 05) keys on a stable substring of it, so it must not drift.
|
||||
CORRECTION_INSTRUCTION: str = (
|
||||
"Your previous reply contained raw tool-call markup, which is not "
|
||||
"interpreted here. Answer the user's question directly in plain "
|
||||
"text — no tool syntax."
|
||||
)
|
||||
|
||||
|
||||
class MalformedReplyError(LLMError):
|
||||
"""The model kept replying in raw tool-scaffolding (phase 71).
|
||||
|
||||
Raised ONLY by the recovery policy — :func:`run_agent` (grounded
|
||||
path) and ``app.api.chat`` (deflected path) — when the one bounded
|
||||
``tools=None`` recovery still comes back with no visible content.
|
||||
It is never raised from inside a stream, so
|
||||
:func:`app.rag.llm.chat_stream_retried`'s retry-before-first-piece
|
||||
rule never sees it. The API layer catches it BEFORE the generic
|
||||
:class:`LLMError` handler and settles the turn with the dedicated
|
||||
"malformed reply" error frame (no ``done``, no ``query_log`` row).
|
||||
Deterministic only (owner permission 2026-09-03): no model
|
||||
participates in detection or repair.
|
||||
"""
|
||||
|
||||
#: Search caps (owner-locked A5, phase 68): a global per-call match cap
|
||||
#: (across documents, in catalog order) and a per-match-line char limit.
|
||||
SEARCH_MAX_MATCHES = 20
|
||||
@@ -352,10 +402,16 @@ class AgentHolder:
|
||||
rejected calls (unknown tool, unknown/missing arguments or document,
|
||||
already-in-context) do not count. Drives the per-turn log line's
|
||||
``tool_calls=N`` field (task 04).
|
||||
``scaffold_stripped``: how many chars of tool-scaffolding the
|
||||
turn's filters removed across the turn's requests (rounds + the
|
||||
forced final + any recovery, phase 71) — drives the per-turn log
|
||||
line's ``scaffold_stripped=N`` field on grounded turns (the
|
||||
deflected path computes its own total in ``app.api.chat``).
|
||||
"""
|
||||
|
||||
read_docs: list[Document] = field(default_factory=list)
|
||||
tool_calls: int = 0
|
||||
scaffold_stripped: int = 0
|
||||
|
||||
|
||||
def _execute_tool(
|
||||
@@ -476,6 +532,24 @@ async def run_agent(
|
||||
``settings.llm_retry_delay``); a round that already streamed pieces
|
||||
fails the turn as before.
|
||||
|
||||
Scaffolding recovery (phase 71, deterministic only): every request —
|
||||
each round, the forced final, and any recovery — runs its content
|
||||
through a fresh :class:`app.rag.scaffolding.ScaffoldingFilter`. A
|
||||
round that ends with NO visible content but a non-empty strip (the
|
||||
scaffolding was the whole "answer") gets exactly ONE recovery:
|
||||
``tools=None``, :data:`CORRECTION_INSTRUCTION` folded into the
|
||||
original single system message (the rest of the history — user
|
||||
message and tool results — unchanged), a fresh filter, the same
|
||||
retry budget. A clean recovery ends the turn; a second empty reply
|
||||
raises :class:`MalformedReplyError` (terminal — the API layer turns
|
||||
it into the dedicated error frame). A round with visible content
|
||||
plus scaffolding needs no recovery (the clean content stands), and
|
||||
a scaffolding-only round that also carried tool calls needs none
|
||||
(the tool ran) — the policy keys on the no-calls exit only. The
|
||||
per-span strip warning log (each span truncated to 200 chars) is
|
||||
the capture mechanism for new registry entries; *holder* accumulates
|
||||
the turn's ``scaffold_stripped`` total for the API layer's log line.
|
||||
|
||||
``seed_docs`` are the documents the retrieval already put in context
|
||||
(they shape the *system_prompt* the caller built); re-reading one of
|
||||
them is rejected as "Already in your context." — the rejection counts
|
||||
@@ -494,6 +568,12 @@ async def run_agent(
|
||||
rounds = 0
|
||||
while True:
|
||||
calls: list[ToolCallPiece] = []
|
||||
# Phase 71: one fresh filter per round (one per model request —
|
||||
# the retry attempts of this logical request share it: a restart
|
||||
# only happens while the filter was never fed). Content-only:
|
||||
# thinking pieces pass through raw.
|
||||
round_filter = ScaffoldingFilter()
|
||||
round_content = 0 # visible (clean) content chars this round
|
||||
# Phase 48: bind the round's stream so a consumer abandon
|
||||
# (GeneratorExit into the yield below) tears down the in-flight
|
||||
# model stream deterministically — not GC-dependent. Phase 67:
|
||||
@@ -511,16 +591,97 @@ async def run_agent(
|
||||
tools=tools,
|
||||
retries=settings.llm_retries,
|
||||
delay=settings.llm_retry_delay,
|
||||
scaffolding=round_filter,
|
||||
)
|
||||
try:
|
||||
async for piece in stream:
|
||||
if isinstance(piece, ToolCallPiece):
|
||||
calls.append(piece)
|
||||
elif isinstance(piece, StreamPiece) and piece.kind == "content":
|
||||
round_content += len(piece.text)
|
||||
yield piece
|
||||
finally:
|
||||
await stream.aclose()
|
||||
# Phase 71: the per-strip-event capture log — one warning per
|
||||
# stripped span, truncated to 200 chars (how a new scaffolding
|
||||
# format gets captured and added to the registry) — and the turn
|
||||
# total for the API layer's ``scaffold_stripped=N`` log field.
|
||||
holder.scaffold_stripped += round_filter.stripped_chars
|
||||
for span in round_filter.stripped_spans:
|
||||
logger.warning(
|
||||
"agent: stripped %d chars of tool-scaffolding in round %d: %r",
|
||||
len(span),
|
||||
rounds + 1,
|
||||
span[:200],
|
||||
)
|
||||
if not calls:
|
||||
return # the answer was streamed
|
||||
if round_content > 0:
|
||||
return # the answer was streamed (the clean content stands)
|
||||
if round_filter.stripped_chars == 0:
|
||||
# Empty/thinking-only answer — today's behavior, unchanged
|
||||
# (the UI handles it); the guardrail keys on a strip.
|
||||
return
|
||||
# Phase 71: the scaffolding was the whole "answer" — the ONE
|
||||
# bounded recovery (a fixed policy, not a conversation):
|
||||
# ``tools=None``, the correction folded into the ORIGINAL
|
||||
# single system message (provider-safe — the user message and
|
||||
# any tool history stay in place), a fresh filter, the same
|
||||
# phase-67 retry budget.
|
||||
logger.warning(
|
||||
"agent: round %d was pure tool-scaffolding (%d chars stripped) "
|
||||
"— running the one bounded recovery",
|
||||
rounds + 1,
|
||||
round_filter.stripped_chars,
|
||||
)
|
||||
messages_recovered = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": system_prompt + "\n" + CORRECTION_INSTRUCTION,
|
||||
},
|
||||
*messages[1:],
|
||||
]
|
||||
recovery_filter = ScaffoldingFilter()
|
||||
recovered = chat_stream_retried(
|
||||
llm,
|
||||
cast("list[dict[str, str]]", messages_recovered),
|
||||
tools=None,
|
||||
retries=settings.llm_retries,
|
||||
delay=settings.llm_retry_delay,
|
||||
scaffolding=recovery_filter,
|
||||
)
|
||||
recovery_content = 0
|
||||
try:
|
||||
async for piece in recovered:
|
||||
if isinstance(piece, StreamPiece) and piece.kind == "content":
|
||||
recovery_content += len(piece.text)
|
||||
yield piece
|
||||
finally:
|
||||
await recovered.aclose()
|
||||
holder.scaffold_stripped += recovery_filter.stripped_chars
|
||||
for span in recovery_filter.stripped_spans:
|
||||
logger.warning(
|
||||
"agent: stripped %d chars of tool-scaffolding in the "
|
||||
"recovery after round %d: %r",
|
||||
len(span),
|
||||
rounds + 1,
|
||||
span[:200],
|
||||
)
|
||||
if recovery_content > 0:
|
||||
return # the recovery answered — the turn ends
|
||||
# The second empty reply is terminal (at most one recovery per
|
||||
# turn). Raised OUTSIDE the stream, so chat_stream_retried's
|
||||
# retry rule never sees it; the API layer catches it before
|
||||
# the generic LLMError handler.
|
||||
logger.warning(
|
||||
"agent: the recovery reply was still empty "
|
||||
"(scaffold_stripped=%d) — settling with a malformed-reply error",
|
||||
holder.scaffold_stripped,
|
||||
)
|
||||
raise MalformedReplyError(
|
||||
f"the model answered in raw tool-scaffolding twice in a row "
|
||||
f"(round {rounds + 1} plus one recovery) — no clean answer "
|
||||
"to stream"
|
||||
)
|
||||
call = calls[0] # a stream can carry several calls; run the first
|
||||
result = _execute_tool(db, call, seed_docs, holder)
|
||||
rounds += 1 # every call the model emits consumes a round
|
||||
@@ -558,17 +719,46 @@ async def run_agent(
|
||||
# teardown as the loop rounds (consumer abandon mid-final
|
||||
# answer must still close the model's stream). Phase 67: the
|
||||
# forced call retries under the same locked-A2 rule as the
|
||||
# loop rounds.
|
||||
# loop rounds. Phase 71: the forced final runs through a
|
||||
# fresh filter too — raw scaffolding can never reach the
|
||||
# user from ANY grounded request.
|
||||
final_filter = ScaffoldingFilter()
|
||||
final = chat_stream_retried(
|
||||
llm,
|
||||
cast("list[dict[str, str]]", messages),
|
||||
tools=None,
|
||||
retries=settings.llm_retries,
|
||||
delay=settings.llm_retry_delay,
|
||||
scaffolding=final_filter,
|
||||
)
|
||||
final_content = 0
|
||||
try:
|
||||
async for piece in final:
|
||||
if isinstance(piece, StreamPiece) and piece.kind == "content":
|
||||
final_content += len(piece.text)
|
||||
yield piece
|
||||
finally:
|
||||
await final.aclose()
|
||||
# Phase 71: the same capture log + turn total; a
|
||||
# scaffolding-only forced final (this turn used no recovery,
|
||||
# so nothing is doubled up) settles with the same terminal
|
||||
# malformed-reply error rather than a silently empty answer.
|
||||
holder.scaffold_stripped += final_filter.stripped_chars
|
||||
for span in final_filter.stripped_spans:
|
||||
logger.warning(
|
||||
"agent: stripped %d chars of tool-scaffolding in the "
|
||||
"forced final answer (round %d): %r",
|
||||
len(span),
|
||||
rounds + 1,
|
||||
span[:200],
|
||||
)
|
||||
if final_content == 0 and final_filter.stripped_chars > 0:
|
||||
logger.warning(
|
||||
"agent: the forced final answer was pure tool-scaffolding "
|
||||
"— settling with a malformed-reply error"
|
||||
)
|
||||
raise MalformedReplyError(
|
||||
"the forced final answer was raw tool-scaffolding — no "
|
||||
"clean answer to stream"
|
||||
)
|
||||
return
|
||||
|
||||
+48
-16
@@ -23,13 +23,18 @@ import json
|
||||
import logging
|
||||
from collections.abc import AsyncGenerator
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, cast
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
|
||||
from openai import AsyncOpenAI, AsyncStream
|
||||
from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
|
||||
|
||||
from app.config import Settings, get_settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Phase 71: the filter type is only needed for typing (the module
|
||||
# stays import-graph-clean; callers pass their own instances).
|
||||
from app.rag.scaffolding import ScaffoldingFilter
|
||||
|
||||
logger = logging.getLogger("app.llm")
|
||||
|
||||
|
||||
@@ -338,6 +343,7 @@ class LLMClient:
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
scaffolding: ScaffoldingFilter | None = None,
|
||||
) -> AsyncGenerator[StreamPiece | ToolCallPiece, None]:
|
||||
"""Stream assistant pieces from the chat model (PLAN A5/A15, phase 17).
|
||||
|
||||
@@ -365,13 +371,26 @@ class LLMClient:
|
||||
``id`` and ``function.name`` on the first partial and
|
||||
``function.arguments`` in fragments — which are accumulated into
|
||||
one :class:`ToolCallPiece` per call, yielded in index order at
|
||||
stream end (or immediately once a chunk carries
|
||||
``finish_reason="tool_calls"``). Malformed ``arguments`` JSON
|
||||
raises :class:`LLMError`. Wire convention verified live against
|
||||
stream end (after the stream's chunks are exhausted — aipi ends
|
||||
the stream at ``finish_reason="tool_calls"``, so this is the
|
||||
wire's emission point). Malformed ``arguments`` JSON raises
|
||||
:class:`LLMError`. Wire convention verified live against
|
||||
aipi's ``turbo`` on 2026-08-26 via
|
||||
``uv run python -m scripts.llm_probe --tools`` (phase 37, task 01:
|
||||
``probe: turbo tool_calls=supported 2026-08-26``).
|
||||
|
||||
Scaffolding guardrail (phase 71): the caller may pass a
|
||||
``ScaffoldingFilter`` — one per request, caller-owned (this
|
||||
method never creates or resets one). When present, only
|
||||
**content** is filtered: ``delta.content`` is fed through the
|
||||
filter and only the clean text is yielded (an empty clean
|
||||
result yields **no** piece — no empty ``delta`` frames); thinking
|
||||
pieces are never filtered (the scratchpad stays raw, phase 17).
|
||||
At stream end the filter's held tail is flushed to a content
|
||||
piece **before** any tool-call materialization (content-
|
||||
before-tools wire convention). ``None`` (the default) keeps
|
||||
today's byte-identical raw path for callers that opt out.
|
||||
|
||||
Any failure (network, HTTP, malformed stream) surfaces as
|
||||
:class:`LLMError` so the API layer can turn it into an SSE
|
||||
``error`` event instead of a hung request.
|
||||
@@ -405,7 +424,6 @@ class LLMClient:
|
||||
await self._client.chat.completions.create(**kwargs),
|
||||
)
|
||||
calls: dict[int, _ToolCallSlot] = {}
|
||||
emitted = False
|
||||
async for chunk in stream:
|
||||
if not chunk.choices:
|
||||
continue
|
||||
@@ -436,16 +454,22 @@ class LLMClient:
|
||||
yield StreamPiece("thinking", reasoning)
|
||||
content = delta.content
|
||||
if content:
|
||||
yield StreamPiece("content", content)
|
||||
if (
|
||||
calls
|
||||
and not emitted
|
||||
and getattr(choice, "finish_reason", None) == "tool_calls"
|
||||
):
|
||||
for piece in _materialize_tool_calls(calls):
|
||||
yield piece
|
||||
emitted = True
|
||||
if calls and not emitted:
|
||||
if scaffolding is not None:
|
||||
# Phase 71: content only — an empty clean result
|
||||
# yields nothing (no empty delta frames).
|
||||
cleaned = scaffolding.feed(content)
|
||||
if cleaned:
|
||||
yield StreamPiece("content", cleaned)
|
||||
else:
|
||||
yield StreamPiece("content", content)
|
||||
# Phase 71: flush the filter's held tail at stream end, BEFORE
|
||||
# any tool-call materialization — flushed-tail content precedes
|
||||
# ToolCallPieces (content-before-tools wire convention).
|
||||
if scaffolding is not None:
|
||||
tail = scaffolding.flush()
|
||||
if tail:
|
||||
yield StreamPiece("content", tail)
|
||||
if calls:
|
||||
for piece in _materialize_tool_calls(calls):
|
||||
yield piece
|
||||
except LLMError:
|
||||
@@ -469,6 +493,7 @@ async def chat_stream_retried(
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
retries: int = 0,
|
||||
delay: float = 0.0,
|
||||
scaffolding: ScaffoldingFilter | None = None,
|
||||
) -> AsyncGenerator[StreamPiece | ToolCallPiece | RetryPiece, None]:
|
||||
"""Stream a chat turn, retrying a dead endpoint (phase 67).
|
||||
|
||||
@@ -493,6 +518,13 @@ async def chat_stream_retried(
|
||||
The request is restarted byte-identical: ``chat_stream`` is stateless,
|
||||
so every attempt is opened with the SAME *messages*/*tools*.
|
||||
|
||||
Scaffolding guardrail (phase 71): *scaffolding* is passed through to
|
||||
every attempt's ``chat_stream``. The SAME caller-owned filter object
|
||||
across the retry attempts of one logical request is safe by
|
||||
construction: a restarted attempt only happens while no piece was
|
||||
emitted, i.e. the filter was never fed (its pending buffer is still
|
||||
empty).
|
||||
|
||||
Teardown (phase 48, extended): every attempt's stream is explicitly
|
||||
closed in a ``finally`` — normal exhaustion, a terminal
|
||||
:class:`LLMError`, and a consumer abandon (``GeneratorExit`` mid-attempt
|
||||
@@ -502,7 +534,7 @@ async def chat_stream_retried(
|
||||
max_attempts = retries + 1
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
emitted = False
|
||||
stream = llm.chat_stream(messages, tools=tools)
|
||||
stream = llm.chat_stream(messages, tools=tools, scaffolding=scaffolding)
|
||||
try:
|
||||
async for piece in stream:
|
||||
emitted = True
|
||||
|
||||
+25
-5
@@ -30,7 +30,19 @@ Agent tools (phase 37; phase 70: the copy teaches the harness-aligned
|
||||
may extend its context through the three server-side tools (round-
|
||||
capped, see :mod:`app.rag.agent`; the cap is the bound and this section
|
||||
does not re-state it, phase 45). The LOW/deflection prompt never
|
||||
carries it and stays byte-identical to the pre-phase text.
|
||||
carries it (phase 71: the LOW prompt's only addition is the
|
||||
plain-text line below — it still has no ``<tools>`` section).
|
||||
|
||||
Deflection plain-text line (phase 71, owner-permitted 2026-09-03):
|
||||
the otherwise-locked ``LOW`` prompt gains exactly one instruction
|
||||
line — "Reply in plain text only — you have no tools in this mode."
|
||||
— appended to the ``DEFLECT_MODE`` body: a deflected turn offers no
|
||||
tools, so any tool markup there is always wrong, and the line closes
|
||||
the door at the prompt (the deterministic filter + one bounded
|
||||
recovery in :mod:`app.rag.scaffolding` / :mod:`app.rag.agent` is the
|
||||
backstop). The ``DEFLECT_MODE`` marker and everything else in the
|
||||
prompt stay put — the E2E mock LLM keys on the marker's *presence*,
|
||||
not the wording, so that contract is unchanged.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -83,9 +95,10 @@ _KB_INTRO = (
|
||||
#: not re-state it, phase 45). Appended after the mode body
|
||||
#: (``<documents>``), so the instructions are the last thing the model
|
||||
#: reads. The LOW/deflection prompt never carries it — a deflection has
|
||||
#: no grounded context to extend — and stays byte-identical to the
|
||||
#: pre-phase text. The E2E mock keys off the ``<tools>`` marker's
|
||||
#: *presence*, not this wording.
|
||||
#: no grounded context to extend (phase 71: the LOW prompt's only
|
||||
#: addition is the plain-text line in :func:`build_deflect_prompt`).
|
||||
#: The E2E mock keys off the ``<tools>`` marker's *presence*, not this
|
||||
#: wording.
|
||||
TOOLS_SECTION: str = (
|
||||
"<tools>\n"
|
||||
"You may extend your context with three tools. `ls` lists the "
|
||||
@@ -224,7 +237,10 @@ def build_deflect_prompt(
|
||||
|
||||
Section order (phase 31): ``<relevance>`` → ``<knowledge_base>`` →
|
||||
``<tuning>`` → ``DEFLECT_MODE`` body; empty steering/overview omit
|
||||
their section, keeping the prompt byte-identical to the pre-phase text.
|
||||
their section, keeping the prompt byte-identical to the pre-phase
|
||||
text. The body ends with the phase-71 plain-text line (owner-
|
||||
permitted 2026-09-03 — the LOW prompt's only change): a deflected
|
||||
turn offers no tools, so any tool markup there is always wrong.
|
||||
"""
|
||||
weak = "\n".join(f"- {t}" for t in titles) if titles else "(nothing close at all)"
|
||||
mid = "\n".join(
|
||||
@@ -239,5 +255,9 @@ def build_deflect_prompt(
|
||||
+ "DEFLECT_MODE: retrieval was weak — the titles below are the closest "
|
||||
"your notes come to the question. They are titles only; do not pretend "
|
||||
"they answer it. Use them to propose 2-3 alternative questions.\n"
|
||||
# Phase 71 (owner-permitted 2026-09-03): the one plain-text line
|
||||
# — prevention at the prompt. The E2E mock keys on the
|
||||
# DEFLECT_MODE marker's presence, so appending is safe.
|
||||
"Reply in plain text only — you have no tools in this mode.\n"
|
||||
+ weak
|
||||
)
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Deterministic tool-scaffolding guardrail — pattern registry + streaming
|
||||
filter (phase 71, task 01).
|
||||
|
||||
The ``lite`` chat model occasionally emits its own chat-template
|
||||
tool-scaffolding as plain answer text (incident 2026-09-03: a deflected
|
||||
round streamed ``<|tool_call_start|>[read(path='…')]<|tool_call_end|>``
|
||||
into the UI although no tools were offered). This module is the
|
||||
detection half of the guardrail: a fixed registry of *observed*
|
||||
scaffolding forms and a streaming state machine that strips them from
|
||||
``delta.content`` as it flows. Pure Python — no I/O, no logging, no
|
||||
model: the strip *warning* log and the one bounded recovery are emitted
|
||||
by the integration layer (phase 71, tasks 02–03), so this module stays
|
||||
unit-testable in isolation.
|
||||
|
||||
Content only: thinking pieces are the model's raw reasoning by design
|
||||
(phase 17) and are never filtered — the guardrail protects the answer,
|
||||
not the scratchpad.
|
||||
"""
|
||||
import re
|
||||
|
||||
__all__ = ["SCAFFOLD_PATTERNS", "ScaffoldingFilter"]
|
||||
|
||||
#: The known scaffolding forms. The registry is the extension point: a
|
||||
#: new entry needs an observed capture (the strip warning log, task 03)
|
||||
#: + a unit fixture in ``tests/unit/test_scaffolding_filter.py`` — no
|
||||
#: speculative entries. Every entry here traces to the 2026-09-03
|
||||
#: incident (the span form is the deflected round's raw text; the
|
||||
#: standalone siblings are from the same tokenizer family).
|
||||
SCAFFOLD_PATTERNS: tuple[re.Pattern, ...] = (
|
||||
# The observed span (incident 2026-09-03) — non-greedy, so multiple
|
||||
# spans in one buffer each strip to their own end token.
|
||||
re.compile(r"<\|tool_call_start\|>[\s\S]*?<\|tool_call_end\|>"),
|
||||
# Standalone sibling token (same tokenizer family).
|
||||
re.compile(r"<\|tool_calls\|>"),
|
||||
# Standalone sibling token (same tokenizer family).
|
||||
re.compile(r"<\|tool_call\|>"),
|
||||
)
|
||||
|
||||
_SPAN_START = "<|tool_call_start|>"
|
||||
_SPAN_END = "<|tool_call_end|>"
|
||||
#: The literal openings a live (possibly incomplete) match can begin with.
|
||||
_OPENINGS: tuple[str, ...] = (_SPAN_START, "<|tool_calls|>", "<|tool_call|>")
|
||||
#: Boundedness bound: in NORMAL state the held tail is at most one char
|
||||
#: short of the longest opening (an open span is unbounded, but it is
|
||||
#: being *stripped*, never emitted).
|
||||
_MAX_OPENING = max(len(opening) for opening in _OPENINGS)
|
||||
|
||||
|
||||
def _leftmost_match(buf: str) -> re.Match | None:
|
||||
"""The leftmost complete match among :data:`SCAFFOLD_PATTERNS`, or
|
||||
None when the buffer holds no complete scaffolding form.
|
||||
|
||||
(Two patterns cannot match at the same start position — their
|
||||
literals diverge after ``<|tool_call`` — so the leftmost start is
|
||||
unambiguous.)
|
||||
"""
|
||||
best: re.Match | None = None
|
||||
for pattern in SCAFFOLD_PATTERNS:
|
||||
match = pattern.search(buf)
|
||||
if match is not None and (best is None or match.start() < best.start()):
|
||||
best = match
|
||||
return best
|
||||
|
||||
|
||||
def _hold_index(buf: str) -> int:
|
||||
"""Index where the live hold-tail begins; everything from it stays
|
||||
pending (called only after the strip loop found no complete match).
|
||||
|
||||
The tail is the leftmost of two candidates: an **open span** (a
|
||||
start token with no end token after it — the span may continue in
|
||||
future chunks, so everything from that start token onward is held)
|
||||
or a **live prefix** (the longest suffix that is a proper prefix of
|
||||
any opening literal — the token may complete in future chunks).
|
||||
Everything before the leftmost candidate is safe to emit: no
|
||||
complete match remains, and no match can grow from earlier text,
|
||||
since a complete opening earlier in the buffer would already have
|
||||
matched (span) or be a candidate in its own right (standalone).
|
||||
"""
|
||||
if not buf:
|
||||
return 0
|
||||
hold = len(buf)
|
||||
start = buf.find(_SPAN_START)
|
||||
if start != -1 and buf.find(_SPAN_END, start + 1) == -1:
|
||||
hold = start
|
||||
for k in range(min(len(buf), _MAX_OPENING - 1), 0, -1):
|
||||
suffix = buf[-k:]
|
||||
if any(k < len(opening) and opening.startswith(suffix) for opening in _OPENINGS):
|
||||
hold = min(hold, len(buf) - k)
|
||||
break
|
||||
return hold
|
||||
|
||||
|
||||
class ScaffoldingFilter:
|
||||
"""Streaming state machine over a model **content** stream (phase 71).
|
||||
|
||||
One instance per model request (callers create fresh instances —
|
||||
the phase-67 retry-after-dead-attempt case is safe: a dead attempt
|
||||
never fed the filter). Feed the raw ``delta.content`` chunks; each
|
||||
:meth:`feed` returns the clean text safe to emit *now*; :meth:`flush`
|
||||
emits the tail at stream end.
|
||||
|
||||
``feed`` appends the chunk to the internal pending buffer, then
|
||||
(a) repeatedly takes the leftmost complete match among
|
||||
:data:`SCAFFOLD_PATTERNS` — drops it (counting it into
|
||||
:attr:`stripped_chars`) and continues — until none remains; then
|
||||
(b) checks the buffer tail for a **live prefix** via
|
||||
:func:`_hold_index`: the longest suffix that is a proper prefix of
|
||||
any opening literal, or an **open span** (a start token with no end
|
||||
token yet — everything from that start token onward is held, since
|
||||
the span may continue in future chunks). Everything before the held
|
||||
tail is emitted; the held tail becomes the new pending state.
|
||||
Bounded: the held tail in NORMAL state is ≤ the longest opening
|
||||
token minus one char; in an open span it is unbounded but is being
|
||||
*stripped*, never emitted.
|
||||
|
||||
``flush`` (end of stream) emits the pending tail **as-is**: a
|
||||
partial marker at EOF is content, not scaffolding — a documented,
|
||||
pinned choice (a stream that ends mid-``<|tool_call_st`` must not
|
||||
be silently eaten, and a lone ``<|tool_call_end|>`` without a start
|
||||
is prose the user sent or the model produced outside a span).
|
||||
|
||||
The stripped spans themselves are exposed read-only
|
||||
(:attr:`stripped_spans`) so the integration layer can log one
|
||||
warning per strip event (phase 71 task 03) — that log line is how
|
||||
a *new* scaffolding format gets captured and added to the registry.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._pending = ""
|
||||
self._stripped_chars = 0
|
||||
self._stripped_spans: list[str] = []
|
||||
|
||||
@property
|
||||
def stripped_chars(self) -> int:
|
||||
"""Total characters removed so far (read by the caller after the
|
||||
round/turn — the strip warning log, phase 71 task 03)."""
|
||||
return self._stripped_chars
|
||||
|
||||
@property
|
||||
def stripped_spans(self) -> list[str]:
|
||||
"""The raw spans removed so far, in stream order (one entry per
|
||||
strip event — the integration layer's per-span warning log
|
||||
truncates each to 200 chars). Read-only: callers must not
|
||||
mutate the filter's state."""
|
||||
return list(self._stripped_spans)
|
||||
|
||||
def feed(self, chunk: str) -> str:
|
||||
"""Append *chunk* to the pending buffer; return the clean text
|
||||
safe to emit now (possibly ``""`` — e.g. while an open span or a
|
||||
partial marker is still held). An empty chunk is a no-op."""
|
||||
if not chunk:
|
||||
return ""
|
||||
buf = self._strip_complete(self._pending + chunk)
|
||||
hold = _hold_index(buf)
|
||||
self._pending = buf[hold:]
|
||||
return buf[:hold]
|
||||
|
||||
def flush(self) -> str:
|
||||
"""End of stream: emit the pending tail as-is and reset it."""
|
||||
tail = self._pending
|
||||
self._pending = ""
|
||||
return tail
|
||||
|
||||
def _strip_complete(self, buf: str) -> str:
|
||||
"""Drop leftmost-complete matches until none remains (step a)."""
|
||||
while (match := _leftmost_match(buf)) is not None:
|
||||
self._stripped_chars += match.end() - match.start()
|
||||
self._stripped_spans.append(match.group(0))
|
||||
buf = buf[: match.start()] + buf[match.end():]
|
||||
return buf
|
||||
Reference in New Issue
Block a user