feat(agent): strip raw tool-scaffolding from streamed answers — deterministic filter with one bounded recovery
This commit is contained in:
+139
-5
@@ -91,6 +91,30 @@ the same budget, and the deflected answer stream goes through
|
||||
The per-turn log line records ``retries=N`` after ``total_ms=N`` (0
|
||||
when nothing was retried — the field is uniform across all turn
|
||||
shapes).
|
||||
|
||||
Tool-scaffolding guardrail (phase 71, deterministic only — owner
|
||||
permission 2026-09-03: "deterministic guardrails only right now, forget
|
||||
using a model for that"): the raw chat-template tokens
|
||||
``<|tool_call_start|>…<|tool_call_end|>`` the ``lite`` model sometimes
|
||||
emits as plain answer text can never reach the user. The DEFLECTED
|
||||
path's request runs ``delta.content`` through a caller-owned
|
||||
``ScaffoldingFilter`` (content only — thinking stays raw); when the
|
||||
filter wipes the whole reply (visible content 0, ``stripped_chars > 0``)
|
||||
the turn gets exactly ONE bounded recovery: the same messages with
|
||||
``agent.CORRECTION_INSTRUCTION`` folded into the single system prompt,
|
||||
``tools=None``, a fresh filter, the same phase-67 retry budget, streamed
|
||||
through the same piece loop (extracted as the inner ``_pump`` helper).
|
||||
A recovery that also comes back empty — and any grounded round where the
|
||||
recovery policy in ``run_agent`` fails — settles with the dedicated
|
||||
structured ``error`` frame (``MalformedReplyError`` caught before the
|
||||
generic ``LLMError`` handler): no ``done``, no ``query_log`` row,
|
||||
byte-for-byte the existing terminal-error shape. The per-turn log line
|
||||
records ``scaffold_stripped=N`` after ``retries=N`` — the sum across the
|
||||
turn's requests (grounded: the agent's rounds + forced final + any
|
||||
recovery, via the holder; deflected: this turn's filters), 0 on clean
|
||||
turns (the field is uniform, the phase-67 ``retries=N`` pattern); the
|
||||
recovery does not bump ``retries=N`` (it is not a phase-67
|
||||
endpoint-retry).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -110,7 +134,12 @@ from app.api.steering import load_steering_notes
|
||||
from app.config import Settings, get_settings
|
||||
from app.db import db_available, get_db
|
||||
from app.models import Document, QueryLog
|
||||
from app.rag.agent import AgentHolder, run_agent
|
||||
from app.rag.agent import (
|
||||
CORRECTION_INSTRUCTION, # phase 71: the harness-owned recovery line
|
||||
AgentHolder,
|
||||
MalformedReplyError, # phase 71: the recovery policy's terminal signal
|
||||
run_agent,
|
||||
)
|
||||
from app.rag.llm import (
|
||||
EmbeddingError,
|
||||
LLMClient,
|
||||
@@ -123,6 +152,7 @@ from app.rag.llm import (
|
||||
from app.rag.overview import load_kb_overview
|
||||
from app.rag.prompts import build_deflect_prompt, build_high_prompt
|
||||
from app.rag.retriever import RetrievedChunk, retrieve, select_documents, weak_hit_titles
|
||||
from app.rag.scaffolding import ScaffoldingFilter # phase 71: the streaming filter
|
||||
from app.rag.suggestions import derive_suggestions
|
||||
from app.schemas import (
|
||||
ChatDoneEvent,
|
||||
@@ -373,19 +403,27 @@ async def chat(
|
||||
# ``tools=None`` request anyway (the kill switch).
|
||||
holder = AgentHolder()
|
||||
answer_stream: AsyncIterator[StreamPiece | ToolCallPiece | RetryPiece]
|
||||
deflected_filter: ScaffoldingFilter | None = None
|
||||
if plan.deflected:
|
||||
# Phase 67: the deflected stream goes through the retry
|
||||
# primitive — a dead endpoint is restarted (SSE ``retry``
|
||||
# frames) only before its first piece (locked A2); the
|
||||
# grounded path stays a plain ``run_agent`` call (task 03
|
||||
# makes IT retry internally) — its ``RetryPiece``s flow
|
||||
# through the shared piece loop below.
|
||||
# through the shared piece loop below. Phase 71: the
|
||||
# request's content also runs through a caller-owned
|
||||
# filter (one per request) — a scaffolding-only reply
|
||||
# streams zero ``delta`` frames instead of raw tokens, and
|
||||
# the filter's ``stripped_chars`` drives the recovery
|
||||
# decision after the piece loop.
|
||||
deflected_filter = ScaffoldingFilter()
|
||||
answer_stream = chat_stream_retried(
|
||||
llm,
|
||||
messages,
|
||||
tools=None,
|
||||
retries=settings.llm_retries,
|
||||
delay=settings.llm_retry_delay,
|
||||
scaffolding=deflected_filter,
|
||||
)
|
||||
else:
|
||||
answer_stream = run_agent(
|
||||
@@ -398,8 +436,19 @@ async def chat(
|
||||
holder=holder,
|
||||
)
|
||||
thinking_chars = 0
|
||||
try:
|
||||
async for piece in answer_stream: # StreamPiece | ToolCallPiece | RetryPiece
|
||||
content_chars = 0 # phase 71: the turn's visible (clean) content
|
||||
scaffold_stripped = 0 # phase 71: sum across the turn's requests
|
||||
|
||||
async def _pump(
|
||||
pieces: AsyncIterator[StreamPiece | ToolCallPiece | RetryPiece],
|
||||
) -> AsyncIterator[str]:
|
||||
"""One request's piece loop (phase 71 extraction): the
|
||||
thinking/tool/retry/delta handling shared by the turn's
|
||||
first pass and — deflected path only — the one bounded
|
||||
recovery. Behavior-preserving for the first pass (pinned
|
||||
by the existing integration suite)."""
|
||||
nonlocal thinking_chars, content_chars, retries_used
|
||||
async for piece in pieces: # StreamPiece | ToolCallPiece | RetryPiece
|
||||
if isinstance(piece, ToolCallPiece):
|
||||
# Phase 37 (PLAN §4 extension; phase 70): one SSE
|
||||
# ``tool`` frame per model-requested call.
|
||||
@@ -435,7 +484,91 @@ async def chat(
|
||||
if settings.stream_thinking:
|
||||
yield sse_event(ChatThinkingEvent(text=piece.text).model_dump())
|
||||
else:
|
||||
content_chars += len(piece.text)
|
||||
yield sse_event({"type": "delta", "text": piece.text})
|
||||
|
||||
try:
|
||||
async for frame in _pump(answer_stream):
|
||||
yield frame
|
||||
if plan.deflected and deflected_filter is not None:
|
||||
scaffold_stripped = deflected_filter.stripped_chars
|
||||
# Phase 71: the deflected reply's visible content was
|
||||
# wiped by the filter (the scaffolding was the whole
|
||||
# "answer") — the ONE bounded recovery: the same
|
||||
# messages with the correction folded into the single
|
||||
# system prompt, ``tools=None``, a FRESH filter, the
|
||||
# same phase-67 retry budget, streamed through the
|
||||
# same piece loop. A round with real visible content
|
||||
# needs no recovery (the clean content stands).
|
||||
if content_chars == 0 and deflected_filter.stripped_chars > 0:
|
||||
logger.warning(
|
||||
"chat: deflected reply was pure tool-scaffolding "
|
||||
"(%d chars stripped) — running the one bounded "
|
||||
"recovery",
|
||||
deflected_filter.stripped_chars,
|
||||
)
|
||||
recovery_filter = ScaffoldingFilter()
|
||||
recovery_stream = chat_stream_retried(
|
||||
llm,
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
plan.system_prompt + "\n"
|
||||
+ CORRECTION_INSTRUCTION
|
||||
),
|
||||
},
|
||||
*messages[1:],
|
||||
],
|
||||
tools=None,
|
||||
retries=settings.llm_retries,
|
||||
delay=settings.llm_retry_delay,
|
||||
scaffolding=recovery_filter,
|
||||
)
|
||||
async for frame in _pump(recovery_stream):
|
||||
yield frame
|
||||
scaffold_stripped += recovery_filter.stripped_chars
|
||||
if content_chars == 0:
|
||||
# The second empty reply is terminal (at most
|
||||
# one recovery per turn) — the dedicated error
|
||||
# frame below (no done, no query_log row).
|
||||
logger.warning(
|
||||
"chat: the recovery reply was still empty "
|
||||
"(scaffold_stripped=%d) — settling with a "
|
||||
"malformed-reply error",
|
||||
scaffold_stripped,
|
||||
)
|
||||
raise MalformedReplyError(
|
||||
"the deflected model answered in raw "
|
||||
"tool-scaffolding twice in a row — no "
|
||||
"clean answer to stream"
|
||||
)
|
||||
else:
|
||||
# Grounded turns: the agent's rounds + forced final +
|
||||
# any recovery already accumulated the turn total on
|
||||
# the holder (the deflected fallback is 0 — the agent
|
||||
# never runs, so this branch is grounded-only).
|
||||
scaffold_stripped = holder.scaffold_stripped
|
||||
except MalformedReplyError as e:
|
||||
# Phase 71: the recovery policy's terminal signal —
|
||||
# caught BEFORE the generic LLMError handler (it
|
||||
# subclasses it), so the dedicated copy reaches the UI;
|
||||
# the generic "dropped the connection" copy stays for
|
||||
# transport failures.
|
||||
logger.error(
|
||||
"chat: malformed reply after the one bounded recovery "
|
||||
"question=%r total_ms=%d — %s",
|
||||
request.message,
|
||||
int((time.monotonic() - started) * 1000),
|
||||
e,
|
||||
)
|
||||
settled = True # terminal: the error frame settles the turn
|
||||
yield sse_event(
|
||||
ChatErrorEvent(
|
||||
detail="The model returned a malformed reply — please try again."
|
||||
).model_dump()
|
||||
)
|
||||
return
|
||||
except LLMError as e:
|
||||
logger.error(
|
||||
"chat: LLM stream failed question=%r total_ms=%d — %s",
|
||||
@@ -504,7 +637,7 @@ async def chat(
|
||||
logger.info(
|
||||
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d summary_hits=%d tuning=%d "
|
||||
"kb_chars=%d threshold=%.2f deflected=%s sources=%r thinking_chars=%d "
|
||||
"tool_calls=%d total_ms=%d retries=%d",
|
||||
"tool_calls=%d total_ms=%d retries=%d scaffold_stripped=%d",
|
||||
request.message,
|
||||
embed_ms,
|
||||
plan.top_score,
|
||||
@@ -519,6 +652,7 @@ async def chat(
|
||||
holder.tool_calls,
|
||||
total_ms,
|
||||
retries_used,
|
||||
scaffold_stripped,
|
||||
)
|
||||
settled = True # terminal: the done frame settles the turn
|
||||
yield sse_event(
|
||||
|
||||
Reference in New Issue
Block a user