feat(chat): stream model thinking over SSE and show it in a collapsible block

This commit is contained in:
2026-08-24 09:52:27 -04:00
parent cbc263a4b2
commit b16deb2b1d
18 changed files with 1045 additions and 63 deletions
+34 -5
View File
@@ -8,6 +8,13 @@ persona prompt (PLAN §6) → ``turbo`` streamed as ``delta`` events → final
Mid-stream failures become a structured ``error`` event; a pre-stream DB
outage is a plain 503 JSON.
Thinking (phase 17, PLAN §4 extension, owner permission 2026-08-23): the
model's reasoning arrives ahead of the answer and is streamed as
``thinking`` events before the ``delta`` events of the same turn. Each
turn's thinking is counted in the per-turn log line
(``thinking_chars=N``); ``BOR_STREAM_THINKING=0`` suppresses the
``thinking`` frames (the pieces are still counted).
Honesty gate (A8, revised 2026-08-21): LOW — deflection — only when the
best cosine is strictly below ``BOR_RELEVANCE_THRESHOLD`` **and** no
candidate chunk FTS-matches the question (``fts_hits == 0``). A
@@ -39,11 +46,22 @@ from app.api.steering import load_steering_notes
from app.config import Settings, get_settings
from app.db import db_available, get_db
from app.models import Document, QueryLog
from app.rag.llm import EmbeddingError, LLMClient, LLMError
from app.rag.llm import (
EmbeddingError,
LLMClient,
LLMError,
StreamPiece, # noqa: F401 (phase 17 typing: chat_stream yields StreamPiece)
)
from app.rag.prompts import build_deflect_prompt, build_high_prompt
from app.rag.retriever import RetrievedChunk, retrieve, select_documents, weak_hit_titles
from app.rag.suggestions import derive_suggestions
from app.schemas import ChatDoneEvent, ChatErrorEvent, ChatRequest, SourceRef
from app.schemas import (
ChatDoneEvent,
ChatErrorEvent,
ChatRequest,
ChatThinkingEvent,
SourceRef,
)
logger = logging.getLogger("app.chat")
router = APIRouter(tags=["chat"])
@@ -202,9 +220,19 @@ async def chat(
]
# 3. Stream the answer (grounded, or an honest deflection).
# Phase 17: thinking pieces stream as ``thinking`` events
# ahead of the ``delta`` events (PLAN §4 extension); the
# kill-switch (``BOR_STREAM_THINKING=0``) suppresses the
# frames, not the counting.
thinking_chars = 0
try:
async for piece in llm.chat_stream(messages):
yield sse_event({"type": "delta", "text": piece})
async for piece in llm.chat_stream(messages): # StreamPiece (phase 17)
if piece.kind == "thinking":
thinking_chars += len(piece.text)
if settings.stream_thinking:
yield sse_event(ChatThinkingEvent(text=piece.text).model_dump())
else:
yield sse_event({"type": "delta", "text": piece.text})
except LLMError as e:
logger.error(
"chat: LLM stream failed question=%r total_ms=%d — %s",
@@ -239,7 +267,7 @@ async def chat(
logger.info(
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d tuning=%d threshold=%.2f "
"deflected=%s sources=%r total_ms=%d",
"deflected=%s sources=%r thinking_chars=%d total_ms=%d",
request.message,
embed_ms,
plan.top_score,
@@ -248,6 +276,7 @@ async def chat(
settings.relevance_threshold,
plan.deflected,
source_paths,
thinking_chars,
total_ms,
)
yield sse_event(