feat(chat): stream model thinking over SSE and show it in a collapsible block
This commit is contained in:
+34
-5
@@ -8,6 +8,13 @@ persona prompt (PLAN §6) → ``turbo`` streamed as ``delta`` events → final
|
||||
Mid-stream failures become a structured ``error`` event; a pre-stream DB
|
||||
outage is a plain 503 JSON.
|
||||
|
||||
Thinking (phase 17, PLAN §4 extension, owner permission 2026-08-23): the
|
||||
model's reasoning arrives ahead of the answer and is streamed as
|
||||
``thinking`` events before the ``delta`` events of the same turn. Each
|
||||
turn's thinking is counted in the per-turn log line
|
||||
(``thinking_chars=N``); ``BOR_STREAM_THINKING=0`` suppresses the
|
||||
``thinking`` frames (the pieces are still counted).
|
||||
|
||||
Honesty gate (A8, revised 2026-08-21): LOW — deflection — only when the
|
||||
best cosine is strictly below ``BOR_RELEVANCE_THRESHOLD`` **and** no
|
||||
candidate chunk FTS-matches the question (``fts_hits == 0``). A
|
||||
@@ -39,11 +46,22 @@ from app.api.steering import load_steering_notes
|
||||
from app.config import Settings, get_settings
|
||||
from app.db import db_available, get_db
|
||||
from app.models import Document, QueryLog
|
||||
from app.rag.llm import EmbeddingError, LLMClient, LLMError
|
||||
from app.rag.llm import (
|
||||
EmbeddingError,
|
||||
LLMClient,
|
||||
LLMError,
|
||||
StreamPiece, # noqa: F401 (phase 17 typing: chat_stream yields StreamPiece)
|
||||
)
|
||||
from app.rag.prompts import build_deflect_prompt, build_high_prompt
|
||||
from app.rag.retriever import RetrievedChunk, retrieve, select_documents, weak_hit_titles
|
||||
from app.rag.suggestions import derive_suggestions
|
||||
from app.schemas import ChatDoneEvent, ChatErrorEvent, ChatRequest, SourceRef
|
||||
from app.schemas import (
|
||||
ChatDoneEvent,
|
||||
ChatErrorEvent,
|
||||
ChatRequest,
|
||||
ChatThinkingEvent,
|
||||
SourceRef,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("app.chat")
|
||||
router = APIRouter(tags=["chat"])
|
||||
@@ -202,9 +220,19 @@ async def chat(
|
||||
]
|
||||
|
||||
# 3. Stream the answer (grounded, or an honest deflection).
|
||||
# Phase 17: thinking pieces stream as ``thinking`` events
|
||||
# ahead of the ``delta`` events (PLAN §4 extension); the
|
||||
# kill-switch (``BOR_STREAM_THINKING=0``) suppresses the
|
||||
# frames, not the counting.
|
||||
thinking_chars = 0
|
||||
try:
|
||||
async for piece in llm.chat_stream(messages):
|
||||
yield sse_event({"type": "delta", "text": piece})
|
||||
async for piece in llm.chat_stream(messages): # StreamPiece (phase 17)
|
||||
if piece.kind == "thinking":
|
||||
thinking_chars += len(piece.text)
|
||||
if settings.stream_thinking:
|
||||
yield sse_event(ChatThinkingEvent(text=piece.text).model_dump())
|
||||
else:
|
||||
yield sse_event({"type": "delta", "text": piece.text})
|
||||
except LLMError as e:
|
||||
logger.error(
|
||||
"chat: LLM stream failed question=%r total_ms=%d — %s",
|
||||
@@ -239,7 +267,7 @@ async def chat(
|
||||
|
||||
logger.info(
|
||||
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d tuning=%d threshold=%.2f "
|
||||
"deflected=%s sources=%r total_ms=%d",
|
||||
"deflected=%s sources=%r thinking_chars=%d total_ms=%d",
|
||||
request.message,
|
||||
embed_ms,
|
||||
plan.top_score,
|
||||
@@ -248,6 +276,7 @@ async def chat(
|
||||
settings.relevance_threshold,
|
||||
plan.deflected,
|
||||
source_paths,
|
||||
thinking_chars,
|
||||
total_ms,
|
||||
)
|
||||
yield sse_event(
|
||||
|
||||
Reference in New Issue
Block a user