feat(rag): retry a failed LLM request before the first token lands — BOR_LLM_RETRIES/BOR_LLM_RETRY_DELAY with a live 'retrying' status
This commit is contained in:
+103
-24
@@ -68,9 +68,27 @@ byte-identical to the pre-phase path (A8):** the LOW prompt never
|
||||
carries tools, and with ``agent_max_rounds`` at **0** ``run_agent``
|
||||
makes exactly one ``tools=None`` request, reproducing the pre-phase
|
||||
behavior (the kill switch).
|
||||
|
||||
LLM retries (phase 67, ``BOR_LLM_RETRIES`` / ``BOR_LLM_RETRY_DELAY``,
|
||||
owner-locked 2026-09-01): when the aipi endpoint dies before a request
|
||||
has streamed its first output frame (locked A2), the request is
|
||||
restarted — up to ``llm_retries`` times (default 3), a flat
|
||||
``llm_retry_delay`` (default 5 s) between attempts. Every restart is
|
||||
announced with an SSE ``retry`` frame (``{"type": "retry",
|
||||
"attempt": n, "max_attempts": N}`` — the attempt about to be tried,
|
||||
1-based, ahead of the pre-retry wait) so the UI can show the transient
|
||||
"Communication interrupted — retrying (n of N)…" status (locked A4);
|
||||
exhaustion and any failure after the first frame keep the existing
|
||||
terminal ``error`` frames. The pre-stream question embedding retries on
|
||||
the same budget, and the deflected answer stream goes through
|
||||
``chat_stream_retried`` (the agent loop retries per round — task 03).
|
||||
The per-turn log line records ``retries=N`` after ``total_ms=N`` (0
|
||||
when nothing was retried — the field is uniform across all turn
|
||||
shapes).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
@@ -91,8 +109,10 @@ from app.rag.llm import (
|
||||
EmbeddingError,
|
||||
LLMClient,
|
||||
LLMError,
|
||||
RetryPiece, # phase 67: one LLM request restart (an SSE retry frame)
|
||||
StreamPiece, # type of the answer pieces streamed by the agent loop
|
||||
ToolCallPiece, # phase 37: one model-requested tool call
|
||||
chat_stream_retried, # phase 67: the retry-before-first-piece primitive
|
||||
)
|
||||
from app.rag.overview import load_kb_overview
|
||||
from app.rag.prompts import build_deflect_prompt, build_high_prompt
|
||||
@@ -102,6 +122,7 @@ from app.schemas import (
|
||||
ChatDoneEvent,
|
||||
ChatErrorEvent,
|
||||
ChatRequest,
|
||||
ChatRetryEvent,
|
||||
ChatThinkingEvent,
|
||||
ChatToolEvent,
|
||||
SourceRef,
|
||||
@@ -246,34 +267,66 @@ async def chat(
|
||||
# consumer went away before any terminal frame; it must not
|
||||
# yield (GeneratorExit handling).
|
||||
settled = False
|
||||
retries_used = 0 # phase 67: LLM requests restarted this turn (log line)
|
||||
try:
|
||||
settings = get_settings()
|
||||
|
||||
# 1. Embed the question.
|
||||
# Phase 67: a dead embeddings endpoint is retried before any
|
||||
# frame has left the server — up to ``llm_retries`` restarts,
|
||||
# a flat ``llm_retry_delay`` between attempts, one SSE
|
||||
# ``retry`` frame per restart (the UI shows the transient
|
||||
# "retrying" status, not an error — locked A4). The final
|
||||
# failure keeps the EXISTING terminal ``error`` frame (the
|
||||
# copy reads correctly after N tries); ``llm_retries=0`` is
|
||||
# byte-identical to the pre-phase-67 single attempt.
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
question_vec = await llm.embed_one(request.message)
|
||||
except EmbeddingError as e:
|
||||
embed_ms = int((time.monotonic() - t0) * 1000)
|
||||
total_ms = int((time.monotonic() - started) * 1000)
|
||||
logger.error(
|
||||
"chat: question=%r embed_ms=%d total_ms=%d — embedding failed: %s",
|
||||
request.message,
|
||||
embed_ms,
|
||||
total_ms,
|
||||
e,
|
||||
)
|
||||
settled = True # terminal: the error frame settles the turn
|
||||
yield sse_event(
|
||||
ChatErrorEvent(
|
||||
detail="I couldn't reach the embedding model — please try again."
|
||||
).model_dump()
|
||||
)
|
||||
return
|
||||
max_attempts = settings.llm_retries + 1
|
||||
attempt = 1
|
||||
while True:
|
||||
try:
|
||||
question_vec = await llm.embed_one(request.message)
|
||||
break
|
||||
except EmbeddingError as e:
|
||||
embed_ms = int((time.monotonic() - t0) * 1000)
|
||||
if attempt >= max_attempts:
|
||||
total_ms = int((time.monotonic() - started) * 1000)
|
||||
logger.error(
|
||||
"chat: question=%r embed_ms=%d total_ms=%d — embedding failed: %s",
|
||||
request.message,
|
||||
embed_ms,
|
||||
total_ms,
|
||||
e,
|
||||
)
|
||||
settled = True # terminal: the error frame settles the turn
|
||||
yield sse_event(
|
||||
ChatErrorEvent(
|
||||
detail="I couldn't reach the embedding model — please try again."
|
||||
).model_dump()
|
||||
)
|
||||
return
|
||||
logger.warning(
|
||||
"chat: question=%r embedding failed (attempt %d/%d) — "
|
||||
"retrying in %.1fs: %s",
|
||||
request.message,
|
||||
attempt,
|
||||
max_attempts,
|
||||
settings.llm_retry_delay,
|
||||
e,
|
||||
)
|
||||
retries_used += 1
|
||||
yield sse_event(
|
||||
ChatRetryEvent(
|
||||
attempt=attempt + 1, max_attempts=max_attempts
|
||||
).model_dump()
|
||||
)
|
||||
await asyncio.sleep(settings.llm_retry_delay)
|
||||
attempt += 1
|
||||
embed_ms = int((time.monotonic() - t0) * 1000)
|
||||
|
||||
# 2. Retrieve top-K chunks, load the owner's steering notes
|
||||
# (phase 15), then the honesty gate (A8) picks the HIGH
|
||||
# (grounded) or LOW (deflected) prompt + context.
|
||||
settings = get_settings()
|
||||
try:
|
||||
steering_notes = load_steering_notes(db)
|
||||
# KB overview (phase 31): one indexed PK lookup per turn —
|
||||
@@ -313,9 +366,21 @@ async def chat(
|
||||
# ``agent_max_rounds=0`` ``run_agent`` is a single
|
||||
# ``tools=None`` request anyway (the kill switch).
|
||||
holder = AgentHolder()
|
||||
answer_stream: AsyncIterator[StreamPiece | ToolCallPiece]
|
||||
answer_stream: AsyncIterator[StreamPiece | ToolCallPiece | RetryPiece]
|
||||
if plan.deflected:
|
||||
answer_stream = llm.chat_stream(messages)
|
||||
# Phase 67: the deflected stream goes through the retry
|
||||
# primitive — a dead endpoint is restarted (SSE ``retry``
|
||||
# frames) only before its first piece (locked A2); the
|
||||
# grounded path stays a plain ``run_agent`` call (task 03
|
||||
# makes IT retry internally) — its ``RetryPiece``s flow
|
||||
# through the shared piece loop below.
|
||||
answer_stream = chat_stream_retried(
|
||||
llm,
|
||||
messages,
|
||||
tools=None,
|
||||
retries=settings.llm_retries,
|
||||
delay=settings.llm_retry_delay,
|
||||
)
|
||||
else:
|
||||
answer_stream = run_agent(
|
||||
llm,
|
||||
@@ -328,7 +393,7 @@ async def chat(
|
||||
)
|
||||
thinking_chars = 0
|
||||
try:
|
||||
async for piece in answer_stream: # StreamPiece | ToolCallPiece
|
||||
async for piece in answer_stream: # StreamPiece | ToolCallPiece | RetryPiece
|
||||
if isinstance(piece, ToolCallPiece):
|
||||
# Phase 37 (PLAN §4 extension): one SSE ``tool``
|
||||
# frame per model-requested call; ``argument`` is
|
||||
@@ -345,6 +410,19 @@ async def chat(
|
||||
).model_dump()
|
||||
)
|
||||
continue
|
||||
if isinstance(piece, RetryPiece):
|
||||
# Phase 67: the answer stream was restarted before
|
||||
# its first piece (locked A2) — a transient status
|
||||
# frame, never an error. No other state changes:
|
||||
# the thinking/clock/timeout handling is the
|
||||
# client's job.
|
||||
retries_used += 1
|
||||
yield sse_event(
|
||||
ChatRetryEvent(
|
||||
attempt=piece.attempt, max_attempts=piece.max_attempts
|
||||
).model_dump()
|
||||
)
|
||||
continue
|
||||
if piece.kind == "thinking":
|
||||
thinking_chars += len(piece.text)
|
||||
if settings.stream_thinking:
|
||||
@@ -419,7 +497,7 @@ async def chat(
|
||||
logger.info(
|
||||
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d summary_hits=%d tuning=%d "
|
||||
"kb_chars=%d threshold=%.2f deflected=%s sources=%r thinking_chars=%d "
|
||||
"tool_calls=%d total_ms=%d",
|
||||
"tool_calls=%d total_ms=%d retries=%d",
|
||||
request.message,
|
||||
embed_ms,
|
||||
plan.top_score,
|
||||
@@ -433,6 +511,7 @@ async def chat(
|
||||
thinking_chars,
|
||||
holder.tool_calls,
|
||||
total_ms,
|
||||
retries_used,
|
||||
)
|
||||
settled = True # terminal: the done frame settles the turn
|
||||
yield sse_event(
|
||||
|
||||
Reference in New Issue
Block a user