Experimental/brain-of-reese
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rag): retry a failed LLM request before the first token lands — BOR_LLM_RETRIES/BOR_LLM_RETRY_DELAY with a live 'retrying' status

Browse Source
This commit is contained in:
ducoterra
2026-09-02 10:52:38 -04:00
parent f04ddbe1f8
commit 88293ed02f
44 changed files with 2488 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/chat.py
+103 -24
View File
@@ -68,9 +68,27 @@ byte-identical to the pre-phase path (A8):** the LOW prompt never
carries tools, and with ``agent_max_rounds`` at **0** ``run_agent``
makes exactly one ``tools=None`` request, reproducing the pre-phase
behavior (the kill switch).
LLM retries (phase 67, ``BOR_LLM_RETRIES`` / ``BOR_LLM_RETRY_DELAY``,
owner-locked 2026-09-01): when the aipi endpoint dies before a request
has streamed its first output frame (locked A2), the request is
restarted — up to ``llm_retries`` times (default 3), a flat
``llm_retry_delay`` (default 5 s) between attempts. Every restart is
announced with an SSE ``retry`` frame (``{"type": "retry",
"attempt": n, "max_attempts": N}`` — the attempt about to be tried,
1-based, ahead of the pre-retry wait) so the UI can show the transient
"Communication interrupted — retrying (n of N)…" status (locked A4);
exhaustion and any failure after the first frame keep the existing
terminal ``error`` frames. The pre-stream question embedding retries on
the same budget, and the deflected answer stream goes through
``chat_stream_retried`` (the agent loop retries per round — task 03).
The per-turn log line records ``retries=N`` after ``total_ms=N`` (0
when nothing was retried — the field is uniform across all turn
shapes).
"""
from __future__ import annotations
import asyncio
import json
import logging
import time
@@ -91,8 +109,10 @@ from app.rag.llm import (
EmbeddingError,
LLMClient,
LLMError,
RetryPiece, # phase 67: one LLM request restart (an SSE retry frame)
StreamPiece, # type of the answer pieces streamed by the agent loop
ToolCallPiece, # phase 37: one model-requested tool call
chat_stream_retried, # phase 67: the retry-before-first-piece primitive
)
from app.rag.overview import load_kb_overview
from app.rag.prompts import build_deflect_prompt, build_high_prompt
@@ -102,6 +122,7 @@ from app.schemas import (
ChatDoneEvent,
ChatErrorEvent,
ChatRequest,
ChatRetryEvent,
ChatThinkingEvent,
ChatToolEvent,
SourceRef,
@@ -246,34 +267,66 @@ async def chat(
# consumer went away before any terminal frame; it must not
# yield (GeneratorExit handling).
settled = False
retries_used = 0 # phase 67: LLM requests restarted this turn (log line)
try:
settings = get_settings()
# 1. Embed the question.
# Phase 67: a dead embeddings endpoint is retried before any
# frame has left the server — up to ``llm_retries`` restarts,
# a flat ``llm_retry_delay`` between attempts, one SSE
# ``retry`` frame per restart (the UI shows the transient
# "retrying" status, not an error — locked A4). The final
# failure keeps the EXISTING terminal ``error`` frame (the
# copy reads correctly after N tries); ``llm_retries=0`` is
# byte-identical to the pre-phase-67 single attempt.
t0 = time.monotonic()
try:
question_vec = await llm.embed_one(request.message)
except EmbeddingError as e:
embed_ms = int((time.monotonic() - t0) * 1000)
total_ms = int((time.monotonic() - started) * 1000)
logger.error(
"chat: question=%r embed_ms=%d total_ms=%d — embedding failed: %s",
request.message,
embed_ms,
total_ms,
e,
)
settled = True # terminal: the error frame settles the turn
yield sse_event(
ChatErrorEvent(
detail="I couldn't reach the embedding model — please try again."
).model_dump()
)
return
max_attempts = settings.llm_retries + 1
attempt = 1
while True:
try:
question_vec = await llm.embed_one(request.message)
break
except EmbeddingError as e:
embed_ms = int((time.monotonic() - t0) * 1000)
if attempt >= max_attempts:
total_ms = int((time.monotonic() - started) * 1000)
logger.error(
"chat: question=%r embed_ms=%d total_ms=%d — embedding failed: %s",
request.message,
embed_ms,
total_ms,
e,
)
settled = True # terminal: the error frame settles the turn
yield sse_event(
ChatErrorEvent(
detail="I couldn't reach the embedding model — please try again."
).model_dump()
)
return
logger.warning(
"chat: question=%r embedding failed (attempt %d/%d) — "
"retrying in %.1fs: %s",
request.message,
attempt,
max_attempts,
settings.llm_retry_delay,
e,
)
retries_used += 1
yield sse_event(
ChatRetryEvent(
attempt=attempt + 1, max_attempts=max_attempts
).model_dump()
)
await asyncio.sleep(settings.llm_retry_delay)
attempt += 1
embed_ms = int((time.monotonic() - t0) * 1000)
# 2. Retrieve top-K chunks, load the owner's steering notes
# (phase 15), then the honesty gate (A8) picks the HIGH
# (grounded) or LOW (deflected) prompt + context.
settings = get_settings()
try:
steering_notes = load_steering_notes(db)
# KB overview (phase 31): one indexed PK lookup per turn —
@@ -313,9 +366,21 @@ async def chat(
# ``agent_max_rounds=0`` ``run_agent`` is a single
# ``tools=None`` request anyway (the kill switch).
holder = AgentHolder()
answer_stream: AsyncIterator[StreamPiece | ToolCallPiece]
answer_stream: AsyncIterator[StreamPiece | ToolCallPiece | RetryPiece]
if plan.deflected:
answer_stream = llm.chat_stream(messages)
# Phase 67: the deflected stream goes through the retry
# primitive — a dead endpoint is restarted (SSE ``retry``
# frames) only before its first piece (locked A2); the
# grounded path stays a plain ``run_agent`` call (task 03
# makes IT retry internally) — its ``RetryPiece``s flow
# through the shared piece loop below.
answer_stream = chat_stream_retried(
llm,
messages,
tools=None,
retries=settings.llm_retries,
delay=settings.llm_retry_delay,
)
else:
answer_stream = run_agent(
llm,
@@ -328,7 +393,7 @@ async def chat(
)
thinking_chars = 0
try:
async for piece in answer_stream: # StreamPiece | ToolCallPiece
async for piece in answer_stream: # StreamPiece | ToolCallPiece | RetryPiece
if isinstance(piece, ToolCallPiece):
# Phase 37 (PLAN §4 extension): one SSE ``tool``
# frame per model-requested call; ``argument`` is
@@ -345,6 +410,19 @@ async def chat(
).model_dump()
)
continue
if isinstance(piece, RetryPiece):
# Phase 67: the answer stream was restarted before
# its first piece (locked A2) — a transient status
# frame, never an error. No other state changes:
# the thinking/clock/timeout handling is the
# client's job.
retries_used += 1
yield sse_event(
ChatRetryEvent(
attempt=piece.attempt, max_attempts=piece.max_attempts
).model_dump()
)
continue
if piece.kind == "thinking":
thinking_chars += len(piece.text)
if settings.stream_thinking:
@@ -419,7 +497,7 @@ async def chat(
logger.info(
"question=%r embed_ms=%d top_score=%.3f fts_hits=%d summary_hits=%d tuning=%d "
"kb_chars=%d threshold=%.2f deflected=%s sources=%r thinking_chars=%d "
"tool_calls=%d total_ms=%d",
"tool_calls=%d total_ms=%d retries=%d",
request.message,
embed_ms,
plan.top_score,
@@ -433,6 +511,7 @@ async def chat(
thinking_chars,
holder.tool_calls,
total_ms,
retries_used,
)
settled = True # terminal: the done frame settles the turn
yield sse_event(