feat(rag): retry a failed LLM request before the first token lands — BOR_LLM_RETRIES/BOR_LLM_RETRY_DELAY with a live 'retrying' status

app/rag/llm.py

@@ -18,6 +18,7 @@ vectors that pgvector rejects.
 """
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 from collections.abc import AsyncGenerator
@@ -82,6 +83,25 @@ class ToolCallPiece:
     arguments: dict[str, Any]
 
 
+@dataclass(frozen=True)
+class RetryPiece:
+    """One LLM request retry that is about to start (phase 67, locked A2).
+
+    ``attempt`` is the 1-based number of the attempt that is about to be
+    tried — the one AFTER the attempt that just failed (a first-attempt
+    failure carries ``attempt=2``, so the API's SSE ``retry`` frame reads
+    "retrying (2 of N)" — the same convention the endpoint's embedding
+    retry loop uses, phase 67 task 02); ``max_attempts`` is the total
+    attempt budget (``llm_retries + 1``). One piece per wait: the API
+    layer turns it into an SSE ``retry`` frame, and it always precedes
+    the pre-retry sleep so the frame reaches the client before the wait
+    starts.
+    """
+
+    attempt: int
+    max_attempts: int
+
+
 @dataclass
 class _ToolCallSlot:
     """Mutable accumulator for one streamed tool call (phase 37, private).
@@ -442,6 +462,80 @@ class LLMClient:
                 await stream.close()
 
 
+async def chat_stream_retried(
+    llm: LLMClient,
+    messages: list[dict[str, str]],
+    *,
+    tools: list[dict[str, Any]] | None = None,
+    retries: int = 0,
+    delay: float = 0.0,
+) -> AsyncGenerator[StreamPiece | ToolCallPiece | RetryPiece, None]:
+    """Stream a chat turn, retrying a dead endpoint (phase 67).
+
+    Wraps :meth:`LLMClient.chat_stream` with the retry-before-first-piece
+    rule (owner-locked A2): a request is restarted **only** while no output
+    piece (thinking/tool/delta) has been yielded for it. Once pieces have
+    flowed, an :class:`LLMError` is re-raised unchanged — a partial answer
+    is never redone, and the API layer's terminal ``error`` frame applies.
+    This primitive is the ONLY place that rule lives (the chat endpoint
+    and the agent loop both build on it).
+
+    Up to *retries* restarts after the initial attempt (``retries + 1``
+    attempts total; ``retries=0`` is exactly one attempt with no
+    :class:`RetryPiece` — the pre-phase-67 kill-switch path). Each restart
+    is preceded by one :class:`RetryPiece` (``attempt`` = the 1-based
+    number of the attempt about to be tried — the failed attempt + 1 —
+    and ``max_attempts`` = ``retries + 1``) and a flat
+    ``asyncio.sleep(delay)`` — the TODO-locked fixed interval, no
+    backoff. The ``RetryPiece`` always precedes its sleep: the API frame
+    must reach the client before the wait starts.
+
+    The request is restarted byte-identical: ``chat_stream`` is stateless,
+    so every attempt is opened with the SAME *messages*/*tools*.
+
+    Teardown (phase 48, extended): every attempt's stream is explicitly
+    closed in a ``finally`` — normal exhaustion, a terminal
+    :class:`LLMError`, and a consumer abandon (``GeneratorExit`` mid-attempt
+    or during the pre-retry sleep) all run it, so an abandoned turn never
+    leaves the endpoint's stream open.
+    """
+    max_attempts = retries + 1
+    for attempt in range(1, max_attempts + 1):
+        emitted = False
+        stream = llm.chat_stream(messages, tools=tools)
+        try:
+            async for piece in stream:
+                emitted = True
+                yield piece
+        except LLMError as e:
+            if emitted:
+                # Locked A2: tokens already flowed — the failure is
+                # terminal, never redo a partial answer.
+                raise
+            if attempt >= max_attempts:
+                # Retries exhausted — the API layer turns this into the
+                # terminal error frame.
+                raise
+            logger.warning(
+                "llm stream failed before the first piece (attempt %d/%d) — "
+                "retrying in %.1fs: %s",
+                attempt,
+                max_attempts,
+                delay,
+                e,
+            )
+            yield RetryPiece(attempt + 1, max_attempts)
+            await asyncio.sleep(delay)
+        else:
+            # A fully consumed attempt — the turn is done; the loop must
+            # NOT open another stream for an attempt that never failed.
+            return
+        finally:
+            # Phase 48 teardown for THIS attempt's stream (a no-op once the
+            # stream already ended; the real close on a consumer abandon).
+            await stream.aclose()
+
+
 async def check_models(llm: LLMClient) -> None:
     """Verify the models a sync needs (embed + summary) before any
     expensive work; raise ModelUnavailableError naming the model.