feat(rag): retry a failed LLM request before the first token lands — BOR_LLM_RETRIES/BOR_LLM_RETRY_DELAY with a live 'retrying' status

tests/e2e/mock_llm.py

@@ -112,6 +112,36 @@ Implements just enough of the aipi surface:
     answer; the E2E asks it against an on-topic fixture (HIGH gate) and
     asserts non-deflection.
 
+  Failure injection (phase 67, LLM retry, TODO.md L3) — deterministic
+  dead-endpoint behavior for the retry E2E suite (``tests/e2e/
+test_llm_retry.py``). The mock is single-conversation per e2e server, so
+  the sequences are driven by module-level counters that reset per
+  trigger phrase after the success they guard (a second question with
+  the same trigger re-drives the sequence from zero):
+  - user message containing ``fail then answer`` (``RETRY_TRIGGER``):
+    the first ``RETRY_DEAD_ATTEMPTS`` (2) app-level streaming attempts
+    respond 500 (JSON body, like a dead proxy) and the third streams
+    the normal composed answer — 2 = 1 original attempt + 1 retry under
+    the default ``BOR_LLM_RETRIES=3``, so a suite exercises a real
+    retry without waiting for the 4-attempt exhaustion. Counted in
+    APP-LEVEL attempts, not raw HTTP POSTs: while the endpoint stays
+    dead, the openai SDK's default policy (max_retries=2 — the app's
+    ``LLMClient`` keeps it) re-POSTs a 500'd streaming request twice
+    before surfacing the error, so each dead attempt costs exactly 3
+    POSTs (``_HTTPS_PER_DEAD_ATTEMPT``).
+  - user message containing ``always fail``
+    (``ALWAYS_FAIL_TRIGGER``): EVERY streaming chat/completions request
+    responds 500 — the retry-budget exhaustion path (the terminal
+    error banner in the UI).
+  - embeddings request whose input contains ``embed fail once``
+    (``EMBED_FAIL_TRIGGER``): the FIRST such request responds 500, the
+    next returns the normal bag-of-words vector — the endpoint's
+    pre-stream embedding retry loop. Raw httpx on the client side (no
+    SDK-level retries), so one POST per attempt: the counter is per
+    POST here, unlike the chat counter above.
+  Non-streaming requests (document summaries, KB overview) never 500 —
+  the retry scope is the chat turn only (owner-locked A1).
+
   ``max_tokens`` is honored deterministically (token ≈ whitespace word),
   like a real endpoint: an answer longer than the cap is truncated. This
   is what makes the phase-11 truncation regression observable.
@@ -129,7 +159,7 @@ import uuid
 from typing import Any
 
 from fastapi import FastAPI
-from fastapi.responses import StreamingResponse
+from fastapi.responses import JSONResponse, StreamingResponse
 
 app = FastAPI()
 
@@ -265,6 +295,76 @@ TABLE_ANSWER = (
     "| value-one | value-two | value-three | value-four | value-five |"
 )
 
+# ---------------------------------------------------------------------------
+# Phase 67 (LLM retry, TODO.md L3): deterministic failure injection
+# ---------------------------------------------------------------------------
+
+#: A user message containing this substring (case-insensitive) gets
+#: ``RETRY_DEAD_ATTEMPTS`` dead streaming attempts (500, JSON body) before
+#: the normal composed answer streams — 1 original attempt + 1 retry under
+#: the default ``BOR_LLM_RETRIES=3`` (see the module docstring).
+RETRY_TRIGGER = "fail then answer"
+#: App-level attempts the endpoint stays dead for before the answer.
+RETRY_DEAD_ATTEMPTS = 2
+
+#: A user message containing this substring (case-insensitive) makes
+#: EVERY streaming chat/completions request respond 500 — the
+#: retry-budget exhaustion path (the terminal error banner in the UI).
+ALWAYS_FAIL_TRIGGER = "always fail"
+
+#: An embeddings request whose input contains this substring
+#: (case-insensitive) 500s on its FIRST POST; the next returns the normal
+#: bag-of-words vector — the endpoint's pre-stream embedding retry loop.
+EMBED_FAIL_TRIGGER = "embed fail once"
+
+#: One DEAD app-level chat attempt costs exactly this many HTTP POSTs
+#: while the endpoint stays down: the openai SDK's default policy
+#: (max_retries=2 — the app's ``LLMClient`` keeps it) re-POSTs a 500'd
+#: streaming request twice before surfacing the error to
+#: ``chat_stream_retried``. The failure counters below therefore count
+#: app-level attempts (groups of this size), not raw POSTs — the visible
+#: sequence (one SSE ``retry`` frame after each dead attempt, the answer
+#: on the third) stays deterministic regardless of the SDK's internal
+#: backoff pacing.
+_HTTPS_PER_DEAD_ATTEMPT = 3
+
+#: Module-level failure counters — the mock is single-conversation per
+#: e2e server. Keyed by trigger phrase (reset per trigger): the number
+#: of matching POSTs served so far. Each sequence resets after the
+#: success it guards, so a second question carrying the same trigger
+#: re-drives the failure sequence from zero.
+_fail_posts: dict[str, int] = {}
+
+
+def _llm_500(why: str) -> JSONResponse:
+    """A dead-proxy 500 with a JSON error body (phase 67 injection)."""
+    return JSONResponse(
+        status_code=500,
+        content={
+            "error": {
+                "message": f"upstream connection reset ({why})",
+                "type": "proxy_error",
+            }
+        },
+    )
+
+
+def _bump_fail(key: str) -> int:
+    n = _fail_posts.get(key, 0) + 1
+    _fail_posts[key] = n
+    return n
+
+
+def _chat_dead(key: str, dead_attempts: int) -> bool:
+    """Bump *key*'s counter; True while the endpoint stays dead.
+
+    Counted in app-level attempts (see ``_HTTPS_PER_DEAD_ATTEMPT``): the
+    first ``dead_attempts * _HTTPS_PER_DEAD_ATTEMPT`` POSTs 500 and the
+    next attempt's first POST streams (the caller resets the counter on
+    the success).
+    """
+    return _bump_fail(key) <= dead_attempts * _HTTPS_PER_DEAD_ATTEMPT
+
 
 #: The agent's ``read_document`` tool-result prefix (app.rag.agent
 #: ``_execute_tool``): ``"Document <source/path>:\n<content>"``.
@@ -656,11 +756,22 @@ def models() -> dict[str, Any]:
 
 
 @app.post("/v1/embeddings")
-def embeddings(body: dict[str, Any]) -> dict[str, Any]:
+def embeddings(body: dict[str, Any]) -> Any:  # dict, or a 500 (phase 67)
     raw = body.get("input")
     if isinstance(raw, str):
         raw = [raw]
     inputs: list[Any] = list(raw) if isinstance(raw, list) else []
+    # Phase 67 (embedding retry): the first embeddings request whose
+    # input carries the marker 500s; the next returns the normal
+    # bag-of-words vector (see the module docstring). Raw httpx on the
+    # client side — no SDK-level retries — so one POST per app attempt:
+    # the counter is per POST here (unlike the chat counter below).
+    joined = " ".join(str(t) for t in inputs if isinstance(t, str)).lower()
+    if EMBED_FAIL_TRIGGER in joined:
+        n = _bump_fail(EMBED_FAIL_TRIGGER)
+        if n == 1:
+            return _llm_500(EMBED_FAIL_TRIGGER)
+        _fail_posts[EMBED_FAIL_TRIGGER] = 0  # the vector went out — restart
     data = [
         {"object": "embedding", "index": i, "embedding": embed_text(t)}
         for i, t in enumerate(inputs)
@@ -827,6 +938,16 @@ def chat_completions(body: dict[str, Any]) -> Any:
     # the flow handles streaming requests; a non-streaming marker request
     # (never issued by the app) falls through to the regular answer.
     if body.get("stream"):
+        # Phase 67 (LLM retry): deterministic failure injection — see
+        # the module docstring. Checked before the marker tool flow: the
+        # injection markers never combine with the tool-flow markers in
+        # any suite, and a dead endpoint answers nothing (no flow).
+        if ALWAYS_FAIL_TRIGGER in user_lower:
+            return _llm_500(ALWAYS_FAIL_TRIGGER)
+        if RETRY_TRIGGER in user_lower:
+            if _chat_dead(RETRY_TRIGGER, RETRY_DEAD_ATTEMPTS):
+                return _llm_500(RETRY_TRIGGER)
+            _fail_posts[RETRY_TRIGGER] = 0  # the answer streamed — restart
         flow = _tool_flow(body)
         if flow is not None:
             if flow[0] == "list":