feat(sync): fail fast with a modal when a model is unavailable
TODO.md L4: with a dead model endpoint the sync discovered it only mid-import, after slow clones — and a tooltip on the button is not a readable error. - app/rag/llm.py: ModelUnavailableError + check_models(llm) — a tiny pre-sync probe (one short embedding + one 1-token-scale completion) that fails naming the unavailable model (embed first, then the summary model); the sync sanitizer still masks credentials. - app/api/sync.py: the probe is step 1 of _run_sync — before source resolution and before any clone_or_pull; a model failure is just another 'failed' state (no new endpoint, A10/A12 untouched). - frontend/assets/header.js: applySyncFailure now also opens the module-owned error modal (every page carrying #sync-btn, zero page-markup changes): lazily built backdrop + role=alertdialog panel, error text via textContent, close via button / Esc / backdrop, focus in-and-out to #sync-btn (with a body→#sync-btn fallback — the run's disabled button drops focus to <body>). - frontend/assets/styles.css: the modal on the phase-08 error palette (z-index above the header, .is-open open/close, reduced-motion stilling, 44px close target). - Tests: probe unit tests (both up / embed down / summary down / custom model names), sync integration (fail-fast before any clone, probe-before-effective_sources ordering, credential masking, healthy regression), the phase-41 source pins, and the story E2E (two module apps on distinct ports — dead endpoint on a closed loopback port vs session mock: ≤10 s fail-fast + modal contract, all three dismissal paths with focus out to #sync-btn, button title/.is-error + Sources banner untouched, healthy phase-32 lifecycle regression to 'Synced HH:MM'). E2E (isolation): test_sync_model_down.py 4/4, test_sync_button.py 3/3, test_git_sources_admin.py 6/6, test_local_directory_sources.py 3/3; unit+integration 721 passed, app/ coverage 99%; ruff + pyright clean.
This commit is contained in:
+14
-6
@@ -15,25 +15,29 @@ stale button state (§7.4 adaptation, phase locked decisions).
|
||||
Pipeline (the canonical "mirror the sources" action — phase locked
|
||||
decisions):
|
||||
|
||||
1. resolve the effective sources — the ``git_sources`` DB rows (git
|
||||
1. verify ``embed`` + summary model availability — fail fast before
|
||||
any clone (:func:`app.rag.llm.check_models`, phase 41): a dead
|
||||
model endpoint aborts the run naming the unavailable model, before
|
||||
source resolution or any ``clone_or_pull``;
|
||||
2. resolve the effective sources — the ``git_sources`` DB rows (git
|
||||
**and** local, phase 38), else the ``BOR_GIT_SOURCES`` fallback
|
||||
(git-only)
|
||||
(:func:`app.rag.git_sources.effective_sources`, shared with the
|
||||
CLI) — empty on both origins (no git rows, no local rows, no env
|
||||
URLs) fails loudly (``no sources configured (git or local)``)
|
||||
instead of silently importing the legacy local directories;
|
||||
2. per resolved row: ``kind=git`` → :func:`scripts.git_sync.clone_or_pull`
|
||||
3. per resolved row: ``kind=git`` → :func:`scripts.git_sync.clone_or_pull`
|
||||
into ``BOR_SOURCES_DIR/<repo-name>/`` (phase 28 — reused, not
|
||||
re-implemented); ``kind=local`` → the stored directory, re-verified
|
||||
``.is_dir()`` **at sync time** (it may have moved/deleted since
|
||||
add-time) — a missing directory raises ``local source missing:
|
||||
<path>``; a failing clone or a missing local dir aborts before any
|
||||
import;
|
||||
3. ``import_sources(..., prune=True)`` over the single combined list
|
||||
4. ``import_sources(..., prune=True)`` over the single combined list
|
||||
(git checkouts + local dirs) — prune so files deleted upstream or
|
||||
out of a local dir leave the index (pruning covers the union; the
|
||||
CLI's no-prune default is unchanged);
|
||||
4. when the import changed the KB (added + updated > 0),
|
||||
5. when the import changed the KB (added + updated > 0),
|
||||
``regenerate_overview`` refreshes the single ``kb_overview`` row
|
||||
(phase 31 trigger, best-effort inside).
|
||||
|
||||
@@ -57,7 +61,7 @@ from app.core.auth import require_admin
|
||||
from app.db import SessionLocal
|
||||
from app.rag.git_sources import effective_sources
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.llm import LLMClient
|
||||
from app.rag.llm import LLMClient, check_models
|
||||
from app.rag.overview import regenerate_overview
|
||||
from scripts.git_sync import GitSyncError, clone_or_pull
|
||||
from scripts.import_docs import repo_name
|
||||
@@ -153,6 +157,11 @@ async def _run_sync() -> None:
|
||||
_status.error = None
|
||||
try:
|
||||
settings = get_settings()
|
||||
# Step 1 (phase 41): fail fast — verify both models the sync
|
||||
# needs (embed + summary) before source resolution or any
|
||||
# clone. The client is reused for the import + overview below.
|
||||
llm = LLMClient()
|
||||
await check_models(llm)
|
||||
# The background task has no request session: open a short-lived
|
||||
# one around the shared phase-35/38 resolver (DB rows of both
|
||||
# kinds win; the BOR_GIT_SOURCES git list is a fallback while
|
||||
@@ -189,7 +198,6 @@ async def _run_sync() -> None:
|
||||
if not path.is_dir():
|
||||
raise GitSyncError(f"local source missing: {path}")
|
||||
sources.append(path)
|
||||
llm = LLMClient()
|
||||
summary: ImportSummary = await import_sources(sources, llm, prune=True)
|
||||
overview = False
|
||||
if summary.added + summary.updated > 0:
|
||||
|
||||
@@ -44,6 +44,15 @@ class LLMError(RuntimeError):
|
||||
"""The chat-completions endpoint failed (network, HTTP, or mid-stream)."""
|
||||
|
||||
|
||||
class ModelUnavailableError(LLMError):
|
||||
"""One of the models a sync needs is unreachable (phase 41 probe).
|
||||
|
||||
Raised by :func:`check_models` when the pre-sync probe finds the
|
||||
embedding or summary model down; the message names the model so the
|
||||
admin can fix the right thing.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StreamPiece:
|
||||
"""One piece of a streamed chat turn (phase 17, PLAN §4 extension).
|
||||
@@ -408,3 +417,33 @@ class LLMClient:
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001 — wrap transport-level failures
|
||||
raise LLMError(f"chat stream from {self.settings.llm_base_url} failed: {e}") from e
|
||||
|
||||
|
||||
async def check_models(llm: LLMClient) -> None:
|
||||
"""Verify the models a sync needs (embed + summary) before any
|
||||
expensive work; raise ModelUnavailableError naming the model.
|
||||
|
||||
The probe is deliberately tiny — one short embedding
|
||||
(``sync model check``) and one 1-token-scale completion (``ping``)
|
||||
— so a dead endpoint is discovered cheaper than a single git
|
||||
clone. The sync sanitizer downstream (``app.api.sync
|
||||
._sanitize_error``) still masks any credentials embedded in the
|
||||
wrapped error text, so the raw endpoint URL in the original
|
||||
exception is safe to include.
|
||||
"""
|
||||
embed_model = llm.settings.llm_embed_model
|
||||
try:
|
||||
await llm.embed_one("sync model check")
|
||||
except Exception as e: # noqa: BLE001 — wrap EmbeddingError + transport failures
|
||||
raise ModelUnavailableError(
|
||||
f"The embedding model ('{embed_model}') is not available — "
|
||||
f"check the model endpoint and retry. ({e})"
|
||||
) from e
|
||||
summary_model = llm.settings.llm_summary_model
|
||||
try:
|
||||
await llm.chat([{"role": "user", "content": "ping"}])
|
||||
except Exception as e: # noqa: BLE001 — wrap LLMError + transport failures
|
||||
raise ModelUnavailableError(
|
||||
f"The summary model ('{summary_model}') is not available — "
|
||||
f"check the model endpoint and retry. ({e})"
|
||||
) from e
|
||||
|
||||
Reference in New Issue
Block a user