feat(sync): fail fast with a modal when a model is unavailable

TODO.md L4: with a dead model endpoint the sync discovered it only mid-import, after slow clones — and a tooltip on the button is not a readable error. - app/rag/llm.py: ModelUnavailableError + check_models(llm) — a tiny pre-sync probe (one short embedding + one 1-token-scale completion) that fails naming the unavailable model (embed first, then the summary model); the sync sanitizer still masks credentials. - app/api/sync.py: the probe is step 1 of _run_sync — before source resolution and before any clone_or_pull; a model failure is just another 'failed' state (no new endpoint, A10/A12 untouched). - frontend/assets/header.js: applySyncFailure now also opens the module-owned error modal (every page carrying #sync-btn, zero page-markup changes): lazily built backdrop + role=alertdialog panel, error text via textContent, close via button / Esc / backdrop, focus in-and-out to #sync-btn (with a body→#sync-btn fallback — the run's disabled button drops focus to <body>). - frontend/assets/styles.css: the modal on the phase-08 error palette (z-index above the header, .is-open open/close, reduced-motion stilling, 44px close target). - Tests: probe unit tests (both up / embed down / summary down / custom model names), sync integration (fail-fast before any clone, probe-before-effective_sources ordering, credential masking, healthy regression), the phase-41 source pins, and the story E2E (two module apps on distinct ports — dead endpoint on a closed loopback port vs session mock: ≤10 s fail-fast + modal contract, all three dismissal paths with focus out to #sync-btn, button title/.is-error + Sources banner untouched, healthy phase-32 lifecycle regression to 'Synced HH:MM'). E2E (isolation): test_sync_model_down.py 4/4, test_sync_button.py 3/3, test_git_sources_admin.py 6/6, test_local_directory_sources.py 3/3; unit+integration 721 passed, app/ coverage 99%; ruff + pyright clean.
2026-08-27 23:44:35 -04:00
parent 6f9e033117
commit 6cf1df9bf2
9 changed files with 1203 additions and 12 deletions
@@ -29,6 +29,16 @@ test), so DB-over-env and the env fallback go through the actual
 indirection; the env list is driven by a fresh ``Settings`` on the
 resolver's module (the dev ``.env`` never leaks in).

+Phase 41: the pre-sync model probe (``check_models``) — a dead model
+endpoint fails the run **before any clone** (the model-naming error
+lands in the ``failed`` state verbatim; the sanitizer is a no-op for
+it); the probe runs before ``effective_sources``; a real probe against
+a stubbed dead ``LLMClient`` names the embed model **and** masks the
+credentials wrapped from the endpoint URL. Where the runner keeps a
+real ``LLMClient`` the probe is stubbed (:func:`_stub_probe`) so no
+test ever hits the network; the ``_real_llm`` tests get a passing
+probe from ``FakeEmbedder.embed_one``/``chat``.
+
 The git / import / overview layers are monkeypatched in ``app.api.sync``
 (same fake style as ``test_import_docs_git.py``) — no real git, no LLM:
 the runner's state machine and HTTP surface are under test.
@@ -47,6 +57,7 @@ import time
 from collections.abc import Iterator
 from datetime import UTC, datetime
 from pathlib import Path
+from typing import Any

 import pytest
 from fastapi.testclient import TestClient
@@ -59,7 +70,7 @@ from app.main import app as fastapi_app
 from app.models import GitSource
 from app.rag import git_sources as git_sources_resolver
 from app.rag.importer import ImportSummary
-from app.rag.llm import EmbeddingError, LLMClient
+from app.rag.llm import EmbeddingError, LLMClient, ModelUnavailableError
 from scripts.git_sync import GitSyncError
 from tests.conftest import ADMIN_PASSWORD
 from tests.fakes import FakeEmbedder
@@ -142,10 +153,25 @@ def clean_documents(db: Session) -> Iterator[None]:

 def _real_llm(monkeypatch: pytest.MonkeyPatch) -> None:
    """The pipeline's ``LLMClient`` becomes the deterministic in-process
-    ``FakeEmbedder`` (real import, no network)."""
+    ``FakeEmbedder`` (real import, no network). ``FakeEmbedder``
+    implements ``embed_one`` + ``chat``, so the phase-41 probe passes
+    against it without a stub."""
    monkeypatch.setattr(sync_api, "LLMClient", lambda: FakeEmbedder())


+def _stub_probe(monkeypatch: pytest.MonkeyPatch) -> list[Any]:
+    """Stub the phase-41 model probe (a real ``LLMClient`` in the runner
+    would hit the network). Returns the clients the probe received, so
+    tests can assert the probe and the import share one client."""
+    seen: list[Any] = []
+
+    async def fake_check_models(llm: LLMClient) -> None:
+        seen.append(llm)
+
+    monkeypatch.setattr(sync_api, "check_models", fake_check_models)
+    return seen
+
+
 def _login(client: TestClient) -> None:
    r = client.post("/api/login", json={"password": ADMIN_PASSWORD})
    assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}"
@@ -250,6 +276,7 @@ def test_admin_sync_success_reports_full_detail(
    )
    clone_calls, fake_clone = _fake_clone()
    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    probe_seen = _stub_probe(monkeypatch)  # real LLMClient — probe stubbed
    summary = ImportSummary(
        files=5, added=1, updated=2, unchanged=2, pruned=3, errors=0,
        chunks=11, embed_batches=4, summaries=1, summary_errors=0,
@@ -294,6 +321,9 @@ def test_admin_sync_success_reports_full_detail(
    assert isinstance(fake_import.llms[0], LLMClient)
    # Overview: refreshed (added + updated > 0) with the same client.
    assert fake_overview.llms == [fake_import.llms[0]]
+    # Phase 41: the probe ran first and got the very client the import
+    # and the overview reuse.
+    assert probe_seen == [fake_import.llms[0]]


 def test_unchanged_kb_skips_overview_refresh(
@@ -310,6 +340,7 @@ def test_unchanged_kb_skips_overview_refresh(
    )
    _, fake_clone = _fake_clone()
    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    _stub_probe(monkeypatch)
    summary = ImportSummary(files=7, added=0, updated=0, unchanged=7, pruned=0)
    fake_import = FakeImportSources(summary)
    monkeypatch.setattr(sync_api, "import_sources", fake_import)
@@ -341,6 +372,7 @@ def test_double_trigger_while_running_returns_409(
    )
    _, fake_clone = _fake_clone()
    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    _stub_probe(monkeypatch)
    # The in-flight run takes a while (asyncio.sleep) so the second POST
    # lands while it is still running.
    fake_import = FakeImportSources(ImportSummary(files=1, added=1), delay=0.5)
@@ -387,6 +419,7 @@ def test_git_failure_marks_failed_and_skips_import(
        )

    monkeypatch.setattr(sync_api, "clone_or_pull", failing_clone)
+    _stub_probe(monkeypatch)
    fake_import = FakeImportSources(ImportSummary())
    monkeypatch.setattr(sync_api, "import_sources", fake_import)
    fake_overview = FakeOverview(ok=True)
@@ -423,6 +456,7 @@ def test_no_sources_configured_fails_loudly(
    )
    clone_calls, fake_clone = _fake_clone()
    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    _stub_probe(monkeypatch)
    fake_import = FakeImportSources(ImportSummary())
    monkeypatch.setattr(sync_api, "import_sources", fake_import)

@@ -454,6 +488,7 @@ def test_db_rows_win_over_env(
    )
    clone_calls, fake_clone = _fake_clone()
    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    _stub_probe(monkeypatch)
    fake_import = FakeImportSources(ImportSummary(files=1, added=1))
    monkeypatch.setattr(sync_api, "import_sources", fake_import)
    monkeypatch.setattr(sync_api, "regenerate_overview", FakeOverview(ok=True))
@@ -486,6 +521,7 @@ def test_env_fallback_when_table_empty(
    )
    clone_calls, fake_clone = _fake_clone()
    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    _stub_probe(monkeypatch)
    fake_import = FakeImportSources(ImportSummary(files=1, added=1))
    monkeypatch.setattr(sync_api, "import_sources", fake_import)
    monkeypatch.setattr(sync_api, "regenerate_overview", FakeOverview(ok=True))
@@ -618,6 +654,7 @@ def test_missing_local_dir_fails_loudly_and_imports_nothing(
    )
    clone_calls, fake_clone = _fake_clone()
    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    _stub_probe(monkeypatch)
    fake_import = FakeImportSources(ImportSummary())
    monkeypatch.setattr(sync_api, "import_sources", fake_import)
    fake_overview = FakeOverview(ok=True)
@@ -647,6 +684,7 @@ def test_import_error_is_reported_with_credentials_masked(
    )
    _, fake_clone = _fake_clone()
    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    _stub_probe(monkeypatch)

    async def failing_import(
        sources: list[Path],
@@ -671,3 +709,135 @@ def test_import_error_is_reported_with_credentials_masked(
    assert "*****@aipi.reeseapps.com" in body["error"]  # credentials masked
    assert "user:secret" not in body["error"]
    assert "connection refused" in body["error"]  # the reason survives
+
+
+# --- phase 41: model probe (fail fast before any clone) --------------------
+
+
+class _DeadEmbedder:
+    """Probe fake whose embedding call dies like a dead endpoint — with a
+    credential-bearing URL in the wrapped error (the sanitizer must
+    mask it). ``chat`` is recorded: it must never be reached."""
+
+    def __init__(self) -> None:
+        self.settings = Settings(_env_file=None)  # pyright: ignore[reportCallIssue]
+        self.embed_batches = 0
+        self.chat_calls: list[list[dict[str, str]]] = []
+
+    async def embed_one(self, text: str) -> list[float]:
+        raise EmbeddingError(
+            "embeddings request to https://user:secret@aipi.reeseapps.com/v1 "
+            "failed: connection refused"
+        )
+
+    async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str:
+        self.chat_calls.append(list(messages))
+        return "pong"
+
+
+def test_model_down_fails_fast_before_any_clone(
+    sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
+) -> None:
+    """A dead model endpoint fails the run **before any clone**: the
+    model-naming error lands in the ``failed`` state verbatim (the
+    sanitizer is a no-op for it), and clone + import never run."""
+    repo_url = f"file://{tmp_path / 'repo.git'}"
+    _seed(db, repo_url)
+    _stub_env(monkeypatch)
+    monkeypatch.setattr(
+        sync_api,
+        "get_settings",
+        lambda: _settings(sources_dir=str(tmp_path / "bor")),
+    )
+    clone_calls, fake_clone = _fake_clone()
+    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    fake_import = FakeImportSources(ImportSummary())
+    monkeypatch.setattr(sync_api, "import_sources", fake_import)
+    message = (
+        "The embedding model ('embed') is not available — "
+        "check the model endpoint and retry."
+    )
+
+    async def dead_probe(llm: LLMClient) -> None:
+        raise ModelUnavailableError(message)
+
+    monkeypatch.setattr(sync_api, "check_models", dead_probe)
+
+    _login(sync_client)
+    assert sync_client.post("/api/sync").status_code == 202
+
+    body = _poll(sync_client, "failed")
+    assert body["error"] == message  # the model is named, verbatim
+    # The sanitizer must leave the model-naming message untouched.
+    assert sync_api._sanitize_error(body["error"]) == body["error"]
+    assert body["detail"] == {}
+    assert clone_calls == []  # fail fast: before any clone
+    assert fake_import.sources == []  # and before any import
+
+
+def test_probe_names_dead_embed_model_and_masks_credentials(
+    sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path
+) -> None:
+    """The **real** probe against a stubbed dead client: the error names
+    the embedding model, and the credentials wrapped from the endpoint
+    URL are masked by the sync sanitizer (the reason survives)."""
+    repo_url = f"file://{tmp_path / 'repo.git'}"
+    _seed(db, repo_url)
+    _stub_env(monkeypatch)
+    monkeypatch.setattr(
+        sync_api,
+        "get_settings",
+        lambda: _settings(sources_dir=str(tmp_path / "bor")),
+    )
+    dead = _DeadEmbedder()
+    monkeypatch.setattr(sync_api, "LLMClient", lambda: dead)
+    clone_calls, fake_clone = _fake_clone()
+    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+    fake_import = FakeImportSources(ImportSummary())
+    monkeypatch.setattr(sync_api, "import_sources", fake_import)
+
+    _login(sync_client)
+    assert sync_client.post("/api/sync").status_code == 202
+
+    body = _poll(sync_client, "failed")
+    assert "The embedding model ('embed') is not available" in body["error"]
+    assert "*****@aipi.reeseapps.com" in body["error"]  # credentials masked
+    assert "user:secret" not in body["error"]
+    assert "connection refused" in body["error"]  # the reason survives
+    assert dead.chat_calls == []  # embed died first — chat never probed
+    assert clone_calls == []
+    assert fake_import.sources == []
+
+
+def test_probe_runs_before_source_resolution(
+    sync_client: TestClient, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """Ordering: the probe is the **first** pipeline step — it runs
+    before ``effective_sources`` (and, transitively, before any
+    clone)."""
+    order: list[str] = []
+
+    async def spy_probe(llm: LLMClient) -> None:
+        order.append("probe")
+
+    def spy_effective_sources(session: Session) -> tuple[list[GitSource], str]:
+        order.append("effective_sources")
+        raise GitSyncError("short-circuit after the ordering spy")
+
+    monkeypatch.setattr(sync_api, "check_models", spy_probe)
+    monkeypatch.setattr(sync_api, "effective_sources", spy_effective_sources)
+    monkeypatch.setattr(
+        sync_api,
+        "get_settings",
+        lambda: _settings(sources_dir=str(tmp_path / "bor")),
+    )
+    clone_calls, fake_clone = _fake_clone()
+    monkeypatch.setattr(sync_api, "clone_or_pull", fake_clone)
+
+    _login(sync_client)
+    assert sync_client.post("/api/sync").status_code == 202
+    body = _poll(sync_client, "failed")
+
+    assert order == ["probe", "effective_sources"]
+    assert clone_calls == []
+    assert "short-circuit" in body["error"]  # the spy aborted the run