feat(sync): fail fast with a modal when a model is unavailable
TODO.md L4: with a dead model endpoint the sync discovered it only mid-import, after slow clones — and a tooltip on the button is not a readable error. - app/rag/llm.py: ModelUnavailableError + check_models(llm) — a tiny pre-sync probe (one short embedding + one 1-token-scale completion) that fails naming the unavailable model (embed first, then the summary model); the sync sanitizer still masks credentials. - app/api/sync.py: the probe is step 1 of _run_sync — before source resolution and before any clone_or_pull; a model failure is just another 'failed' state (no new endpoint, A10/A12 untouched). - frontend/assets/header.js: applySyncFailure now also opens the module-owned error modal (every page carrying #sync-btn, zero page-markup changes): lazily built backdrop + role=alertdialog panel, error text via textContent, close via button / Esc / backdrop, focus in-and-out to #sync-btn (with a body→#sync-btn fallback — the run's disabled button drops focus to <body>). - frontend/assets/styles.css: the modal on the phase-08 error palette (z-index above the header, .is-open open/close, reduced-motion stilling, 44px close target). - Tests: probe unit tests (both up / embed down / summary down / custom model names), sync integration (fail-fast before any clone, probe-before-effective_sources ordering, credential masking, healthy regression), the phase-41 source pins, and the story E2E (two module apps on distinct ports — dead endpoint on a closed loopback port vs session mock: ≤10 s fail-fast + modal contract, all three dismissal paths with focus out to #sync-btn, button title/.is-error + Sources banner untouched, healthy phase-32 lifecycle regression to 'Synced HH:MM'). E2E (isolation): test_sync_model_down.py 4/4, test_sync_button.py 3/3, test_git_sources_admin.py 6/6, test_local_directory_sources.py 3/3; unit+integration 721 passed, app/ coverage 99%; ruff + pyright clean.
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
"""Unit tests: the phase-41 pre-sync model probe (``check_models``).
|
||||
|
||||
The probe verifies both models a sync needs — ``embed`` (one short
|
||||
embedding) and the summary model ``lite`` (one 1-token-scale
|
||||
completion) — before any expensive work, and fails with a
|
||||
: class:`ModelUnavailableError` that **names the dead model**. The fake
|
||||
client is duck-typed (``embed_one`` / ``chat`` + ``settings``), same
|
||||
shape as :class:`tests.fakes.FakeEmbedder` — no network, no httpx
|
||||
transport.
|
||||
|
||||
Covered paths (keep ``app/`` >90%):
|
||||
- both models up → returns, both methods called exactly once;
|
||||
- embedding probe raises → ``ModelUnavailableError`` naming the
|
||||
**embedding model**, and the summary probe is **never** reached;
|
||||
- summary probe raises → ``ModelUnavailableError`` naming the
|
||||
**summary model**;
|
||||
- message content: model name present, "not available" wording, the
|
||||
original exception chained as ``__cause__``;
|
||||
- the error is an ``LLMError`` (the sync's generic failure catch).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Settings
|
||||
from app.rag.llm import (
|
||||
EmbeddingError,
|
||||
LLMClient,
|
||||
LLMError,
|
||||
ModelUnavailableError,
|
||||
check_models,
|
||||
)
|
||||
|
||||
|
||||
class _ProbeClient:
|
||||
"""Duck-typed :class:`app.rag.llm.LLMClient` stand-in for the probe."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embed_error: Exception | None = None,
|
||||
chat_error: Exception | None = None,
|
||||
embed_model: str = "embed",
|
||||
summary_model: str = "lite",
|
||||
) -> None:
|
||||
kwargs: dict[str, Any] = {
|
||||
"_env_file": None,
|
||||
"llm_embed_model": embed_model,
|
||||
"llm_summary_model": summary_model,
|
||||
}
|
||||
self.settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
self.embed_error = embed_error
|
||||
self.chat_error = chat_error
|
||||
self.embed_texts: list[str] = []
|
||||
self.chat_messages: list[list[dict[str, str]]] = []
|
||||
|
||||
async def embed_one(self, text: str) -> list[float]:
|
||||
self.embed_texts.append(text)
|
||||
if self.embed_error is not None:
|
||||
raise self.embed_error
|
||||
return [0.0] * 768
|
||||
|
||||
async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str:
|
||||
self.chat_messages.append(list(messages))
|
||||
if self.chat_error is not None:
|
||||
raise self.chat_error
|
||||
return "pong"
|
||||
|
||||
|
||||
def _run_check(llm: _ProbeClient) -> Any:
|
||||
# The probe is duck-typed at the ``embed_one`` / ``chat`` surface — the
|
||||
# cast keeps pyright honest about the narrow stand-in.
|
||||
return asyncio.run(check_models(cast("LLMClient", llm)))
|
||||
|
||||
|
||||
# --- both models up --------------------------------------------------------
|
||||
|
||||
|
||||
def test_both_models_up_returns_and_calls_both() -> None:
|
||||
llm = _ProbeClient()
|
||||
|
||||
assert _run_check(llm) is None
|
||||
assert llm.embed_texts == ["sync model check"] # the tiny probe input
|
||||
assert llm.chat_messages == [[{"role": "user", "content": "ping"}]]
|
||||
|
||||
|
||||
def test_chat_probe_defaults_to_summary_model_setting() -> None:
|
||||
"""The probe calls ``chat`` without a model override — the client's
|
||||
``llm_summary_model`` default (``lite``) is what gets pinged."""
|
||||
llm = _ProbeClient()
|
||||
|
||||
_run_check(llm)
|
||||
# ``chat`` received exactly the probe messages; the model default is
|
||||
# resolved inside LLMClient.chat from settings.llm_summary_model.
|
||||
assert llm.settings.llm_summary_model == "lite"
|
||||
assert llm.chat_messages == [[{"role": "user", "content": "ping"}]]
|
||||
|
||||
|
||||
# --- embedding model down --------------------------------------------------
|
||||
|
||||
|
||||
def test_embed_down_names_embed_model_and_never_reaches_chat() -> None:
|
||||
boom = EmbeddingError(
|
||||
"embeddings request to https://aipi.reeseapps.com/v1 failed: "
|
||||
"connection refused"
|
||||
)
|
||||
llm = _ProbeClient(embed_error=boom)
|
||||
|
||||
with pytest.raises(ModelUnavailableError) as excinfo:
|
||||
_run_check(llm)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "'embed'" in message # the embedding model is named
|
||||
assert "not available" in message
|
||||
assert excinfo.value.__cause__ is boom # the original failure is chained
|
||||
assert llm.chat_messages == [] # the summary probe is never reached
|
||||
assert isinstance(excinfo.value, LLMError) # the sync's generic catch
|
||||
|
||||
|
||||
def test_embed_down_wraps_any_exception_not_just_embedding_error() -> None:
|
||||
boom = RuntimeError("transport exploded")
|
||||
llm = _ProbeClient(embed_error=boom)
|
||||
|
||||
with pytest.raises(ModelUnavailableError) as excinfo:
|
||||
_run_check(llm)
|
||||
|
||||
assert "The embedding model ('embed') is not available" in str(excinfo.value)
|
||||
assert excinfo.value.__cause__ is boom
|
||||
assert llm.chat_messages == []
|
||||
|
||||
|
||||
# --- summary model down ----------------------------------------------------
|
||||
|
||||
|
||||
def test_chat_down_names_summary_model() -> None:
|
||||
boom = LLMError(
|
||||
"chat completion from https://aipi.reeseapps.com/v1 failed: "
|
||||
"connection refused"
|
||||
)
|
||||
llm = _ProbeClient(chat_error=boom)
|
||||
|
||||
with pytest.raises(ModelUnavailableError) as excinfo:
|
||||
_run_check(llm)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "'lite'" in message # the summary model is named
|
||||
assert "not available" in message
|
||||
assert excinfo.value.__cause__ is boom
|
||||
assert llm.embed_texts == ["sync model check"] # the embed probe passed
|
||||
assert isinstance(excinfo.value, LLMError)
|
||||
|
||||
|
||||
def test_chat_down_wraps_any_exception() -> None:
|
||||
llm = _ProbeClient(chat_error=ValueError("stream died"))
|
||||
|
||||
with pytest.raises(ModelUnavailableError) as excinfo:
|
||||
_run_check(llm)
|
||||
|
||||
assert "The summary model ('lite') is not available" in str(excinfo.value)
|
||||
|
||||
|
||||
# --- message content (modal-readable wording) -------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("client", "model"),
|
||||
[
|
||||
(_ProbeClient(embed_error=EmbeddingError("down")), "embed"),
|
||||
(_ProbeClient(chat_error=LLMError("down")), "lite"),
|
||||
],
|
||||
ids=["embed-down", "summary-down"],
|
||||
)
|
||||
def test_message_content_mentions_model_and_availability(
|
||||
client: _ProbeClient, model: str
|
||||
) -> None:
|
||||
with pytest.raises(ModelUnavailableError) as excinfo:
|
||||
_run_check(client)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert f"('{model}')" in message # the model name, quoted
|
||||
assert "not available" in message # the modal-readable wording
|
||||
assert "check the model endpoint" in message # actionable hint
|
||||
|
||||
|
||||
def test_custom_model_names_are_surfaced_verbatim() -> None:
|
||||
"""The probe names whatever the settings configure, not hardcoded
|
||||
model strings."""
|
||||
llm = _ProbeClient(
|
||||
embed_error=EmbeddingError("down"),
|
||||
embed_model="bge-m3",
|
||||
summary_model="hermes-lite",
|
||||
)
|
||||
|
||||
with pytest.raises(ModelUnavailableError) as excinfo:
|
||||
_run_check(llm)
|
||||
|
||||
assert "'bge-m3'" in str(excinfo.value)
|
||||
Reference in New Issue
Block a user