"""Unit tests: the phase-41 pre-sync model probe (``check_models``). The probe verifies both models a sync needs — ``embed`` (one short embedding) and the summary model ``lite`` (one 1-token-scale completion) — before any expensive work, and fails with a : class:`ModelUnavailableError` that **names the dead model**. The fake client is duck-typed (``embed_one`` / ``chat`` + ``settings``), same shape as :class:`tests.fakes.FakeEmbedder` — no network, no httpx transport. Covered paths (keep ``app/`` >90%): - both models up → returns, both methods called exactly once; - embedding probe raises → ``ModelUnavailableError`` naming the **embedding model**, and the summary probe is **never** reached; - summary probe raises → ``ModelUnavailableError`` naming the **summary model**; - message content: model name present, "not available" wording, the original exception chained as ``__cause__``; - the error is an ``LLMError`` (the sync's generic failure catch). """ from __future__ import annotations import asyncio from typing import Any, cast import pytest from app.config import Settings from app.rag.llm import ( EmbeddingError, LLMClient, LLMError, ModelUnavailableError, check_models, ) class _ProbeClient: """Duck-typed :class:`app.rag.llm.LLMClient` stand-in for the probe.""" def __init__( self, embed_error: Exception | None = None, chat_error: Exception | None = None, embed_model: str = "embed", summary_model: str = "lite", ) -> None: kwargs: dict[str, Any] = { "_env_file": None, "llm_embed_model": embed_model, "llm_summary_model": summary_model, } self.settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] self.embed_error = embed_error self.chat_error = chat_error self.embed_texts: list[str] = [] self.chat_messages: list[list[dict[str, str]]] = [] async def embed_one(self, text: str) -> list[float]: self.embed_texts.append(text) if self.embed_error is not None: raise self.embed_error return [0.0] * 768 async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str: self.chat_messages.append(list(messages)) if self.chat_error is not None: raise self.chat_error return "pong" def _run_check(llm: _ProbeClient) -> Any: # The probe is duck-typed at the ``embed_one`` / ``chat`` surface — the # cast keeps pyright honest about the narrow stand-in. return asyncio.run(check_models(cast("LLMClient", llm))) # --- both models up -------------------------------------------------------- def test_both_models_up_returns_and_calls_both() -> None: llm = _ProbeClient() assert _run_check(llm) is None assert llm.embed_texts == ["sync model check"] # the tiny probe input assert llm.chat_messages == [[{"role": "user", "content": "ping"}]] def test_chat_probe_defaults_to_summary_model_setting() -> None: """The probe calls ``chat`` without a model override — the client's ``llm_summary_model`` default (``lite``) is what gets pinged.""" llm = _ProbeClient() _run_check(llm) # ``chat`` received exactly the probe messages; the model default is # resolved inside LLMClient.chat from settings.llm_summary_model. assert llm.settings.llm_summary_model == "lite" assert llm.chat_messages == [[{"role": "user", "content": "ping"}]] # --- embedding model down -------------------------------------------------- def test_embed_down_names_embed_model_and_never_reaches_chat() -> None: boom = EmbeddingError( "embeddings request to https://aipi.reeseapps.com/v1 failed: " "connection refused" ) llm = _ProbeClient(embed_error=boom) with pytest.raises(ModelUnavailableError) as excinfo: _run_check(llm) message = str(excinfo.value) assert "'embed'" in message # the embedding model is named assert "not available" in message assert excinfo.value.__cause__ is boom # the original failure is chained assert llm.chat_messages == [] # the summary probe is never reached assert isinstance(excinfo.value, LLMError) # the sync's generic catch def test_embed_down_wraps_any_exception_not_just_embedding_error() -> None: boom = RuntimeError("transport exploded") llm = _ProbeClient(embed_error=boom) with pytest.raises(ModelUnavailableError) as excinfo: _run_check(llm) assert "The embedding model ('embed') is not available" in str(excinfo.value) assert excinfo.value.__cause__ is boom assert llm.chat_messages == [] # --- summary model down ---------------------------------------------------- def test_chat_down_names_summary_model() -> None: boom = LLMError( "chat completion from https://aipi.reeseapps.com/v1 failed: " "connection refused" ) llm = _ProbeClient(chat_error=boom) with pytest.raises(ModelUnavailableError) as excinfo: _run_check(llm) message = str(excinfo.value) assert "'lite'" in message # the summary model is named assert "not available" in message assert excinfo.value.__cause__ is boom assert llm.embed_texts == ["sync model check"] # the embed probe passed assert isinstance(excinfo.value, LLMError) def test_chat_down_wraps_any_exception() -> None: llm = _ProbeClient(chat_error=ValueError("stream died")) with pytest.raises(ModelUnavailableError) as excinfo: _run_check(llm) assert "The summary model ('lite') is not available" in str(excinfo.value) # --- message content (modal-readable wording) ------------------------------- @pytest.mark.parametrize( ("client", "model"), [ (_ProbeClient(embed_error=EmbeddingError("down")), "embed"), (_ProbeClient(chat_error=LLMError("down")), "lite"), ], ids=["embed-down", "summary-down"], ) def test_message_content_mentions_model_and_availability( client: _ProbeClient, model: str ) -> None: with pytest.raises(ModelUnavailableError) as excinfo: _run_check(client) message = str(excinfo.value) assert f"('{model}')" in message # the model name, quoted assert "not available" in message # the modal-readable wording assert "check the model endpoint" in message # actionable hint def test_custom_model_names_are_surfaced_verbatim() -> None: """The probe names whatever the settings configure, not hardcoded model strings.""" llm = _ProbeClient( embed_error=EmbeddingError("down"), embed_model="bge-m3", summary_model="hermes-lite", ) with pytest.raises(ModelUnavailableError) as excinfo: _run_check(llm) assert "'bge-m3'" in str(excinfo.value)