brain-of-reese/tests/unit/test_sync_model_probe.py

"""Unit tests: the phase-41 pre-sync model probe (``check_models``).

The probe verifies both models a sync needs — ``embed`` (one short
embedding) and the summary model ``lite`` (one 1-token-scale
completion) — before any expensive work, and fails with a
: class:`ModelUnavailableError` that **names the dead model**. The fake
client is duck-typed (``embed_one`` / ``chat`` + ``settings``), same
shape as :class:`tests.fakes.FakeEmbedder` — no network, no httpx
transport.

Covered paths (keep ``app/`` >90%):
- both models up → returns, both methods called exactly once;
- embedding probe raises → ``ModelUnavailableError`` naming the
  **embedding model**, and the summary probe is **never** reached;
- summary probe raises → ``ModelUnavailableError`` naming the
  **summary model**;
- message content: model name present, "not available" wording, the
  original exception chained as ``__cause__``;
- the error is an ``LLMError`` (the sync's generic failure catch).
"""
from __future__ import annotations

import asyncio
from typing import Any, cast

import pytest

from app.config import Settings
from app.rag.llm import (
    EmbeddingError,
    LLMClient,
    LLMError,
    ModelUnavailableError,
    check_models,
)


class _ProbeClient:
    """Duck-typed :class:`app.rag.llm.LLMClient` stand-in for the probe."""

    def __init__(
        self,
        embed_error: Exception | None = None,
        chat_error: Exception | None = None,
        embed_model: str = "embed",
        summary_model: str = "lite",
    ) -> None:
        kwargs: dict[str, Any] = {
            "_env_file": None,
            "llm_embed_model": embed_model,
            "llm_summary_model": summary_model,
        }
        self.settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
        self.embed_error = embed_error
        self.chat_error = chat_error
        self.embed_texts: list[str] = []
        self.chat_messages: list[list[dict[str, str]]] = []

    async def embed_one(self, text: str) -> list[float]:
        self.embed_texts.append(text)
        if self.embed_error is not None:
            raise self.embed_error
        return [0.0] * 768

    async def chat(self, messages: list[dict[str, str]], model: str | None = None) -> str:
        self.chat_messages.append(list(messages))
        if self.chat_error is not None:
            raise self.chat_error
        return "pong"


def _run_check(llm: _ProbeClient) -> Any:
    # The probe is duck-typed at the ``embed_one`` / ``chat`` surface — the
    # cast keeps pyright honest about the narrow stand-in.
    return asyncio.run(check_models(cast("LLMClient", llm)))


# --- both models up --------------------------------------------------------


def test_both_models_up_returns_and_calls_both() -> None:
    llm = _ProbeClient()

    assert _run_check(llm) is None
    assert llm.embed_texts == ["sync model check"]  # the tiny probe input
    assert llm.chat_messages == [[{"role": "user", "content": "ping"}]]


def test_chat_probe_defaults_to_summary_model_setting() -> None:
    """The probe calls ``chat`` without a model override — the client's
    ``llm_summary_model`` default (``lite``) is what gets pinged."""
    llm = _ProbeClient()

    _run_check(llm)
    # ``chat`` received exactly the probe messages; the model default is
    # resolved inside LLMClient.chat from settings.llm_summary_model.
    assert llm.settings.llm_summary_model == "lite"
    assert llm.chat_messages == [[{"role": "user", "content": "ping"}]]


# --- embedding model down --------------------------------------------------


def test_embed_down_names_embed_model_and_never_reaches_chat() -> None:
    boom = EmbeddingError(
        "embeddings request to https://aipi.reeseapps.com/v1 failed: "
        "connection refused"
    )
    llm = _ProbeClient(embed_error=boom)

    with pytest.raises(ModelUnavailableError) as excinfo:
        _run_check(llm)

    message = str(excinfo.value)
    assert "'embed'" in message  # the embedding model is named
    assert "not available" in message
    assert excinfo.value.__cause__ is boom  # the original failure is chained
    assert llm.chat_messages == []  # the summary probe is never reached
    assert isinstance(excinfo.value, LLMError)  # the sync's generic catch


def test_embed_down_wraps_any_exception_not_just_embedding_error() -> None:
    boom = RuntimeError("transport exploded")
    llm = _ProbeClient(embed_error=boom)

    with pytest.raises(ModelUnavailableError) as excinfo:
        _run_check(llm)

    assert "The embedding model ('embed') is not available" in str(excinfo.value)
    assert excinfo.value.__cause__ is boom
    assert llm.chat_messages == []


# --- summary model down ----------------------------------------------------


def test_chat_down_names_summary_model() -> None:
    boom = LLMError(
        "chat completion from https://aipi.reeseapps.com/v1 failed: "
        "connection refused"
    )
    llm = _ProbeClient(chat_error=boom)

    with pytest.raises(ModelUnavailableError) as excinfo:
        _run_check(llm)

    message = str(excinfo.value)
    assert "'lite'" in message  # the summary model is named
    assert "not available" in message
    assert excinfo.value.__cause__ is boom
    assert llm.embed_texts == ["sync model check"]  # the embed probe passed
    assert isinstance(excinfo.value, LLMError)


def test_chat_down_wraps_any_exception() -> None:
    llm = _ProbeClient(chat_error=ValueError("stream died"))

    with pytest.raises(ModelUnavailableError) as excinfo:
        _run_check(llm)

    assert "The summary model ('lite') is not available" in str(excinfo.value)


# --- message content (modal-readable wording) -------------------------------


@pytest.mark.parametrize(
    ("client", "model"),
    [
        (_ProbeClient(embed_error=EmbeddingError("down")), "embed"),
        (_ProbeClient(chat_error=LLMError("down")), "lite"),
    ],
    ids=["embed-down", "summary-down"],
)
def test_message_content_mentions_model_and_availability(
    client: _ProbeClient, model: str
) -> None:
    with pytest.raises(ModelUnavailableError) as excinfo:
        _run_check(client)

    message = str(excinfo.value)
    assert f"('{model}')" in message  # the model name, quoted
    assert "not available" in message  # the modal-readable wording
    assert "check the model endpoint" in message  # actionable hint


def test_custom_model_names_are_surfaced_verbatim() -> None:
    """The probe names whatever the settings configure, not hardcoded
    model strings."""
    llm = _ProbeClient(
        embed_error=EmbeddingError("down"),
        embed_model="bge-m3",
        summary_model="hermes-lite",
    )

    with pytest.raises(ModelUnavailableError) as excinfo:
        _run_check(llm)

    assert "'bge-m3'" in str(excinfo.value)