"""Phase 41 E2E (Playwright): model-down sync fails fast with a modal. Story: ``.agent/user_stories/sync-model-fail-fast.md`` Run in isolation (DB must be up: ``podman compose up -d db``; git on PATH — a documented environment prerequisite, phase 28): uv run pytest tests/e2e/test_sync_model_down.py -v --no-cov The story gate proves the whole story in the browser against a **dead model endpoint**: * the pre-sync probe (``app.rag.llm.check_models``) fails the run **before source resolution and before any clone** — a click against the dead endpoint settles retry-ready with a model-naming modal in a short wall-clock budget (≤ ~10 s, vs the phase-32 healthy run's generous 60 s budget); * the modal contract (AC 2): ``role="alertdialog"`` + ``aria-modal="true"`` + the "Sync failed" title + the sanitized, model-naming error text, dismissed via the × button, ``Esc``, or a backdrop click — focus in-and-out to ``#sync-btn``; * the existing surfaces stay (AC 4): the button's failed-state ``title`` / ``aria-label`` / ``.is-error`` affordance and the Sources page's ``#sync-error-banner`` (rendered off ``bor:sync-status``); * the healthy pipeline is untouched (AC 5): a **second** module app (session mock, real ``file://`` clone → import → overview) still runs the full phase-32 lifecycle to "Synced HH:MM" — the probe did not break the happy path. Two module-scoped apps on distinct ports (the ``test_sync_button.py`` module-app pattern): the dead-model app on ``APP_PORT + 41`` with ``BOR_LLM_BASE_URL=http://127.0.0.1:9/v1`` (a closed port — an instant connection refused on loopback) pointed at a **real** ``file://`` fixture repo, so a regressed, non-fail-fast run would spend real time cloning before failing; and the healthy app on ``APP_PORT`` (the session mock — the conftest session app never boots in this isolated run, so no port clash). Test → story mapping (Playwright Mapping Rule): 1. ``test_model_down_fails_fast_with_modal`` → AC 1 + 2 (fast fail before any clone; the modal with the model-naming error) 2. ``test_modal_dismissal`` → AC 2 (× / Esc / backdrop, one fresh failure per path, focus back to ``#sync-btn`` each time) 3. ``test_sync_error_surfaces_unaffected`` → AC 4 (button title/``.is-error``; the Sources banner off ``bor:sync-status``) 4. ``test_healthy_sync_still_succeeds`` → AC 5 (phase 32/35/38 regression: counts + the idempotent second run) """ from __future__ import annotations import os import re import subprocess import sys import time from collections.abc import Iterator from pathlib import Path import pytest from playwright.sync_api import Page, expect from sqlalchemy import text from app.db import SessionLocal from app.models import KbOverview from e2e.auth_helpers import login from e2e.conftest import ( ADMIN_PASSWORD, APP_PORT, SESSION_SECRET, USE_REAL_LLM, _wait_http, ) REPO = Path(__file__).resolve().parents[2] APP_URL = f"http://127.0.0.1:{APP_PORT}" #: The dead-model app's port — distinct so the isolated run never #: clashes with the session app (``APP_PORT``) or the mock (8901). DEAD_PORT = APP_PORT + 41 DEAD_URL = f"http://127.0.0.1:{DEAD_PORT}" #: A closed port on loopback — an instant connection refused, so the #: probe (and therefore the whole run) dies within milliseconds. DEAD_LLM_URL = "http://127.0.0.1:9/v1" #: The fixture doc, same shape as ``test_sync_button.py``'s (a #: regressed, non-fail-fast run would clone + import it — then still #: fail, but only after spending real clone time). FIXTURE_DOC = "notes/sync-fixture.md" SENTINEL = "RESE-MDOWN-SENTINEL-41a7" #: "Synced HH:MM" — the local-time last-result label (header.js's #: fmtSyncTime), any hour/minute. SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}") #: The fail-fast budget: probe dies on a connection refused, the 2 s #: poll observes the ``failed`` state — seconds, not the healthy run's #: 60 s (a regressed run that cloned first would blow this budget). FAIL_TIMEOUT_MS = 10_000 #: Real git clone + embed against the mock LLM (phase-32 budget). SYNC_TIMEOUT_MS = 60_000 def _git(cwd: Path, *args: str) -> None: """Run git in *cwd*; a non-zero exit fails the fixture loudly.""" proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) if proc.returncode != 0: raise AssertionError(f"git {' '.join(args)} failed: {proc.stderr.strip()}") def _build_fixture_repo(root: Path) -> Path: """A real one-commit git repo the sync would clone (built under *root* with real ``git`` subprocess calls — module-safe, no network).""" repo = root / "homelab-notes" (repo / "notes").mkdir(parents=True) (repo / "notes" / "sync-fixture.md").write_text( "# Model-down sync fixture note\n" "\n" "One small note that exists only to prove the phase-41 fail-fast\n" "story end to end: with the model endpoint dead the sync must\n" "fail before this repo is ever cloned.\n" "\n" f"Marker: {SENTINEL}\n", encoding="utf-8", ) _git(repo, "init", "-q") _git(repo, "add", "-A") _git( repo, "-c", "user.email=e@x", "-c", "user.name=t", "-c", "commit.gpgsign=false", # the fixture commit never signs "commit", "-qm", "one", ) assert (repo / ".git").is_dir() return repo def _app_env(port: int, llm_base_url: str, repo: Path) -> dict[str, str]: """Per-module app env (the conftest pattern): admin auth, static frontend, mock-calibrated threshold, its own checkout dir, and the story's subject — one ``file://`` fixture repo as the source.""" env = dict(os.environ) env.pop("DEBUGPY", None) env["BOR_ENVIRONMENT"] = "e2e" env["BOR_STATIC_DIR"] = str(REPO / "frontend") env["BOR_LLM_BASE_URL"] = llm_base_url # Mock-calibrated threshold (conftest pattern) — keeps the healthy # app's deterministic gate behavior; production default stays 0.62. env["BOR_RELEVANCE_THRESHOLD"] = "0.30" env.setdefault( "BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", ) # Phase 16: admin auth must be set or create_app() refuses to boot. env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD env["BOR_SESSION_SECRET"] = SESSION_SECRET env["BOR_GIT_SOURCES"] = f"file://{repo}" env["BOR_SOURCES_DIR"] = str(repo.parent / "checkouts") return env def _boot_app(env: dict[str, str], port: int, health_url: str) -> subprocess.Popen: """Boot the app under test and block until its health endpoint is up.""" proc = subprocess.Popen( [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], cwd=REPO, env=env, ) _wait_http(health_url) return proc @pytest.fixture(scope="module") def dead_git_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: """The dead app's configured source — a real repo it must NEVER clone (the probe fails first). Its own tmp root keeps the checkout dir apart from the healthy app's.""" return _build_fixture_repo(tmp_path_factory.mktemp("mdown_dead")) @pytest.fixture(scope="module") def healthy_git_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: """The healthy app's configured source — the probe passes, the real pipeline runs: clone → import → overview (phase 32/35/38 shape).""" return _build_fixture_repo(tmp_path_factory.mktemp("mdown_healthy")) @pytest.fixture(scope="module") def dead_app_server(dead_git_repo: Path) -> Iterator[str]: """The dead-model app: ``BOR_LLM_BASE_URL`` points at a closed loopback port, so ``check_models`` dies on a connection refused — before source resolution, before any clone.""" proc = _boot_app( _app_env(DEAD_PORT, DEAD_LLM_URL, dead_git_repo), DEAD_PORT, f"{DEAD_URL}/api/health", ) try: yield DEAD_URL finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() @pytest.fixture(scope="module") def healthy_app_server(mock_llm: int, healthy_git_repo: Path) -> Iterator[str]: """The healthy app: same pipeline, live session mock — proves the phase-41 probe did not break the happy path.""" llm = "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else f"http://127.0.0.1:{mock_llm}/v1" proc = _boot_app(_app_env(APP_PORT, llm, healthy_git_repo), APP_PORT, f"{APP_URL}/api/health") try: yield APP_URL finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() @pytest.fixture(scope="module") def dead_app_url(dead_app_server: str) -> str: return dead_app_server @pytest.fixture(scope="module") def healthy_app_url(healthy_app_server: str) -> str: return healthy_app_server @pytest.fixture(scope="module") def app_url(healthy_app_url: str) -> str: """Shadow the session fixture (the conftest session app must never boot in this isolated run — it would clash on ``APP_PORT``); ``db_ready`` health-checks the healthy module app instead.""" return healthy_app_url def _truncate_kb() -> None: """Fresh KB per test (the E2E isolation pattern): any counts and the ``kb_overview`` row the healthy run produces are its own doing.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log, kb_overview")) db.commit() @pytest.fixture(autouse=True) def _clean_kb(db_ready: None) -> Iterator[None]: _truncate_kb() yield def _overview_row() -> KbOverview | None: with SessionLocal() as db: return db.get(KbOverview, 1) def _dismiss_reattached_modal(page: Page) -> None: """Close the modal the page load RE-ATTACHED to (phase 32/41 behavior): the dead app's status is still ``failed`` from an earlier test in this module, so a fresh page load re-enters the failed state — button affordances restored, modal open, focus in the dialog. Dismiss it (Esc) for a clean trigger state.""" modal = page.locator(".sync-modal") expect(modal).to_be_visible(timeout=5_000) expect(page.locator(".sync-modal-close")).to_be_focused() page.keyboard.press("Escape") expect(modal).to_be_hidden() def _trigger_dead_failure(page: Page, app_url: str) -> None: """One fresh fail-fast failure against the dead-model app: click ``#sync-btn`` → running → the probe dies → the run settles retry-ready with the modal open (≤ the 10 s fail-fast budget).""" btn = page.locator("#sync-btn") btn.click() expect(page.locator("#sync-label")).to_have_text("Syncing…", timeout=5_000) expect(page.locator(".sync-modal")).to_be_visible(timeout=FAIL_TIMEOUT_MS) expect(btn).to_be_enabled(timeout=FAIL_TIMEOUT_MS) expect(page.locator("#sync-label")).to_have_text("Sync sources") # --- 1. Fail fast + the model-naming modal --------------------------------- def test_model_down_fails_fast_with_modal( page: Page, dead_app_url: str, db_ready: None ) -> None: """AC 1 + 2 — with a dead model endpoint the click settles retry-ready **and** the modal is visible within a short wall-clock budget (≤ ~10 s, vs the healthy run's 60 s): the run failed before any clone. The error names the embedding model; the dialog carries the alertdialog/aria-modal contract.""" login(page, dead_app_url) btn = page.locator("#sync-btn") expect(btn).to_be_visible() start = time.monotonic() _trigger_dead_failure(page, dead_app_url) elapsed = time.monotonic() - start # Fail-fast: seconds, not a clone-then-import budget. assert elapsed < 10.0, ( f"the dead-model sync took {elapsed:.1f}s — a non-fail-fast run " "(clones before probing) would blow this budget" ) # The server state agrees: failed, model-naming error. body = page.request.get(f"{dead_app_url}/api/sync/status").json() assert body["state"] == "failed", body assert "embedding model" in body["error"], body["error"] assert "'embed'" in body["error"], body["error"] assert "not available" in body["error"], body["error"] # The modal contract (AC 2). modal = page.get_by_role("alertdialog") expect(modal).to_be_visible() expect(modal).to_have_attribute("aria-modal", "true") expect(page.locator("#sync-modal-title")).to_have_text("Sync failed") error = page.locator("#sync-modal-error") expect(error).to_contain_text("embedding model") expect(error).to_contain_text("'embed'") expect(error).to_contain_text("not available") # The close control is present and focused (focus moved IN). expect(page.locator(".sync-modal-close")).to_be_focused() # --- 2. Dismissal: × / Esc / backdrop, focus out to #sync-btn --------------- def test_modal_dismissal(page: Page, dead_app_url: str, db_ready: None) -> None: """AC 2 — one fresh failure per dismissal path: the × button, ``Esc``, and a backdrop click (never the panel). Each open focuses the close button; each close returns focus to ``#sync-btn`` — the control that started the run.""" login(page, dead_app_url) _dismiss_reattached_modal(page) # the prior test's failure is still last btn = page.locator("#sync-btn") modal = page.locator(".sync-modal") close_btn = page.locator(".sync-modal-close") # Path 1: the × button. _trigger_dead_failure(page, dead_app_url) expect(close_btn).to_be_focused() close_btn.click() expect(modal).to_be_hidden() expect(btn).to_be_focused() # focus returned to #sync-btn # Path 2: Esc. _trigger_dead_failure(page, dead_app_url) expect(close_btn).to_be_focused() page.keyboard.press("Escape") expect(modal).to_be_hidden() expect(btn).to_be_focused() # Path 3: a click on the backdrop itself (the dimmed area — the # panel is centered, so a corner click can never land on it). _trigger_dead_failure(page, dead_app_url) expect(close_btn).to_be_focused() page.locator(".sync-modal-backdrop").click(position={"x": 10, "y": 10}) expect(modal).to_be_hidden() expect(btn).to_be_focused() # --- 3. The secondary surfaces are untouched -------------------------------- def test_sync_error_surfaces_unaffected( page: Page, dead_app_url: str, db_ready: None ) -> None: """AC 4 — after a failure the button keeps its failed-state ``title`` / ``aria-label`` / ``.is-error`` affordance (the non-Sources visibility), and on ``/sources.html`` (same dead-model app) the ``#sync-error-banner`` still renders the error off ``bor:sync-status`` — the phase-32/34 contract the modal is additive to.""" # Trigger the failure on the chat page (the modal is module-owned — # it follows the button onto every page). The re-attach modal from # the prior test's last failure closes first (clean trigger state). login(page, dead_app_url, next="/") _dismiss_reattached_modal(page) _trigger_dead_failure(page, dead_app_url) btn = page.locator("#sync-btn") expect(btn).to_have_attribute("title", re.compile("embedding model")) expect(btn).to_have_attribute("aria-label", re.compile("embedding model")) expect(btn).to_have_class(re.compile("is-error")) # A clean navigation: the Sources page re-attaches to the failed # run and renders the banner off the event. page.goto(f"{dead_app_url}/sources.html") banner = page.locator("#sync-error-banner") expect(banner).to_be_visible() expect(banner).to_have_attribute("role", "alert") expect(page.locator("#sync-error-text")).to_contain_text("embedding model") expect(page.locator("#sync-error-text")).to_contain_text("'embed'") # --- 4. The healthy pipeline is untouched (phase 32 regression) ------------- def test_healthy_sync_still_succeeds( page: Page, healthy_app_url: str, db_ready: None ) -> None: """AC 5 — with a live model the probe passes and the full phase-32/35/38 pipeline runs unchanged: real ``file://`` clone → import (prune) → KB overview regeneration → "Synced HH:MM" + the counts, then the idempotent second run ("0 added · 1 unchanged").""" login(page, healthy_app_url) # lands on /sources.html (the button's home) btn = page.locator("#sync-btn") expect(btn).to_be_visible() expect(page.locator("#sync-label")).to_have_text("Sync sources") # --- run 1: probe passes; clone + import + overview ------------------ btn.click() expect(btn).to_be_disabled() expect(page.locator("#sync-label")).to_have_text("Syncing…") expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) expect(btn).to_be_enabled() # never stale — re-enabled at the terminal state expect(page.locator("#sync-result")).to_have_text("1 added") # The REAL clone was imported: the fixture path is in the Sources # table (the sentinel lives inside it). expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1) # Phase-31 regeneration ran (the import changed the KB): the single # kb_overview row is fresh and non-empty (DB check — truncated # before this test, so it is the sync's own doing). row = _overview_row() assert row is not None and row.content.strip(), ( "kb_overview must be regenerated by a successful, KB-changing sync" ) # --- run 2: idempotent pull + hash skip ------------------------------- btn.click() expect(btn).to_be_disabled() expect(page.locator("#sync-label")).to_have_text("Syncing…") expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) expect(btn).to_be_enabled() expect(page.locator("#sync-result")).to_have_text("0 added · 1 unchanged") expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1) # And no modal ever opened (no failure on the healthy app). expect(page.locator(".sync-modal")).to_have_count(0)