brain-of-reese/tests/e2e/test_sync_model_down.py

"""Phase 41 E2E (Playwright): model-down sync fails fast with a modal.

Story: ``.agents/user_stories/sync-model-fail-fast.md``
Run in isolation (DB must be up: ``podman compose up -d db``; git on
PATH — a documented environment prerequisite, phase 28):

    uv run pytest tests/e2e/test_sync_model_down.py -v --no-cov

The story gate proves the whole story in the browser against a
**dead model endpoint**:

* the pre-sync probe (``app.rag.llm.check_models``) fails the run
  **before source resolution and before any clone** — a click against
  the dead endpoint settles retry-ready with a model-naming modal in
  a short wall-clock budget (≤ ~10 s, vs the phase-32 healthy run's
  generous 60 s budget);
* the modal contract (AC 2): ``role="alertdialog"`` +
  ``aria-modal="true"`` + the "Sync failed" title + the sanitized,
  model-naming error text, dismissed via the × button, ``Esc``, or a
  backdrop click — focus in-and-out to ``#sync-btn``;
* the existing surfaces stay (AC 4): the button's failed-state
  ``title`` / ``aria-label`` / ``.is-error`` affordance and the
  Sources page's ``#sync-error-banner`` (rendered off
  ``bor:sync-status``);
* the healthy pipeline is untouched (AC 5): a **second** module app
  (session mock, real ``file://`` clone → import → overview) still
  runs the full phase-32 lifecycle to "Synced HH:MM" — the probe did
  not break the happy path.

Two module-scoped apps on distinct ports (the ``test_sync_button.py``
module-app pattern): the dead-model app on ``APP_PORT + 41`` with
``BOR_LLM_BASE_URL=http://127.0.0.1:9/v1`` (a closed port — an
instant connection refused on loopback) pointed at a **real**
``file://`` fixture repo, so a regressed, non-fail-fast run would
spend real time cloning before failing; and the healthy app on
``APP_PORT`` (the session mock — the conftest session app never boots
in this isolated run, so no port clash).

Test → story mapping (Playwright Mapping Rule):
1. ``test_model_down_fails_fast_with_modal`` → AC 1 + 2 (fast fail
   before any clone; the modal with the model-naming error)
2. ``test_modal_dismissal`` → AC 2 (× / Esc / backdrop, one fresh
   failure per path, focus back to ``#sync-btn`` each time)
3. ``test_sync_error_surfaces_unaffected`` → AC 4 (button
   title/``.is-error``; the Sources banner off ``bor:sync-status``)
4. ``test_healthy_sync_still_succeeds`` → AC 5 (phase 32/35/38
   regression: counts + the idempotent second run)
"""
from __future__ import annotations

import os
import re
import subprocess
import sys
import time
from collections.abc import Iterator
from pathlib import Path

import pytest
from playwright.sync_api import Page, expect
from sqlalchemy import text

from app.db import SessionLocal
from app.models import KbOverview
from e2e.auth_helpers import login
from e2e.conftest import (
    ADMIN_PASSWORD,
    SESSION_SECRET,
    USE_REAL_LLM,
    _wait_http,
)
from e2e.sync_helpers import counts_line

REPO = Path(__file__).resolve().parents[2]

# Phase 79 (task 04, full inventory): the conftest session app owns its
# port in a combined run — this module's healthy app binds its own port
# instead (a same-port second uvicorn dies on bind and would drive the
# wrong server); the dead-model app stays derived from it. Env-
# overridable.
APP_PORT = int(os.environ.get("E2E_APP_PORT_SYNCDOWN", "8134"))
APP_URL = f"http://127.0.0.1:{APP_PORT}"
#: The dead-model app's port — distinct so the isolated run never
#: clashes with the session app (``APP_PORT``) or the mock (8901).
DEAD_PORT = APP_PORT + 41
DEAD_URL = f"http://127.0.0.1:{DEAD_PORT}"
#: A closed port on loopback — an instant connection refused, so the
#: probe (and therefore the whole run) dies within milliseconds.
DEAD_LLM_URL = "http://127.0.0.1:9/v1"

#: The fixture doc, same shape as ``test_sync_button.py``'s (a
#: regressed, non-fail-fast run would clone + import it — then still
#: fail, but only after spending real clone time).
FIXTURE_DOC = "notes/sync-fixture.md"
SENTINEL = "RESE-MDOWN-SENTINEL-41a7"

#: "Synced HH:MM" — the local-time last-result label (header.js's
#: fmtSyncTime), any hour/minute.
SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}")

#: The fail-fast budget: probe dies on a connection refused, the 2 s
#: poll observes the ``failed`` state — seconds, not the healthy run's
#: 60 s (a regressed run that cloned first would blow this budget).
FAIL_TIMEOUT_MS = 10_000
#: Real git clone + embed against the mock LLM (phase-32 budget).
SYNC_TIMEOUT_MS = 60_000


def _git(cwd: Path, *args: str) -> None:
    """Run git in *cwd*; a non-zero exit fails the fixture loudly."""
    proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True)
    if proc.returncode != 0:
        raise AssertionError(f"git {' '.join(args)} failed: {proc.stderr.strip()}")


def _build_fixture_repo(root: Path) -> Path:
    """A real one-commit git repo the sync would clone (built under
    *root* with real ``git`` subprocess calls — module-safe, no
    network)."""
    repo = root / "homelab-notes"
    (repo / "notes").mkdir(parents=True)
    (repo / "notes" / "sync-fixture.md").write_text(
        "# Model-down sync fixture note\n"
        "\n"
        "One small note that exists only to prove the phase-41 fail-fast\n"
        "story end to end: with the model endpoint dead the sync must\n"
        "fail before this repo is ever cloned.\n"
        "\n"
        f"Marker: {SENTINEL}\n",
        encoding="utf-8",
    )
    _git(repo, "init", "-q")
    _git(repo, "add", "-A")
    _git(
        repo,
        "-c", "user.email=e@x", "-c", "user.name=t",
        "-c", "commit.gpgsign=false",  # the fixture commit never signs
        "commit", "-qm", "one",
    )
    assert (repo / ".git").is_dir()
    return repo


def _app_env(port: int, llm_base_url: str, repo: Path) -> dict[str, str]:
    """Per-module app env (the conftest pattern): admin auth, static
    frontend, mock-calibrated threshold, its own checkout dir, and the
    story's subject — one ``file://`` fixture repo as the source."""
    env = dict(os.environ)
    env.pop("DEBUGPY", None)
    env["BOR_ENVIRONMENT"] = "e2e"
    env["BOR_STATIC_DIR"] = str(REPO / "frontend")
    env["BOR_LLM_BASE_URL"] = llm_base_url
    # Mock-calibrated threshold (conftest pattern) — keeps the healthy
    # app's deterministic gate behavior; production default stays 0.62.
    env["BOR_RELEVANCE_THRESHOLD"] = "0.30"
    env.setdefault(
        "BOR_DATABASE_URL",
        "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese",
    )
    # Phase 16: admin auth must be set or create_app() refuses to boot.
    env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD
    env["BOR_SESSION_SECRET"] = SESSION_SECRET
    env["BOR_GIT_SOURCES"] = f"file://{repo}"
    env["BOR_SOURCES_DIR"] = str(repo.parent / "checkouts")
    return env


def _boot_app(env: dict[str, str], port: int, health_url: str) -> subprocess.Popen:
    """Boot the app under test and block until its health endpoint is up."""
    proc = subprocess.Popen(
        [sys.executable, "-m", "uvicorn", "app.main:app",
         "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
        cwd=REPO,
        env=env,
    )
    _wait_http(health_url)
    return proc


@pytest.fixture(scope="module")
def dead_git_repo(tmp_path_factory: pytest.TempPathFactory) -> Path:
    """The dead app's configured source — a real repo it must NEVER
    clone (the probe fails first). Its own tmp root keeps the checkout
    dir apart from the healthy app's."""
    return _build_fixture_repo(tmp_path_factory.mktemp("mdown_dead"))


@pytest.fixture(scope="module")
def healthy_git_repo(tmp_path_factory: pytest.TempPathFactory) -> Path:
    """The healthy app's configured source — the probe passes, the real
    pipeline runs: clone → import → overview (phase 32/35/38 shape)."""
    return _build_fixture_repo(tmp_path_factory.mktemp("mdown_healthy"))


@pytest.fixture(scope="module")
def dead_app_server(dead_git_repo: Path) -> Iterator[str]:
    """The dead-model app: ``BOR_LLM_BASE_URL`` points at a closed
    loopback port, so ``check_models`` dies on a connection refused —
    before source resolution, before any clone."""
    proc = _boot_app(
        _app_env(DEAD_PORT, DEAD_LLM_URL, dead_git_repo),
        DEAD_PORT,
        f"{DEAD_URL}/api/health",
    )
    try:
        yield DEAD_URL
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            proc.kill()


@pytest.fixture(scope="module")
def healthy_app_server(mock_llm: int, healthy_git_repo: Path) -> Iterator[str]:
    """The healthy app: same pipeline, live session mock — proves the
    phase-41 probe did not break the happy path."""
    llm = "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else f"http://127.0.0.1:{mock_llm}/v1"
    proc = _boot_app(_app_env(APP_PORT, llm, healthy_git_repo), APP_PORT, f"{APP_URL}/api/health")
    try:
        yield APP_URL
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            proc.kill()


@pytest.fixture(scope="module")
def dead_app_url(dead_app_server: str) -> str:
    return dead_app_server


@pytest.fixture(scope="module")
def healthy_app_url(healthy_app_server: str) -> str:
    return healthy_app_server


@pytest.fixture(scope="module")
def app_url(healthy_app_url: str) -> str:
    """Shadow the session fixture (the conftest session app must never
    boot in this isolated run — it would clash on ``APP_PORT``);
    ``db_ready`` health-checks the healthy module app instead."""
    return healthy_app_url


def _truncate_kb() -> None:
    """Fresh KB per test (the E2E isolation pattern): any counts and the
    ``kb_overview`` row the healthy run produces are its own doing.
    ``git_sources`` too — the shared Postgres may carry leftover
    registry rows that would take precedence over the apps' env
    fallback repos."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log, kb_overview, git_sources"))
        db.commit()


@pytest.fixture(autouse=True)
def _clean_kb(db_ready: None) -> Iterator[None]:
    _truncate_kb()
    yield


def _overview_row() -> KbOverview | None:
    with SessionLocal() as db:
        return db.get(KbOverview, 1)


# The git fixture repo's directory name — the source name the import
# records for it (the repo_name basename rule, phase 28).
GIT_SOURCE_NAME = "homelab-notes"


def _go_top(page: Page) -> None:
    """Back to the top level (no-op when already there — the
    breadcrumb is hidden at the top)."""
    if not page.locator("#kb-crumb").is_hidden():
        page.locator("#kb-crumb a.kb-crumb-link").first.click()


def _expect_fixture_row(page: Page) -> None:
    """The REAL clone was imported — the fixture path is listed. Phase
    97: the catalog is the drill-down tree, so drill source → folder
    ``notes`` first (the flat all-docs row is the same row, one level
    deeper)."""
    _go_top(page)
    page.click(f'#folders-tbody a.folder-link:text-is("{GIT_SOURCE_NAME}")')
    page.click('#folders-tbody a.folder-link:text-is("notes")')
    expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1)


def _dismiss_reattached_modal(page: Page) -> None:
    """Close the modal the page load RE-ATTACHED to (phase 32/41
    behavior): the dead app's status is still ``failed`` from an
    earlier test in this module, so a fresh page load re-enters the
    failed state — button affordances restored, modal open, focus in
    the dialog. Dismiss it (Esc) for a clean trigger state."""
    modal = page.locator(".sync-modal")
    expect(modal).to_be_visible(timeout=5_000)
    expect(page.locator(".sync-modal-close")).to_be_focused()
    page.keyboard.press("Escape")
    expect(modal).to_be_hidden()


def _trigger_dead_failure(page: Page, app_url: str) -> None:
    """One fresh fail-fast failure against the dead-model app: click
    ``#sync-btn`` → running → the probe dies → the run settles
    retry-ready with the modal open (≤ the 10 s fail-fast budget)."""
    btn = page.locator("#sync-btn")
    btn.click()
    expect(page.locator("#sync-label")).to_have_text("Syncing…", timeout=5_000)
    expect(page.locator(".sync-modal")).to_be_visible(timeout=FAIL_TIMEOUT_MS)
    expect(btn).to_be_enabled(timeout=FAIL_TIMEOUT_MS)
    expect(page.locator("#sync-label")).to_have_text("Sync sources")


# --- 1. Fail fast + the model-naming modal ---------------------------------


def test_model_down_fails_fast_with_modal(
    page: Page, dead_app_url: str, db_ready: None
) -> None:
    """AC 1 + 2 — with a dead model endpoint the click settles
    retry-ready **and** the modal is visible within a short wall-clock
    budget (≤ ~10 s, vs the healthy run's 60 s): the run failed before
    any clone. The error names the embedding model; the dialog carries
    the alertdialog/aria-modal contract."""
    login(page, dead_app_url)
    btn = page.locator("#sync-btn")
    expect(btn).to_be_visible()

    start = time.monotonic()
    _trigger_dead_failure(page, dead_app_url)
    elapsed = time.monotonic() - start
    # Fail-fast: seconds, not a clone-then-import budget.
    assert elapsed < 10.0, (
        f"the dead-model sync took {elapsed:.1f}s — a non-fail-fast run "
        "(clones before probing) would blow this budget"
    )

    # The server state agrees: failed, model-naming error.
    body = page.request.get(f"{dead_app_url}/api/sync/status").json()
    assert body["state"] == "failed", body
    assert "embedding model" in body["error"], body["error"]
    assert "'embed'" in body["error"], body["error"]
    assert "not available" in body["error"], body["error"]

    # The modal contract (AC 2).
    modal = page.get_by_role("alertdialog")
    expect(modal).to_be_visible()
    expect(modal).to_have_attribute("aria-modal", "true")
    expect(page.locator("#sync-modal-title")).to_have_text("Sync failed")
    error = page.locator("#sync-modal-error")
    expect(error).to_contain_text("embedding model")
    expect(error).to_contain_text("'embed'")
    expect(error).to_contain_text("not available")
    # The close control is present and focused (focus moved IN).
    expect(page.locator(".sync-modal-close")).to_be_focused()


# --- 2. Dismissal: × / Esc / backdrop, focus out to #sync-btn ---------------


def test_modal_dismissal(page: Page, dead_app_url: str, db_ready: None) -> None:
    """AC 2 — one fresh failure per dismissal path: the × button,
    ``Esc``, and a backdrop click (never the panel). Each open focuses
    the close button; each close returns focus to ``#sync-btn`` — the
    control that started the run."""
    login(page, dead_app_url)
    _dismiss_reattached_modal(page)  # the prior test's failure is still last
    btn = page.locator("#sync-btn")
    modal = page.locator(".sync-modal")
    close_btn = page.locator(".sync-modal-close")

    # Path 1: the × button.
    _trigger_dead_failure(page, dead_app_url)
    expect(close_btn).to_be_focused()
    close_btn.click()
    expect(modal).to_be_hidden()
    expect(btn).to_be_focused()  # focus returned to #sync-btn

    # Path 2: Esc.
    _trigger_dead_failure(page, dead_app_url)
    expect(close_btn).to_be_focused()
    page.keyboard.press("Escape")
    expect(modal).to_be_hidden()
    expect(btn).to_be_focused()

    # Path 3: a click on the backdrop itself (the dimmed area — the
    # panel is centered, so a corner click can never land on it).
    _trigger_dead_failure(page, dead_app_url)
    expect(close_btn).to_be_focused()
    page.locator(".sync-modal-backdrop").click(position={"x": 10, "y": 10})
    expect(modal).to_be_hidden()
    expect(btn).to_be_focused()


# --- 3. The secondary surfaces are untouched --------------------------------


def test_sync_error_surfaces_unaffected(
    page: Page, dead_app_url: str, db_ready: None
) -> None:
    """AC 4 — after a failure the button keeps its failed-state
    ``title`` / ``aria-label`` / ``.is-error`` affordance (the
    non-Sources visibility), and on ``/sources.html`` (same dead-model
    app) the ``#sync-error-banner`` still renders the error off
    ``bor:sync-status`` — the phase-32/34 contract the modal is
    additive to."""
    # Trigger the failure on the Sources page (the button's home since
    # the owner rework 2026-08-28; the modal is page-local there). The
    # re-attach modal from the prior test's last failure closes first
    # (clean trigger state).
    login(page, dead_app_url)
    _dismiss_reattached_modal(page)
    _trigger_dead_failure(page, dead_app_url)

    btn = page.locator("#sync-btn")
    expect(btn).to_have_attribute("title", re.compile("embedding model"))
    expect(btn).to_have_attribute("aria-label", re.compile("embedding model"))
    expect(btn).to_have_class(re.compile("is-error"))

    # A clean navigation: the Sources page re-attaches to the failed
    # run and renders the banner off the event.
    page.goto(f"{dead_app_url}/sources.html")
    banner = page.locator("#sync-error-banner")
    expect(banner).to_be_visible()
    expect(banner).to_have_attribute("role", "alert")
    expect(page.locator("#sync-error-text")).to_contain_text("embedding model")
    expect(page.locator("#sync-error-text")).to_contain_text("'embed'")


# --- 4. The healthy pipeline is untouched (phase 32 regression) -------------


def test_healthy_sync_still_succeeds(
    page: Page, healthy_app_url: str, db_ready: None
) -> None:
    """AC 5 — with a live model the probe passes and the full
    phase-32/35/38 pipeline runs unchanged: real ``file://`` clone →
    import (prune) → KB overview regeneration → "Synced HH:MM" + the
    counts, then the idempotent second run ("0 added · 1 unchanged")."""
    login(page, healthy_app_url)  # lands on /sources.html (the button's home)

    btn = page.locator("#sync-btn")
    expect(btn).to_be_visible()
    expect(page.locator("#sync-label")).to_have_text("Sync sources")

    # --- run 1: probe passes; clone + import + overview ------------------
    btn.click()
    expect(btn).to_be_disabled()
    expect(page.locator("#sync-label")).to_have_text("Syncing…")
    expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS)
    expect(btn).to_be_enabled()  # never stale — re-enabled at the terminal state
    # The counts line rides the box (phase 150 — the sole sync surface).
    expect(page.locator("#sync-progress-text")).to_have_text(counts_line("1 added"))

    # The REAL clone was imported: the fixture path is in the Sources
    # table (the sentinel lives inside it) — via the drill-down tree
    # (phase 97).
    _expect_fixture_row(page)

    # Phase-31 regeneration ran (the import changed the KB): the single
    # kb_overview row is fresh and non-empty (DB check — truncated
    # before this test, so it is the sync's own doing).
    row = _overview_row()
    assert row is not None and row.content.strip(), (
        "kb_overview must be regenerated by a successful, KB-changing sync"
    )

    # --- run 2: idempotent pull + hash skip -------------------------------
    btn.click()
    expect(btn).to_be_disabled()
    expect(page.locator("#sync-label")).to_have_text("Syncing…")
    expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS)
    expect(btn).to_be_enabled()
    expect(page.locator("#sync-progress-text")).to_have_text(counts_line("0 added · 1 unchanged"))
    # The doc survived the prune re-import — via the drill-down tree
    # (phase 97).
    _expect_fixture_row(page)

    # And no modal ever opened (no failure on the healthy app).
    expect(page.locator(".sync-modal")).to_have_count(0)