feat(admin): one-click sources sync — admin-only button triggers git clone/pull + re-import + KB overview refresh with polled live status

tests/e2e/test_sync_button.py

@@ -0,0 +1,283 @@
+"""Phase 32 E2E (Playwright): the admin-only "Sync sources" button.
+
+Story: ``.agent/user_stories/admin-sync-button.md``
+Run in isolation (DB must be up: ``podman compose up -d db``; git on
+PATH — a documented environment prerequisite, phase 28):
+
+    uv run pytest tests/e2e/test_sync_button.py -v --no-cov
+
+The story gate runs the **real** sync path end to end — a real
+``git clone`` of a local ``file://`` fixture repo (deterministic, no
+network), a real import against the mock LLM, a real KB-overview
+regeneration — plus the admin-only visibility and the full button
+lifecycle (§7.4: "Syncing…" → "Synced HH:MM" + counts, the idempotent
+second run, the 409 double trigger).
+
+Per-module app env (the E2E conftest pattern, module-scoped): this
+story's app boots with ``BOR_GIT_SOURCES=file://<fixture repo>`` and
+its own ``BOR_SOURCES_DIR`` — the session app (no git sources) is
+never started in this isolated run, so no port clash.
+
+Test → story mapping (Playwright Mapping Rule):
+1. ``test_anonymous_sees_no_button`` → AC 1 + 3 (hidden button, 403s)
+2. ``test_admin_sync_lifecycle`` → AC 2 + 4 (full lifecycle + idempotent
+   second run + the fresh ``kb_overview`` row)
+3. ``test_double_trigger_409`` → AC 2 (one sync at a time)
+"""
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+import time
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+import pytest
+from playwright.sync_api import Page, expect
+from sqlalchemy import text
+
+from app.db import SessionLocal
+from app.models import KbOverview
+from e2e.auth_helpers import login
+from e2e.conftest import (
+    ADMIN_PASSWORD,
+    APP_PORT,
+    SESSION_SECRET,
+    USE_REAL_LLM,
+    _wait_http,
+)
+
+REPO = Path(__file__).resolve().parents[2]
+APP_URL = f"http://127.0.0.1:{APP_PORT}"
+
+#: The unique sentinel inside the fixture doc (task 03 step 1) — its
+#: import into the Sources table proves the REAL clone was indexed.
+SENTINEL = "RESE-SYNC-SENTINEL-9b2c"
+FIXTURE_DOC = "notes/sync-fixture.md"
+
+#: "Synced HH:MM" — the local-time last-result label (sources.js's
+#: fmtSyncTime), any hour/minute.
+SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}")
+
+#: Real git clone + embed against the mock LLM — generous budget
+#: (task 03: the sync can legitimately take a while, no client timeout).
+SYNC_TIMEOUT_MS = 60_000
+
+
+def _git(cwd: Path, *args: str) -> None:
+    """Run git in *cwd*; a non-zero exit fails the fixture loudly."""
+    proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise AssertionError(f"git {' '.join(args)} failed: {proc.stderr.strip()}")
+
+
+@pytest.fixture(scope="module")
+def sync_git_repo(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    """A real one-commit git repo the sync must clone (task 03 step 1).
+
+    ``tmp_path`` is function-scoped while the module-scoped app fixture
+    needs the repo for the module's lifetime, so it is built under
+    ``tmp_path_factory`` (the same pytest-managed temp area, module-safe)
+    via real ``git`` subprocess calls.
+    """
+    root = tmp_path_factory.mktemp("sync_git")
+    repo = root / "homelab-notes"
+    (repo / "notes").mkdir(parents=True)
+    (repo / "notes" / "sync-fixture.md").write_text(
+        "# Sync fixture note\n"
+        "\n"
+        "One small note that exists only to prove the admin sync button\n"
+        "end to end: a real git clone, a real import, a real KB overview\n"
+        "regeneration.\n"
+        "\n"
+        f"Marker: {SENTINEL}\n",
+        encoding="utf-8",
+    )
+    _git(repo, "init", "-q")
+    _git(repo, "add", "-A")
+    _git(
+        repo,
+        "-c", "user.email=e@x", "-c", "user.name=t",
+        "-c", "commit.gpgsign=false",  # the fixture commit never signs
+        "commit", "-qm", "one",
+    )
+    assert (repo / ".git").is_dir()
+    return repo
+
+
+@pytest.fixture(scope="module")
+def app_server(mock_llm: int, sync_git_repo: Path) -> Iterator[str]:
+    """The real app under test — per-module env: the sync's subject is a
+    real ``file://`` git source with its own checkout dir (the conftest
+    session app boots without ``BOR_GIT_SOURCES`` and is never started
+    in this isolated run)."""
+    env = dict(os.environ)
+    env.pop("DEBUGPY", None)
+    env["BOR_ENVIRONMENT"] = "e2e"
+    env["BOR_STATIC_DIR"] = str(REPO / "frontend")
+    env["BOR_LLM_BASE_URL"] = (
+        "https://aipi.reeseapps.com/v1"
+        if USE_REAL_LLM
+        else f"http://127.0.0.1:{mock_llm}/v1"
+    )
+    # Mock-calibrated threshold (conftest pattern) — keeps every story
+    # suite's deterministic gate behavior; production default stays 0.62.
+    env["BOR_RELEVANCE_THRESHOLD"] = "0.30"
+    env.setdefault(
+        "BOR_DATABASE_URL",
+        "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese",
+    )
+    # Phase 16: admin auth must be set or create_app() refuses to boot.
+    env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD
+    env["BOR_SESSION_SECRET"] = SESSION_SECRET
+    # Phase 32: the sync button's subject — one real local repo.
+    env["BOR_GIT_SOURCES"] = f"file://{sync_git_repo}"
+    env["BOR_SOURCES_DIR"] = str(sync_git_repo.parent / "checkouts")
+    proc = subprocess.Popen(
+        [sys.executable, "-m", "uvicorn", "app.main:app",
+         "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"],
+        cwd=REPO,
+        env=env,
+    )
+    try:
+        _wait_http(f"{APP_URL}/api/health")
+        yield APP_URL
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+
+@pytest.fixture(scope="module")
+def app_url(app_server: str) -> str:
+    return app_server
+
+
+def _truncate_kb() -> None:
+    """Fresh KB per test (the E2E isolation pattern): the sync's counts
+    and the ``kb_overview`` row must be the sync's own doing."""
+    with SessionLocal() as db:
+        db.execute(text("TRUNCATE chunks, documents, query_log, kb_overview"))
+        db.commit()
+
+
+@pytest.fixture(autouse=True)
+def _clean_kb(db_ready: None) -> Iterator[None]:
+    _truncate_kb()
+    yield
+
+
+def _overview_row() -> KbOverview | None:
+    with SessionLocal() as db:
+        return db.get(KbOverview, 1)
+
+
+def _wait_sync_done(page: Page, app_url: str, timeout_s: float = 60.0) -> dict[str, Any]:
+    """Poll the (cookie-authenticated) status endpoint until the run
+    reaches a terminal state — exactly what the UI's 2 s poll loop
+    observes."""
+    deadline = time.monotonic() + timeout_s
+    body: dict[str, Any] = {}
+    while time.monotonic() < deadline:
+        r = page.request.get(f"{app_url}/api/sync/status")
+        assert r.status == 200
+        body = r.json()
+        if body["state"] in ("success", "failed"):
+            return body
+        time.sleep(0.5)
+    raise AssertionError(f"sync did not reach a terminal state: {body}")
+
+
+# --- 1. Anonymous: no button, locked endpoints ----------------------------
+
+
+def test_anonymous_sees_no_button(page: Page, app_url: str, db_ready: None) -> None:
+    """AC 1 + 3 — anonymous: ``#sync-btn`` never leaves ``hidden`` (the
+    whoami reveal is admin-only, so for a logged-out visitor it is
+    absent from the *revealed* DOM) and both sync endpoints answer
+    403 — the admin-only surface (A10 extended, phase 16 pattern)."""
+    page.goto(f"{app_url}/sources.html")
+    expect(page.locator("#sync-btn")).to_be_hidden()
+    # The same whoami gate that hides the button also hides the admin
+    # nav link and shows the catalog sign-in gate.
+    expect(page.locator("#nav-sources")).to_be_hidden()
+    expect(page.locator("#sources-gate")).to_be_visible()
+    assert page.request.get(f"{app_url}/api/sync/status").status == 403
+    assert page.request.post(f"{app_url}/api/sync").status == 403
+
+
+# --- 2. Admin: the full lifecycle against the real sync path --------------
+
+
+def test_admin_sync_lifecycle(page: Page, app_url: str, db_ready: None) -> None:
+    """AC 2 + 4 — click → "Syncing…" (disabled) → "Synced HH:MM" + the
+    counts, against the REAL pipeline (git clone of the ``file://``
+    fixture → import with prune → KB overview regeneration, mock LLM).
+    Then the idempotent second run: fast-forward pull + sha256 hash
+    skip → "0 added · 1 unchanged"."""
+    login(page, app_url)  # lands on /sources.html (the button's home)
+
+    btn = page.locator("#sync-btn")
+    expect(btn).to_be_visible()
+    expect(page.locator("#sync-label")).to_have_text("Sync sources")
+
+    # --- run 1: clone + import + overview --------------------------------
+    btn.click()
+    expect(btn).to_be_disabled()
+    expect(page.locator("#sync-label")).to_have_text("Syncing…")
+    expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS)
+    expect(btn).to_be_enabled()  # never stale — re-enabled at the terminal state
+    # The fixture doc (one A9-format file) is the only change.
+    expect(page.locator("#sync-result")).to_have_text("1 added")
+
+    # The REAL clone was imported: the fixture path is in the Sources
+    # table (the sentinel lives inside it).
+    expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1)
+
+    # Phase-31 regeneration ran (the import changed the KB): the single
+    # kb_overview row is fresh and non-empty (DB check — truncated
+    # before this test, so it is the sync's own doing).
+    row = _overview_row()
+    assert row is not None and row.content.strip(), (
+        "kb_overview must be regenerated by a successful, KB-changing sync"
+    )
+
+    # --- run 2: idempotent pull + hash skip -------------------------------
+    btn.click()
+    expect(btn).to_be_disabled()
+    expect(page.locator("#sync-label")).to_have_text("Syncing…")
+    expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS)
+    expect(btn).to_be_enabled()
+    # Nothing re-embedded (sha256 delta) — the no-op run announces the
+    # unchanged count instead of an empty live region.
+    expect(page.locator("#sync-result")).to_have_text("0 added · 1 unchanged")
+    # The doc survived the prune re-import (its file is still in the repo).
+    expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1)
+
+
+# --- 3. Concurrency: one sync at a time ------------------------------------
+
+
+def test_double_trigger_409(page: Page, app_url: str, db_ready: None) -> None:
+    """AC 2 — a second trigger while a run is in flight gets 409
+    ("a sync is already running"); the UI adopts the in-flight run
+    instead of starting a second one. The E2E path adds that the REAL
+    run behind the 409 still completes (the adopted run is the one and
+    only run)."""
+    login(page, app_url)
+
+    first = page.request.post(f"{app_url}/api/sync")
+    assert first.status == 202, first.text
+    second = page.request.post(f"{app_url}/api/sync")
+    assert second.status == 409, second.text
+    assert second.json()["detail"] == "a sync is already running"
+
+    # The adopted (single) run still completes successfully.
+    body = _wait_sync_done(page, app_url)
+    assert body["state"] == "success", body
+    assert body["detail"]["added"] == 1  # the fixture doc, fresh after the truncate