brain-of-reese/tests/e2e/test_local_directory_sources.py

"""Phase 38 story E2E (Playwright): local directory sources.

Story: ``.agent/user_stories/local-directory-sources.md``
Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_local_directory_sources.py -v --no-cov

The story gate for the **local directory** kind of the admin-managed
source registry (phase 38): the admin adds an existing, non-git
directory on the same admin page as the git repos (phase 35), and the
real Sync button (phase 32) imports it — with add-time fail-loud
validation (a missing/relative path is rejected inline, naming the
path) and union pruning (a file deleted from the directory leaves the
index on the next sync; removing the row stops it being a source).

The fixture is a **host temp dir** (``tmp_path_factory`` — the app
server runs on the same host, so the path is visible to it) containing
one plain ``.md`` with a distinctive sentinel token. No git anywhere in
this suite (the directory is deliberately NOT a git repo — that is the
point of the story), so no ``BOR_GIT_SOURCES`` and no clone: the sync
pipeline under test is the ``kind=local`` branch (direct directory
walk, re-verified ``.is_dir()`` at sync time) with prune over the union
(the KB was truncated, so the fixture file is the only thing the sync
can import — and the only thing it can prune).

Per-module app env (the conftest pattern, module-scoped — as in
``test_sync_button.py``): this story's app boots WITHOUT
``BOR_GIT_SOURCES`` (the env fallback is git-only by the phase's locked
decision — local directories are DB-registered, no env var), so an
empty table means "no sources configured" until the admin adds the
directory through the real page.

Contract under test:

* anonymous: the sign-in gate (the phase-16/35 ``#git-sources-gate``
  pattern), the manager hidden (list + BOTH add forms inert), NO
  ``/api/git-sources`` call, and 403 on the source routes + the sync
  trigger (the phase-35 regression assertions, A10);
* admin: a missing path (``/nonexistent/bor-e2e``) 422s inline naming
  the path with no row added and the button never stale; the host temp
  dir adds (201 → row with the **Local** badge + the full path in a
  mono cell, input cleared, button re-enabled); the same path again
  409s inline ("already exists", path named) with no second row;
* admin: the header **Sync** button (the phase-32 lifecycle, "Syncing…"
  → "Synced HH:MM") imports the fixture file — it appears in
  ``GET /api/docs`` (and its sentinel is in ``GET
  /api/documents/content``); deleting the file and syncing again prunes
  it (``pruned: 1``, gone from ``GET /api/docs`` — union prune); then
  removing the row on the page makes it disappear (accept the confirm;
  the empty state returns);
* the new local form's a11y basics (UI Structure Check, AGENTS.md rule
  5): labeled input, role=alert error line, ≥44px target, 3px
  focus-visible outline.

Test → story mapping (Playwright Mapping Rule):
1. ``test_anonymous_soft_gate_and_403s``
2. ``test_admin_add_missing_path_then_dir_then_duplicate``
3. ``test_admin_sync_imports_fixture_prunes_after_delete_removes_row``
"""
from __future__ import annotations

import os
import re
import subprocess
import sys
import time
from collections.abc import Iterator
from pathlib import Path
from typing import Any

import pytest
from playwright.sync_api import Dialog, Page, expect
from sqlalchemy import text

from app.db import SessionLocal
from e2e.auth_helpers import login
from e2e.conftest import (
    ADMIN_PASSWORD,
    APP_PORT,
    SESSION_SECRET,
    USE_REAL_LLM,
    _wait_http,
)

REPO = Path(__file__).resolve().parents[2]
APP_URL = f"http://127.0.0.1:{APP_PORT}"

GIT_SOURCES_URL = "/git-sources.html"

#: The unique sentinel inside the fixture doc — its import into the KB
#: (visible via GET /api/docs + /api/documents/content) proves the real
#: sync walked the local directory.
SENTINEL = "RESE-LOCAL-DIR-TOKEN-4d7e"
FIXTURE_REL = "notes/bor-local-fixture.md"

#: A path that must NOT exist on the host — the add-time 422 subject.
MISSING_PATH = "/nonexistent/bor-e2e"

#: "Synced HH:MM" — the local-time last-result label (header.js's
#: fmtSyncTime), any hour/minute.
SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}")

#: A single-file import against the mock LLM is fast, but the sync runs
#: the full pipeline (verify → walk → embed → overview) — same generous
#: budget as test_sync_button.py, no client-side hard timeout.
SYNC_TIMEOUT_MS = 60_000


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture(scope="module")
def local_dir(tmp_path_factory: pytest.TempPathFactory) -> Path:
    """The story's ``~/Notes``: a plain (non-git) directory the admin
    registers as a source. Built under ``tmp_path_factory`` (module
    lifetime, like ``test_sync_button.py``'s fixture repo) and holding
    one A9-format fixture doc with the sentinel token. The app server
    runs on the same host, so this path is visible to it."""
    root = tmp_path_factory.mktemp("bor_local_dir") / "notes-dir"
    (root / "notes").mkdir(parents=True)
    (root / FIXTURE_REL).write_text(
        "# Local directory fixture\n"
        "\n"
        "One small note that exists only to prove the local-directory\n"
        "source story end to end: the admin adds this directory on the\n"
        "git sources page, the real Sync button walks it and imports it,\n"
        "and deleting the file + syncing again prunes it (union prune).\n"
        "\n"
        f"Marker: {SENTINEL}\n",
        encoding="utf-8",
    )
    assert (root / FIXTURE_REL).is_file()
    assert not (root / ".git").exists()  # the story: NOT a git repo
    return root


@pytest.fixture(scope="module")
def app_server(mock_llm: int, local_dir: Path) -> Iterator[str]:
    """The real app under test — per-module env: NO ``BOR_GIT_SOURCES``
    (the env fallback is git-only; local directories are DB-registered)
    and a scratch ``BOR_SOURCES_DIR`` (no git row ever syncs here, it is
    set for hygiene). The session app is never started in this isolated
    run, so no port clash."""
    env = dict(os.environ)
    env.pop("DEBUGPY", None)
    env["BOR_ENVIRONMENT"] = "e2e"
    env["BOR_STATIC_DIR"] = str(REPO / "frontend")
    env["BOR_LLM_BASE_URL"] = (
        "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else f"http://127.0.0.1:{mock_llm}/v1"
    )
    # Mock-calibrated threshold (conftest pattern) — no chat turn is ever
    # sent in this suite, but the app boots with the same env shape.
    env["BOR_RELEVANCE_THRESHOLD"] = "0.30"
    env.setdefault(
        "BOR_DATABASE_URL",
        "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese",
    )
    # Phase 16: admin auth must be set or create_app() refuses to boot.
    env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD
    env["BOR_SESSION_SECRET"] = SESSION_SECRET
    # The repo's .env file carries the owner's BOR_GIT_SOURCES (the app
    # reads it from cwd) — override it with an EMPTY value (the env var
    # beats the .env file): the story is local-kind only, the env
    # fallback stays git-only, and an empty table + empty fallback must
    # mean "no sources configured" until the admin adds the directory.
    env["BOR_GIT_SOURCES"] = ""
    env["BOR_SOURCES_DIR"] = str(local_dir.parent / "checkouts")
    proc = subprocess.Popen(
        [sys.executable, "-m", "uvicorn", "app.main:app",
         "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"],
        cwd=REPO,
        env=env,
    )
    try:
        _wait_http(f"{APP_URL}/api/health")
        yield APP_URL
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            proc.kill()


@pytest.fixture(scope="module")
def app_url(app_server: str) -> str:
    return app_server


def _truncate_all() -> None:
    """Fresh registry + KB per test (the E2E isolation pattern): the
    sync's counts and every GET /api/docs assertion must be this test's
    own doing. The E2E suites share one Postgres, and a leftover
    git_sources row would flip the sync from "no sources configured" to
    importing another suite's source (or a leftover document would show
    up in the docs list the pruned-union assertions inspect)."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log, kb_overview, git_sources"))
        db.commit()


@pytest.fixture(autouse=True)
def _clean(db_ready: None) -> Iterator[None]:
    _truncate_all()
    yield
    _truncate_all()


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _admin_git_sources_page(page: Page, app_url: str) -> None:
    """Real form login landing on the sources page (admin settled:
    Sign out visible, the manager revealed by the page module)."""
    login(page, app_url, next=GIT_SOURCES_URL)
    expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000)
    expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000)
    expect(page.locator("#git-sources-gate")).to_be_hidden()
    expect(page.locator("#git-sources-content")).to_be_visible()


def _add_local_dir(page: Page, path: str) -> None:
    """Add a local directory through the real page form and wait for the
    new row (the 201 → reload → row lifecycle of git-sources.js)."""
    page.fill("#local-source-path", path)
    page.click("#local-source-add")
    expect(
        page.locator("#git-sources-tbody tr", has_text=path)
    ).to_have_count(1, timeout=30_000)


def _click_sync(page: Page) -> None:
    """The phase-32 button lifecycle: click → disabled + "Syncing…" →
    "Synced HH:MM" (re-enabled — never stale). The server status poll
    underneath is what the 2 s UI loop observes."""
    btn = page.locator("#sync-btn")
    expect(btn).to_be_visible()
    btn.click()
    expect(btn).to_be_disabled()
    expect(page.locator("#sync-label")).to_have_text("Syncing…")
    expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS)
    expect(btn).to_be_enabled()


def _wait_sync_done(page: Page, app_url: str, timeout_s: float = 60.0) -> dict[str, Any]:
    """Poll the (cookie-authenticated) status endpoint until the run
    reaches a terminal state — exactly what the UI's 2 s poll loop
    observes (test_sync_button.py's helper)."""
    deadline = time.monotonic() + timeout_s
    body: dict[str, Any] = {}
    while time.monotonic() < deadline:
        r = page.request.get(f"{app_url}/api/sync/status")
        assert r.status == 200
        body = r.json()
        if body["state"] in ("success", "failed"):
            return body
        time.sleep(0.5)
    raise AssertionError(f"sync did not reach a terminal state: {body}")


def _docs(page: Page, app_url: str) -> list[dict[str, Any]]:
    """GET /api/docs as the signed-in page (admin cookie rides along)."""
    r = page.request.get(f"{app_url}/api/docs")
    assert r.status == 200, r.text
    return r.json()["documents"]


# ---------------------------------------------------------------------------
# 1. Anonymous: the soft gate, inert manager, no API calls, 403s
# ---------------------------------------------------------------------------


def test_anonymous_soft_gate_and_403s(
    page: Page, app_url: str, db_ready: None
) -> None:
    """The phase-16/35 gate on this page (regression through the phase-38
    form): anonymous visitors see the sign-in gate and a fully hidden
    manager (list + git form + local form), the page never calls the
    admin API, and every admin route 403s (A10)."""
    page.set_default_timeout(30_000)

    api_calls: list[str] = []
    page.on(
        "request",
        lambda r: api_calls.append(r.url)
        if "/api/git-sources" in r.url
        else None,
    )

    page.goto(app_url + GIT_SOURCES_URL)
    expect(page.locator("#sign-in-link")).to_be_visible(timeout=15_000)
    expect(page.locator("#sign-out-btn")).to_be_hidden()

    gate = page.locator("#git-sources-gate")
    expect(gate).to_be_visible()
    expect(gate).to_contain_text("Sign in to manage the git sources")

    # The manager is absent/inert: list, BOTH add forms, env note — all
    # inside the hidden #git-sources-content.
    expect(page.locator("#git-sources-content")).to_be_hidden()
    expect(page.locator("#git-sources-table")).to_be_hidden()
    expect(page.locator("#git-source-form")).to_be_hidden()
    expect(page.locator("#local-source-form")).to_be_hidden()
    expect(page.locator("#git-sources-env-note")).to_be_hidden()

    # The gate never called the admin API…
    assert api_calls == [], f"anonymous page called the git sources API: {api_calls}"
    # …and the API 403s anonymous callers (the phase-35 assertions):
    # all three source routes, for BOTH kinds, plus the sync trigger.
    assert page.request.get(f"{app_url}/api/git-sources").status == 403
    assert (
        page.request.post(
            f"{app_url}/api/git-sources", data={"url": "https://example.com/x.git"}
        ).status
        == 403
    )
    assert (
        page.request.post(
            f"{app_url}/api/git-sources", data={"kind": "local", "path": "/tmp"}
        ).status
        == 403
    )
    assert (
        page.request.delete(
            f"{app_url}/api/git-sources/00000000-0000-0000-0000-000000000000"
        ).status
        == 403
    )
    assert page.request.post(f"{app_url}/api/sync").status == 403


# ---------------------------------------------------------------------------
# 2. Admin: add validation (missing path, dir, duplicate) + local form
#    a11y basics
# ---------------------------------------------------------------------------


def test_admin_add_missing_path_then_dir_then_duplicate(
    page: Page, app_url: str, local_dir: Path, db_ready: None
) -> None:
    """Add-time fail-loud validation on the real page: a missing path
    422s inline NAMING the path (no row, never-stale button, the input
    survives for one edit); the host temp dir adds (row with the Local
    badge + full path, input cleared); the same path again 409s inline
    ("already exists", path named, no second row)."""
    page.set_default_timeout(30_000)
    _admin_git_sources_page(page, app_url)
    expect(page.locator("#git-sources-tbody tr")).to_have_count(0)

    error = page.locator("#local-source-error")
    add_btn = page.locator("#local-source-add")

    # --- missing path: inline 422 naming it, NO row, button recovers ---
    page.fill("#local-source-path", MISSING_PATH)
    add_btn.click()
    expect(error).to_be_visible(timeout=30_000)
    assert error.get_attribute("role") == "alert"
    expect(error).to_contain_text(MISSING_PATH)
    expect(error).to_contain_text("not a directory")
    expect(page.locator("#git-sources-tbody tr")).to_have_count(0)
    expect(add_btn).to_be_enabled()
    expect(add_btn).to_have_text("Add directory")
    expect(page.locator("#local-source-path")).to_have_value(MISSING_PATH)

    # --- the temp dir: 201 → the row appears with the Local badge ------
    _add_local_dir(page, str(local_dir))
    row = page.locator("#git-sources-tbody tr", has_text=str(local_dir))
    expect(row).to_have_count(1)
    badge = row.locator("span.git-source-kind")
    expect(badge).to_have_text("Local")
    expect(badge).to_have_class(re.compile(r"\bis-local\b"))
    # The mono cell carries the full path (rendered as text)…
    expect(row.locator("td.git-source-url-cell code")).to_have_text(str(local_dir))
    # …and the row's Remove button is labeled with the kind + path.
    expect(row.locator(".git-source-remove")).to_have_attribute(
        "aria-label", f"Remove local source: {local_dir}"
    )
    # The 201 cleared the input and re-enabled the button (never stale).
    expect(page.locator("#local-source-path")).to_have_value("")
    expect(add_btn).to_be_enabled()
    expect(add_btn).to_have_text("Add directory")
    # The API agrees: kind=local with the stored (expanded) path.
    r = page.request.get(f"{app_url}/api/git-sources")
    assert r.status == 200, r.text
    body = r.json()
    assert body["from_env"] is False
    assert [
        (s["kind"], s["path"]) for s in body["sources"]
    ] == [("local", str(local_dir))]

    # --- duplicate: inline 409 naming the path, NO second row -----------
    page.fill("#local-source-path", str(local_dir))
    add_btn.click()
    expect(error).to_be_visible(timeout=30_000)
    expect(error).to_contain_text("already exists")
    expect(error).to_contain_text(str(local_dir))
    expect(page.locator("#git-sources-tbody tr")).to_have_count(1)
    expect(add_btn).to_be_enabled()
    expect(add_btn).to_have_text("Add directory")
    expect(page.locator("#local-source-path")).to_have_value(str(local_dir))

    # --- the new form's a11y basics (UI Structure Check, AGENTS.md 5) ---
    expect(page.get_by_label("Add a local directory")).to_have_count(1)
    box = add_btn.bounding_box()
    assert box is not None and box["height"] >= 44, f"target too small: {box}"
    page.focus("#local-source-path")
    outline = page.evaluate(
        "() => getComputedStyle(document.querySelector('#local-source-path')).outlineWidth"
    )
    assert outline == "3px", f"focus-visible outline missing: {outline!r}"


# ---------------------------------------------------------------------------
# 3. Admin: the real Sync imports the fixture; deleting the file +
#    syncing again prunes it (union prune); removing the row ends it
# ---------------------------------------------------------------------------


def test_admin_sync_imports_fixture_prunes_after_delete_removes_row(
    page: Page, app_url: str, local_dir: Path, db_ready: None
) -> None:
    """The phase-32 button drives the phase-38 pipeline: the header Sync
    imports the local directory's fixture file (GET /api/docs shows it,
    the sentinel is in its content); deleting the file and syncing again
    prunes it (``pruned: 1`` — prune over the union); then removing the
    row on the page makes it disappear (the empty state returns)."""
    page.set_default_timeout(30_000)
    _admin_git_sources_page(page, app_url)

    # Fresh registry (the autouse fixture truncated it) — add the source
    # through the real page, then run the sync lifecycle against it.
    _add_local_dir(page, str(local_dir))

    # --- run 1: the real sync walks the local dir and imports the file -
    _click_sync(page)
    body = _wait_sync_done(page, app_url)
    assert body["state"] == "success", body
    assert body["detail"]["added"] == 1, body["detail"]
    assert body["detail"]["pruned"] == 0, body["detail"]

    # The fixture doc is in GET /api/docs…
    docs = _docs(page, app_url)
    fixture_docs = [d for d in docs if d["path"] == FIXTURE_REL]
    assert len(fixture_docs) == 1, f"fixture doc missing from /api/docs: {docs}"
    assert fixture_docs[0]["source"] == local_dir.name
    # …and its content carries the sentinel (the walk imported THIS file).
    content = page.request.get(
        f"{app_url}/api/documents/content"
        f"?source={local_dir.name}&path={FIXTURE_REL}"
    )
    assert content.status == 200, content.text
    assert SENTINEL in content.json()["content"]

    # --- run 2: file deleted → the next sync prunes it (union prune) ---
    (local_dir / FIXTURE_REL).unlink()
    _click_sync(page)
    body = _wait_sync_done(page, app_url)
    assert body["state"] == "success", body
    assert body["detail"]["pruned"] == 1, body["detail"]
    assert body["detail"]["added"] == 0, body["detail"]
    docs = _docs(page, app_url)
    assert [d for d in docs if d["path"] == FIXTURE_REL] == [], (
        f"fixture doc survived the prune: {docs}"
    )

    # --- remove the row: accept the confirm → it disappears ------------
    removes: list[str] = []
    page.on(
        "request",
        lambda r: removes.append(r.url)
        if r.method == "DELETE" and "/api/git-sources/" in r.url
        else None,
    )

    def handle_dialog(dialog: Dialog) -> None:
        # "Remove this local source…? Its documents stay indexed until
        # the next sync prunes them." — accept it.
        dialog.accept()

    page.on("dialog", handle_dialog)
    try:
        row = page.locator("#git-sources-tbody tr", has_text=str(local_dir))
        row.locator(".git-source-remove").click()
        expect(page.locator("#git-sources-tbody tr")).to_have_count(0, timeout=30_000)
        expect(page.locator("#git-sources-empty")).to_be_visible()
    finally:
        page.remove_listener("dialog", handle_dialog)
    assert len(removes) == 1, f"expected one DELETE, saw: {removes}"
    # The registry is empty again — and with no env git list, a further
    # sync would fail loudly ("no sources configured (git or local)").
    r = page.request.get(f"{app_url}/api/git-sources")
    assert r.json() == {"sources": [], "from_env": True}