"""Phase 38 story E2E (Playwright): local directory sources. Story: ``.agents/user_stories/local-directory-sources.md`` Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_local_directory_sources.py -v --no-cov **Phase-49 rewrite (owner permission 2026-08-28): the page form this story drove is GONE.** The "Add a local directory" form (``#local-source-form``, phase 38) was removed from ``/git-sources.html`` and replaced by the archive upload form (``#archive-upload-form``, phase 49 — its own story E2E is ``test_archive_upload_sources.py``). The phase-38 ACCEPTANCE stands — a local directory can still be registered, imported, pruned, and removed — so this suite now adds local sources through the **authenticated API** the page's own JS no longer calls (``POST /api/git-sources {"kind": "local", "path": …}``, the contract the page used to wrap; the admin cookie rides the browser context via ``page.request``). Do NOT "restore" a form here: adding a plain directory by hand is an API-only operation now. The story gate for the **local directory** kind of the admin-managed source registry (phase 38): the admin registers an existing, non-git directory (API — see above), and the real Sync button (phase 32) imports it — with add-time fail-loud validation (a missing/relative path is rejected with 422, naming the path in the JSON detail) and union pruning (a file deleted from the directory leaves the index on the next sync; removing the row stops it being a source). The fixture is a **host temp dir** (``tmp_path_factory`` — the app server runs on the same host, so the path is visible to it) containing one plain ``.md`` with a distinctive sentinel token. No git anywhere in this suite (the directory is deliberately NOT a git repo — that is the point of the story), so no ``BOR_GIT_SOURCES`` and no clone: the sync pipeline under test is the ``kind=local`` branch (direct directory walk, re-verified ``.is_dir()`` at sync time) with prune over the union (the KB was truncated, so the fixture file is the only thing the sync can import — and the only thing it can prune). Per-module app env (the conftest pattern, module-scoped — as in ``test_sync_button.py``): this story's app boots WITHOUT ``BOR_GIT_SOURCES`` (the env fallback is git-only by the phase's locked decision — local directories are DB-registered, no env var), so an empty table means "no sources configured" until the admin adds the directory through the real page. Contract under test (local adds are API-driven — phase-49 rewrite): * anonymous: the sign-in gate (the phase-16/35 ``#git-sources-gate`` pattern), the manager hidden (list + git add form + archive upload form inert), NO ``/api/git-sources`` call, and 403 on the source routes (incl. the phase-49 upload route) + the sync trigger (the phase-35 regression assertions, A10); * admin: a missing path (``/nonexistent/bor-e2e``) 422s with the JSON detail NAMING the path ("not a directory") and no row added; the host temp dir adds (201 → the row renders with the **Local** badge + the full path in a mono cell once the list re-renders); the same path again 409s ("already exists", path named) with no second row; * admin: the **Sync** button on the Sources page (the phase-32 lifecycle, "Syncing…" → "Synced HH:MM") imports the fixture file — it appears in ``GET /api/docs`` (and its sentinel is in ``GET /api/documents/content``); deleting the file and syncing again prunes it (``pruned: 1``, gone from ``GET /api/docs`` — union prune); then removing the row on the page makes it disappear (accept the confirm; the empty state returns). * (The phase-38 form's a11y assertions moved with the form: the upload form's UI Structure Check lives in the phase-49 story E2E.) Test → story mapping (Playwright Mapping Rule): 1. ``test_anonymous_soft_gate_and_403s`` 2. ``test_admin_add_missing_path_then_dir_then_duplicate`` 3. ``test_admin_sync_imports_fixture_prunes_after_delete_removes_row`` """ from __future__ import annotations import os import re import subprocess import sys import time from collections.abc import Iterator from pathlib import Path from typing import Any import pytest from playwright.sync_api import Page, expect from sqlalchemy import text from app.db import SessionLocal from e2e.auth_helpers import login from e2e.conftest import ( ADMIN_PASSWORD, APP_PORT, SESSION_SECRET, USE_REAL_LLM, _wait_http, ) REPO = Path(__file__).resolve().parents[2] APP_URL = f"http://127.0.0.1:{APP_PORT}" GIT_SOURCES_URL = "/git-sources.html" #: The unique sentinel inside the fixture doc — its import into the KB #: (visible via GET /api/docs + /api/documents/content) proves the real #: sync walked the local directory. SENTINEL = "RESE-LOCAL-DIR-TOKEN-4d7e" FIXTURE_REL = "notes/bor-local-fixture.md" #: A path that must NOT exist on the host — the add-time 422 subject. MISSING_PATH = "/nonexistent/bor-e2e" #: "Synced HH:MM" — the local-time last-result label (header.js's #: fmtSyncTime), any hour/minute. SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}") #: A single-file import against the mock LLM is fast, but the sync runs #: the full pipeline (verify → walk → embed → overview) — same generous #: budget as test_sync_button.py, no client-side hard timeout. SYNC_TIMEOUT_MS = 60_000 # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest.fixture(scope="module") def local_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: """The story's ``~/Notes``: a plain (non-git) directory the admin registers as a source. Built under ``tmp_path_factory`` (module lifetime, like ``test_sync_button.py``'s fixture repo) and holding one A9-format fixture doc with the sentinel token. The app server runs on the same host, so this path is visible to it.""" root = tmp_path_factory.mktemp("bor_local_dir") / "notes-dir" (root / "notes").mkdir(parents=True) (root / FIXTURE_REL).write_text( "# Local directory fixture\n" "\n" "One small note that exists only to prove the local-directory\n" "source story end to end: the admin adds this directory on the\n" "git sources page, the real Sync button walks it and imports it,\n" "and deleting the file + syncing again prunes it (union prune).\n" "\n" f"Marker: {SENTINEL}\n", encoding="utf-8", ) assert (root / FIXTURE_REL).is_file() assert not (root / ".git").exists() # the story: NOT a git repo return root @pytest.fixture(scope="module") def app_server(mock_llm: int, local_dir: Path) -> Iterator[str]: """The real app under test — per-module env: NO ``BOR_GIT_SOURCES`` (the env fallback is git-only; local directories are DB-registered) and a scratch ``BOR_SOURCES_DIR`` (no git row ever syncs here, it is set for hygiene). The session app is never started in this isolated run, so no port clash.""" env = dict(os.environ) env.pop("DEBUGPY", None) env["BOR_ENVIRONMENT"] = "e2e" env["BOR_STATIC_DIR"] = str(REPO / "frontend") env["BOR_LLM_BASE_URL"] = ( "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else f"http://127.0.0.1:{mock_llm}/v1" ) # Mock-calibrated threshold (conftest pattern) — no chat turn is ever # sent in this suite, but the app boots with the same env shape. env["BOR_RELEVANCE_THRESHOLD"] = "0.30" env.setdefault( "BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", ) # Phase 16: admin auth must be set or create_app() refuses to boot. env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD env["BOR_SESSION_SECRET"] = SESSION_SECRET # The repo's .env file carries the owner's BOR_GIT_SOURCES (the app # reads it from cwd) — override it with an EMPTY value (the env var # beats the .env file): the story is local-kind only, the env # fallback stays git-only, and an empty table + empty fallback must # mean "no sources configured" until the admin adds the directory. env["BOR_GIT_SOURCES"] = "" env["BOR_SOURCES_DIR"] = str(local_dir.parent / "checkouts") proc = subprocess.Popen( [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], cwd=REPO, env=env, ) try: _wait_http(f"{APP_URL}/api/health") yield APP_URL finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() @pytest.fixture(scope="module") def app_url(app_server: str) -> str: return app_server def _truncate_all() -> None: """Fresh registry + KB per test (the E2E isolation pattern): the sync's counts and every GET /api/docs assertion must be this test's own doing. The E2E suites share one Postgres, and a leftover git_sources row would flip the sync from "no sources configured" to importing another suite's source (or a leftover document would show up in the docs list the pruned-union assertions inspect).""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log, kb_overview, git_sources")) db.commit() @pytest.fixture(autouse=True) def _clean(db_ready: None) -> Iterator[None]: _truncate_all() yield _truncate_all() # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _admin_git_sources_page(page: Page, app_url: str) -> None: """Real form login landing on the sources page (admin settled: Sign out visible, the manager revealed by the page module).""" login(page, app_url, next=GIT_SOURCES_URL) expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000) expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000) expect(page.locator("#git-sources-gate")).to_be_hidden() expect(page.locator("#git-sources-content")).to_be_visible() def _add_local_dir_api(page: Page, app_url: str, path: str) -> None: """Register a local directory through the authenticated API (phase-49 rewrite: the page form is gone — the ``kind=local`` POST contract the page used to wrap is unchanged, and the admin cookie rides the browser context, cf. test_git_sources_admin.py). ``data`` with a dict is JSON-serialized by Playwright's Python API (there is no ``json=`` kwarg — the JS API's shape is ``json``). 201 is the only success — the caller re-renders the page when it needs the row in the table.""" r = page.request.post( f"{app_url}/api/git-sources", data={"kind": "local", "path": path} ) assert r.status == 201, f"expected 201 for {path}: {r.status} {r.text}" def _click_sync(page: Page, app_url: str) -> None: """The phase-32 button lifecycle: click → disabled + "Syncing…" → "Synced HH:MM" (re-enabled — never stale). The server status poll underneath is what the 2 s UI loop observes. The button's home is the Sources page (owner rework 2026-08-28 — it left the shared navbar), so the helper visits it first.""" page.goto(app_url + "/sources.html") btn = page.locator("#sync-btn") expect(btn).to_be_visible() btn.click() expect(btn).to_be_disabled() expect(page.locator("#sync-label")).to_have_text("Syncing…") expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) expect(btn).to_be_enabled() def _wait_sync_done(page: Page, app_url: str, timeout_s: float = 60.0) -> dict[str, Any]: """Poll the (cookie-authenticated) status endpoint until the run reaches a terminal state — exactly what the UI's 2 s poll loop observes (test_sync_button.py's helper).""" deadline = time.monotonic() + timeout_s body: dict[str, Any] = {} while time.monotonic() < deadline: r = page.request.get(f"{app_url}/api/sync/status") assert r.status == 200 body = r.json() if body["state"] in ("success", "failed"): return body time.sleep(0.5) raise AssertionError(f"sync did not reach a terminal state: {body}") def _docs(page: Page, app_url: str) -> list[dict[str, Any]]: """GET /api/docs as the signed-in page (admin cookie rides along).""" r = page.request.get(f"{app_url}/api/docs") assert r.status == 200, r.text return r.json()["documents"] # --------------------------------------------------------------------------- # 1. Anonymous: the soft gate, inert manager, no API calls, 403s # --------------------------------------------------------------------------- def test_anonymous_soft_gate_and_403s( page: Page, app_url: str, db_ready: None ) -> None: """The phase-16/35 gate on this page (regression through the phase-49 form swap): anonymous visitors see the sign-in gate and a fully hidden manager (list + git form + upload form — the phase-38 local form is gone), the page never calls the admin API, and every admin route 403s (A10).""" page.set_default_timeout(30_000) api_calls: list[str] = [] page.on( "request", lambda r: api_calls.append(r.url) if "/api/git-sources" in r.url else None, ) page.goto(app_url + GIT_SOURCES_URL) expect(page.locator("#sign-in-link")).to_be_visible(timeout=15_000) expect(page.locator("#sign-out-btn")).to_be_hidden() gate = page.locator("#git-sources-gate") expect(gate).to_be_visible() expect(gate).to_contain_text("Sign in to manage the git sources") # The manager is absent/inert: list, git add form, the phase-49 # archive upload form, env note — all inside the hidden # #git-sources-content. expect(page.locator("#git-sources-content")).to_be_hidden() expect(page.locator("#git-sources-table")).to_be_hidden() expect(page.locator("#git-source-form")).to_be_hidden() expect(page.locator("#archive-upload-form")).to_be_hidden() expect(page.locator("#git-sources-env-note")).to_be_hidden() # The phase-38 local-directory form is GONE (phase-49 rewrite) — # the upload form replaced it. expect(page.locator("#local-source-form")).to_have_count(0) expect(page.locator("#local-source-path")).to_have_count(0) expect(page.locator("#local-source-add")).to_have_count(0) # The gate never called the admin API… assert api_calls == [], f"anonymous page called the git sources API: {api_calls}" # …and the API 403s anonymous callers (the phase-35 assertions, # plus the phase-49 upload route): all four source routes, for # BOTH kinds, plus the sync trigger. assert page.request.get(f"{app_url}/api/git-sources").status == 403 assert ( page.request.post( f"{app_url}/api/git-sources", data={"url": "https://example.com/x.git"} ).status == 403 ) assert ( page.request.post( f"{app_url}/api/git-sources", data={"kind": "local", "path": "/tmp"} ).status == 403 ) assert ( page.request.delete( f"{app_url}/api/git-sources/00000000-0000-0000-0000-000000000000" ).status == 403 ) # The phase-49 upload route 403s too (require_admin runs before the # multipart body is ever parsed — the body here is a stand-in JSON # payload, not a real multipart upload). assert ( page.request.post( f"{app_url}/api/git-sources/upload", data={"file": ""} ).status == 403 ) assert page.request.post(f"{app_url}/api/sync").status == 403 # --------------------------------------------------------------------------- # 2. Admin: add validation (missing path, dir, duplicate) — API-driven # (phase-49 rewrite: the form this drove is gone) # --------------------------------------------------------------------------- def test_admin_add_missing_path_then_dir_then_duplicate( page: Page, app_url: str, local_dir: Path, db_ready: None ) -> None: """Add-time fail-loud validation (phase-49 rewrite: the page form is gone, so the same contract is asserted on the API body the page used to render): a missing path 422s NAMING the path in the JSON detail (no row added); the host temp dir adds (201 → the row renders with the Local badge + full path once the list re-renders); the same path again 409s ("already exists", path named, no second row).""" page.set_default_timeout(30_000) _admin_git_sources_page(page, app_url) expect(page.locator("#git-sources-tbody tr")).to_have_count(0) # --- missing path: 422 naming it in the JSON detail, NO row -------- r = page.request.post( f"{app_url}/api/git-sources", data={"kind": "local", "path": MISSING_PATH} ) assert r.status == 422, r.text detail = r.json()["detail"] assert MISSING_PATH in detail, f"detail does not name the path: {detail!r}" assert "not a directory" in detail r = page.request.get(f"{app_url}/api/git-sources") assert r.status == 200, r.text assert r.json()["sources"] == [] # --- the temp dir: 201 → the row renders with the Local badge ------ _add_local_dir_api(page, app_url, str(local_dir)) # The API add is invisible to the open page (its JS no longer adds # local dirs) — re-render the list, exactly as a fresh visit would. page.reload() expect(page.locator("#git-sources-content")).to_be_visible(timeout=30_000) row = page.locator("#git-sources-tbody tr", has_text=str(local_dir)) expect(row).to_have_count(1, timeout=30_000) badge = row.locator("span.git-source-kind") expect(badge).to_have_text("Local") expect(badge).to_have_class(re.compile(r"\bis-local\b")) # The mono cell carries the full path (rendered as text)... expect(row.locator("td.git-source-url-cell code")).to_have_text(str(local_dir)) # …and the row's Remove button is labeled with the kind + path. expect(row.locator(".git-source-remove")).to_have_attribute( "aria-label", f"Remove local source: {local_dir}" ) # The API agrees: kind=local with the stored (expanded) path. r = page.request.get(f"{app_url}/api/git-sources") assert r.status == 200, r.text body = r.json() assert body["from_env"] is False assert [ (s["kind"], s["path"]) for s in body["sources"] ] == [("local", str(local_dir))] # --- duplicate: 409 naming the path, NO second row ----------------- r = page.request.post( f"{app_url}/api/git-sources", data={"kind": "local", "path": str(local_dir)} ) assert r.status == 409, r.text detail = r.json()["detail"] assert "already exists" in detail assert str(local_dir) in detail r = page.request.get(f"{app_url}/api/git-sources") assert r.status == 200, r.text assert len(r.json()["sources"]) == 1 # the row was NOT duplicated # --------------------------------------------------------------------------- # 3. Admin: the real Sync imports the fixture; deleting the file + # syncing again prunes it (union prune); removing the row ends it # --------------------------------------------------------------------------- def test_admin_sync_imports_fixture_prunes_after_delete_removes_row( page: Page, app_url: str, local_dir: Path, db_ready: None ) -> None: """The phase-32 button drives the phase-38 pipeline: the Sources- page Sync imports the local directory's fixture file (GET /api/docs shows it, the sentinel is in its content); deleting the file and syncing again prunes it (``pruned: 1`` — prune over the union); then removing the row on the page makes it disappear (the empty state returns). The local add is API-driven (phase-49 rewrite).""" page.set_default_timeout(30_000) _admin_git_sources_page(page, app_url) # Fresh registry (the autouse fixture truncated it) — register the # source via the API (the page form is gone), then run the sync # lifecycle against it. _add_local_dir_api(page, app_url, str(local_dir)) # --- run 1: the real sync walks the local dir and imports the file - _click_sync(page, app_url) body = _wait_sync_done(page, app_url) assert body["state"] == "success", body assert body["detail"]["added"] == 1, body["detail"] assert body["detail"]["pruned"] == 0, body["detail"] # The fixture doc is in GET /api/docs… docs = _docs(page, app_url) fixture_docs = [d for d in docs if d["path"] == FIXTURE_REL] assert len(fixture_docs) == 1, f"fixture doc missing from /api/docs: {docs}" assert fixture_docs[0]["source"] == local_dir.name # …and its content carries the sentinel (the walk imported THIS file). content = page.request.get( f"{app_url}/api/documents/content" f"?source={local_dir.name}&path={FIXTURE_REL}" ) assert content.status == 200, content.text assert SENTINEL in content.json()["content"] # --- run 2: file deleted → the next sync prunes it (union prune) --- (local_dir / FIXTURE_REL).unlink() _click_sync(page, app_url) body = _wait_sync_done(page, app_url) assert body["state"] == "success", body assert body["detail"]["pruned"] == 1, body["detail"] assert body["detail"]["added"] == 0, body["detail"] docs = _docs(page, app_url) assert [d for d in docs if d["path"] == FIXTURE_REL] == [], ( f"fixture doc survived the prune: {docs}" ) # --- remove the row: the phase-69 in-app confirmation modal -------- # (window.confirm is retired — the row's Remove button opens the # #remove-confirm-dialog, which names the source and states the # full-removal policy; "Remove source" runs the server-side DELETE.) # Back on the manager page (the sync clicks visited the Sources page). page.goto(app_url + GIT_SOURCES_URL) removes: list[str] = [] page.on( "request", lambda r: removes.append(r.url) if r.method == "DELETE" and "/api/git-sources/" in r.url else None, ) row = page.locator("#git-sources-tbody tr", has_text=str(local_dir)) row.locator(".git-source-remove").click() dialog = page.locator("#remove-confirm-dialog") expect(dialog).to_be_visible() expect(dialog.locator("#remove-confirm-source")).to_contain_text(str(local_dir)) # Confirm: the full cleanup runs server-side; the row disappears. dialog.locator("#remove-confirm-remove").click() expect(page.locator("#git-sources-tbody tr")).to_have_count(0, timeout=30_000) expect(page.locator("#git-sources-empty")).to_be_visible() assert len(removes) == 1, f"expected one DELETE, saw: {removes}" # The registry is empty again — and with no env git list, a further # sync would fail loudly ("no sources configured (git or local)"). r = page.request.get(f"{app_url}/api/git-sources") assert r.json() == {"sources": [], "from_env": True}