"""Phase 89 story E2E (Playwright): per-source ignore paths — the box on the Sources page excludes files/folders from the import (``/git-sources.html``). Story source: ``TODO.md`` L3 (owner roadmap confirmation 2026-09-08 — TODO-derived, no separate user-story file). Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov The story gate proves the whole item end to end through the REAL page + REAL API + REAL sync pipeline (the in-app Sync's background task, mock LLM). **No git, no network** — the suite's single source is a local directory row (``kind="local"``, phase 38), so the sync walks the fixture dir directly: no clone, no pull, no remote of any kind. The module app boots with one deterministic ``BOR_GIT_SOURCES`` env URL (the ``test_git_sources_admin.py`` module-env pattern) that is NEVER synced or cloned: every sync in this suite runs with the local DB row in place (DB rows win over the env fallback), and the one env-fallback test never triggers a sync. Contract under test: * anonymous: the sign-in gate (the ``#git-sources-gate`` pattern), the manager hidden, NO ``/api/git-sources`` call on load, 403 on ``GET``/``POST /api/git-sources`` AND ``PATCH /api/git-sources/{id}`` (the ``test_git_sources_admin.py`` anonymous pin, extended to the phase-89 route); * admin: the per-row "Ignore paths" button opens the page-local alertdialog (the phase-69 ``#remove-confirm-dialog`` idiom); one path per line; Save → ``PATCH`` 200 → the row shows the "N ignored" count tag; the list round-trips through ``GET /api/git-sources``; * the sync honors the list: with ``ignore/`` set, nothing under ``ignore/`` is indexed — no ``documents`` rows (and therefore no chunks/embeddings, no summary calls) — and the sync's ``files`` count excludes the ignored files from the walk; * the spec's no-mid-path rule: ``myfile.md`` ignores the root-level ``myfile.md`` only, never ``sub/myfile.md``; saving the box REPLACES the list (A5 — the previous test's list is gone); * A2: a previously indexed file that newly matches an ignore pattern is PRUNED from the KB on the next sync (``detail.pruned``); * a11y + the error path: the dialog is a real ``role="alertdialog"``, the textarea has a visible label, focus lands on Cancel, Escape closes and returns focus to the trigger; a 501-char entry 422s with the fixed detail, the textarea content is KEPT and the Save button re-enables with its "Save" label; the happy path heals the error state; * A3: env-fallback rows (table empty → ``BOR_GIT_SOURCES``) render the "from .env" tag with NO "Ignore paths" button and the env note. Test → contract mapping: 1. ``test_anonymous_gate_and_403s`` 2. ``test_ignore_box_excludes_from_import`` 3. ``test_prefix_rule_no_mid_path`` 4. ``test_newly_ignored_file_is_pruned`` 5. ``test_editor_a11y_and_error_path`` 6. ``test_env_fallback_rows_have_no_box`` """ from __future__ import annotations import os import subprocess import sys import time import uuid from collections.abc import Iterator from pathlib import Path from typing import Any import pytest from playwright.sync_api import Page, expect from sqlalchemy import text from app.db import SessionLocal from e2e.auth_helpers import login from e2e.conftest import ( ADMIN_PASSWORD, SESSION_SECRET, USE_REAL_LLM, _wait_http, ) REPO = Path(__file__).resolve().parents[2] # Phase 79 (task 04, full inventory): the conftest session app owns its # port in a combined run — this module app binds its own port instead # (a same-port second uvicorn dies on bind and would drive the wrong # server). Env-overridable. APP_PORT = int(os.environ.get("E2E_APP_PORT_IGNORE", "8140")) APP_URL = f"http://127.0.0.1:{APP_PORT}" GIT_SOURCES_URL = "/git-sources.html" #: The module app's ``BOR_GIT_SOURCES`` — one deterministic URL that is #: NEVER cloned (the env-fallback test reads it; the sync tests all run #: with the local DB row in place, so the env list never reaches a #: clone). ENV_SOURCE = "https://github.com/reese/env-alpha.git" #: The sentinel inside the fixture's keeper doc — proof the (non- #: ignored) markdown path was really imported. SENTINEL = "RESE-IGNORE-KEEP-7f3a" #: The six fixture files (source-relative POSIX paths — exactly the #: strings ``documents.path`` stores). KEEP_MD = "keep.md" NOTES_YAML = "notes.yaml" ROOT_MYFILE = "myfile.md" SUB_MYFILE = "sub/myfile.md" IGNORE_SECRET = "ignore/secret.md" IGNORE_DEEP = "ignore/deep/x.txt" ALL_SIX = (KEEP_MD, NOTES_YAML, ROOT_MYFILE, SUB_MYFILE, IGNORE_SECRET, IGNORE_DEEP) #: ``POST /api/sync`` → terminal ``GET /api/sync/status`` (the #: test_sync_button.py polling idiom) — real import of ≤6 small files #: against the mock LLM; generous budget. SYNC_TIMEOUT_S = 60.0 SYNC_TICK_S = 2.0 # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest.fixture(scope="module") def source_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: """The local source's on-disk tree (task 06 step 1). Module-scoped (the same reasoning as ``test_sync_button.py``'s module-scoped fixture note): ``tmp_path`` is function-scoped while the module-scoped app + the seeded row reference the dir for the module's lifetime, so it is built under ``tmp_path_factory`` (the same pytest-managed temp area, module-safe). Plain files — no git: a ``kind="local"`` row is walked directly by the sync (phase 38). """ root = tmp_path_factory.mktemp("ignore_src") (root / "sub").mkdir() (root / "ignore" / "deep").mkdir(parents=True) (root / KEEP_MD).write_text( "# Keep\n" "\n" "The keeper doc — the one markdown file the ignore list must\n" "never touch.\n" f"\n" f"Marker: {SENTINEL}\n", encoding="utf-8", ) # Non-markdown → the phase-30 lite-summary path (proves "not # summarized" applies to the non-md branch too when ignored). (root / NOTES_YAML).write_text( "title: Notes\nitems:\n - one\n - two\n", encoding="utf-8" ) (root / ROOT_MYFILE).write_text( "# Root myfile\n\nRoot-level file — matched by the bare " "``myfile.md`` prefix (the spec's own example).\n", encoding="utf-8", ) (root / SUB_MYFILE).write_text( "# Sub myfile\n\nA namesake in a subdirectory — NEVER matched " "by ``myfile.md`` (no mid-path matching).\n", encoding="utf-8", ) (root / IGNORE_SECRET).write_text( "# Secret\n\nIgnored by the ``ignore/`` prefix.\n", encoding="utf-8" ) (root / IGNORE_DEEP).write_text("deep ignored text\n", encoding="utf-8") return root @pytest.fixture(scope="module") def app_server(mock_llm: int, source_dir: Path) -> Iterator[str]: """The real app under test — per-module env (the conftest pattern, module-scoped): one deterministic ``BOR_GIT_SOURCES`` URL (the env fallback's subject — never cloned), module-scratch checkouts/ upload dirs, the mock LLM. No git anywhere in this suite's path.""" env = dict(os.environ) env.pop("DEBUGPY", None) env["BOR_ENVIRONMENT"] = "e2e" env["BOR_STATIC_DIR"] = str(REPO / "frontend") env["BOR_LLM_BASE_URL"] = ( "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else f"http://127.0.0.1:{mock_llm}/v1" ) # Mock-calibrated threshold (conftest pattern) — no chat turn is # ever sent in this suite, but the app boots with the same env # shape. env["BOR_RELEVANCE_THRESHOLD"] = "0.30" env.setdefault( "BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", ) # Phase 16: admin auth must be set or create_app() refuses to boot. env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD env["BOR_SESSION_SECRET"] = SESSION_SECRET # The empty-table env fallback (only read while ``git_sources`` is # EMPTY — the env-fallback test; never cloned in this suite). env["BOR_GIT_SOURCES"] = ENV_SOURCE # Module-scratch dirs (never reached by this suite's local-row # syncs — kept explicit so a shared checkouts dir cannot leak rows # into the walk). scratch = source_dir.parent env["BOR_SOURCES_DIR"] = str(scratch / "checkouts") env["BOR_UPLOAD_DIR"] = str(scratch / "uploads") proc = subprocess.Popen( [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], cwd=REPO, env=env, ) try: _wait_http(f"{APP_URL}/api/health") yield APP_URL finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() @pytest.fixture(scope="module") def app_url(app_server: str) -> str: return app_server def _truncate_all() -> None: """Fresh registry + KB per test (the E2E isolation pattern, the test_sync_button.py module-env DSN): the sync's counts and the catalog must be each test's own doing.""" with SessionLocal() as db: db.execute( text( "TRUNCATE chunks, documents, query_log, kb_overview, " "git_sources" ) ) db.commit() @pytest.fixture(autouse=True) def _clean(db_ready: None) -> Iterator[None]: """This suite owns ``git_sources`` AND the KB tables (the E2E isolation pattern): suites run in isolation but share one Postgres, and a leftover row would flip the app from the ``BOR_GIT_SOURCES`` env fallback to the DB list (and a leftover document would skew the prune counts). Empty BOTH before and after every test.""" _truncate_all() yield _truncate_all() # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _admin_git_sources_page(page: Page, app_url: str) -> None: """Real form login landing on the git sources page (admin settled: Sign out visible, the manager revealed by the page module).""" login(page, app_url, next=GIT_SOURCES_URL) expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000) expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000) expect(page.locator("#git-sources-gate")).to_be_hidden() expect(page.locator("#git-sources-content")).to_be_visible() def _seed_local_source(page: Page, app_url: str, source_dir: Path) -> str: """(Re)seed the stored row for the module fixture dir via the REAL admin API (201) — ``kind="local"`` (the POST contract is unchanged by phase 89; no ``ignore_paths`` → the stored list is ``[]``) — then reload the view so the page's table re-fetches the row (the API seed is server-side: the mounted view does not know about it without a re-show). Returns the stored (expanded) path — the row's ``url`` location value, the same string the page renders and aria-labels.""" r = page.request.post( f"{app_url}/api/git-sources", data={"kind": "local", "path": str(source_dir)}, ) assert r.status == 201, r.text body = r.json() assert body["ignore_paths"] == [] # the default list, round-tripped page.reload() _admin_git_sources_page(page, app_url) return body["url"] def run_sync(page: Page, app_url: str) -> dict[str, Any]: """``POST /api/sync`` → poll ``GET /api/sync/status`` to a terminal state (the test_sync_button.py polling idiom — ~2 s ticks, 60 s budget). Returns the terminal status body.""" r = page.request.post(f"{app_url}/api/sync") assert r.status == 202, r.text deadline = time.monotonic() + SYNC_TIMEOUT_S body: dict[str, Any] = {} while True: s = page.request.get(f"{app_url}/api/sync/status") assert s.status == 200, s.text body = s.json() if body["state"] in ("success", "failed"): return body assert time.monotonic() < deadline, ( f"sync did not reach a terminal state: {body}" ) time.sleep(SYNC_TICK_S) def _catalog_paths(page: Page, app_url: str) -> list[str]: """The RAG catalog's data source (``GET /api/docs`` — the Sources page's table): every ``documents.path`` (source-relative POSIX).""" r = page.request.get(f"{app_url}/api/docs") assert r.status == 200, r.text return [d["path"] for d in r.json()["documents"]] def _row(page: Page, value: str) -> Any: """The table row whose mono location cell shows ``value``.""" return page.locator("#git-sources-tbody tr", has_text=value) def _open_ignore_editor(page: Page, value: str) -> Any: """Click the row's "Ignore paths" button; the alertdialog opens. Returns the trigger button (focus returns to it on close).""" btn = _row(page, value).locator(".git-source-ignore") expect(btn).to_have_count(1) btn.click() dialog = page.locator("#ignore-editor-dialog") expect(dialog).to_be_visible(timeout=30_000) return btn def _save_ignore_list(page: Page, value: str, entry: str) -> None: """Open the row's box, replace its content with one line (``entry`` — ``""`` clears) and Save; wait for the A5 round-trip: the dialog closes and the row shows the "1 ignored" / cleared count state.""" _open_ignore_editor(page, value) page.fill("#ignore-editor-textarea", entry) page.click("#ignore-editor-save") if entry.strip(): expect(_row(page, value).locator(".git-source-ignore-count")).to_have_text( "1 ignored", timeout=30_000 ) else: expect(_row(page, value).locator(".git-source-ignore-count")).to_have_count(0) expect(page.locator("#ignore-editor-dialog")).to_be_hidden() # --------------------------------------------------------------------------- # 1. Anonymous: gate, inert manager, no API calls, 403s (incl. PATCH) # --------------------------------------------------------------------------- def test_anonymous_gate_and_403s(page: Page, app_url: str, db_ready: None) -> None: page.set_default_timeout(30_000) # Track every /api/git-sources request the page itself makes — the # gate must be reached WITHOUT touching the admin API (the # test_git_sources_admin.py pattern). api_calls: list[str] = [] page.on( "request", lambda r: api_calls.append(r.url) if "/api/git-sources" in r.url else None, ) page.goto(app_url + GIT_SOURCES_URL) expect(page.locator("#sign-in-link")).to_be_visible(timeout=15_000) # The sign-in gate (the #sources-gate pattern, phase 16)… gate = page.locator("#git-sources-gate") expect(gate).to_be_visible() expect(gate).to_contain_text("Sign in to manage the git sources") expect( gate.locator("a[href='/login.html?next=/git-sources.html']") ).to_have_count(1) # …and the manager is absent/inert: table + form + env note, all # inside the hidden #git-sources-content. expect(page.locator("#git-sources-content")).to_be_hidden() expect(page.locator("#git-sources-table")).to_be_hidden() expect(page.locator("#git-source-form")).to_be_hidden() expect(page.locator("#git-sources-env-note")).to_be_hidden() # The gate never called the admin API… assert api_calls == [], f"anonymous page called the git sources API: {api_calls}" # …and the API 403s anonymous callers on GET/POST AND the # phase-89 PATCH (the page context has no cookie — the # test_git_sources_admin.py anonymous pin, extended). assert page.request.get(f"{app_url}/api/git-sources").status == 403 assert ( page.request.post( f"{app_url}/api/git-sources", data={"url": ENV_SOURCE} ).status == 403 ) assert ( page.request.patch( f"{app_url}/api/git-sources/{uuid.uuid4()}", data={"ignore_paths": ["ignore/"]}, ).status == 403 ) # --------------------------------------------------------------------------- # 2. The box sets the list; the sync honors it (nothing under ignore/ # embedded/summarized; the walk's files count excludes it) # --------------------------------------------------------------------------- def test_ignore_box_excludes_from_import( page: Page, app_url: str, db_ready: None, source_dir: Path ) -> None: page.set_default_timeout(30_000) _admin_git_sources_page(page, app_url) # Seed the row (no list) through the real API… stored_path = _seed_local_source(page, app_url, source_dir) expect(page.locator("#git-sources-tbody tr")).to_have_count(1) # …open the box, type "ignore/" (one line), Save → 200 → the row's # "1 ignored" count tag lands (the A5 round-trip through GET). _save_ignore_list(page, stored_path, "ignore/") r = page.request.get(f"{app_url}/api/git-sources") assert r.status == 200, r.text body = r.json() # A1: the raw box line is stored NORMALIZED ("ignore/" → "ignore") # — the round-trip is canonical. assert [s["ignore_paths"] for s in body["sources"]] == [["ignore"]] assert body["from_env"] is False # The real sync (mock LLM): success, and the walk counted the # remaining FOUR files — the two ignore/ files never entered it # (no documents → no chunks/embeddings, no summary calls for # them). (6 fixture files − 2 ignored = 4; the task's "5" is a # slip against its own six-file tree — the exclusion is what is # under test, and it holds either way.) sync = run_sync(page, app_url) assert sync["state"] == "success", sync detail = sync["detail"] assert detail["files"] == 4, detail assert detail["added"] == 4, detail assert detail["errors"] == 0, detail # The catalog holds exactly the four non-ignored paths. paths = _catalog_paths(page, app_url) for p in (KEEP_MD, NOTES_YAML, ROOT_MYFILE, SUB_MYFILE): assert p in paths, f"{p} missing from the catalog: {paths}" assert not any(p.startswith("ignore/") for p in paths), ( f"ignored paths leaked into the catalog: {paths}" ) # --------------------------------------------------------------------------- # 3. The spec's no-mid-path rule: myfile.md ignores the root-level file # only; the box REPLACES the previous test's list (A5) # --------------------------------------------------------------------------- def test_prefix_rule_no_mid_path( page: Page, app_url: str, db_ready: None, source_dir: Path ) -> None: page.set_default_timeout(30_000) _admin_git_sources_page(page, app_url) stored_path = _seed_local_source(page, app_url, source_dir) # Bare filename prefix (no leading/trailing slash): the spec's own # example — matches the root-level file, never the subdirectory # namesake. _save_ignore_list(page, stored_path, "myfile.md") sync = run_sync(page, app_url) assert sync["state"] == "success", sync assert sync["detail"]["files"] == 5, sync["detail"] paths = _catalog_paths(page, app_url) # No mid-path matching: the namesake in sub/ is indexed… assert SUB_MYFILE in paths, f"sub/myfile.md was wrongly ignored: {paths}" # …the root-level file is NOT… assert ROOT_MYFILE not in paths, f"root myfile.md must be ignored: {paths}" # …and the ignore/ files ARE indexed this time — the box replaced # (A5), not appended to, the previous test's "ignore/" list # (regression guard). assert IGNORE_SECRET in paths, f"ignore/secret.md must be indexed: {paths}" assert IGNORE_DEEP in paths, f"ignore/deep/x.txt must be indexed: {paths}" # --------------------------------------------------------------------------- # 4. A2: a previously indexed file that newly matches is pruned # --------------------------------------------------------------------------- def test_newly_ignored_file_is_pruned( page: Page, app_url: str, db_ready: None, source_dir: Path ) -> None: page.set_default_timeout(30_000) _admin_git_sources_page(page, app_url) stored_path = _seed_local_source(page, app_url, source_dir) # Sync 1 — no list: all six files are indexed. first = run_sync(page, app_url) assert first["state"] == "success", first assert first["detail"]["files"] == 6, first["detail"] assert first["detail"]["added"] == 6, first["detail"] paths = _catalog_paths(page, app_url) for p in ALL_SIX: assert p in paths, f"{p} missing after the first sync: {paths}" # New list: "ignore/" — the two previously indexed files now match. _save_ignore_list(page, stored_path, "ignore/") # Sync 2 — the prune=True run deletes the newly ignored documents # (the seen-set mechanism, the A9 out-of-scope-junk precedent). second = run_sync(page, app_url) assert second["state"] == "success", second detail = second["detail"] assert detail["pruned"] == 2, detail assert detail["files"] == 4, detail paths = _catalog_paths(page, app_url) for p in (KEEP_MD, NOTES_YAML, ROOT_MYFILE, SUB_MYFILE): assert p in paths, f"{p} missing after the prune: {paths}" assert not any(p.startswith("ignore/") for p in paths), ( f"newly ignored paths survived the prune: {paths}" ) # --------------------------------------------------------------------------- # 5. The editor's a11y + the 422 error path (and the happy-path heal) # --------------------------------------------------------------------------- def test_editor_a11y_and_error_path( page: Page, app_url: str, db_ready: None, source_dir: Path ) -> None: page.set_default_timeout(30_000) _admin_git_sources_page(page, app_url) stored_path = _seed_local_source(page, app_url, source_dir) # The trigger is the labeled per-row button (left of Remove)… row = _row(page, stored_path) btn = row.locator(".git-source-ignore") expect(btn).to_have_count(1) expect(btn).to_have_attribute( "aria-label", f"Edit ignored paths for local source: {stored_path}" ) expect(row.locator(".git-source-ignore-count")).to_have_count(0) # no list yet # …and it opens a real alertdialog (the phase-69 idiom) with the # visible label and the Cancel-safe focus default. btn.click() dialog = page.locator("#ignore-editor-dialog") expect(dialog).to_be_visible(timeout=30_000) assert dialog.get_attribute("role") == "alertdialog" assert dialog.get_attribute("aria-modal") == "true" # exact: the row button's aria-label ("Edit ignored paths for …") # contains the substring — the visible