From 8c706259e98b264b0a7f25693567363204e36b04 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Wed, 9 Sep 2026 01:45:42 -0400 Subject: [PATCH] phase: 89_source_ignore_paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All verification complete — TODO.md was already cleared in the roadmap commit; the two extra unit-test diffs are necessary fake-signature adaptations for the new keywords. Everything is green, no fixes were needed. ## Phase 89 — final verification pass: ALL GREEN **Verified (all 6 task files present in `complete/`):** - `git_sources.ignore_paths` JSONB column + migration 0013; `alembic downgrade -1 && upgrade head` round-trips (head `0013`) - Importer: `normalize_ignore_path`/`is_ignored`/`_ignore_for_root`, `ignore` in walk + progress pre-walk, `ignore_by_root` in `import_sources` - API: GET/POST carry list; admin-only `PATCH` (replace, 404/422 fixed details, anonymous 403) - Pipelines wired: `_run_sync`, `_run_upload` re-upload, `scripts/import_docs.py` - Sources-page box: dialog, §7.4 save lifecycle, `N ignored` tag, a11y; env rows get no box **Test/lint results:** - `uv run pytest` → 1808 passed - `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90%) - `uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov` → 6 passed (isolated, DB up) - Regressions in isolation: `test_git_sources_admin` 6, `test_archive_upload_sources` 5, `test_sync_button` 3, `test_smoke` 3 — all passed - `uv run ruff check . && uv run pyright` → clean (0 errors) **Completion criteria:** box→PATCH 200→count+GET round-trip ✅ · sync excludes `ignore/` (no docs/chunks/embeddings/summaries) + prunes newly-ignored (pruned==2) ✅ · no-mid-path rule E2E ✅ · PATCH 404/422/replace/clear/403 ✅ · full gate green ✅ · commit + phase move left to harness per rules. **Deviations:** none blocking — E2E pins `files == 4` (overview's "5" was an off-by-one vs its own 6-file tree, documented in-test); `tests/unit/test_importer.py` + `test_sync_button.py` test-double fakes extended for the new keywords (needed for the suite to stay green). **Next pending phase:** none — `todo/` holds only this phase. --- .../89_source_ignore_paths/00_phase.md | 0 .../01_ignore_paths_column.md | 0 .../02_ignore_prefix_matching.md | 0 .../03_ignore_paths_api.md | 0 .../04_ignore_paths_in_import_pipelines.md | 0 .../05_ignore_paths_sources_ui.md | 0 .../06_e2e_ignore_paths.md | 0 .../89_source_ignore_paths__00_phase.a1.err | 0 .../89_source_ignore_paths__00_phase.a1.md | 23 + ..._source_ignore_paths__00_phase.a1.validate | 87 +++ ...gnore_paths__01_ignore_paths_column.a1.err | 0 ...ignore_paths__01_ignore_paths_column.a1.md | 19 + ..._paths__01_ignore_paths_column.a1.validate | 86 +++ ...re_paths__02_ignore_prefix_matching.a1.err | 0 ...ore_paths__02_ignore_prefix_matching.a1.md | 10 + ...ths__02_ignore_prefix_matching.a1.validate | 86 +++ ...e_ignore_paths__03_ignore_paths_api.a1.err | 0 ...ce_ignore_paths__03_ignore_paths_api.a1.md | 17 + ...ore_paths__03_ignore_paths_api.a1.validate | 86 +++ ...04_ignore_paths_in_import_pipelines.a1.err | 0 ..._04_ignore_paths_in_import_pipelines.a1.md | 19 + ...nore_paths_in_import_pipelines.a1.validate | 86 +++ ...e_paths__05_ignore_paths_sources_ui.a1.err | 0 ...re_paths__05_ignore_paths_sources_ui.a1.md | 18 + ...hs__05_ignore_paths_sources_ui.a1.validate | 87 +++ ...e_ignore_paths__06_e2e_ignore_paths.a1.err | 0 ...ce_ignore_paths__06_e2e_ignore_paths.a1.md | 11 + ...ore_paths__06_e2e_ignore_paths.a1.validate | 87 +++ .../versions/0013_git_source_ignore_paths.py | 48 ++ app/api/git_sources.py | 132 +++- app/api/sync.py | 29 +- app/models.py | 14 +- app/rag/importer.py | 76 ++- app/schemas.py | 32 +- frontend/assets/git-sources.js | 228 +++++++ frontend/assets/styles.css | 215 ++++++ frontend/index.html | 38 ++ scripts/import_docs.py | 50 +- tests/e2e/test_source_ignore_paths.py | 640 ++++++++++++++++++ tests/integration/test_git_sources_api.py | 230 ++++++- tests/integration/test_git_sources_upload.py | 90 +++ tests/integration/test_import_docs_git.py | 125 +++- tests/integration/test_importer_ignore.py | 202 ++++++ tests/integration/test_sync_api.py | 132 ++++ tests/unit/test_importer.py | 9 +- tests/unit/test_importer_ignore.py | 180 +++++ tests/unit/test_models.py | 41 +- tests/unit/test_source_ignore_paths.py | 546 +++++++++++++++ tests/unit/test_sync_button.py | 1 + 49 files changed, 3717 insertions(+), 63 deletions(-) rename .agents/phases/{todo => complete}/89_source_ignore_paths/00_phase.md (100%) rename .agents/phases/{todo => complete}/89_source_ignore_paths/01_ignore_paths_column.md (100%) rename .agents/phases/{todo => complete}/89_source_ignore_paths/02_ignore_prefix_matching.md (100%) rename .agents/phases/{todo => complete}/89_source_ignore_paths/03_ignore_paths_api.md (100%) rename .agents/phases/{todo => complete}/89_source_ignore_paths/04_ignore_paths_in_import_pipelines.md (100%) rename .agents/phases/{todo => complete}/89_source_ignore_paths/05_ignore_paths_sources_ui.md (100%) rename .agents/phases/{todo => complete}/89_source_ignore_paths/06_e2e_ignore_paths.md (100%) create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.err create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.md create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.validate create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.err create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.md create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.validate create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.err create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.md create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.validate create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.err create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.md create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.validate create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.err create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.md create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.validate create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.err create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.md create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.validate create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.err create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.md create mode 100644 .agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.validate create mode 100644 alembic/versions/0013_git_source_ignore_paths.py create mode 100644 tests/e2e/test_source_ignore_paths.py create mode 100644 tests/integration/test_importer_ignore.py create mode 100644 tests/unit/test_importer_ignore.py create mode 100644 tests/unit/test_source_ignore_paths.py diff --git a/.agents/phases/todo/89_source_ignore_paths/00_phase.md b/.agents/phases/complete/89_source_ignore_paths/00_phase.md similarity index 100% rename from .agents/phases/todo/89_source_ignore_paths/00_phase.md rename to .agents/phases/complete/89_source_ignore_paths/00_phase.md diff --git a/.agents/phases/todo/89_source_ignore_paths/01_ignore_paths_column.md b/.agents/phases/complete/89_source_ignore_paths/01_ignore_paths_column.md similarity index 100% rename from .agents/phases/todo/89_source_ignore_paths/01_ignore_paths_column.md rename to .agents/phases/complete/89_source_ignore_paths/01_ignore_paths_column.md diff --git a/.agents/phases/todo/89_source_ignore_paths/02_ignore_prefix_matching.md b/.agents/phases/complete/89_source_ignore_paths/02_ignore_prefix_matching.md similarity index 100% rename from .agents/phases/todo/89_source_ignore_paths/02_ignore_prefix_matching.md rename to .agents/phases/complete/89_source_ignore_paths/02_ignore_prefix_matching.md diff --git a/.agents/phases/todo/89_source_ignore_paths/03_ignore_paths_api.md b/.agents/phases/complete/89_source_ignore_paths/03_ignore_paths_api.md similarity index 100% rename from .agents/phases/todo/89_source_ignore_paths/03_ignore_paths_api.md rename to .agents/phases/complete/89_source_ignore_paths/03_ignore_paths_api.md diff --git a/.agents/phases/todo/89_source_ignore_paths/04_ignore_paths_in_import_pipelines.md b/.agents/phases/complete/89_source_ignore_paths/04_ignore_paths_in_import_pipelines.md similarity index 100% rename from .agents/phases/todo/89_source_ignore_paths/04_ignore_paths_in_import_pipelines.md rename to .agents/phases/complete/89_source_ignore_paths/04_ignore_paths_in_import_pipelines.md diff --git a/.agents/phases/todo/89_source_ignore_paths/05_ignore_paths_sources_ui.md b/.agents/phases/complete/89_source_ignore_paths/05_ignore_paths_sources_ui.md similarity index 100% rename from .agents/phases/todo/89_source_ignore_paths/05_ignore_paths_sources_ui.md rename to .agents/phases/complete/89_source_ignore_paths/05_ignore_paths_sources_ui.md diff --git a/.agents/phases/todo/89_source_ignore_paths/06_e2e_ignore_paths.md b/.agents/phases/complete/89_source_ignore_paths/06_e2e_ignore_paths.md similarity index 100% rename from .agents/phases/todo/89_source_ignore_paths/06_e2e_ignore_paths.md rename to .agents/phases/complete/89_source_ignore_paths/06_e2e_ignore_paths.md diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.err b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.md b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.md new file mode 100644 index 0000000..979a434 --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.md @@ -0,0 +1,23 @@ +All verification complete — TODO.md was already cleared in the roadmap commit; the two extra unit-test diffs are necessary fake-signature adaptations for the new keywords. Everything is green, no fixes were needed. + +## Phase 89 — final verification pass: ALL GREEN + +**Verified (all 6 task files present in `complete/`):** +- `git_sources.ignore_paths` JSONB column + migration 0013; `alembic downgrade -1 && upgrade head` round-trips (head `0013`) +- Importer: `normalize_ignore_path`/`is_ignored`/`_ignore_for_root`, `ignore` in walk + progress pre-walk, `ignore_by_root` in `import_sources` +- API: GET/POST carry list; admin-only `PATCH` (replace, 404/422 fixed details, anonymous 403) +- Pipelines wired: `_run_sync`, `_run_upload` re-upload, `scripts/import_docs.py` +- Sources-page box: dialog, §7.4 save lifecycle, `N ignored` tag, a11y; env rows get no box + +**Test/lint results:** +- `uv run pytest` → 1808 passed +- `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90%) +- `uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov` → 6 passed (isolated, DB up) +- Regressions in isolation: `test_git_sources_admin` 6, `test_archive_upload_sources` 5, `test_sync_button` 3, `test_smoke` 3 — all passed +- `uv run ruff check . && uv run pyright` → clean (0 errors) + +**Completion criteria:** box→PATCH 200→count+GET round-trip ✅ · sync excludes `ignore/` (no docs/chunks/embeddings/summaries) + prunes newly-ignored (pruned==2) ✅ · no-mid-path rule E2E ✅ · PATCH 404/422/replace/clear/403 ✅ · full gate green ✅ · commit + phase move left to harness per rules. + +**Deviations:** none blocking — E2E pins `files == 4` (overview's "5" was an off-by-one vs its own 6-file tree, documented in-test); `tests/unit/test_importer.py` + `test_sync_button.py` test-double fakes extended for the new keywords (needed for the suite to stay green). + +**Next pending phase:** none — `todo/` holds only this phase. diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.validate b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.validate new file mode 100644 index 0000000..8bb6e3b --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__00_phase.a1.validate @@ -0,0 +1,87 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 27%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 39%] +........................................................................ [ 43%] +........................................................................ [ 47%] +........................................................................ [ 51%] +........................................................................ [ 55%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 67%] +........................................................................ [ 71%] +........................................................................ [ 75%] +........................................................................ [ 79%] +........................................................................ [ 83%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 95%] +........................................................................ [ 99%] +........ [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 178 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 235 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 101 0 100% +app/api/tokens.py 28 0 100% +app/config.py 141 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 19 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 64 0 100% +app/models.py 95 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 222 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 217 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 237 0 100% +-------------------------------------------------- +TOTAL 3279 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.err b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.md b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.md new file mode 100644 index 0000000..c96cde0 --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.md @@ -0,0 +1,19 @@ +All green. Task 01 complete. + +## Report + +**Implemented (task 01 — storage half of per-source ignore paths):** +- `app/models.py`: `GitSource.ignore_paths` — JSONB NOT NULL, `default=list` (INSERT-time) + server default `'[]'` (verbatim task spec); `text` added to the sqlalchemy imports; module-docstring `git_sources` bullet updated (phase 89). +- `alembic/versions/0013_git_source_ignore_paths.py` (new): `add_column` JSONB/`'[]'`/NOT NULL, `drop_column` downgrade — 0007 house-header style, revises 0012. +- `tests/unit/test_models.py` (extended, house pattern): column exists / NOT NULL / server default `arg.text == "'[]'"` (narrowed via `DefaultClause` + `TextClause` for pyright), plus the Python-side default contract test. + +**Gates (exact commands):** +- `uv run alembic upgrade head` → `0012 -> 0013` clean; `alembic downgrade -1 && alembic upgrade head` round-trips; head = `0013`. +- `uv run pytest tests/integration/test_git_sources_api.py` → 23 passed, **unchanged** (dev DB `git_sources` empty; `'[]'` server default covers existing rows; a live ORM-insert probe confirmed `[]` is stored — probe row deleted). +- `uv run pytest --cov=app --cov-report=term-missing` → **1748 passed**, TOTAL **99%** (>90% ✓, `app/models.py` 100%). +- `uv run ruff check . && uv run pyright` → clean (0 errors). +- `git diff --stat`: only `app/models.py`, the new migration, and the unit test. + +**Notable deviation:** the task's literal pin `GitSource(...).ignore_paths == []` cannot hold — SQLAlchemy 2.0 `default=list` is an INSERT-time `CallableColumnDefault` (unflushed attr is `None`; the builtin is wrapped in a ctx lambda, so `arg is list` also fails). The test instead pins the INSERT-time default resolving to `[]`; real-DB behavior is covered by the API integration suite. Model code matches the spec verbatim. + +**Next pending task:** `.agents/phases/todo/89_source_ignore_paths/02_ignore_prefix_matching.md` diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.validate b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.validate new file mode 100644 index 0000000..8f2cd83 --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__01_ignore_paths_column.a1.validate @@ -0,0 +1,86 @@ +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 12%] +........................................................................ [ 16%] +........................................................................ [ 20%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 32%] +........................................................................ [ 37%] +........................................................................ [ 41%] +........................................................................ [ 45%] +........................................................................ [ 49%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 61%] +........................................................................ [ 65%] +........................................................................ [ 70%] +........................................................................ [ 74%] +........................................................................ [ 78%] +........................................................................ [ 82%] +........................................................................ [ 86%] +........................................................................ [ 90%] +........................................................................ [ 94%] +........................................................................ [ 98%] +.................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 178 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 212 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 98 0 100% +app/api/tokens.py 28 0 100% +app/config.py 141 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 19 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 64 0 100% +app/models.py 95 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 222 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 180 3 98% +app/rag/llm.py 217 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 232 0 100% +-------------------------------------------------- +TOTAL 3238 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.err b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.md b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.md new file mode 100644 index 0000000..adfb481 --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.md @@ -0,0 +1,10 @@ +All green. Task complete. + +**Report** + +- Implemented in `app/rag/importer.py`: `normalize_ignore_path` / `is_ignored` / `_ignore_for_root` helpers (spec A1 raw-prefix semantics, single normalization choke point); `iter_importable_files(..., ignore=())` walk-level skip; `import_sources(..., ignore_by_root=None)` — pre-walk and processing loop use the same per-root tuple, `seen`/`_prune` untouched so newly-ignored files are pruned on next `prune=True` run (A2); docstrings per house style. +- New `tests/unit/test_importer_ignore.py` (24 tests): normalization table, spec examples verbatim (`/my/files/` ≡ `my/files/` ≡ `my/files`; `myfile.txt` root-level only, no mid-path), raw-prefix A1 edge (`my/files2`, `ab.md`), empty-prefix no-match, `iter_importable_files` on a 5-file tmp tree incl. the "walk receives already-normalized tuples" contract pin, `_ignore_for_root` normalize/drop/missing/None. +- New `tests/integration/test_importer_ignore.py` (5 tests): ignored files never indexed (no doc/chunk rows, no embed or chat call, kept `.txt` still summarized); A2 prune (`pruned == 1`, row gone, pattern given without trailing slash); progress `total` == 2 excludes ignored; no-map byte-identical (`files == 4`); unlisted second root imports everything. +- Test / lint / coverage: `uv run pytest tests/unit/test_importer_ignore.py tests/integration/test_importer_ignore.py -v` → 29 passed; `uv run pytest -q` → exit 0 (1777 collected, no failures); `uv run pytest --cov=app --cov-report=term-missing -q` → TOTAL 99% (>90% gate); `uv run ruff check .` → clean; `uv run pyright` → 0 errors. +- Notable: (1) fixed the phase-64 counting-sentinel in `tests/unit/test_importer.py` to accept/forward the new `ignore` keyword (its monkeypatch stub broke on the new call site — the one extra file beyond the task's stated diff scope); (2) prune test uses a 3-file tree (one file under `ignore/`) so `pruned == 1` pins exactly, as the task specifies. +- Next pending task: `.agents/phases/todo/89_source_ignore_paths/03_ignore_paths_api.md`. diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.validate b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.validate new file mode 100644 index 0000000..e2c79f0 --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__02_ignore_prefix_matching.a1.validate @@ -0,0 +1,86 @@ +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 12%] +........................................................................ [ 16%] +........................................................................ [ 20%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 32%] +........................................................................ [ 36%] +........................................................................ [ 40%] +........................................................................ [ 44%] +........................................................................ [ 48%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 68%] +........................................................................ [ 72%] +........................................................................ [ 76%] +........................................................................ [ 81%] +........................................................................ [ 85%] +........................................................................ [ 89%] +........................................................................ [ 93%] +........................................................................ [ 97%] +................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 178 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 212 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 98 0 100% +app/api/tokens.py 28 0 100% +app/config.py 141 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 19 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 64 0 100% +app/models.py 95 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 222 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 217 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 232 0 100% +-------------------------------------------------- +TOTAL 3248 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.err b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.md b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.md new file mode 100644 index 0000000..e3e2ec7 --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.md @@ -0,0 +1,17 @@ +Everything is green. Final report: + +**Task 03 (phase 89) complete — ignore-list API** + +- `app/schemas.py`: `GitSourceIn.ignore_paths` (optional, raw lines), `GitSourceOut`/`GitSourceRow.ignore_paths` (non-null; env rows `[]`), new `GitSourceIgnoreIn` (required list, A5) +- `app/api/git_sources.py`: shared `_validate_ignore_paths` (normalize + A4 fixed-detail 422s, empty-check first; constants 200/500), GET reports per-row list, POST stores normalized for both kinds, new admin-only `PATCH /api/git-sources/{id}` (replace semantics, 404/422/403) placed after POST before `/upload`; module docstring updated +- `tests/integration/test_git_sources_api.py`: 10 new tests (normalize round-trip both kinds, absent→`[]`, POST+PATCH 422 exact details, missing-field 422, 404, invalid id, replace+clear, A4 boundaries, row-unchanged on reject); existing shape pins updated additively, anonymous test covers PATCH + +**Gates:** +- `uv run pytest` → 1787 passed +- `uv run pytest --cov=app --cov-report=term-missing` → TOTAL 99% (>90% ✓; `git_sources.py` + `schemas.py` 100%) +- `uv run pytest tests/integration/test_git_sources_api.py --no-cov` → 33 passed +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors + +**Notes:** no defects found in prior tasks; diff limited to the 3 task files. `.agents/PLAN.md` is absent in this repo — `00_phase.md` design (locked A1–A5) was the design source; no deviations. + +**Next pending task:** `.agents/phases/todo/89_source_ignore_paths/04_ignore_paths_in_import_pipelines.md` diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.validate b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.validate new file mode 100644 index 0000000..79d3b3a --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__03_ignore_paths_api.a1.validate @@ -0,0 +1,86 @@ +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 12%] +........................................................................ [ 16%] +........................................................................ [ 20%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 32%] +........................................................................ [ 36%] +........................................................................ [ 40%] +........................................................................ [ 44%] +........................................................................ [ 48%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 68%] +........................................................................ [ 72%] +........................................................................ [ 76%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 92%] +........................................................................ [ 96%] +........................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 178 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 232 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 98 0 100% +app/api/tokens.py 28 0 100% +app/config.py 141 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 19 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 64 0 100% +app/models.py 95 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 222 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 217 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 237 0 100% +-------------------------------------------------- +TOTAL 3273 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.err b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.md b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.md new file mode 100644 index 0000000..aa66f46 --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.md @@ -0,0 +1,19 @@ +Task 04 complete. Report: + +**Implemented** +- `app/api/sync.py::_run_sync` — per-row loop now also builds `ignore_by_root` (keyed by the same `str(root)` the importer sees; `setdefault().extend()` union on root-string collision) and passes it to `import_sources`; module docstring step 4 updated +- `app/api/git_sources.py::_run_upload` — step 6 captures the row's saved `ignore_paths` into a plain local (both existing/new-row branches, before `db.close()`); step 8 passes `ignore_by_root={str(final_dir): ignore_paths}`; route docstring steps 6/8 updated +- `scripts/import_docs.py` — `_resolve_sources` returns `(sources, ignore_by_root)` (manual `--source` and legacy fallback → `{}`); `main` unpacks and passes the map; module + function docstrings updated +- Tests: real-import sync test (ignored file excluded from KB + `files` count), fake-import union/empty-map wiring tests, CLI rows-branch/manual map tests, re-upload-respects-PATCHed-list test (pruned on re-scan) + new-source-no-list regression; `ignore_by_root` kwarg added to the 4 `import_sources` fakes + +**Results** +- `uv run pytest -q` → 1795 passed, exit 0 (incl. `tests/integration/test_sync_api.py test_import_docs_git.py test_git_sources_upload.py` + `tests/unit/test_sync_button.py`: 105 passed) +- `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90%); `app/api/sync.py` + `app/api/git_sources.py` 100% +- `uv run ruff check . && uv run pyright` → clean +- E2E regressions in isolation (`test_git_sources_admin`, `test_archive_upload_sources`, `test_sync_button`, `test_smoke`) → 17 passed + +**Decisions** +- One-line mechanical addition to `tests/unit/test_sync_button.py` fake signature (outside the task's listed files) — required to keep the suite green once the runner passes the new kwarg +- Union-order test pins distinct `added_at` (same-timestamp rows tie-break on random uuid) + +**Next pending task:** `05_ignore_paths_sources_ui.md` diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.validate b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.validate new file mode 100644 index 0000000..7029c1c --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__04_ignore_paths_in_import_pipelines.a1.validate @@ -0,0 +1,86 @@ +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 12%] +........................................................................ [ 16%] +........................................................................ [ 20%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 32%] +........................................................................ [ 36%] +........................................................................ [ 40%] +........................................................................ [ 44%] +........................................................................ [ 48%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 68%] +........................................................................ [ 72%] +........................................................................ [ 76%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 92%] +........................................................................ [ 96%] +................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 178 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 235 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 101 0 100% +app/api/tokens.py 28 0 100% +app/config.py 141 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 19 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 64 0 100% +app/models.py 95 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 222 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 217 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 237 0 100% +-------------------------------------------------- +TOTAL 3279 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.err b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.md b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.md new file mode 100644 index 0000000..cb07816 --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.md @@ -0,0 +1,18 @@ +**Task 05 complete — per-row "Ignore paths" box on the Sources page.** + +Implemented: +- `frontend/index.html`: `#ignore-editor-dialog` (phase-69 `#remove-confirm-dialog` pattern — alertdialog, dim backdrop, visible label, mono textarea, `role="alert"` error line, Cancel/Save) placed after the remove dialog in `#git-sources-content` +- `frontend/assets/git-sources.js`: per-row "Ignore paths" button (stored rows only, left of Remove, textContent-only), `N ignored` count tag, `openIgnoreEditor`/`closeIgnoreEditor`/`saveIgnorePaths` with the §7.4 "Saving…" lifecycle, focus-on-Cancel, Escape/Cancel/backdrop close with focus return, blank-line dropping, A5-replace PATCH, LAST-announcement order; module docstring phase-89 bullet +- `frontend/assets/styles.css`: `.ignore-editor-*` dialog family (house tokens, no blur, ≥44px, brand Save / ghost Cancel) + `.git-source-ignore` / `.git-source-ignore-count` row chrome +- `tests/unit/test_source_ignore_paths.py` (new): 13 source-level pins (markup, lifecycle, row guard, CSS, id no-collision) + +Results: +- `uv run pytest tests/unit/test_source_ignore_paths.py --no-cov` → 13 passed +- `uv run pytest --cov=app --cov-report=term-missing` → **1808 passed, TOTAL 99%** (>90%) +- `uv run ruff check . && uv run pyright` → clean (0 errors) +- E2E regressions in isolation: `test_git_sources_admin.py` 6 passed; `test_smoke.py` + `test_archive_upload_sources.py` + `test_sync_button.py` 11 passed +- Headless manual smoke: add local source → open box → save (blank line dropped) → `2 ignored` tag + announcer → refresh prefills (normalized) → Escape closes with focus return → empty box clears (A5) — all verified, source cleaned up + +Decisions: Escape/Cancel/backdrop are no-ops mid-PATCH (in-flight guard, remove-dialog idiom); Save uses the `.new-chat-btn` brand hover pair; count tag = ink-on-bg chip (16.7:1). One bug found+fixed in my own test (f-string doubled the announce phrase). + +Next pending task: `.agents/phases/todo/89_source_ignore_paths/06_e2e_ignore_paths.md`. diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.validate b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.validate new file mode 100644 index 0000000..8bb6e3b --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__05_ignore_paths_sources_ui.a1.validate @@ -0,0 +1,87 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 27%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 39%] +........................................................................ [ 43%] +........................................................................ [ 47%] +........................................................................ [ 51%] +........................................................................ [ 55%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 67%] +........................................................................ [ 71%] +........................................................................ [ 75%] +........................................................................ [ 79%] +........................................................................ [ 83%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 95%] +........................................................................ [ 99%] +........ [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 178 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 235 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 101 0 100% +app/api/tokens.py 28 0 100% +app/config.py 141 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 19 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 64 0 100% +app/models.py 95 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 222 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 217 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 237 0 100% +-------------------------------------------------- +TOTAL 3279 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.err b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.md b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.md new file mode 100644 index 0000000..15df18d --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.md @@ -0,0 +1,11 @@ +**Task 06 complete — report** + +- Implemented `tests/e2e/test_source_ignore_paths.py` (new story suite, 6 tests): anonymous gate + 403s (incl. new PATCH), box→sync exclusion (`ignore/` never walked/embedded/summarized, `files` count excludes it), spec's no-mid-path rule (`sub/myfile.md` kept, root `myfile.md` ignored, list *replaces* per A5), A2 prune of newly-ignored files (`pruned == 2`), editor a11y + 422 path (501-char entry, textarea kept, Save re-enabled, happy-path heal), env-fallback rows with no box + env note. Module app on port 8140 with `kind="local"` fixture dir (6 files) — no git, no network. +- Results (all green): + - `uv run pytest --cov=app --cov-report=term-missing` → 1808 passed, TOTAL **99%** (>90%) + - `uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov` → 6 passed (run twice, stable) + - Regression E2E in isolation: `test_git_sources_admin.py` 6 passed, `test_archive_upload_sources.py` 5 passed, `test_sync_button.py` 3 passed, `test_smoke.py` 3 passed + - `uv run ruff check . && uv run pyright` → All checks passed / 0 errors +- Deviations: (1) task's `files == 5` assertion is an arithmetic slip against its own 6-file tree (6−2=4) — test asserts 4, documented in-code; (2) stored-list round-trip asserts the A1-normalized value (`ignore` for box line `ignore/`); (3) no `git add`/`commit` run — left in working tree per executor rules (harness makes the one atomic `--no-gpg-sign` commit; `TODO.md` already exactly `# TODO`). +- Diff hygiene: matches step-3 file set (the small `tests/unit/test_importer.py` / `test_sync_button.py` modifications pre-date this task, from completed tasks 02/04). +- Next pending task: none — this was the last task of phase 89; the pipeline gate moves the phase dir and commits. diff --git a/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.validate b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.validate new file mode 100644 index 0000000..8bb6e3b --- /dev/null +++ b/.agents/reports/89_source_ignore_paths/89_source_ignore_paths__06_e2e_ignore_paths.a1.validate @@ -0,0 +1,87 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 27%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 39%] +........................................................................ [ 43%] +........................................................................ [ 47%] +........................................................................ [ 51%] +........................................................................ [ 55%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 67%] +........................................................................ [ 71%] +........................................................................ [ 75%] +........................................................................ [ 79%] +........................................................................ [ 83%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 95%] +........................................................................ [ 99%] +........ [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 178 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 7 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 235 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 101 0 100% +app/api/tokens.py 28 0 100% +app/config.py 141 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 108 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 19 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 64 0 100% +app/models.py 95 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 222 0 100% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 217 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 237 0 100% +-------------------------------------------------- +TOTAL 3279 12 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +validation OK diff --git a/alembic/versions/0013_git_source_ignore_paths.py b/alembic/versions/0013_git_source_ignore_paths.py new file mode 100644 index 0000000..c260352 --- /dev/null +++ b/alembic/versions/0013_git_source_ignore_paths.py @@ -0,0 +1,48 @@ +"""git_sources.ignore_paths: per-source ignore-path list (phase 89) + +Revision ID: 0013 +Revises: 0012 +Create Date: 2026-09-08 + +Phase 89 (per-source ignore paths: the owner types a list of +source-relative path prefixes per source — one path per line — into the +box on the Sources page; the importer never walks, embeds, or summarizes +a file whose source-relative path starts with one of them; A1 raw prefix +semantics, A4 limits, A5 replace semantics; one additive, reversible +column, A13): + +* ``git_sources.ignore_paths`` — JSONB NOT NULL with server default + ``'[]'``: the normalized, non-empty source-relative path prefixes. A + file is ignored when its source-relative POSIX path starts with any + entry (no mid-path matching, no globs). Existing rows read ``[]`` — + every pre-phase-89 row imports exactly as before. +""" +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision = "0013" +down_revision = "0012" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "git_sources", + sa.Column( + "ignore_paths", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'[]'"), + nullable=False, + ), + ) + + +def downgrade() -> None: + # Safe order: the column is the only 0013 artefact — dropping it + # leaves 0012's schema byte-identical (A13, fully reversible). + op.drop_column("git_sources", "ignore_paths") diff --git a/app/api/git_sources.py b/app/api/git_sources.py index 2acdca0..92394af 100644 --- a/app/api/git_sources.py +++ b/app/api/git_sources.py @@ -14,10 +14,18 @@ fallback is active). Routes: ``GET`` (DB rows oldest-first, or the env list with ``from_env: true`` while the table is empty; rows carry ``kind`` + -``path``, git rows — and env rows — report ``path: null``), ``POST`` +``path``, git rows — and env rows — report ``path: null``; every row +reports ``ignore_paths`` — DB rows their stored normalized list, env +rows ``[]``, phase 89), ``POST`` (201, validated create; ``kind`` selects the validation: git → exactly the phase-35 URL contract, local → an existing absolute directory, else -422 naming the path), ``POST /upload`` (phase 49, backgrounded in phase +422 naming the path; both kinds accept ``ignore_paths`` — optional, +absent → ``[]``, stored normalized, phase 89), +``PATCH /{source_id}`` (phase 89, A5 — replace one source's ignore +list: 404 unknown id, the required body list is normalized + A4- +validated with fixed-detail 422s and REPLACES the row's list wholesale +— an empty list clears all; 200 → the ``GitSourceOut`` shape), +``POST /upload`` (phase 49, backgrounded in phase 64 task 03 — admin archive upload: the ``.tar``/``.tar.gz``/``.tgz``/ ``.zip`` name/format gate + the 1 MiB-chunk receive with the ``upload_max_mb`` cap run **inline** and answered 202 the moment the @@ -90,7 +98,7 @@ from app.rag.archive_upload import ( swap_in, unpack_archive, ) -from app.rag.importer import import_sources +from app.rag.importer import import_sources, normalize_ignore_path from app.rag.llm import LLMClient, check_models from app.rag.overview import regenerate_overview from app.rag.source_removal import ( @@ -101,6 +109,7 @@ from app.rag.source_removal import ( ) from app.rag.sources_meta import bump_sources_version from app.schemas import ( + GitSourceIgnoreIn, GitSourceIn, GitSourceList, GitSourceOut, @@ -172,6 +181,14 @@ _upload_status = UploadStatus() #: accepted shapes are exactly these four prefixes. URL_RE = re.compile(r"^(https?://|ssh://|git@)") +#: Phase 89, A4 — the per-source ignore-list limits, enforced in +#: :func:`_validate_ignore_paths` (shared by POST and PATCH): at most +#: 200 entries, each ≤500 chars after normalization. The 422 details +#: are fixed strings that never echo the input (the router's +#: credential-safety discipline, applied for consistency). +MAX_IGNORE_PATHS = 200 +MAX_IGNORE_PATH_LENGTH = 500 + @router.get("", response_model=GitSourceList) def list_git_sources( @@ -181,10 +198,13 @@ def list_git_sources( DB rows ordered by ``(added_at, id)`` (oldest first, id tie-break for same-timestamp inserts) with ``from_env: false`` — each row carries - its ``kind`` and, for local rows, the stored ``path`` (git rows and - env rows report ``path: null``); while the table is empty, the + its ``kind``, its ``ignore_paths`` (phase 89 — the stored, + normalized list; ``or []`` guards a row that predated the column), + and, for local rows, the stored ``path`` (git rows and env rows + report ``path: null``); while the table is empty, the ``BOR_GIT_SOURCES`` env URLs as git rows (the env fallback is - git-only) with null ``id``/``added_at`` and ``from_env: true``. + git-only) with null ``id``/``added_at``, ``ignore_paths: []`` (no + DB row to store a list on), and ``from_env: true``. """ rows = db.scalars( select(GitSource).order_by(GitSource.added_at.asc(), GitSource.id.asc()) @@ -200,6 +220,7 @@ def list_git_sources( url=row.url, path=row.path, added_at=row.added_at, + ignore_paths=row.ignore_paths or [], ) for row in rows ], @@ -207,7 +228,9 @@ def list_git_sources( ) return GitSourceList( sources=[ - GitSourceRow(id=None, kind="git", url=url, path=None, added_at=None) + GitSourceRow( + id=None, kind="git", url=url, path=None, added_at=None, ignore_paths=[] + ) for url in get_settings().git_source_list ], from_env=True, @@ -236,9 +259,35 @@ def create_git_source( Wrong field combinations (git without url, local without path, both kinds' fields) are 422 with fixed, input-free details. + + ``ignore_paths`` (phase 89) — optional, both kinds: the raw box + lines are normalized + A4-validated (``_validate_ignore_paths`` — + the fixed-detail 422s) and the normalized list is what is stored and + reported. """ row = _create_git_row(payload, db) if payload.kind == "git" else _create_local_row(payload, db) - return GitSourceOut(id=row.id, url=row.url, added_at=row.added_at) + return GitSourceOut( + id=row.id, url=row.url, added_at=row.added_at, ignore_paths=row.ignore_paths + ) + + +def _validate_ignore_paths(raw: list[str] | None) -> list[str]: + """Normalize + enforce the A4 limits (phase 89); the fixed 422 + details never echo the input (the credential-safety discipline, + applied for consistency). + + Shared by POST and PATCH. The empty check runs FIRST: a + whitespace-only entry must 422, not be silently dropped (the UI + drops blank lines client-side; the API stays defensive). + """ + entries = [normalize_ignore_path(e) for e in (raw or [])] + if any(not e for e in entries): + raise HTTPException(status_code=422, detail="ignore paths must be non-empty") + if len(entries) > MAX_IGNORE_PATHS: + raise HTTPException(status_code=422, detail="a source has at most 200 ignore paths") + if any(len(e) > MAX_IGNORE_PATH_LENGTH for e in entries): + raise HTTPException(status_code=422, detail="an ignore path exceeds 500 characters") + return entries def _commit_new(row: GitSource, duplicate_detail: str, db: Session) -> GitSource: @@ -270,7 +319,13 @@ def _create_git_row(payload: GitSourceIn, db: Session) -> GitSource: if db.scalar(select(GitSource).where(GitSource.url == url)) is not None: raise HTTPException(status_code=409, detail="a git source with this URL already exists") return _commit_new( - GitSource(url=url, kind="git"), "a git source with this URL already exists", db + GitSource( + url=url, + kind="git", + ignore_paths=_validate_ignore_paths(payload.ignore_paths), + ), + "a git source with this URL already exists", + db, ) @@ -296,12 +351,41 @@ def _create_local_row(payload: GitSourceIn, db: Session) -> GitSource: # rows carry the expanded path there too — git URL shapes and absolute # paths cannot collide). return _commit_new( - GitSource(url=path, kind="local", path=path), + GitSource( + url=path, + kind="local", + path=path, + ignore_paths=_validate_ignore_paths(payload.ignore_paths), + ), f"a local source with this path already exists: {path}", db, ) +@router.patch("/{source_id}", response_model=GitSourceOut) +def patch_git_source( + source_id: uuid.UUID, + payload: GitSourceIgnoreIn, + db: Session = Depends(get_db), # noqa: B008 +) -> GitSourceOut: + """Replace one source's ignore list (phase 89, A5). + + 404 unknown id. The body list (required) is normalized + + A4-validated (fixed 422 details) and REPLACES the row's list + wholesale — an empty list clears all. Returns the updated + row's public shape (id, url, added_at, ignore_paths). + """ + row = db.get(GitSource, source_id) + if row is None: + raise HTTPException(status_code=404, detail="git source not found") + row.ignore_paths = _validate_ignore_paths(payload.ignore_paths) + db.commit() + db.refresh(row) + return GitSourceOut( + id=row.id, url=row.url, added_at=row.added_at, ignore_paths=row.ignore_paths + ) + + @router.post("/upload", response_model=UploadAccepted, status_code=202) async def upload_archive( file: UploadFile = File(...), # noqa: B008 @@ -339,14 +423,18 @@ async def upload_archive( 6. upsert the row by ``path`` (``kind='local'``; an existing row is left as-is — ``added_at`` preserved — and the unique index is the backstop: a concurrent insert lands ``failed`` with - ``a local source with this path already exists: ``); + ``a local source with this path already exists: ``); the + row's saved ``ignore_paths`` are captured for the scan (phase + 89: a re-upload of an existing source honors the list the owner + already saved); 7. fail-fast ``check_models`` — ``ModelUnavailableError`` → ``failed`` with the sanitized message (the phase-49 503 becomes a status state, A5); the folder/row are already committed, so the next sync/re-upload retries idempotently; - 8. ``import_sources([folder], llm, prune=True, progress=)`` - + the change-gated ``regenerate_overview`` — the hook feeds the - status ``current_file`` / ``files_done`` / ``files_total``; + 8. ``import_sources([folder], llm, prune=True, progress=, + ignore_by_root={folder: row's list})`` (phase 89) + the + change-gated ``regenerate_overview`` — the hook feeds the status + ``current_file`` / ``files_done`` / ``files_total``; 9. one INFO log line (PLAN §9 / AGENTS.md rule 10 — ``total_ms`` is the background run's duration); 10. ``success`` — ``detail`` = the ``UploadOut`` fields. @@ -502,8 +590,10 @@ async def _run_upload( path = str(final_dir) db = SessionLocal() try: - if db.scalar(select(GitSource).where(GitSource.path == path)) is None: - db.add(GitSource(url=path, kind="local", path=path)) + row = db.scalar(select(GitSource).where(GitSource.path == path)) + if row is None: + row = GitSource(url=path, kind="local", path=path) + db.add(row) try: db.commit() except IntegrityError: @@ -511,6 +601,11 @@ async def _run_upload( raise ValueError( f"a local source with this path already exists: {path}" ) from None + # Phase 89: the row's saved ignore list, copied to plain + # values while the row is still usable in this session — a + # re-upload of an existing source honors the list the owner + # already saved; a fresh row has no list yet. + ignore_paths = list(row.ignore_paths or []) finally: db.close() # Step 7 — fail-fast models (phase 41): ``ModelUnavailableError`` @@ -530,7 +625,10 @@ async def _run_upload( _upload_status.files_done = done _upload_status.files_total = total - summary = await import_sources([final_dir], llm, prune=True, progress=_hook) + summary = await import_sources( + [final_dir], llm, prune=True, progress=_hook, + ignore_by_root={str(final_dir): ignore_paths}, + ) overview = False if summary.added + summary.updated > 0: overview = await regenerate_overview(llm) diff --git a/app/api/sync.py b/app/api/sync.py index aabbb35..c342d76 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -34,9 +34,11 @@ decisions): ``; a failing clone or a missing local dir aborts before any import; 4. ``import_sources(..., prune=True)`` over the single combined list - (git checkouts + local dirs) — prune so files deleted upstream or - out of a local dir leave the index (pruning covers the union; the - CLI's no-prune default is unchanged); + (git checkouts + local dirs), honoring each row's ``ignore_paths`` + (phase 89 — the per-root ignore map is built in the same per-row + loop as the source list) — prune so files deleted upstream, out of + a local dir, or newly matching an ignore pattern leave the index + (pruning covers the union; the CLI's no-prune default is unchanged); 5. when the import changed the KB (added + updated > 0), ``regenerate_overview`` refreshes the single ``kb_overview`` row (phase 31 trigger, best-effort inside); @@ -214,19 +216,26 @@ async def _run_sync() -> None: ) sources_root = Path(settings.sources_dir).expanduser() sources: list[Path] = [] + ignore_by_root: dict[str, list[str]] = {} for row in rows: if row.kind == "git": - sources.append(clone_or_pull(row.url, sources_root / repo_name(row.url))) + root = clone_or_pull(row.url, sources_root / repo_name(row.url)) else: # kind=local — the stored expanded path (phase 38 also # mirrors it in the NOT-NULL ``url`` location column, the # ``or`` keeps the type checker honest); re-verified at # sync time because the directory may have moved or been # deleted since add-time. - path = Path(row.path or row.url).expanduser() - if not path.is_dir(): - raise GitSyncError(f"local source missing: {path}") - sources.append(path) + root = Path(row.path or row.url).expanduser() + if not root.is_dir(): + raise GitSyncError(f"local source missing: {root}") + sources.append(root) + # Phase 89: the row's ignore list, keyed by the SAME root + # string the importer sees; two rows sharing a root string + # get the union (extend, not replace) — the sibling/repo-name + # edge. + if row.ignore_paths: + ignore_by_root.setdefault(str(root), []).extend(row.ignore_paths) # Phase 64 (task 02): the per-file progress hook — the status # endpoint reports the file being processed right now. The # closure captures the module ``_status`` exactly like the state @@ -236,7 +245,9 @@ async def _run_sync() -> None: _status.files_done = done _status.files_total = total - summary: ImportSummary = await import_sources(sources, llm, prune=True, progress=_hook) + summary: ImportSummary = await import_sources( + sources, llm, prune=True, progress=_hook, ignore_by_root=ignore_by_root + ) overview = False if summary.added + summary.updated > 0: overview = await regenerate_overview(llm) diff --git a/app/models.py b/app/models.py index 58cde51..fd07491 100644 --- a/app/models.py +++ b/app/models.py @@ -16,7 +16,9 @@ Data model — see ``.agents/PLAN.md`` §Data Model: * ``git_sources`` — admin-managed source registry (git URLs + local directories) the Sync button and import_docs import (phase 35; ``kind`` discriminator added in - phase 38). + phase 38; ``ignore_paths`` (phase 89 — JSONB list + of normalized path prefixes, server default + ``'[]'``)). * ``saved_chats`` — owner-saved chat conversations: one row per explicitly Saved conversation (auto-``title`` + the ``bor.chat.v1`` message list as JSONB, phase @@ -72,6 +74,7 @@ from sqlalchemy import ( Text, UniqueConstraint, func, + text, ) from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -226,6 +229,15 @@ class GitSource(Base): #: Absolute directory of a ``local`` source; NULL for git rows. #: Unique — Postgres treats NULLs as distinct under a unique index. path: Mapped[str | None] = mapped_column(Text, unique=True) + #: Per-source ignore paths (phase 89, A1/A4): the normalized, + #: non-empty source-relative path prefixes the owner types into the + #: box on the Sources page. A file is ignored when its + #: source-relative POSIX path starts with any entry (raw prefix — + #: no mid-path matching, no globs). Server default '[]' — every + #: pre-phase-89 row imports exactly as before. + ignore_paths: Mapped[list] = mapped_column( + JSONB, default=list, server_default=text("'[]'"), nullable=False + ) added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) diff --git a/app/rag/importer.py b/app/rag/importer.py index f3f3c82..bb61529 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -118,17 +118,63 @@ class ImportSummary: ) +def normalize_ignore_path(entry: str) -> str: + """One ignore-path entry → canonical form (phase 89, A1). + + Trim surrounding whitespace, then strip ALL leading/trailing + ``/`` — so ``"/my/files/"``, ``"my/files/"`` and ``"my/files"`` + all become ``"my/files"``. ``""`` / ``"//"``, + ``" "`` normalize to ``""`` (callers drop empties). + """ + return entry.strip().strip("/") + + +def is_ignored(rel: str, prefixes: tuple[str, ...]) -> bool: + """Phase 89, A1 — the pure prefix rule, nothing else. + + ``rel`` is the source-relative POSIX path WITHOUT a leading + slash (the ``documents.path`` string). Match = ``rel`` STARTS + WITH a normalized entry: raw string prefix — deliberately NO + component-boundary check (``"my/files"`` also matches + ``"my/files2/x.md"``) and NO mid-path matching (``"myfile.txt"`` + matches ``"myfile.txt"`` but not ``"some/path/myfile.txt"``). + """ + return any(rel.startswith(p) for p in prefixes) + + +def _ignore_for_root( + root: Path, ignore_by_root: dict[str, list[str]] | None +) -> tuple[str, ...]: + """The normalized, non-empty prefix tuple for one root (phase 89). + + Keyed by ``str(root)`` — the root string exactly as the caller + passed it in ``sources`` (unambiguous when two rows share a + source *name* but different dirs). Callers may pass RAW box + lines: the importer normalizes + drops empties here, the single + choke point — stored lists (already normalized) normalize to + themselves. + """ + raw = (ignore_by_root or {}).get(str(root)) or [] + return tuple(p for p in (normalize_ignore_path(e) for e in raw) if p) + + def iter_importable_files( root: Path, extensions: frozenset[str], excluded: frozenset[str] = EXCLUDED_DIRS, + ignore: tuple[str, ...] = (), ) -> list[Path]: """All importable files under *root* (sorted), per the A9 scope rules. *extensions* is a set of lowercased dotted suffixes (``{'.md', '.py'}``). Skips: any path with a dot-prefixed component (hidden dirs/files — vendored caches like ``.esphome/.espressif/**``) and the well-known - non-content directories in *excluded*. + non-content directories in *excluded*. *ignore* (phase 89, A1) is a + tuple of ALREADY-normalized, non-empty source-relative path prefixes + (the importer's ``_ignore_for_root`` is the normalization choke point + — raw box lines never reach this function): a file is skipped when its + source-relative POSIX path starts with any entry; the default ``()`` + keeps every existing caller byte-identical. """ if not root.is_dir(): return [] @@ -139,6 +185,8 @@ def iter_importable_files( rel = path.relative_to(root) if any(part.startswith(".") or part in excluded for part in rel.parts): continue + if ignore and is_ignored(rel.as_posix(), ignore): + continue if path.suffix.lower() not in extensions: continue files.append(path) @@ -153,6 +201,7 @@ async def import_sources( limit: int | None = None, session: Session | None = None, progress: Callable[[str, str, int, int], None] | None = None, + ignore_by_root: dict[str, list[str]] | None = None, ) -> ImportSummary: """Import every A9-format file under *sources* (see module docstring). @@ -172,6 +221,18 @@ async def import_sources( ``limit``, the hook still fires per processed file only — ``done`` never exceeds the limit, but ``total`` stays the full pre-walk count (an incomplete walk must not misreport the denominator). + + ``ignore_by_root`` (phase 89, A1/A2) maps ``str(root)`` — the root path + string exactly as passed in *sources* — to that source's RAW ignore-path + lines (the importer normalizes them via ``_ignore_for_root``, the single + choke point): matching files are never walked, so they are never + embedded and never summarized, and the progress pre-walk uses the same + per-root tuple as the processing loop, so ``total`` never counts them. + A file that newly matches a pattern simply never enters ``seen``, so the + next ``prune=True`` run deletes its row automatically (A2 — the A9 + junk-precedent). ``None`` (the default) changes nothing: the map is read + per root, unlisted roots get an empty tuple, and every existing caller + behaves byte-identically. """ if limit is not None and limit <= 0: raise ValueError("limit must be >= 1") @@ -188,7 +249,13 @@ async def import_sources( total = 0 if progress is not None: for root in sources: - total += len(iter_importable_files(root, llm.settings.import_extension_set)) + total += len( + iter_importable_files( + root, + llm.settings.import_extension_set, + ignore=_ignore_for_root(root, ignore_by_root), + ) + ) try: for root in sources: if not root.is_dir(): @@ -198,7 +265,10 @@ async def import_sources( break source = root.name source_names.add(source) - for path in iter_importable_files(root, llm.settings.import_extension_set): + ignore = _ignore_for_root(root, ignore_by_root) + for path in iter_importable_files( + root, llm.settings.import_extension_set, ignore=ignore + ): if limit is not None and summary.files >= limit: break rel = path.relative_to(root).as_posix() diff --git a/app/schemas.py b/app/schemas.py index f875b3b..67aa946 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -327,11 +327,18 @@ class GitSourceIn(BaseModel): Trimmed here; the API layer then ``expanduser()``s it and requires an absolute existing directory (else 422 naming the path — the path is not a secret, unlike a git URL) and no ``url``. + + ``ignore_paths`` (phase 89) is optional at create time (absent → + ``[]``) and carries the RAW box lines — trimming/normalization happens + in the API layer, not the schema, so the A4 422 details stay fixed + strings (the router's credential-safety discipline, applied for + consistency). """ kind: Literal["git", "local"] = "git" url: str | None = Field(default=None, min_length=1, max_length=500) path: str | None = Field(default=None, min_length=1, max_length=2000) + ignore_paths: list[str] | None = Field(default=None) @field_validator("url", mode="before") @classmethod @@ -352,12 +359,14 @@ class GitSourceOut(BaseModel): ``kind=local`` rows, the stored (expanded) directory path — the phase-35 response shape is unchanged by phase 38, so a local 201 reports its path in ``url`` and the full row (``kind`` + ``path``) - via ``GET``. + via ``GET``. ``ignore_paths`` (phase 89) is the stored, normalized + list — non-null (a row created without it reports ``[]``). """ id: uuid.UUID | None url: str added_at: datetime | None + ignore_paths: list[str] class GitSourceRow(BaseModel): @@ -369,7 +378,9 @@ class GitSourceRow(BaseModel): ``path: null``; local rows carry ``path`` (the absolute directory, expanded) and the same string in ``url`` (the table's NOT-NULL location column). ``id`` / ``added_at`` are nullable: env-fallback - rows (table empty) carry neither. + rows (table empty) carry neither. ``ignore_paths`` (phase 89) is the + row's stored, normalized list — env-fallback rows (no DB row to + store a list on) report ``[]``. """ id: uuid.UUID | None @@ -377,6 +388,23 @@ class GitSourceRow(BaseModel): url: str path: str | None added_at: datetime | None + ignore_paths: list[str] + + +class GitSourceIgnoreIn(BaseModel): + """``PATCH /api/git-sources/{source_id}`` body (phase 89, A5). + + ``ignore_paths`` is REQUIRED — the box's lines in REPLACE semantics: + the body list (normalized + A4-validated in the API layer, with the + fixed-detail 422s) becomes the row's whole list — an empty list + clears all; an absent field is a 422 with the model's own detail. + Entries are RAW box lines: normalization + the A4 limits are + enforced in the API layer so the 422 details stay fixed strings + (the router's credential-safety discipline, applied for + consistency). + """ + + ignore_paths: list[str] class GitSourceList(BaseModel): diff --git a/frontend/assets/git-sources.js b/frontend/assets/git-sources.js index 896eccc..212c3fd 100644 --- a/frontend/assets/git-sources.js +++ b/frontend/assets/git-sources.js @@ -143,6 +143,35 @@ * gate and never fetches /api/git-sources (the Sources-page gate * pattern). * + * Phase 89 (task 05) — the per-source ignore-paths editor: every + * STORED row (s.id truthy — env-fallback rows carry NO button, A3: + * nothing is stored to edit) gets an "Ignore paths" button in the + * actions cell (left of Remove) that opens the page-local alertdialog + * (#ignore-editor-dialog — the EXACT #remove-confirm-dialog pattern, + * phase 69): #ignore-editor-source names the source (textContent ONLY + * — the same `value` expression makeRow uses — URLs may embed + * user:pass@ credentials, phase 32), the mono textarea prefills the + * row's stored list (one path per line — the phase-89 A1 prefix rule, + * stated in plain words in the helper copy), and focus lands on + * Cancel (the safe default); Escape / Cancel / the dim backdrop close + * as CANCEL (no request — focus returns to the row's trigger button). + * "Save" runs the §7.4 in-flight lifecycle: both buttons disable + + * the save relabels "Saving…" while the PATCH + * /api/git-sources/{id} is out (blank lines in the box are separators, + * not entries — split + trim + drop empty client-side; the server + * still rejects empty entries defensively, A4). 200 → close (focus + * return), loadSources (the row's "N ignored" count tag lands — the + * A5 replace semantics round-trip through GET), announce — the update + * confirmation is the LAST announcement (the reload's "N sources + * listed." must not overwrite it); non-2xx → the in-dialog + * role="alert" line (the server detail, apiDetail 422-shape-aware) + + * the buttons restored, the dialog STAYS open and the textarea + * content is KEPT (the instruction survives — tuning-form + * convention); network failure → the fixed reachable? line, same + * restore. Rows whose list is non-empty show the "N ignored" count + * tag next to the location (text + a distinct background — + * never color alone, WCAG 1.4.1). + * * Scope boundary (phase locked decisions): adding a git repo does * NOT clone — the sync service (server-side) does that. Removing a * source, however, performs the FULL cleanup server-side (phase 69): @@ -203,6 +232,17 @@ export async function mount(root) { const removeError = root.querySelector("#remove-confirm-error"); const removeCancelBtn = root.querySelector("#remove-confirm-cancel"); const removeRemoveBtn = root.querySelector("#remove-confirm-remove"); + /* Phase 89: the per-source ignore-paths editor — the SAME page-local + alertdialog idiom as the remove-confirm modal (phase 69): static + markup in the shell's Sources view; this module owns the open / + cancel / save lifecycle. */ + const ignoreDialog = root.querySelector("#ignore-editor-dialog"); + const ignoreBackdrop = root.querySelector(".ignore-editor-backdrop"); + const ignoreSourceEl = root.querySelector("#ignore-editor-source"); + const ignoreTextarea = root.querySelector("#ignore-editor-textarea"); + const ignoreErrorEl = root.querySelector("#ignore-editor-error"); + const ignoreCancelBtn = root.querySelector("#ignore-editor-cancel"); + const ignoreSaveBtn = root.querySelector("#ignore-editor-save"); /* Polite live region: the screen-reader confirmation for loads, adds, and removals (the phase-15 announcer pattern). */ @@ -321,6 +361,16 @@ export async function mount(root) { const code = document.createElement("code"); code.textContent = value; // rendered as text, never as HTML urlTd.append(badge, code); + /* Phase 89: the "N ignored" count tag — stored rows with a + non-empty list only (text + a distinct background, never color + alone — WCAG 1.4.1). The list round-trips: (s.ignore_paths || + []) is the same expression openIgnoreEditor prefills from. */ + if (s.id && (s.ignore_paths || []).length > 0) { + const count = document.createElement("span"); + count.className = "git-source-ignore-count"; + count.textContent = `${s.ignore_paths.length} ignored`; + urlTd.append(count); + } tr.appendChild(urlTd); const addedTd = document.createElement("td"); @@ -330,6 +380,22 @@ export async function mount(root) { const actTd = document.createElement("td"); actTd.className = "git-source-actions-cell"; if (s.id) { + /* Phase 89: the per-row ignore-paths editor trigger (A3: stored + rows only — env-fallback rows fall through to the "from + .env" tag below). The aria-label is the ONLY place `value` + appears (setAttribute — never innerHTML); the text label is + static. Sits LEFT of Remove — the non-destructive action + first. */ + const ignoreBtn = document.createElement("button"); + ignoreBtn.type = "button"; + ignoreBtn.className = "git-source-ignore"; + ignoreBtn.setAttribute( + "aria-label", + `Edit ignored paths for ${kindLabel} source: ${value}`, + ); + ignoreBtn.textContent = "Ignore paths"; + ignoreBtn.addEventListener("click", () => openIgnoreEditor(s, ignoreBtn)); + actTd.appendChild(ignoreBtn); const btn = document.createElement("button"); btn.type = "button"; btn.className = "git-source-remove"; @@ -514,6 +580,168 @@ export async function mount(root) { if (removeBackdrop) removeBackdrop.addEventListener("click", cancelRemoveConfirm); if (removeRemoveBtn) removeRemoveBtn.addEventListener("click", confirmRemove); + /* ---------- ignore paths (PATCH /api/git-sources/{id}) — the per-row editor ---------- + * Phase 89: a stored row's "Ignore paths" button (makeRow — left of + * Remove) opens the page-local alertdialog + * (#ignore-editor-dialog — the EXACT #remove-confirm-dialog idiom, + * phase 69) via openIgnoreEditor(s, triggerBtn): #ignore-editor- + * source names the source (textContent ONLY — the same `value` + * expression makeRow uses: s.path ?? s.url for local rows, s.url + * for git — URLs may embed user:pass@ credentials, phase 32), the + * mono textarea PREFILLS the row's stored list (one path per line + * — the phase-89 A1 prefix rule, stated in plain words in the + * helper copy), the error line clears, and focus lands on Cancel + * (the safe default). Escape / Cancel / the dim backdrop close as + * CANCEL: no request, focus returns to the row's trigger button. + * + * "Save" (saveIgnorePaths) runs the §7.4 never-stale lifecycle: + * both buttons disable and the save relabels "Saving…" while the + * PATCH is out — a blank line in the box is a separator, not an + * entry (split + trim + drop empty lines client-side; the server + * still rejects empty entries defensively, A4). 200 → close + * (focus return) → loadSources (the "N ignored" count tag lands — + * the A5 replace round-trips through GET) → announce (the update + * confirmation is the LAST announcement — the reload's "N sources + * listed." must not overwrite it); non-2xx → the in-dialog + * role="alert" line (the server detail, apiDetail 422-shape-aware) + * + the buttons restored — the dialog STAYS open and the textarea + * content is KEPT (the instruction survives — the tuning-form + * convention); network failure → the fixed reachable? line, same + * restore. Env-fallback rows (id null) carry NO button (A3 — + * nothing is stored to edit). */ + let ignoreTarget = null; // the row object of the open editor + let ignoreTriggerBtn = null; // the row's button — focus returns here on close + let ignoreInFlight = false; // §7.4: a PATCH is out (both buttons disabled) + + function openIgnoreEditor(s, triggerBtn) { + if (!ignoreDialog || !ignoreTextarea) return; // defensive — the markup ships with the page + if (ignoreInFlight) return; // one editor at a time + const isLocal = s.kind === "local"; + // The same `value` expression makeRow uses — textContent ONLY. + if (ignoreSourceEl) ignoreSourceEl.textContent = isLocal ? s.path ?? s.url : s.url; + ignoreTextarea.value = (s.ignore_paths || []).join("\n"); + if (ignoreErrorEl) { + ignoreErrorEl.textContent = ""; + ignoreErrorEl.hidden = true; // a new attempt starts clean + } + ignoreInFlight = false; + ignoreTarget = s; + ignoreTriggerBtn = triggerBtn; // recorded for the focus return on close + ignoreDialog.hidden = false; + document.addEventListener("keydown", onIgnoreDialogKeydown); + // Cancel is the safe default — focus lands on it (the remove- + // dialog precedent; visibly: the global 3px :focus-visible + // outline). + if (ignoreCancelBtn) ignoreCancelBtn.focus(); + } + + /* Any close (cancel, success): hide the dialog, reset the textarea + + error line, clear the target, detach the keydown handling, and + return focus to the row's "Ignore paths" button (WCAG 2.1). */ + function closeIgnoreEditor() { + if (!ignoreDialog) return; + ignoreDialog.hidden = true; + ignoreInFlight = false; + if (ignoreTextarea) ignoreTextarea.value = ""; + if (ignoreErrorEl) { + ignoreErrorEl.textContent = ""; + ignoreErrorEl.hidden = true; + } + if (ignoreCancelBtn) ignoreCancelBtn.disabled = false; + if (ignoreSaveBtn) { + ignoreSaveBtn.disabled = false; + ignoreSaveBtn.textContent = "Save"; + } + document.removeEventListener("keydown", onIgnoreDialogKeydown); + const trigger = ignoreTriggerBtn; + ignoreTarget = null; + ignoreTriggerBtn = null; + if (trigger) trigger.focus(); // focus returns to the row's button + } + + /* Escape / Cancel / backdrop all close as cancel — NO request. A + cancel is a no-op while a PATCH is in flight (no half-cancel of + an in-progress save; the buttons are disabled anyway, the + Escape/backdrop paths need this guard). */ + function cancelIgnoreEditor() { + if (ignoreInFlight) return; + closeIgnoreEditor(); + } + + /* While open (attached in openIgnoreEditor, detached in + closeIgnoreEditor): Escape closes as cancel. */ + function onIgnoreDialogKeydown(e) { + if (e.key === "Escape") { + e.preventDefault(); + cancelIgnoreEditor(); + } + } + + async function saveIgnorePaths() { + if (!ignoreTarget || ignoreInFlight) return; // one request at a time + // The box's lines: one path per line — a blank line is a + // separator, not an entry (trim + drop empty; the server still + // rejects empties defensively, A4). + const lines = ignoreTextarea.value.split("\n").map((l) => l.trim()).filter(Boolean); + const t = ignoreTarget; + const value = t.kind === "local" ? (t.path ?? t.url) : t.url; + ignoreInFlight = true; + if (ignoreErrorEl) { + ignoreErrorEl.textContent = ""; + ignoreErrorEl.hidden = true; + } + if (ignoreCancelBtn) ignoreCancelBtn.disabled = true; + if (ignoreSaveBtn) { + ignoreSaveBtn.disabled = true; + ignoreSaveBtn.textContent = "Saving…"; // §7.4 in-flight label + } + try { + const r = await fetch(`/api/git-sources/${encodeURIComponent(t.id)}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ignore_paths: lines }), + }); + if (r.ok) { + // 200: the server replaced the row's list (A5). + closeIgnoreEditor(); // focus returns to the row's button + await loadSources(); // the "N ignored" count tag lands + // The update confirmation is the LAST announcement: the + // reload's "N sources listed." must not overwrite it (the + // same order as the remove flow). + announce(`Ignored paths updated for ${value}`); + return; + } + // non-2xx: the in-dialog role="alert" line (the server detail) + // — the dialog STAYS open and the textarea content is KEPT: + // the fix is one edit + retry, not a re-type. + if (ignoreErrorEl) { + ignoreErrorEl.textContent = await apiDetail(r, "Could not save the ignored paths — try again."); + ignoreErrorEl.hidden = false; + } + } catch { + if (ignoreErrorEl) { + ignoreErrorEl.textContent = "Could not save the ignored paths — is the app reachable?"; + ignoreErrorEl.hidden = false; + } + } finally { + // Never stale (PLAN §7.4): the failure paths re-enable BOTH + // buttons + relabel the save; the success path already closed + // the dialog (which resets them) — the restore is a no-op + // there. + ignoreInFlight = false; + if (ignoreCancelBtn) ignoreCancelBtn.disabled = false; + if (ignoreSaveBtn) { + ignoreSaveBtn.disabled = false; + ignoreSaveBtn.textContent = "Save"; + } + } + } + + /* The dialog's own buttons (static markup — wired once). */ + if (ignoreCancelBtn) ignoreCancelBtn.addEventListener("click", cancelIgnoreEditor); + if (ignoreBackdrop) ignoreBackdrop.addEventListener("click", cancelIgnoreEditor); + if (ignoreSaveBtn) ignoreSaveBtn.addEventListener("click", saveIgnorePaths); + /* ---------- add (POST /api/git-sources) — the git form ---------- * wireAddForm gives the form the §7.4 never-stale lifecycle: while * the request is out the button disables + relabels "Adding…" and diff --git a/frontend/assets/styles.css b/frontend/assets/styles.css index e267458..c4a5e2a 100644 --- a/frontend/assets/styles.css +++ b/frontend/assets/styles.css @@ -2183,6 +2183,173 @@ details.thinking .thinking-text ul { margin: 0 0 0.5rem; } color: var(--bg); } +/* ---------- Per-source ignore-paths editor (phase 89, task 05) ---------- + The shell's Sources view: the page-local alertdialog that edits + one source's ignore list (one path per line — the phase-89 A1 + prefix rule, stated in plain words in the helper copy). The EXACT + #remove-confirm-dialog overlay contract (phase 69): a fixed + full-viewport dim backdrop + a centered panel (z-index 1000, above + the sticky header (20) + skip-link (100); NO blur — the phase-08 + no-blur perf anchor), scaled to the 46rem chat-column width or the + viewport, whichever is narrower. The box: a VISIBLE block label + (WCAG — never aria-label-only) over a mono textarea (the box is a + data entry, not prose — the var(--mono) stack); the error line is + the err pair (err-ink on err-bg 9.3:1, the err-line border); + Cancel is the ghost ink-soft family (5.1:1 on --surface); Save is + the solid brand family (--bg text on --brand 5.2:1 — the + .new-chat-btn convention; the hover lightens the fill). Both + buttons >=44px; :focus-visible via the global 3px outline rule (no + local suppression — focus lands on Cancel at open and is visible); + no animation (reduced-motion safe by construction). AA pairs: title + / copy / label are --ink (or ink-soft) on --surface (13.8:1 / + 5.1:1); the source value is --ink on --bg (16.7:1). Phase-08 + tokens only; system fonts; no CDN. */ +.ignore-editor { + position: fixed; + inset: 0; + z-index: 1000; + display: flex; /* the panel is the only in-flow child — margin: auto centers it */ +} +/* Explicit (the global [hidden] rule already wins — the documented, + testable contract for the skeleton). */ +.ignore-editor[hidden] { display: none; } + +.ignore-editor-backdrop { + position: fixed; + inset: 0; + /* --bg at 82% — the doc-modal dim, no backdrop-filter (no-blur). */ + background: rgba(15, 10, 10, 0.82); +} + +.ignore-editor-panel { + /* position:relative lifts the panel above the fixed backdrop + (positioned elements paint over in-flow siblings otherwise). */ + position: relative; + z-index: 1; + margin: auto; + width: min(46rem, calc(100vw - 2rem)); + padding: 1.5rem; + background: var(--surface); + border: 1px solid var(--line); + border-radius: var(--radius); + box-shadow: var(--shadow-lg); +} + +.ignore-editor-title { + margin: 0 0 0.75rem; + font-size: 1.25rem; + line-height: 1.3; + color: var(--ink); +} + +/* The source's value (git URL or local path) — mono, wrapped (a long + URL must not overflow the panel), --ink on --bg (16.7:1). */ +.ignore-editor-source { + display: block; + margin: 0 0 0.75rem; + padding: 0.5rem 0.65rem; + font-family: var(--mono); + font-size: 0.85rem; + line-height: 1.5; + color: var(--ink); + background: var(--bg); + border: 1px solid var(--line); + border-radius: var(--radius-sm); + overflow-wrap: anywhere; +} + +.ignore-editor-copy { + margin: 0 0 1rem; + color: var(--ink); /* 13.8:1 on --surface */ +} +.ignore-editor-copy code { font-family: var(--mono); font-size: 0.85em; } + +/* Visible block label (WCAG — the textarea is NEVER labelled only via + aria-label); ink-soft on --surface (5.1:1). */ +.ignore-editor-label { + display: block; + margin-bottom: 0.35rem; + color: var(--ink-soft); + font-size: 0.88rem; + font-weight: 600; +} + +/* The box: full panel width, mono (the var(--mono) stack), theme- + consistent border on --bg; the focus-visible ring is the global + 3px outline (no local suppression); vertical resize for long + lists (up to 200 entries, A4). */ +.ignore-editor-textarea { + display: block; + width: 100%; + min-height: 9rem; + padding: 0.55rem 0.65rem; + font-family: var(--mono); + font-size: 0.85rem; + line-height: 1.5; + color: var(--ink); + background: var(--bg); + border: 1px solid var(--line); + border-radius: var(--radius-sm); + resize: vertical; +} + +/* The in-dialog failure line (role=alert): the err pair (err-ink on + err-bg 9.3:1, the err-line border) — the .remove-confirm-error + language. */ +.ignore-editor-error { + margin: 0.75rem 0 0; + padding: 0.5rem 0.65rem; + color: var(--err-ink); + background: var(--err-bg); + border: 1px solid var(--err-line); + border-radius: var(--radius-sm); +} + +.ignore-editor-actions { + display: flex; + justify-content: flex-end; + gap: 0.6rem; + margin-top: 1.1rem; +} + +/* The two dialog buttons: >=44px targets, the house 3px :focus-visible + via the global outline rule (no local override). */ +.ignore-editor-btn { + min-height: 44px; + min-width: 44px; + padding: 0.55rem 1.1rem; + border-radius: var(--radius-sm); + font: inherit; + font-weight: 600; + font-size: 0.9rem; + white-space: nowrap; + cursor: pointer; +} +.ignore-editor-btn:disabled { opacity: 0.5; cursor: wait; } + +/* Cancel — the ghost ink-soft family (5.1:1 on --surface), like the + remove dialog's Cancel; the brand pair on hover (12.4:1 on + brand-soft). */ +.ignore-editor-cancel { + border: 1px solid var(--line); + background: transparent; + color: var(--ink-soft); +} +.ignore-editor-cancel:hover:not(:disabled) { + background: var(--brand-soft); + color: var(--brand-ink); +} + +/* "Save" — the solid brand family (the .new-chat-btn convention): + --bg text on --brand (5.2:1 — AA); the hover lightens the brand + fill (the .new-chat-btn hover pair). */ +.ignore-editor-save { + border: 0; + background: var(--brand); + color: var(--bg); +} +.ignore-editor-save:hover:not(:disabled) { background: #f55a72; } + /* The list: the Sources page's table pattern — full width in the 72rem frame, surface card, horizontally scrollable wrapper (the URL column never wraps or ellipsizes: long URLs, credentials @@ -2266,6 +2433,54 @@ details.thinking .thinking-text ul { margin: 0 0 0.5rem; } .git-source-remove:hover:not(:disabled) { background: var(--err-bg); color: var(--err-ink); border-color: var(--err-line); } .git-source-remove:disabled { opacity: 0.5; cursor: wait; } +/* Phase 89: the per-row "Ignore paths" trigger (makeRow — left of + Remove): the .git-source-remove idiom (same size/spacing, >=44px + target) in a NEUTRAL secondary fill — it must read distinct from + the destructive Remove (text label included, never icon-only; the + hover takes the brand-soft pair, 12.4:1 — Remove hovers to the err + pair, 9.3:1). */ +.git-source-ignore { + display: inline-flex; + align-items: center; + justify-content: center; + min-height: 44px; + min-width: 44px; + flex: 0 0 auto; + padding: 0.35rem 0.7rem; + margin-right: 0.4rem; + border: 1px solid var(--line); + border-radius: var(--radius-sm); + background: transparent; + color: var(--ink-soft); + font: inherit; + font-weight: 600; + font-size: 0.82rem; + white-space: nowrap; + cursor: pointer; +} +.git-source-ignore:hover:not(:disabled) { + background: var(--brand-soft); + color: var(--brand-ink); +} + +/* Phase 89: the "N ignored" count tag — next to the location + in the Source cell (TEXT + a distinct background, never color + alone — WCAG 1.4.1): --ink on --bg (16.7:1) inside the --surface + card; the line border keeps the chip legible when the row hover + swaps the cell onto --bg too. */ +.git-source-ignore-count { + display: inline-block; + margin-left: 0.55rem; + padding: 0.08rem 0.5rem; + border: 1px solid var(--line); + border-radius: 999px; + font-size: 0.72rem; + font-weight: 600; + color: var(--ink); + background: var(--bg); + white-space: nowrap; +} + /* Env-fallback rows carry no Remove (nothing is stored to remove) — the tag says where the row comes from (brand pair, 6.9:1). */ .git-source-env-tag { diff --git a/frontend/index.html b/frontend/index.html index 448e3a2..804f1dd 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -647,6 +647,44 @@ + + + diff --git a/scripts/import_docs.py b/scripts/import_docs.py index e68c808..83bba71 100644 --- a/scripts/import_docs.py +++ b/scripts/import_docs.py @@ -26,6 +26,11 @@ precedence order: ``~/Deployments``), kept for backwards compatibility (reached only while both the table and ``BOR_GIT_SOURCES`` are empty). +Phase 89: resolution also returns each row's ignore paths, keyed by +the resolved root string (the importer normalizes them); manual +``--source`` dirs and the legacy fallback have no rows, so they import +with no ignore. + Imported formats (PLAN anchor A9, revised; phase 56): the A9 family by default — ``md, markdown, txt, yaml, yml, json, py`` plus the quadlet family and ``j2`` (case-insensitive). ``BOR_IMPORT_EXTENSIONS`` may add @@ -133,9 +138,11 @@ def repo_name(url: str) -> str: return slug or "repo" -def _resolve_sources(cli_sources: list[Path] | None, settings: Settings) -> list[Path]: +def _resolve_sources( + cli_sources: list[Path] | None, settings: Settings +) -> tuple[list[Path], dict[str, list[str]]]: """Resolve the directories to import (phase 28, extended in phases - 35 and 38). + 35 and 38; per-root ignore maps, phase 89). Precedence: ``--source`` (explicit manual paths — always wins) > the effective sources — the ``git_sources`` DB rows (git + local), @@ -148,12 +155,18 @@ def _resolve_sources(cli_sources: list[Path] | None, settings: Settings) -> list stored directory, re-verified ``.is_dir()`` at run time) > the legacy ``DEFAULT_SOURCES``. + Returns ``(sources, ignore_by_root)`` (phase 89): the map is keyed + by the resolved root string, exactly as the importer sees it (two + rows sharing a root string get the union — extend, not replace); + manual ``--source`` dirs and the legacy fallback have no rows, so + they import with an empty map (no ignore). + A :class:`GitSyncError` from a failing clone/pull — or a missing local directory (``local source missing: ``) — propagates to :func:`main`, which aborts the run before importing anything. """ if cli_sources: - return [path.expanduser() for path in cli_sources] + return [path.expanduser() for path in cli_sources], {} db = SessionLocal() try: rows, origin = effective_sources(db) @@ -167,21 +180,28 @@ def _resolve_sources(cli_sources: list[Path] | None, settings: Settings) -> list ) sources_root = Path(settings.sources_dir).expanduser() sources: list[Path] = [] + ignore_by_root: dict[str, list[str]] = {} for row in rows: if row.kind == "git": - sources.append(clone_or_pull(row.url, sources_root / repo_name(row.url))) + root = clone_or_pull(row.url, sources_root / repo_name(row.url)) else: # kind=local — the stored expanded path (phase 38 also # mirrors it in the NOT-NULL ``url`` location column, the # ``or`` keeps the type checker honest); a missing # directory aborts before importing, the same pre-import # fail-loud as a failing git clone. - path = Path(row.path or row.url).expanduser() - if not path.is_dir(): - raise GitSyncError(f"local source missing: {path}") - sources.append(path) - return sources - return [path.expanduser() for path in DEFAULT_SOURCES] + root = Path(row.path or row.url).expanduser() + if not root.is_dir(): + raise GitSyncError(f"local source missing: {root}") + sources.append(root) + # Phase 89: the row's ignore list, keyed by the SAME root + # string the importer sees; two rows sharing a root string + # get the union (extend, not replace) — the sibling/repo-name + # edge. + if row.ignore_paths: + ignore_by_root.setdefault(str(root), []).extend(row.ignore_paths) + return sources, ignore_by_root + return [path.expanduser() for path in DEFAULT_SOURCES], {} def _overview_row_exists() -> bool: @@ -204,9 +224,10 @@ def main(argv: list[str] | None = None) -> int: # Git sources resolve (and clone/pull) *before* any import: a failing # repo aborts the run with a non-zero exit, naming the failure — a bad - # URL must never silently import partial junk. + # URL must never silently import partial junk. The second element is + # the phase-89 per-root ignore map (empty for manual/fallback paths). try: - sources = _resolve_sources(args.source, settings) + sources, ignore_by_root = _resolve_sources(args.source, settings) except GitSyncError as e: print(f"import_docs: source sync failed: {e}", file=sys.stderr) return 1 @@ -248,7 +269,10 @@ def main(argv: list[str] | None = None) -> int: runs and unchanged re-runs never bump. The returned token is the new version, or ``"skipped"``. """ - summary = await import_sources(sources, llm, prune=args.prune, limit=args.limit) + summary = await import_sources( + sources, llm, prune=args.prune, limit=args.limit, + ignore_by_root=ignore_by_root, + ) if args.limit is not None: # An incomplete walk is debug-only — it must never advance # the generation (mirrors the --limit overview skip below). diff --git a/tests/e2e/test_source_ignore_paths.py b/tests/e2e/test_source_ignore_paths.py new file mode 100644 index 0000000..8fec3de --- /dev/null +++ b/tests/e2e/test_source_ignore_paths.py @@ -0,0 +1,640 @@ +"""Phase 89 story E2E (Playwright): per-source ignore paths — the box on +the Sources page excludes files/folders from the import (``/git-sources.html``). + +Story source: ``TODO.md`` L3 (owner roadmap confirmation 2026-09-08 — +TODO-derived, no separate user-story file). +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_source_ignore_paths.py -v --no-cov + +The story gate proves the whole item end to end through the REAL page + +REAL API + REAL sync pipeline (the in-app Sync's background task, mock +LLM). **No git, no network** — the suite's single source is a local +directory row (``kind="local"``, phase 38), so the sync walks the +fixture dir directly: no clone, no pull, no remote of any kind. The +module app boots with one deterministic ``BOR_GIT_SOURCES`` env URL +(the ``test_git_sources_admin.py`` module-env pattern) that is NEVER +synced or cloned: every sync in this suite runs with the local DB row +in place (DB rows win over the env fallback), and the one +env-fallback test never triggers a sync. + +Contract under test: + +* anonymous: the sign-in gate (the ``#git-sources-gate`` pattern), the + manager hidden, NO ``/api/git-sources`` call on load, 403 on + ``GET``/``POST /api/git-sources`` AND ``PATCH /api/git-sources/{id}`` + (the ``test_git_sources_admin.py`` anonymous pin, extended to the + phase-89 route); +* admin: the per-row "Ignore paths" button opens the page-local + alertdialog (the phase-69 ``#remove-confirm-dialog`` idiom); one path + per line; Save → ``PATCH`` 200 → the row shows the "N ignored" count + tag; the list round-trips through ``GET /api/git-sources``; +* the sync honors the list: with ``ignore/`` set, nothing under + ``ignore/`` is indexed — no ``documents`` rows (and therefore no + chunks/embeddings, no summary calls) — and the sync's ``files`` + count excludes the ignored files from the walk; +* the spec's no-mid-path rule: ``myfile.md`` ignores the root-level + ``myfile.md`` only, never ``sub/myfile.md``; saving the box + REPLACES the list (A5 — the previous test's list is gone); +* A2: a previously indexed file that newly matches an ignore pattern + is PRUNED from the KB on the next sync (``detail.pruned``); +* a11y + the error path: the dialog is a real ``role="alertdialog"``, + the textarea has a visible label, focus lands on Cancel, Escape + closes and returns focus to the trigger; a 501-char entry 422s with + the fixed detail, the textarea content is KEPT and the Save button + re-enables with its "Save" label; the happy path heals the error + state; +* A3: env-fallback rows (table empty → ``BOR_GIT_SOURCES``) render the + "from .env" tag with NO "Ignore paths" button and the env note. + +Test → contract mapping: +1. ``test_anonymous_gate_and_403s`` +2. ``test_ignore_box_excludes_from_import`` +3. ``test_prefix_rule_no_mid_path`` +4. ``test_newly_ignored_file_is_pruned`` +5. ``test_editor_a11y_and_error_path`` +6. ``test_env_fallback_rows_have_no_box`` +""" +from __future__ import annotations + +import os +import subprocess +import sys +import time +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest +from playwright.sync_api import Page, expect +from sqlalchemy import text + +from app.db import SessionLocal +from e2e.auth_helpers import login +from e2e.conftest import ( + ADMIN_PASSWORD, + SESSION_SECRET, + USE_REAL_LLM, + _wait_http, +) + +REPO = Path(__file__).resolve().parents[2] + +# Phase 79 (task 04, full inventory): the conftest session app owns its +# port in a combined run — this module app binds its own port instead +# (a same-port second uvicorn dies on bind and would drive the wrong +# server). Env-overridable. +APP_PORT = int(os.environ.get("E2E_APP_PORT_IGNORE", "8140")) +APP_URL = f"http://127.0.0.1:{APP_PORT}" + +GIT_SOURCES_URL = "/git-sources.html" + +#: The module app's ``BOR_GIT_SOURCES`` — one deterministic URL that is +#: NEVER cloned (the env-fallback test reads it; the sync tests all run +#: with the local DB row in place, so the env list never reaches a +#: clone). +ENV_SOURCE = "https://github.com/reese/env-alpha.git" + +#: The sentinel inside the fixture's keeper doc — proof the (non- +#: ignored) markdown path was really imported. +SENTINEL = "RESE-IGNORE-KEEP-7f3a" + +#: The six fixture files (source-relative POSIX paths — exactly the +#: strings ``documents.path`` stores). +KEEP_MD = "keep.md" +NOTES_YAML = "notes.yaml" +ROOT_MYFILE = "myfile.md" +SUB_MYFILE = "sub/myfile.md" +IGNORE_SECRET = "ignore/secret.md" +IGNORE_DEEP = "ignore/deep/x.txt" +ALL_SIX = (KEEP_MD, NOTES_YAML, ROOT_MYFILE, SUB_MYFILE, IGNORE_SECRET, IGNORE_DEEP) + +#: ``POST /api/sync`` → terminal ``GET /api/sync/status`` (the +#: test_sync_button.py polling idiom) — real import of ≤6 small files +#: against the mock LLM; generous budget. +SYNC_TIMEOUT_S = 60.0 +SYNC_TICK_S = 2.0 + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def source_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: + """The local source's on-disk tree (task 06 step 1). + + Module-scoped (the same reasoning as ``test_sync_button.py``'s + module-scoped fixture note): ``tmp_path`` is function-scoped while + the module-scoped app + the seeded row reference the dir for the + module's lifetime, so it is built under ``tmp_path_factory`` (the + same pytest-managed temp area, module-safe). Plain files — no git: + a ``kind="local"`` row is walked directly by the sync (phase 38). + """ + root = tmp_path_factory.mktemp("ignore_src") + (root / "sub").mkdir() + (root / "ignore" / "deep").mkdir(parents=True) + (root / KEEP_MD).write_text( + "# Keep\n" + "\n" + "The keeper doc — the one markdown file the ignore list must\n" + "never touch.\n" + f"\n" + f"Marker: {SENTINEL}\n", + encoding="utf-8", + ) + # Non-markdown → the phase-30 lite-summary path (proves "not + # summarized" applies to the non-md branch too when ignored). + (root / NOTES_YAML).write_text( + "title: Notes\nitems:\n - one\n - two\n", encoding="utf-8" + ) + (root / ROOT_MYFILE).write_text( + "# Root myfile\n\nRoot-level file — matched by the bare " + "``myfile.md`` prefix (the spec's own example).\n", + encoding="utf-8", + ) + (root / SUB_MYFILE).write_text( + "# Sub myfile\n\nA namesake in a subdirectory — NEVER matched " + "by ``myfile.md`` (no mid-path matching).\n", + encoding="utf-8", + ) + (root / IGNORE_SECRET).write_text( + "# Secret\n\nIgnored by the ``ignore/`` prefix.\n", encoding="utf-8" + ) + (root / IGNORE_DEEP).write_text("deep ignored text\n", encoding="utf-8") + return root + + +@pytest.fixture(scope="module") +def app_server(mock_llm: int, source_dir: Path) -> Iterator[str]: + """The real app under test — per-module env (the conftest pattern, + module-scoped): one deterministic ``BOR_GIT_SOURCES`` URL (the env + fallback's subject — never cloned), module-scratch checkouts/ + upload dirs, the mock LLM. No git anywhere in this suite's path.""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = ( + "https://aipi.reeseapps.com/v1" + if USE_REAL_LLM + else f"http://127.0.0.1:{mock_llm}/v1" + ) + # Mock-calibrated threshold (conftest pattern) — no chat turn is + # ever sent in this suite, but the app boots with the same env + # shape. + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + # Phase 16: admin auth must be set or create_app() refuses to boot. + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + # The empty-table env fallback (only read while ``git_sources`` is + # EMPTY — the env-fallback test; never cloned in this suite). + env["BOR_GIT_SOURCES"] = ENV_SOURCE + # Module-scratch dirs (never reached by this suite's local-row + # syncs — kept explicit so a shared checkouts dir cannot leak rows + # into the walk). + scratch = source_dir.parent + env["BOR_SOURCES_DIR"] = str(scratch / "checkouts") + env["BOR_UPLOAD_DIR"] = str(scratch / "uploads") + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], + cwd=REPO, + env=env, + ) + try: + _wait_http(f"{APP_URL}/api/health") + yield APP_URL + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +def _truncate_all() -> None: + """Fresh registry + KB per test (the E2E isolation pattern, the + test_sync_button.py module-env DSN): the sync's counts and the + catalog must be each test's own doing.""" + with SessionLocal() as db: + db.execute( + text( + "TRUNCATE chunks, documents, query_log, kb_overview, " + "git_sources" + ) + ) + db.commit() + + +@pytest.fixture(autouse=True) +def _clean(db_ready: None) -> Iterator[None]: + """This suite owns ``git_sources`` AND the KB tables (the E2E + isolation pattern): suites run in isolation but share one + Postgres, and a leftover row would flip the app from the + ``BOR_GIT_SOURCES`` env fallback to the DB list (and a leftover + document would skew the prune counts). Empty BOTH before and + after every test.""" + _truncate_all() + yield + _truncate_all() + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _admin_git_sources_page(page: Page, app_url: str) -> None: + """Real form login landing on the git sources page (admin settled: + Sign out visible, the manager revealed by the page module).""" + login(page, app_url, next=GIT_SOURCES_URL) + expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000) + expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000) + expect(page.locator("#git-sources-gate")).to_be_hidden() + expect(page.locator("#git-sources-content")).to_be_visible() + + +def _seed_local_source(page: Page, app_url: str, source_dir: Path) -> str: + """(Re)seed the stored row for the module fixture dir via the REAL + admin API (201) — ``kind="local"`` (the POST contract is unchanged + by phase 89; no ``ignore_paths`` → the stored list is ``[]``) — + then reload the view so the page's table re-fetches the row (the + API seed is server-side: the mounted view does not know about it + without a re-show). Returns the stored (expanded) path — the + row's ``url`` location value, the same string the page renders + and aria-labels.""" + r = page.request.post( + f"{app_url}/api/git-sources", + data={"kind": "local", "path": str(source_dir)}, + ) + assert r.status == 201, r.text + body = r.json() + assert body["ignore_paths"] == [] # the default list, round-tripped + page.reload() + _admin_git_sources_page(page, app_url) + return body["url"] + + +def run_sync(page: Page, app_url: str) -> dict[str, Any]: + """``POST /api/sync`` → poll ``GET /api/sync/status`` to a terminal + state (the test_sync_button.py polling idiom — ~2 s ticks, 60 s + budget). Returns the terminal status body.""" + r = page.request.post(f"{app_url}/api/sync") + assert r.status == 202, r.text + deadline = time.monotonic() + SYNC_TIMEOUT_S + body: dict[str, Any] = {} + while True: + s = page.request.get(f"{app_url}/api/sync/status") + assert s.status == 200, s.text + body = s.json() + if body["state"] in ("success", "failed"): + return body + assert time.monotonic() < deadline, ( + f"sync did not reach a terminal state: {body}" + ) + time.sleep(SYNC_TICK_S) + + +def _catalog_paths(page: Page, app_url: str) -> list[str]: + """The RAG catalog's data source (``GET /api/docs`` — the Sources + page's table): every ``documents.path`` (source-relative POSIX).""" + r = page.request.get(f"{app_url}/api/docs") + assert r.status == 200, r.text + return [d["path"] for d in r.json()["documents"]] + + +def _row(page: Page, value: str) -> Any: + """The table row whose mono location cell shows ``value``.""" + return page.locator("#git-sources-tbody tr", has_text=value) + + +def _open_ignore_editor(page: Page, value: str) -> Any: + """Click the row's "Ignore paths" button; the alertdialog opens. + Returns the trigger button (focus returns to it on close).""" + btn = _row(page, value).locator(".git-source-ignore") + expect(btn).to_have_count(1) + btn.click() + dialog = page.locator("#ignore-editor-dialog") + expect(dialog).to_be_visible(timeout=30_000) + return btn + + +def _save_ignore_list(page: Page, value: str, entry: str) -> None: + """Open the row's box, replace its content with one line (``entry`` + — ``""`` clears) and Save; wait for the A5 round-trip: the dialog + closes and the row shows the "1 ignored" / cleared count state.""" + _open_ignore_editor(page, value) + page.fill("#ignore-editor-textarea", entry) + page.click("#ignore-editor-save") + if entry.strip(): + expect(_row(page, value).locator(".git-source-ignore-count")).to_have_text( + "1 ignored", timeout=30_000 + ) + else: + expect(_row(page, value).locator(".git-source-ignore-count")).to_have_count(0) + expect(page.locator("#ignore-editor-dialog")).to_be_hidden() + + +# --------------------------------------------------------------------------- +# 1. Anonymous: gate, inert manager, no API calls, 403s (incl. PATCH) +# --------------------------------------------------------------------------- + + +def test_anonymous_gate_and_403s(page: Page, app_url: str, db_ready: None) -> None: + page.set_default_timeout(30_000) + + # Track every /api/git-sources request the page itself makes — the + # gate must be reached WITHOUT touching the admin API (the + # test_git_sources_admin.py pattern). + api_calls: list[str] = [] + page.on( + "request", + lambda r: api_calls.append(r.url) if "/api/git-sources" in r.url else None, + ) + + page.goto(app_url + GIT_SOURCES_URL) + expect(page.locator("#sign-in-link")).to_be_visible(timeout=15_000) + + # The sign-in gate (the #sources-gate pattern, phase 16)… + gate = page.locator("#git-sources-gate") + expect(gate).to_be_visible() + expect(gate).to_contain_text("Sign in to manage the git sources") + expect( + gate.locator("a[href='/login.html?next=/git-sources.html']") + ).to_have_count(1) + # …and the manager is absent/inert: table + form + env note, all + # inside the hidden #git-sources-content. + expect(page.locator("#git-sources-content")).to_be_hidden() + expect(page.locator("#git-sources-table")).to_be_hidden() + expect(page.locator("#git-source-form")).to_be_hidden() + expect(page.locator("#git-sources-env-note")).to_be_hidden() + + # The gate never called the admin API… + assert api_calls == [], f"anonymous page called the git sources API: {api_calls}" + # …and the API 403s anonymous callers on GET/POST AND the + # phase-89 PATCH (the page context has no cookie — the + # test_git_sources_admin.py anonymous pin, extended). + assert page.request.get(f"{app_url}/api/git-sources").status == 403 + assert ( + page.request.post( + f"{app_url}/api/git-sources", data={"url": ENV_SOURCE} + ).status + == 403 + ) + assert ( + page.request.patch( + f"{app_url}/api/git-sources/{uuid.uuid4()}", + data={"ignore_paths": ["ignore/"]}, + ).status + == 403 + ) + + +# --------------------------------------------------------------------------- +# 2. The box sets the list; the sync honors it (nothing under ignore/ +# embedded/summarized; the walk's files count excludes it) +# --------------------------------------------------------------------------- + + +def test_ignore_box_excludes_from_import( + page: Page, app_url: str, db_ready: None, source_dir: Path +) -> None: + page.set_default_timeout(30_000) + _admin_git_sources_page(page, app_url) + + # Seed the row (no list) through the real API… + stored_path = _seed_local_source(page, app_url, source_dir) + expect(page.locator("#git-sources-tbody tr")).to_have_count(1) + + # …open the box, type "ignore/" (one line), Save → 200 → the row's + # "1 ignored" count tag lands (the A5 round-trip through GET). + _save_ignore_list(page, stored_path, "ignore/") + r = page.request.get(f"{app_url}/api/git-sources") + assert r.status == 200, r.text + body = r.json() + # A1: the raw box line is stored NORMALIZED ("ignore/" → "ignore") + # — the round-trip is canonical. + assert [s["ignore_paths"] for s in body["sources"]] == [["ignore"]] + assert body["from_env"] is False + + # The real sync (mock LLM): success, and the walk counted the + # remaining FOUR files — the two ignore/ files never entered it + # (no documents → no chunks/embeddings, no summary calls for + # them). (6 fixture files − 2 ignored = 4; the task's "5" is a + # slip against its own six-file tree — the exclusion is what is + # under test, and it holds either way.) + sync = run_sync(page, app_url) + assert sync["state"] == "success", sync + detail = sync["detail"] + assert detail["files"] == 4, detail + assert detail["added"] == 4, detail + assert detail["errors"] == 0, detail + + # The catalog holds exactly the four non-ignored paths. + paths = _catalog_paths(page, app_url) + for p in (KEEP_MD, NOTES_YAML, ROOT_MYFILE, SUB_MYFILE): + assert p in paths, f"{p} missing from the catalog: {paths}" + assert not any(p.startswith("ignore/") for p in paths), ( + f"ignored paths leaked into the catalog: {paths}" + ) + + +# --------------------------------------------------------------------------- +# 3. The spec's no-mid-path rule: myfile.md ignores the root-level file +# only; the box REPLACES the previous test's list (A5) +# --------------------------------------------------------------------------- + + +def test_prefix_rule_no_mid_path( + page: Page, app_url: str, db_ready: None, source_dir: Path +) -> None: + page.set_default_timeout(30_000) + _admin_git_sources_page(page, app_url) + stored_path = _seed_local_source(page, app_url, source_dir) + + # Bare filename prefix (no leading/trailing slash): the spec's own + # example — matches the root-level file, never the subdirectory + # namesake. + _save_ignore_list(page, stored_path, "myfile.md") + + sync = run_sync(page, app_url) + assert sync["state"] == "success", sync + assert sync["detail"]["files"] == 5, sync["detail"] + + paths = _catalog_paths(page, app_url) + # No mid-path matching: the namesake in sub/ is indexed… + assert SUB_MYFILE in paths, f"sub/myfile.md was wrongly ignored: {paths}" + # …the root-level file is NOT… + assert ROOT_MYFILE not in paths, f"root myfile.md must be ignored: {paths}" + # …and the ignore/ files ARE indexed this time — the box replaced + # (A5), not appended to, the previous test's "ignore/" list + # (regression guard). + assert IGNORE_SECRET in paths, f"ignore/secret.md must be indexed: {paths}" + assert IGNORE_DEEP in paths, f"ignore/deep/x.txt must be indexed: {paths}" + + +# --------------------------------------------------------------------------- +# 4. A2: a previously indexed file that newly matches is pruned +# --------------------------------------------------------------------------- + + +def test_newly_ignored_file_is_pruned( + page: Page, app_url: str, db_ready: None, source_dir: Path +) -> None: + page.set_default_timeout(30_000) + _admin_git_sources_page(page, app_url) + stored_path = _seed_local_source(page, app_url, source_dir) + + # Sync 1 — no list: all six files are indexed. + first = run_sync(page, app_url) + assert first["state"] == "success", first + assert first["detail"]["files"] == 6, first["detail"] + assert first["detail"]["added"] == 6, first["detail"] + paths = _catalog_paths(page, app_url) + for p in ALL_SIX: + assert p in paths, f"{p} missing after the first sync: {paths}" + + # New list: "ignore/" — the two previously indexed files now match. + _save_ignore_list(page, stored_path, "ignore/") + + # Sync 2 — the prune=True run deletes the newly ignored documents + # (the seen-set mechanism, the A9 out-of-scope-junk precedent). + second = run_sync(page, app_url) + assert second["state"] == "success", second + detail = second["detail"] + assert detail["pruned"] == 2, detail + assert detail["files"] == 4, detail + + paths = _catalog_paths(page, app_url) + for p in (KEEP_MD, NOTES_YAML, ROOT_MYFILE, SUB_MYFILE): + assert p in paths, f"{p} missing after the prune: {paths}" + assert not any(p.startswith("ignore/") for p in paths), ( + f"newly ignored paths survived the prune: {paths}" + ) + + +# --------------------------------------------------------------------------- +# 5. The editor's a11y + the 422 error path (and the happy-path heal) +# --------------------------------------------------------------------------- + + +def test_editor_a11y_and_error_path( + page: Page, app_url: str, db_ready: None, source_dir: Path +) -> None: + page.set_default_timeout(30_000) + _admin_git_sources_page(page, app_url) + stored_path = _seed_local_source(page, app_url, source_dir) + + # The trigger is the labeled per-row button (left of Remove)… + row = _row(page, stored_path) + btn = row.locator(".git-source-ignore") + expect(btn).to_have_count(1) + expect(btn).to_have_attribute( + "aria-label", f"Edit ignored paths for local source: {stored_path}" + ) + expect(row.locator(".git-source-ignore-count")).to_have_count(0) # no list yet + + # …and it opens a real alertdialog (the phase-69 idiom) with the + # visible label and the Cancel-safe focus default. + btn.click() + dialog = page.locator("#ignore-editor-dialog") + expect(dialog).to_be_visible(timeout=30_000) + assert dialog.get_attribute("role") == "alertdialog" + assert dialog.get_attribute("aria-modal") == "true" + # exact: the row button's aria-label ("Edit ignored paths for …") + # contains the substring — the visible