From ad7585d4744525b6ec2676f827f1df44fa09a32f Mon Sep 17 00:00:00 2001 From: ducoterra Date: Fri, 11 Sep 2026 22:48:02 -0400 Subject: [PATCH] phase: 97_kb_tree_catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All completion criteria verified — everything is green, no defects found. Final report: ## Phase 97 final verification pass — ALL GREEN **Verified (no code changes needed):** - `GET /api/docs/tree` (admin), `build_kb_tree` pure builder, `PATCH /api/folders/summary`, migration 0018 (`manually_edited`, head confirmed), generator skip/keep + `kept_manual` stat, RAG tree UI + edit affordance in `sources.js`/`index.html`/`styles.css` - `tests/e2e/test_kb_tree.py`: 8 passed — top level, drill source/folder, edit round-trip, clear, manual-desc-survives-sync, reload fallback, anonymous gate - Integration: tree shape/order/403/empty/indexed-only + PATCH update/create/root/clear/404/403/no-LLM + stat-walk equivalence (in `test_docs_api.py`); 3-field `folder_summaries=` import token preserved **Gates (exact commands):** - `uv run pytest --cov=app --cov-report=term-missing` → **2053 passed**, TOTAL coverage **99%** (>90% ✓) - `uv run ruff check . && uv run pyright` → **All checks passed / 0 errors** - `uv run pytest tests/e2e/test_kb_tree.py -v --no-cov` → **8 passed** in isolation - 30 story/RAG-view E2E suites run **one per process**: all passed, incl. `test_ls_tree_drilldown` (agent `ls` byte-identical ✓), `test_import_documents`, `test_edit_summaries`, `test_admin_auth`, `test_kb_overview` **Completion criteria:** tree view ✓ · edit round-trip + clear ✓ · manual persists/clear resets ✓ · `ls` unchanged ✓ · pytest/coverage/lint ✓ · E2E isolation ✓ · commit — left to harness per protocol (working tree untouched, `git add/commit` not run) **Deviations:** none. **Next pending phase:** none — `todo/` contains only 97 (96 already committed). --- .../97_kb_tree_catalog/00_phase.md | 0 .../97_kb_tree_catalog/01_manual_flag.md | 0 .../97_kb_tree_catalog/02_tree_api.md | 0 .../03_folder_summary_api.md | 0 .../97_kb_tree_catalog/04_tree_ui.md | 0 .../97_kb_tree_catalog/05_folder_edit_ui.md | 0 .../97_kb_tree_catalog/06_e2e_kb_tree.md | 0 .../07_catalog_suites_import_sync.md | 0 .../08_catalog_suites_viewer_nav.md | 0 .../97_kb_tree_catalog__00_phase.a1.err | 0 .../97_kb_tree_catalog__00_phase.a1.md | 18 + .../97_kb_tree_catalog__00_phase.a1.validate | 96 ++ .../97_kb_tree_catalog__01_manual_flag.a1.err | 0 .../97_kb_tree_catalog__01_manual_flag.a1.md | 18 + ...b_tree_catalog__01_manual_flag.a1.validate | 95 ++ .../97_kb_tree_catalog__02_tree_api.a1.err | 0 .../97_kb_tree_catalog__02_tree_api.a1.md | 12 + ...7_kb_tree_catalog__02_tree_api.a1.validate | 95 ++ ...tree_catalog__03_folder_summary_api.a1.err | 0 ..._tree_catalog__03_folder_summary_api.a1.md | 19 + ...catalog__03_folder_summary_api.a1.validate | 96 ++ .../97_kb_tree_catalog__04_tree_ui.a1.err | 0 .../97_kb_tree_catalog__04_tree_ui.a1.md | 15 + ...97_kb_tree_catalog__04_tree_ui.a1.validate | 96 ++ ..._kb_tree_catalog__05_folder_edit_ui.a1.err | 0 ...7_kb_tree_catalog__05_folder_edit_ui.a1.md | 9 + ...ree_catalog__05_folder_edit_ui.a1.validate | 96 ++ .../97_kb_tree_catalog__06_e2e_kb_tree.a1.err | 0 .../97_kb_tree_catalog__06_e2e_kb_tree.a1.md | 15 + ...b_tree_catalog__06_e2e_kb_tree.a1.validate | 96 ++ ...alog__07_catalog_suites_import_sync.a1.err | 0 ...talog__07_catalog_suites_import_sync.a1.md | 12 + ..._07_catalog_suites_import_sync.a1.validate | 96 ++ ...talog__08_catalog_suites_viewer_nav.a1.err | 0 ...atalog__08_catalog_suites_viewer_nav.a1.md | 12 + ...__08_catalog_suites_viewer_nav.a1.validate | 96 ++ .../0018_folder_summary_manual_flag.py | 56 + app/api/docs.py | 334 +++++- app/models.py | 19 + app/rag/folder_summaries.py | 82 +- app/schemas.py | 107 ++ frontend/assets/sources.js | 705 +++++++++++- frontend/assets/styles.css | 155 +++ frontend/index.html | 42 + tests/e2e/test_admin_auth.py | 55 +- tests/e2e/test_archive_upload_sources.py | 6 + tests/e2e/test_chat_persistence.py | 5 +- tests/e2e/test_document_back_navigation.py | 10 +- tests/e2e/test_document_viewer.py | 17 +- tests/e2e/test_edit_summaries.py | 5 + tests/e2e/test_global_tuning.py | 8 +- tests/e2e/test_history_page_width.py | 49 +- tests/e2e/test_import_documents.py | 181 ++- tests/e2e/test_import_extensions_env.py | 36 +- tests/e2e/test_kb_tree.py | 941 +++++++++++++++ tests/e2e/test_ls_tree_drilldown.py | 1 + tests/e2e/test_markdown_tables.py | 8 +- tests/e2e/test_nav_consistency.py | 9 +- tests/e2e/test_nav_switch_keeps_stream.py | 13 +- tests/e2e/test_navbar_refresh.py | 14 +- tests/e2e/test_quadlet_jinja_import.py | 28 +- tests/e2e/test_responsive_polish.py | 37 +- tests/e2e/test_retrieval_quality.py | 7 + tests/e2e/test_shared_header.py | 5 +- tests/e2e/test_sources_midstream_bug.py | 10 +- tests/e2e/test_sticky_navbar.py | 9 +- tests/e2e/test_summary_in_viewer.py | 9 + tests/e2e/test_sync_button.py | 33 +- tests/e2e/test_sync_model_down.py | 32 +- tests/e2e/test_sync_upload_progress.py | 29 +- tests/e2e/test_theme_semantic_completion.py | 49 +- tests/e2e/test_upload_no_scan.py | 33 +- tests/integration/test_docs_api.py | 579 +++++++++- tests/integration/test_migration_0018.py | 223 ++++ .../integration/test_sync_folder_summaries.py | 93 +- tests/unit/test_folder_summaries.py | 126 +- tests/unit/test_frontend_router.py | 47 +- tests/unit/test_frontend_sync_upload.py | 8 +- tests/unit/test_kb_tree_builder.py | 285 +++++ tests/unit/test_kb_tree_ui.py | 1014 +++++++++++++++++ tests/unit/test_sync_button.py | 4 +- 81 files changed, 6299 insertions(+), 211 deletions(-) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/00_phase.md (100%) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/01_manual_flag.md (100%) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/02_tree_api.md (100%) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/03_folder_summary_api.md (100%) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/04_tree_ui.md (100%) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/05_folder_edit_ui.md (100%) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/06_e2e_kb_tree.md (100%) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/07_catalog_suites_import_sync.md (100%) rename .agents/phases/{todo => complete}/97_kb_tree_catalog/08_catalog_suites_viewer_nav.md (100%) create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.validate create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.validate create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.validate create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.validate create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.validate create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.validate create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.validate create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.validate create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.err create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.md create mode 100644 .agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.validate create mode 100644 alembic/versions/0018_folder_summary_manual_flag.py create mode 100644 tests/e2e/test_kb_tree.py create mode 100644 tests/integration/test_migration_0018.py create mode 100644 tests/unit/test_kb_tree_builder.py create mode 100644 tests/unit/test_kb_tree_ui.py diff --git a/.agents/phases/todo/97_kb_tree_catalog/00_phase.md b/.agents/phases/complete/97_kb_tree_catalog/00_phase.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/00_phase.md rename to .agents/phases/complete/97_kb_tree_catalog/00_phase.md diff --git a/.agents/phases/todo/97_kb_tree_catalog/01_manual_flag.md b/.agents/phases/complete/97_kb_tree_catalog/01_manual_flag.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/01_manual_flag.md rename to .agents/phases/complete/97_kb_tree_catalog/01_manual_flag.md diff --git a/.agents/phases/todo/97_kb_tree_catalog/02_tree_api.md b/.agents/phases/complete/97_kb_tree_catalog/02_tree_api.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/02_tree_api.md rename to .agents/phases/complete/97_kb_tree_catalog/02_tree_api.md diff --git a/.agents/phases/todo/97_kb_tree_catalog/03_folder_summary_api.md b/.agents/phases/complete/97_kb_tree_catalog/03_folder_summary_api.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/03_folder_summary_api.md rename to .agents/phases/complete/97_kb_tree_catalog/03_folder_summary_api.md diff --git a/.agents/phases/todo/97_kb_tree_catalog/04_tree_ui.md b/.agents/phases/complete/97_kb_tree_catalog/04_tree_ui.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/04_tree_ui.md rename to .agents/phases/complete/97_kb_tree_catalog/04_tree_ui.md diff --git a/.agents/phases/todo/97_kb_tree_catalog/05_folder_edit_ui.md b/.agents/phases/complete/97_kb_tree_catalog/05_folder_edit_ui.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/05_folder_edit_ui.md rename to .agents/phases/complete/97_kb_tree_catalog/05_folder_edit_ui.md diff --git a/.agents/phases/todo/97_kb_tree_catalog/06_e2e_kb_tree.md b/.agents/phases/complete/97_kb_tree_catalog/06_e2e_kb_tree.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/06_e2e_kb_tree.md rename to .agents/phases/complete/97_kb_tree_catalog/06_e2e_kb_tree.md diff --git a/.agents/phases/todo/97_kb_tree_catalog/07_catalog_suites_import_sync.md b/.agents/phases/complete/97_kb_tree_catalog/07_catalog_suites_import_sync.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/07_catalog_suites_import_sync.md rename to .agents/phases/complete/97_kb_tree_catalog/07_catalog_suites_import_sync.md diff --git a/.agents/phases/todo/97_kb_tree_catalog/08_catalog_suites_viewer_nav.md b/.agents/phases/complete/97_kb_tree_catalog/08_catalog_suites_viewer_nav.md similarity index 100% rename from .agents/phases/todo/97_kb_tree_catalog/08_catalog_suites_viewer_nav.md rename to .agents/phases/complete/97_kb_tree_catalog/08_catalog_suites_viewer_nav.md diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.md new file mode 100644 index 0000000..c570b45 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.md @@ -0,0 +1,18 @@ +All completion criteria verified — everything is green, no defects found. Final report: + +## Phase 97 final verification pass — ALL GREEN + +**Verified (no code changes needed):** +- `GET /api/docs/tree` (admin), `build_kb_tree` pure builder, `PATCH /api/folders/summary`, migration 0018 (`manually_edited`, head confirmed), generator skip/keep + `kept_manual` stat, RAG tree UI + edit affordance in `sources.js`/`index.html`/`styles.css` +- `tests/e2e/test_kb_tree.py`: 8 passed — top level, drill source/folder, edit round-trip, clear, manual-desc-survives-sync, reload fallback, anonymous gate +- Integration: tree shape/order/403/empty/indexed-only + PATCH update/create/root/clear/404/403/no-LLM + stat-walk equivalence (in `test_docs_api.py`); 3-field `folder_summaries=` import token preserved + +**Gates (exact commands):** +- `uv run pytest --cov=app --cov-report=term-missing` → **2053 passed**, TOTAL coverage **99%** (>90% ✓) +- `uv run ruff check . && uv run pyright` → **All checks passed / 0 errors** +- `uv run pytest tests/e2e/test_kb_tree.py -v --no-cov` → **8 passed** in isolation +- 30 story/RAG-view E2E suites run **one per process**: all passed, incl. `test_ls_tree_drilldown` (agent `ls` byte-identical ✓), `test_import_documents`, `test_edit_summaries`, `test_admin_auth`, `test_kb_overview` + +**Completion criteria:** tree view ✓ · edit round-trip + clear ✓ · manual persists/clear resets ✓ · `ls` unchanged ✓ · pytest/coverage/lint ✓ · E2E isolation ✓ · commit — left to harness per protocol (working tree untouched, `git add/commit` not run) + +**Deviations:** none. **Next pending phase:** none — `todo/` contains only 97 (96 already committed). diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.validate new file mode 100644 index 0000000..cd15dfa --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__00_phase.a1.validate @@ -0,0 +1,96 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 49%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 66%] +........................................................................ [ 70%] +........................................................................ [ 73%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 94%] +........................................................................ [ 98%] +..................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 127 1 99% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 306 0 100% +-------------------------------------------------- +TOTAL 3824 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.md new file mode 100644 index 0000000..02c6102 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.md @@ -0,0 +1,18 @@ +Task 01 complete and fully verified. (Note: a prior interrupted run had left the code work in the tree; I audited it against the task file, found and fixed two real defects in its new migration test — a teardown deadlock and a cleanup-before-assert bug — then re-verified everything.) + +**Implemented** +- Migration `0018_folder_summary_manual_flag`: additive `folder_summaries.manually_edited` Boolean NOT NULL default `false`, tested downgrade (A13) +- `FolderSummary.manually_edited` model column + contract docstrings +- Generator: single `existing` row fetch; manual rows SKIPPED on regeneration (`kept_manual`, zero `lite` burn, byte-identical), NEVER pruned; 4-field stats log line; `import_docs` token stays 3 fields +- Tests: 2 new unit tests (manual survives regenerate/prune; flag is the only difference), `test_migration_0018.py` (up/down/round-trip), new sync test (manual row survives a KB-changing sync; token `1/0/0`; 4-field log) + +**Results** +- `uv run pytest --cov=app --cov-report=term-missing` → 1994 passed, **99%** coverage (gate >90%) +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors +- `uv run pytest tests/e2e/test_ls_tree_drilldown.py -v --no-cov` (phase-94 regression) → 3 passed + +**Decisions** +- Fixed defects in the prior run's `test_migration_0018.py`: fixture teardown now `db.rollback()`s before the repair DDL (idle-in-transaction SELECT held ACCESS SHARE, deadlocking the `ALTER TABLE`); moved row-survival assertions before the `_clear_rows` cleanup +- No behavior change to phase 94/96 suites — all green + +**Next pending task:** `.agents/phases/todo/97_kb_tree_catalog/02_tree_api.md` diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.validate new file mode 100644 index 0000000..0594b76 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__01_manual_flag.a1.validate @@ -0,0 +1,95 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 18%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 32%] +........................................................................ [ 36%] +........................................................................ [ 39%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 54%] +........................................................................ [ 57%] +........................................................................ [ 61%] +........................................................................ [ 64%] +........................................................................ [ 68%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 79%] +........................................................................ [ 83%] +........................................................................ [ 86%] +........................................................................ [ 90%] +........................................................................ [ 93%] +........................................................................ [ 97%] +.................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 50 0 100% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 279 0 100% +-------------------------------------------------- +TOTAL 3720 14 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.md new file mode 100644 index 0000000..5e9c26b --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.md @@ -0,0 +1,12 @@ +All gates green. Task 02 is complete — the work from the interrupted prior run was verified end-to-end, with one defect fixed. + +**Report — Phase 97, Task 02 (`GET /api/docs/tree`)** + +- Verified the in-tree implementation of the task: `build_kb_tree` (pure, module-level) + `_folder_counts`/`_level_children`/`_source_node` in `app/api/docs.py`, `KbTree*` schemas (recursive union with `kind` discriminators) in `app/schemas.py`, and the admin-only `GET /api/docs/tree` endpoint composing `list_source_names` + the `/api/docs` query + all `folder_summaries` rows. `GET /api/docs` untouched. +- Verified the unit suite (`tests/unit/test_kb_tree_builder.py`: superset ordering, 0-doc source, nested counts, existence rule, name collision, ordering, summaries, verbatim metadata) including the `group_folder_listing` cross-check at root and a nested level, and the integration extensions in `tests/integration/test_docs_api.py` (shape/order/counts/summaries, 403 anonymous, empty tree, indexed-only source, stat-walk equivalence). +- **Fixed one defect:** `test_docs_tree_empty_registry_and_catalog` failed because the dev `.env`'s `BOR_GIT_SOURCES` fallback activates when `git_sources` is truncated. Patched the resolver module's `get_settings` with fresh `Settings(_env_file=None, git_sources="")` (the `test_sync_api`/`test_git_sources_api` house pattern). +- `uv run pytest --cov=app --cov-report=term-missing` → **2011 passed**, coverage **99%** on `app/` (gate >90%). +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors, 0 warnings. +- `uv run pytest tests/integration/test_docs_api.py -v --no-cov` → 8 passed; `uv run pytest tests/e2e/test_ls_tree_drilldown.py -v --no-cov` → 3 passed (phase-94 regression, isolation). +- No deviations from the phase design; all changes left uncommitted in the working tree per protocol. +- **Next pending task:** `.agents/phases/todo/97_kb_tree_catalog/03_folder_summary_api.md` (`PATCH /api/folders/summary`). diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.validate new file mode 100644 index 0000000..96e3e67 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__02_tree_api.a1.validate @@ -0,0 +1,95 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 32%] +........................................................................ [ 35%] +........................................................................ [ 39%] +........................................................................ [ 42%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 82%] +........................................................................ [ 85%] +........................................................................ [ 89%] +........................................................................ [ 93%] +........................................................................ [ 96%] +................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 104 1 99% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 298 0 100% +-------------------------------------------------- +TOTAL 3793 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.md new file mode 100644 index 0000000..90df0f7 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.md @@ -0,0 +1,19 @@ +Everything is green. Final report: + +**Task 03 — `PATCH /api/folders/summary` (phase 97)** + +Implemented: +- `app/schemas.py`: `FolderSummaryUpdate` / `FolderSummaryResult`, next to the phase-57 `SummaryUpdate`/`SummaryResult` +- `app/api/docs.py`: `PATCH /folders/summary` on the same `kb` router — `require_admin`; source check (registered OR indexed, else 404 `source not found`); folder check (`""` = root, else strict `folder + "/"` prefix existence rule over indexed paths, DB-only, else 404 `folder not found`); non-empty strip → upsert with `manually_edited=True` + fresh UTC `updated_at` (create where no row exists); empty → row deleted, double-clear 200 no-op; no LLM call, docstring + module line record the phase-57 re-embed contrast + +Tests (extensions to `tests/integration/test_docs_api.py`, 8 new): +- update AI row (text replaced, flag flips, stamp advances, in-place second save) + tree round-trip; create in a <2-doc folder; root save on a registered 0-doc source; root round-trip through `GET /api/docs/tree`; clear AI + manual rows, double-clear no-op, tree null; 404s (unknown source, folder-elsewhere ordering, unknown/traversal/name-collision/trailing-slash folders); 403 anonymous **and** live access-token non-admin; source pin — handler source never names `LLMClient` + +Results: +- `uv run pytest tests/integration/test_docs_api.py -v --no-cov` → 16 passed +- `uv run pytest --cov=app` → 2019 passed, TOTAL 99% (>90% gate) +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors + +Notable: folder existence is exact `str.startswith` over fetched paths (no SQL LIKE wildcard risk); the no-LLM pin uses `inspect.getsource` on the handler (the docstring was reworded so it can't trip its own grep). + +Next pending task: `.agents/phases/todo/97_kb_tree_catalog/04_tree_ui.md` diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.validate new file mode 100644 index 0000000..234d783 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__03_folder_summary_api.a1.validate @@ -0,0 +1,96 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 32%] +........................................................................ [ 35%] +........................................................................ [ 39%] +........................................................................ [ 42%] +........................................................................ [ 46%] +........................................................................ [ 49%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 71%] +........................................................................ [ 74%] +........................................................................ [ 78%] +........................................................................ [ 82%] +........................................................................ [ 85%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 96%] +........................................................................ [ 99%] +... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 127 1 99% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 306 0 100% +-------------------------------------------------- +TOTAL 3824 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.md new file mode 100644 index 0000000..46e1a2c --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.md @@ -0,0 +1,15 @@ +All green. Final report: + +**Phase 97, Task 04 (tree UI) — complete** + +- RAG view now renders the drill-down tree: static skeleton in `index.html` (`#kb-crumb`, `#kb-level`, `#folders-table` with Folder|Documents|Description — the ONE table; top level lists the sources, `ls()`-equivalence), after `#stat-cards`, before the unchanged 5-column file table. +- `sources.js`: `loadDocs` → `loadTree` (single `GET /api/docs/tree` fetch, `loadSeq` race token kept); module `kbTree` + `current` drill state; `renderLevel()` (level block hidden when no stored description — the `ls` rule; file table hidden at top), whole-tree `treeStats()` walk, client-side `goTo` navigation (no fetch/URL change), never-stale `resetVanishedLocation()` (vanished source/folder resets to top before render); anonymous branch never fetches. +- `styles.css`: `.kb-crumb*`, `.kb-level`, `.kb-folders-table`, `.folder-link` — house palette only, no new hue, AA pairs, hover+focus drill affordance. +- New `tests/unit/test_kb_tree_ui.py` (21 pins: fetch path, skeleton ids/order, drill transitions, textContent contract, empty-state semantic, reset fallback, the 3 refresh wirings + boot, no-new-hue); updated the `loadDocs`→`loadTree` pins in `test_frontend_router.py` / `test_frontend_sync_upload.py` / `test_sync_button.py`. +- Defect fixed (previously shipped): `tests/e2e/test_ls_tree_drilldown.py` `_truncate_all` lacked `db.commit()` — its isolation TRUNCATE silently rolled back (reproduced live; the house pattern commits). Suite re-verified green with real isolation. + +**Gates:** `uv run pytest` → 2040 passed · `uv run pytest --cov=app --cov-report=term-missing` → TOTAL 99% (>90%) · `uv run ruff check . && uv run pyright` → clean · `tests/e2e/test_ls_tree_drilldown.py` (phase 94) → 3/3 in isolation · live-browser smoke of the new view (top→source→nested drill, breadcrumb up, modal link) passed (throwaway script, deleted). + +**Note:** `tests/e2e/test_edit_summaries.py` (phase 57) is 3/4 — the failing test asserts the flat `#docs-tbody` catalog post-login, the flat-layout assertion the phase overview assigns to task 08; phase-57 behavior itself is unchanged. + +**Next pending task:** `.agents/phases/todo/97_kb_tree_catalog/05_folder_edit_ui.md` diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.validate new file mode 100644 index 0000000..7987478 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__04_tree_ui.a1.validate @@ -0,0 +1,96 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 49%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 60%] +........................................................................ [ 63%] +........................................................................ [ 67%] +........................................................................ [ 70%] +........................................................................ [ 74%] +........................................................................ [ 77%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 95%] +........................................................................ [ 98%] +........................ [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 127 1 99% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 306 0 100% +-------------------------------------------------- +TOTAL 3824 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.md new file mode 100644 index 0000000..183fca1 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.md @@ -0,0 +1,9 @@ +**Task 05 complete — folder description editor (level block + rows)** + +- `frontend/index.html`: static `button.kb-summary-edit#kb-level-edit` ("Edit", type=button) in `#kb-level`, inside a new `.kb-level-body` with the `

` (the editor swaps inside it; the `

` stays put) +- `frontend/assets/sources.js`: ONE shared `wireDescriptionEdit()` (mirrors phase-57 `document.js`) — Edit → `textarea.kb-summary-editor` (prefilled via `.value`) + Save/Cancel + `role="status"` live region → `PATCH /api/folders/summary` `{source, folder_path, summary}` (`""` = source root); 200 → textContent re-render + in-place `kbTree` node update + "Description updated." / "Description cleared."; failure keeps the editor with neutral phase-55 copy; Cancel restores. Row Description cells (`makeDescCell`, both row builders) carry the always-present Edit button (creation where no row is stored); no whoami gate (view is admin-only; endpoint `require_admin` is the API gate). Level editor uses a `getTarget()` getter (persistent block) and a `reset()` handle called on every re-render (§7.4) +- `frontend/assets/styles.css`: `.kb-summary-edit/editor/save/cancel/status` (+`.kb-summary-actions`) — house palette, 8rem-min editor, 24px+ target, no new hue +- `tests/unit/test_kb_tree_ui.py`: 12 new source pins + 3 updated pins for the new row-cell shape (all task-05 Testing & Quality items covered) +- Tests: `uv run pytest` → **2053 passed**; `uv run pytest --cov=app --cov-report=term-missing` → **99%** (gate >90%); `uv run ruff check . && uv run pyright` → **clean** (0 errors); `node --check` on the module + a 24-check functional DOM smoke (row + level: PATCH body, clear, guarded hide, failures, cancel, reset) all pass +- Notable: clearing at the level hides the block after a 2 s guarded beat (phase-57 removal-beat contract — the live-region "Description cleared." must be readable; guard makes a navigate-away/re-create first a no-op). No backend change; no `git` operations (left for the harness) +- Next pending task: `.agents/phases/todo/97_kb_tree_catalog/06_e2e_kb_tree.md` diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.validate new file mode 100644 index 0000000..cd15dfa --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__05_folder_edit_ui.a1.validate @@ -0,0 +1,96 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 49%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 66%] +........................................................................ [ 70%] +........................................................................ [ 73%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 94%] +........................................................................ [ 98%] +..................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 127 1 99% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 306 0 100% +-------------------------------------------------- +TOTAL 3824 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.md new file mode 100644 index 0000000..eac45ff --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.md @@ -0,0 +1,15 @@ +Everything is green. Final report: + +**Task 06 complete — `tests/e2e/test_kb_tree.py` (8 tests, green in isolation ×2, 6.7 s)** + +- **Implemented** + - New dedicated Playwright suite per the task spec: module-scoped app (port 8139, `E2E_APP_PORT_KBTREE`), 2 local sources via authenticated API + real in-process `POST /api/sync` (7 docs; alpha 5 / beta 2), canned `FOLDER_SUMMARY_MODE` rows pinned (all `manually_edited=false`), no mock changes. + - 8 observables: top-level source rows (registry order, counts, root descriptions, hidden file table, stat cards 7 docs + DB-computed chunks) · drill-into-source (breadcrumb aria-current, level block, folder rows, 5-column file row) · drill-into-folder (path title, Source column, hidden `#folders-wrap`, breadcrumb up) · row-surface edit → "Description updated." + SQL `manually_edited=true` · empty save → "Description cleared." + row deleted · level-block edit of the beta root surviving a KB-changing sync (`kept_manual`: manual text+flag kept, siblings regenerated to canned, new doc counts 3) · vanished `two/` + nav re-click → reset to top level (§7.4) · anonymous gate with zero `/api/docs*` requests. + - Defect fix (found while testing tasks 04/05 output): the phase-97 CSS targets `.kb-crumb`/`.kb-level`, but the static `#kb-crumb` nav and `#kb-level` section never carried those classes — dead rules (no breadcrumb flex/gap, no level-block surface panel). Added the classes in `frontend/index.html` (global `[hidden]{display:none!important}` keeps the hidden contract) and updated the two exact-match HTML pins in `tests/unit/test_kb_tree_ui.py`. +- **Gates** + - `uv run pytest tests/e2e/test_kb_tree.py -v --no-cov` → 8 passed (isolated, DB up) + - `uv run pytest` → 2053 passed; `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90%) + - `uv run ruff check .` → clean; `uv run pyright` → 0 errors + - `uv run pytest tests/e2e/test_ls_tree_drilldown.py -v --no-cov` → 3 passed (agent `ls` byte-identical; first run's deadlock was environmental — the owner's `--reload` dev server shares the same Postgres mid-sync, passed after it settled) +- **Note:** the legacy flat-catalog story suites (e.g. `test_admin_auth`, `test_edit_summaries`, `test_import_documents`) are red by the phase-97 UI change — they are the explicit scope of tasks 07/08, untouched here. +- **Next pending task:** `07_catalog_suites_import_sync.md` diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.validate new file mode 100644 index 0000000..cd15dfa --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__06_e2e_kb_tree.a1.validate @@ -0,0 +1,96 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 49%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 66%] +........................................................................ [ 70%] +........................................................................ [ 73%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 94%] +........................................................................ [ 98%] +..................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 127 1 99% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 306 0 100% +-------------------------------------------------- +TOTAL 3824 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.md new file mode 100644 index 0000000..00c09ac --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.md @@ -0,0 +1,12 @@ +Everything is green and complete. Final report: + +**Task 07 — import/sync/upload story suites onto the drill-down catalog — complete** + +- Applied the drill recipe to all 11 listed suites (drill helpers per suite, `:text-is()` house pattern from task 06): per-level row/count re-asserts replace the flat `#docs-tbody` totals (stat cards carry KB totals); `.hidden`/junk absences scoped after a drill; 0-document source rows become the load-settled "nothing indexed" signal where a source is registered (`test_upload_no_scan`, `test_archive_upload_sources`, `test_sync_upload_progress`); `.table-wrap` selects scoped to `:has(#docs-table)` (two wraps now exist). +- `test_import_documents.py` now runs its own module app (port 8140, conftest leak-guard pattern) with `BOR_GIT_SOURCES=""` + truncates `git_sources`/`folder_summaries` — the zero-SOURCES empty state is unreachable on the session app (operator `.env` fallback source would leak a 0-doc row). +- `test_markdown_tables.py` (import family, also red) fixed with the same mechanical drill; source-registry suites (git_sources_admin/local_directory/ignore_paths/removal) verified untouched (21/21). +- Fixed a **pre-existing defect** (proven red at HEAD via worktree): `test_sync_upload_progress::test_sync_live_file_label` — the module app's shared in-memory sync state re-attaches "Synced HH:MM" at boot after test 1's sync (phase-32 contract); the boot precondition now accepts idle-or-re-attached, run assertions unchanged. +- E2E in isolation: import_documents 3/3, admin_auth 6/6, sticky_navbar 3/3, sync_upload_progress 4/4, upload_no_scan 3/3, sync_button 3/3, sync_model_down 4/4, import_extensions_env 2/2, quadlet_jinja 4/4, retrieval_quality 4/4, archive_upload 5/5, markdown_tables 6/6; phase gates green: kb_tree 8/8, ls_tree_drilldown 3/3, read_truncation_cap 3/3. +- `uv run pytest --cov=app --cov-report=term-missing` → **2053 passed, 99%**; `uv run ruff check .` clean; `uv run pyright` → 0 errors. Tests-only: no app/frontend changes from this task. +- Deviation note: `test_edit_summaries` (1 red) is the viewer/editor suite — explicitly task 08's list (flat-table drill), left for it. +- Next pending task: `.agents/phases/todo/97_kb_tree_catalog/08_catalog_suites_viewer_nav.md`. diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.validate new file mode 100644 index 0000000..cd15dfa --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__07_catalog_suites_import_sync.a1.validate @@ -0,0 +1,96 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 49%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 66%] +........................................................................ [ 70%] +........................................................................ [ 73%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 94%] +........................................................................ [ 98%] +..................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 127 1 99% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 306 0 100% +-------------------------------------------------- +TOTAL 3824 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.err b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.md b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.md new file mode 100644 index 0000000..a817033 --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.md @@ -0,0 +1,12 @@ +All validation gates are green. The working tree contains only test-file changes from this task (14 E2E suites) on top of the pre-existing uncommitted tasks 01–07 work — no app/frontend source changes from this task, and no task/phase files touched. + +**Report** + +- **Updated viewer/doc suites onto the drill-down catalog** (drill = only change): `test_document_viewer` (2 drills incl. seeded `notes/xss-fixture`), `test_document_back_navigation` (drill + post-back signal = source row), `test_edit_summaries` (modal-surface drill), `test_summary_in_viewer` (2 drills). +- **Updated UX suite**: `test_responsive_polish` — top-level source-row render signals, `#folders-wrap` card for full-width/in-card-scroll pins, `longkb → deep` drill for the 60-char path row. +- **Updated the six `.first`-wait nav/stream suites** (`test_chat_persistence`, `test_navbar_refresh`, `test_nav_consistency`, `test_nav_switch_keeps_stream`, `test_shared_header`, `test_sources_midstream_bug`): catalog-rendered signal is now the top-level `#folders-tbody` source row. +- **`test_navbar_refresh`**: RAG re-fetch pin tracks the view's new endpoint `/api/docs/tree`. +- **`test_history_page_width`**: 640px in-card-scroll / desktop-width pins now drill `docs → homelab` to the file table (`.docs-table` no longer unique). +- **Pre-existing defects fixed (tests only)**: second shell nav (`#kb-crumb`) broke the "one labeled nav" landmark pins in `test_responsive_polish` + `test_global_tuning` (now pinned by `aria-label='Primary'`); `test_theme_semantic_completion` tool flow was broken by phase 94's registry-driven `ls` + uncontrolled `git_sources` (10-line degenerate loop) — fixed with the phase-94 house recipe (register both sources, TRUNCATE `git_sources`, 3 tool lines). +- **Gates**: `uv run pytest --cov=app --cov-report=term-missing` → exit 0, TOTAL **99%** (>90%); `uv run ruff check . && uv run pyright` → clean (0 errors); full `tests/e2e/` sweep — **all 87 suites green in isolation**; `uv run pytest tests/e2e/test_kb_tree.py -v --no-cov` → 8 passed; `test_ls_tree_drilldown.py` → 3 passed (agent tree byte-identical). +- Next pending task: none in `97_kb_tree_catalog` — this was the phase's last task (phase complete, awaiting harness validation/commit). diff --git a/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.validate b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.validate new file mode 100644 index 0000000..cd15dfa --- /dev/null +++ b/.agents/reports/97_kb_tree_catalog/97_kb_tree_catalog__08_catalog_suites_viewer_nav.a1.validate @@ -0,0 +1,96 @@ +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 10%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 35%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 49%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 66%] +........................................................................ [ 70%] +........................................................................ [ 73%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 94%] +........................................................................ [ 98%] +..................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 181 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 94 0 100% +app/api/docs.py 127 1 99% +app/api/git_sources.py 229 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 29 0 100% +app/api/sync.py 115 0 100% +app/api/tokens.py 28 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 140 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 33 0 100% +app/db.py 21 0 100% +app/main.py 66 0 100% +app/models.py 125 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 316 1 99% +app/rag/archive_upload.py 128 0 100% +app/rag/chunker.py 206 4 98% +app/rag/folder_summaries.py 121 0 100% +app/rag/git_sources.py 14 0 100% +app/rag/importer.py 190 3 98% +app/rag/llm.py 243 1 99% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 88 0 100% +app/rag/retriever.py 150 3 98% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 24 0 100% +app/schemas.py 306 0 100% +-------------------------------------------------- +TOTAL 3824 15 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/alembic/versions/0018_folder_summary_manual_flag.py b/alembic/versions/0018_folder_summary_manual_flag.py new file mode 100644 index 0000000..38be5b7 --- /dev/null +++ b/alembic/versions/0018_folder_summary_manual_flag.py @@ -0,0 +1,56 @@ +"""folder_summaries.manually_edited: owner-edited descriptions (phase 97, task 01) + +Revision ID: 0018 +Revises: 0017 +Create Date: 2026-09-11 + +Phase 97 (the RAG view shows the catalog as the drill-down tree the +agent sees, with editable folder descriptions): the owner can edit — +or clear — any directory's description, just like a file's summary +(phase 57's affordance). The owner's words are OWNER CONTENT: the +sync-time generator must never silently rewrite them. This migration +adds the one flag that makes that enforceable — a single additive, +fully reversible column (A13), no other schema change: + +* ``folder_summaries.manually_edited`` — BOOLEAN NOT NULL, server + default ``false``: ``true`` only while the row's description was + written by the owner (``PATCH /api/folders/summary`` — phase 97, + task 03 — the ONLY writer; the generator never sets it). The + generator's two rules for a manual row (``app.rag.folder_summaries``): + SKIPPED on regeneration (no ``lite`` burn on owner text — counted + ``kept_manual`` in the stats) and NEVER pruned (owner content + persists until the owner clears it — even a manual row for a folder + that dropped below 2 documents). Pre-0018 rows backfill ``false``: + an AI-written row stays an AI row. Clearing the description DELETES + the row — the next KB-changing sync regenerates an AI description + (the reset path; no separate regenerate button in v1). +""" +from __future__ import annotations + +import sqlalchemy as sa + +from alembic import op + +revision = "0018" +down_revision = "0017" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "folder_summaries", + sa.Column( + "manually_edited", + sa.Boolean(), + server_default=sa.text("false"), + nullable=False, + ), + ) + + +def downgrade() -> None: + # The column is the only 0018 artefact — dropping it (the rows and + # their summaries survive) leaves 0017's schema byte-identical + # (A13, fully reversible). + op.drop_column("folder_summaries", "manually_edited") diff --git a/app/api/docs.py b/app/api/docs.py index dc80bb3..c5859d9 100644 --- a/app/api/docs.py +++ b/app/api/docs.py @@ -9,9 +9,25 @@ PATCH /api/documents/summary — the admin summary editor (phase 57): update or clear ``documents.summary`` and re-embed the ``is_summary`` chunk (embed first, mutate second — a failed LLM call leaves the row and chunk untouched; the content chunks are never re-embedded, D4). + +GET /api/docs/tree — the admin's full recursive KB tree in one fetch +(phase 97, task 02): the same drill-down tree the agent's ``ls`` +walks, with the file metadata the RAG view's rows and stat cards need +(the view drills client-side; ``GET /api/docs`` is untouched). + +PATCH /api/folders/summary — the admin folder-description editor +(phase 97, task 03): update / create / clear a stored +``folder_summaries`` row, marking every non-empty save +``manually_edited`` (from this point on the sync-time generator skips +the row and never prunes it — task 01). A pure DB write: NO +LLM/embedding call — a folder description is never embedded (no +chunk, no retrieval role beyond the ``ls`` line), the deliberate +contrast with the phase-57 ``is_summary`` re-embed above. """ from __future__ import annotations +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime from pathlib import Path from fastapi import APIRouter, Depends, HTTPException @@ -21,9 +37,23 @@ from sqlalchemy.orm import Session from app.api.sync import _sanitize_error from app.core.auth import require_admin, require_user from app.db import get_db -from app.models import Chunk, Document +from app.models import Chunk, Document, FolderSummary +from app.rag.agent import list_source_names +from app.rag.folder_summaries import folder_of from app.rag.llm import EmbeddingError, LLMClient -from app.schemas import DocContent, DocList, DocSummary, SummaryResult, SummaryUpdate +from app.schemas import ( + DocContent, + DocList, + DocSummary, + FolderSummaryResult, + FolderSummaryUpdate, + KbTree, + KbTreeFile, + KbTreeFolder, + KbTreeSource, + SummaryResult, + SummaryUpdate, +) router = APIRouter(tags=["kb"]) @@ -184,3 +214,303 @@ async def update_document_summary( return SummaryResult( source=doc.source, path=doc.path, summary=doc.summary, chunks=chunks ) + + +@router.patch("/folders/summary", response_model=FolderSummaryResult) +def update_folder_summary( + payload: FolderSummaryUpdate, + db: Session = Depends(get_db), # noqa: B008 + _admin: None = Depends(require_admin), # noqa: B008 +) -> FolderSummaryResult: + """Update / create / clear a folder's stored description. + + Admin-only: the catalog is admin-only (phase 16) and this gate is + the API-level defense in depth (the RAG view never renders for + anonymous, the endpoint must not lean on that). ``folder_path = + ""`` is the SOURCE ROOT (the phase-94 ``folder_summaries`` + convention — the top-level source summary). + + Checks, in order — DB-only (the ``/documents/content`` rule: no + filesystem access at all, a traversal string such as ``../../etc`` + is simply not a prefix of any indexed path): + + * **Source** — registered (``list_source_names``) OR has indexed + documents → else 404 ``{"detail": "source not found"}``. + * **Folder** — ``""`` is valid for an allowed source; otherwise the + phase-94 existence rule over the source's indexed paths (some + ``Document.path`` starts with ``folder_path + "/"`` — a file + merely sharing the folder's name is NOT a folder) → else 404 + ``{"detail": "folder not found"}``. + + Writes: a non-empty (after ``strip()``) ``summary`` upserts the row + with ``summary = stripped text``, ``manually_edited = True``, and a + fresh UTC ``updated_at`` — a manual description can be CREATED + where no row exists (a < 2-document folder, or the generator's + fail-soft miss), and from this save on the task-01 keep/keep-out + rules apply (the generator skips the row and never prunes it). + An empty/whitespace-only ``summary`` CLEARS instead: the row is + ``db.delete``'d when present (the phase-57 analog — the row may be + AI-written or manual, either way it is gone; the next KB-changing + sync regenerates an AI row — the reset path). A clear with no row + is a 200 no-op. The response echoes the stored text — ``summary`` + null after a clear. + + No LLM/embedding call on this path: a folder description is never + embedded (no chunk, no retrieval role beyond the ``ls`` line) — + the deliberate contrast with the phase-57 ``is_summary`` re-embed + in :func:`update_document_summary`. The no-LLM contract is + source-pinned in the test suite (the handler's source never names + the LLM client). + """ + source = payload.source + folder = payload.folder_path + paths = list(db.scalars(select(Document.path).where(Document.source == source))) + if source not in list_source_names(db) and not paths: + raise HTTPException(status_code=404, detail="source not found") + if folder and not any(p.startswith(folder + "/") for p in paths): + raise HTTPException(status_code=404, detail="folder not found") + row = db.scalar( + select(FolderSummary).where( + FolderSummary.source == source, FolderSummary.folder_path == folder + ) + ) + text = payload.summary.strip() + if text: + if row is None: + row = FolderSummary(source=source, folder_path=folder) + db.add(row) + row.summary = text + row.manually_edited = True + row.updated_at = datetime.now(UTC) + else: + if row is not None: + db.delete(row) + db.commit() + return FolderSummaryResult(source=source, folder_path=folder, summary=text or None) + + +#: One catalogue row the tree builder consumes: +#: ``(source, path, title, chunks, indexed_at)`` — the ``GET /api/docs`` +#: query's columns minus the document ``id`` (the tree has no document +#: ids), in the same ``(source, path)`` order; ``indexed_at`` is the +#: ISO-8601 string the endpoint converts (the builder stays pure over +#: plain types — unit-testable without a DB). +TreeDocRow = tuple[str, str, str, int, str] + +#: One of a source's file rows, already source-scoped: +#: ``(path, title, chunks, indexed_at)``. +TreeFileRow = tuple[str, str, int, str] + + +def _folder_counts( + rows: Sequence[TreeFileRow], +) -> tuple[set[str], dict[str, int]]: + """One source's folders (existence rule) + recursive counts (pure). + + The phase-94 rules, reused verbatim from + :func:`app.rag.agent.group_folder_listing` (ONE concept end to end — + the UI tree is the ``ls`` tree plus file metadata): a folder exists + ⟺ some indexed path starts with ``folder + "/"`` (a slash-boundary + prefix of an indexed path — a document's OWN path is never a + folder; the folder's parent is :func:`folder_of`, the shared + notion, never re-derived). The count of a folder is its recursive + subtree — every path EQUAL to the folder (the file sharing its + name counts, the ``path == folder`` arm) or starting with + ``folder + "/"`` — exactly the set the sync-time folder summary + describes. + + Returns ``(folders, counts)`` — the folder-name set and the + per-folder count map (every folder counts ≥ 1 by construction: its + own descendants, or the file wearing its name, exist). + """ + folders: set[str] = set() + for path, _title, _chunks, _indexed_at in rows: + folder = folder_of(path) + while folder: + folders.add(folder) + folder = folder_of(folder) + counts: dict[str, int] = {folder: 0 for folder in folders} + for path, _title, _chunks, _indexed_at in rows: + if path in folders: + counts[path] += 1 + folder = folder_of(path) + while folder: + counts[folder] += 1 + folder = folder_of(folder) + return folders, counts + + +def _level_children( + source: str, + folder: str, + folders: set[str], + counts: dict[str, int], + rows: Sequence[TreeFileRow], + summaries: Mapping[tuple[str, str], str], +) -> list[KbTreeFolder | KbTreeFile]: + """One level's children (pure): subfolders in path order, then the + direct files in input (catalog) order. + + *folder* is source-relative (``""`` = the source root). A folder + *sub* appears here iff ``folder_of(sub) == folder`` (a DIRECT + subfolder) and it exists (the :func:`_folder_counts` set — the + phase-94 existence rule); a file appears iff + ``folder_of(path) == folder`` (a DIRECT file). The subfolder + order is the sorted (path) order and the file order is the input + (catalog — ``GET /api/docs``) order, both matching + :func:`app.rag.agent.group_folder_listing` level-for-level; the + file list is NOT capped (the ``ls`` 50-line cap is a model-context + budget — the UI is for humans). Recurses one level per call. + """ + children: list[KbTreeFolder | KbTreeFile] = [] + for sub in sorted(g for g in folders if folder_of(g) == folder): + children.append( + KbTreeFolder( + path=sub, + documents=counts[sub], + summary=summaries.get((source, sub)), + children=_level_children(source, sub, folders, counts, rows, summaries), + ) + ) + for path, title, chunks, indexed_at in rows: + if folder_of(path) == folder: + children.append( + KbTreeFile(path=path, title=title, chunks=chunks, indexed_at=indexed_at) + ) + return children + + +def build_kb_tree( + names: Sequence[str], + doc_rows: Sequence[TreeDocRow], + summaries: Mapping[tuple[str, str], str], +) -> list[KbTreeSource]: + """The pure tree builder behind ``GET /api/docs/tree`` (phase 97, + task 02) — module-level and DB-free so unit tests drive it + directly (the house pattern; the endpoint composes the fetches). + + *names* — the registry source names in order (``app.rag.agent. + list_source_names`` — deduped, registry order). *doc_rows* — the + catalogue ``(source, path, title, chunks, indexed_at)`` tuples in + the ``GET /api/docs`` query order (``source, path``). *summaries* — + ``{(source, folder_path): summary}`` over the stored + ``folder_summaries`` rows (``folder_path = ""`` = the source root; + rows for sources the tree does not list are simply never + referenced). + + Shape, per the phase-97 ``00_phase.md`` "The tree endpoint": + + * **Sources** — the registry names first (each ALWAYS present — a + registered 0-document source lists with ``documents: 0`` and no + children), then the distinct indexed sources not in *names* + (alphabetical — the superset rule: the catalog has never hidden + an indexed document, while the agent's ``ls`` keeps listing + registry sources only — unchanged). Every doc source is listed by + construction; the registry order still leads. + * **Folder nodes** — ``path`` source-relative (never ``""`` — the + source node IS the root); direct subfolders only, in path + (sorted) order, a folder existing only under the phase-94 + existence rule; ``documents`` = the recursive subtree count; + ``summary`` = the stored row (AI OR manual — any row) or null; + ``children`` = the folder's own subfolders + direct files, same + shape. + * **File nodes** — direct files only, in input (catalog) order; + ``path`` source-relative; ``title`` / ``chunks`` / ``indexed_at`` + verbatim from the catalogue row. + + ONE concept end to end: the builder reuses + :func:`app.rag.folder_summaries.folder_of` and the phase-94 + existence / count rules, so — for a single-source dataset — its + level equals :func:`app.rag.agent.group_folder_listing`'s output + (same subfolder ``(path, count, summary)`` triples in order, same + file ``(path, title)`` pairs in order — the "UI shows what the + agent sees" cross-check, unit-pinned at the root and a nested + level). + """ + by_source: dict[str, list[TreeFileRow]] = {} + for source, path, title, chunks, indexed_at in doc_rows: + by_source.setdefault(source, []).append((path, title, chunks, indexed_at)) + tree: list[KbTreeSource] = [] + listed: set[str] = set() + for name in names: + if name in listed: # defensive: list_source_names dedupes + continue + listed.add(name) + tree.append(_source_node(name, by_source.get(name, ()), summaries)) + for source in sorted(by_source): + if source not in listed: + tree.append(_source_node(source, by_source[source], summaries)) + return tree + + +def _source_node( + source: str, + rows: Sequence[TreeFileRow], + summaries: Mapping[tuple[str, str], str], +) -> KbTreeSource: + """One source node (pure): whole-source count + the source-root + summary + the root level's children (direct subfolders + direct + files). + + ``documents`` is ``len(rows)`` — the source's WHOLE recursive + count (every one of its documents, the set its stored + ``(source, "")`` summary describes). A source with no rows lists + ``documents: 0`` and no children (the registered 0-document source + — the phase-70/72 invariant, extended by the superset rule). + """ + folders, counts = _folder_counts(rows) + return KbTreeSource( + name=source, + documents=len(rows), + summary=summaries.get((source, "")), + children=_level_children(source, "", folders, counts, rows, summaries), + ) + + +@router.get("/docs/tree", response_model=KbTree) +def list_kb_tree( + db: Session = Depends(get_db), # noqa: B008 + _admin: None = Depends(require_admin), # noqa: B008 +) -> KbTree: + """The full recursive KB tree in ONE fetch (phase 97, task 02). + + Admin-only, like ``GET /api/docs`` — anonymous callers get 403 + ``admin only`` (the RAG view's anonymous gate never fetches the + tree). The RAG view drills CLIENT-side: this is the view's single + fetch, zero per-level requests (the ``00_phase.md`` "The tree + endpoint" contract). + + Composition: the registry source names (``list_source_names`` — + the superset rule's registry half, imported from ``app.rag.agent`` + exactly as ``app/api/chat.py`` does) + the SAME outerjoin/grouped + catalogue query ``GET /api/docs`` runs (the document ``id`` + excluded — the tree has no document ids) + ALL stored + ``folder_summaries`` rows (a bounded select — one row per ≥ 2-doc + folder; rows for sources the tree does not list are never + referenced by the builder) — through the pure + :func:`build_kb_tree`. ``GET /api/docs`` itself is untouched. + """ + names = list_source_names(db) + rows = db.execute( + select( + Document.source, + Document.path, + Document.title, + func.count(Chunk.id).label("chunks"), + Document.indexed_at, + ) + .outerjoin(Chunk, Chunk.document_id == Document.id) + .group_by(Document.id, Document.source, Document.path, Document.title, Document.indexed_at) + .order_by(Document.source, Document.path) + ).all() + doc_rows: list[TreeDocRow] = [ + (source, path, title, chunks, indexed_at.isoformat()) + for source, path, title, chunks, indexed_at in rows + ] + summaries: dict[tuple[str, str], str] = { + (source, folder_path): summary + for source, folder_path, summary in db.execute( + select(FolderSummary.source, FolderSummary.folder_path, FolderSummary.summary) + ).all() + } + return KbTree(sources=build_kb_tree(names, doc_rows, summaries)) diff --git a/app/models.py b/app/models.py index 94c4534..8626bf8 100644 --- a/app/models.py +++ b/app/models.py @@ -478,6 +478,17 @@ class FolderSummary(Base): ``lite`` model wrote at sync time (``FOLDER_SUMMARY_MODE``, ``app.rag.folder_summaries`` — change-gated and fail-soft like the KB overview: an old summary is better than none). + * ``manually_edited`` — ``true`` only while the description is the + OWNER'S words (phase 97, task 01): set ONLY by + ``PATCH /api/folders/summary`` (phase 97, task 03) — the + generator never sets it. The sync-time generator (``app.rag. + folder_summaries``) SKIPS a manual row on regeneration (no + ``lite`` burn on owner text — counted ``kept_manual``) and never + prunes it (owner content persists until cleared — even for a + folder below the 2-document minimum); an owner correction is + never silently rewritten (the phase-97 ``00_phase.md`` decision). + Clearing the description deletes the row — the next KB-changing + sync regenerates an AI description (the reset path). Chat turns only READ these rows (the agent's ``ls`` output, phase 94 task 03) — generation happens at sync time only (task 02). @@ -493,6 +504,14 @@ class FolderSummary(Base): #: The lite-written 1–3 sentence description — never empty (the #: generator validates before storing, ``app.rag.folder_summaries``). summary: Mapped[str] = mapped_column(Text) + #: Owner-edited flag (phase 97, task 01): ``true`` only while the + #: description is the owner's words — set ONLY by + #: ``PATCH /api/folders/summary`` (phase 97, task 03); the + #: generator skips a manual row on regeneration and never prunes + #: it (the class docstring's two rules). + manually_edited: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) #: Fresh UTC stamp on every upsert (the ``kb_overview.updated_at`` #: precedent; the generator always sets it explicitly). updated_at: Mapped[datetime] = mapped_column( diff --git a/app/rag/folder_summaries.py b/app/rag/folder_summaries.py index 9deec33..df79317 100644 --- a/app/rag/folder_summaries.py +++ b/app/rag/folder_summaries.py @@ -28,6 +28,16 @@ single-document folder is fully described by its one file line, so no otherwise go stale), while rows for folders that still have ≥ 2 documents persist (an unchanged folder's summary is still true). +Manual rows (phase 97, task 01): ``manually_edited`` (migration 0018) +marks the descriptions the OWNER edited — ``PATCH /api/folders/ +summary`` (phase 97, task 03) is the ONLY writer. The generator's two +rules for a manual row: it is SKIPPED on regeneration (no ``lite`` +burn on owner text — counted ``kept_manual`` in the stats) and it is +NEVER pruned (owner content persists until the owner clears it — even +for a folder that dropped below the minimum). Clearing the +description DELETES the row, so the next KB-changing sync regenerates +an AI description for that folder (the reset path). + Chat turns never generate folder summaries — the agent's ``ls`` output (phase 94, task 03) only reads the stored rows. Generation is the caller's job at sync time (phase 94, task 02), and :func:`generate_ @@ -399,20 +409,29 @@ async def generate_folder_summaries( FOLDER: one folder's :class:`LLMError` is logged and counted, its previous row (if any) is kept, and the remaining folders still land (a ``lite`` outage must never fail the sync — the KB - is the product, the summaries are auxiliary). With + is the product, the summaries are auxiliary). An EXISTING row + with ``manually_edited`` is SKIPPED instead of regenerated — + no ``lite`` call for owner text (no burn on the owner's words), + the row stays byte-identical (text AND ``updated_at``), and the + skip counts ``kept_manual`` (phase 97, task 01: an owner + correction is never silently rewritten). With ``only_missing=True`` the iteration is restricted to the candidates that have NO stored row (the same gap :func:`missing_folder_summaries` reports, derived from the SAME - catalogue pass — one bounded stored-key select, no second - catalogue query): existing rows stay byte-identical (summary - text AND ``updated_at`` — never re-stamped, even a stale-looking - one; staleness is the changed-KB regeneration's job) and no - ``lite`` call is burned for a folder that already has a summary - — the unchanged-sync self-heal fill (phase 96, task 02). + stored-rows pass — one select, no second catalogue query): + existing rows stay byte-identical (summary text AND + ``updated_at`` — never re-stamped, even a stale-looking one; + staleness is the changed-KB regeneration's job) and no ``lite`` + call is burned for a folder that already has a summary — the + unchanged-sync self-heal fill (phase 96, task 02). 4. DELETE rows whose folder no longer has ≥ 2 documents — a - pruned/renamed folder's summary goes stale and is dropped. - Rows for folders that still qualify persist (regenerated in - step 3 — an unchanged folder's summary is still true). The + pruned/renamed folder's summary goes stale and is dropped — + EXCEPT a manual row: owner content persists until the owner + clears it, even for a folder that dropped below the minimum + (phase 97, task 01; the clear deletes the row, so the next + KB-changing sync regenerates an AI description — the reset + path). Rows for folders that still qualify persist (regenerated + in step 3 — an unchanged folder's summary is still true). The prune pass runs in BOTH modes: under ``only_missing`` on an unchanged catalogue it is a no-op (the invariant kept), and it still drops rows whose folder fell below the minimum. @@ -421,22 +440,37 @@ async def generate_folder_summaries( ``bump_sources_version`` convention: the sync path owns the transaction, so a failed sync rolls the summaries back with it). - Returns the small stats dict ``{"generated", "failed", "pruned"}`` - for the caller's summary-line logging (PLAN §9 ample logging) — - the caller logs the mode, not the generator. + Returns the small stats dict ``{"generated", "failed", "pruned", + "kept_manual"}`` for the caller's summary-line logging (PLAN §9 + ample logging) — the caller logs the mode, not the generator. The + ``import_docs`` summary-line token stays its 3 fields + (``//`` — ``kept_manual`` is a stat, not + a token; phase 97, task 01). """ - stats = {"generated": 0, "failed": 0, "pruned": 0} + stats = {"generated": 0, "failed": 0, "pruned": 0, "kept_manual": 0} if skip: return stats candidates = _candidates(_catalog_rows(db)) + # The existing rows, fetched ONCE (phase 97, task 01): both the + # ``only_missing`` filter and the prune pass key off this single + # {(source, folder_path): row} dict — one fetch, one concept. + existing = { + (row.source, row.folder_path): row + for row in db.execute(select(FolderSummary)).scalars() + } keys = sorted(candidates) if only_missing: - stored = _stored_keys(db) - keys = [key for key in keys if key not in stored] + keys = [key for key in keys if key not in existing] for key in keys: source, folder_path = key + stored = existing.get(key) + if stored is not None and stored.manually_edited: + # Owner-edited description (phase 97, task 01): NEVER + # overwrite it — no lite burn on owner text. + stats["kept_manual"] += 1 + continue docs = candidates[key] try: summary = await summarize_folder(source, folder_path, docs, llm) @@ -449,21 +483,17 @@ async def generate_folder_summaries( _upsert(db, source, folder_path, summary) stats["generated"] += 1 - existing = db.execute( - select(FolderSummary.source, FolderSummary.folder_path) - ).all() - for source, folder_path in existing: - if (source, folder_path) not in candidates: - row = db.get(FolderSummary, (source, folder_path)) - if row is not None: - db.delete(row) - stats["pruned"] += 1 + for key, row in existing.items(): + if key not in candidates and not row.manually_edited: + db.delete(row) + stats["pruned"] += 1 db.flush() logger.info( - "folder_summaries: generated=%d failed=%d pruned=%d", + "folder_summaries: generated=%d failed=%d pruned=%d kept_manual=%d", stats["generated"], stats["failed"], stats["pruned"], + stats["kept_manual"], ) return stats diff --git a/app/schemas.py b/app/schemas.py index 1f2b5dc..700533a 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -244,6 +244,75 @@ class DocList(BaseModel): documents: list[DocSummary] +class KbTreeFile(BaseModel): + """One file node of the KB drill-down tree (phase 97, task 02). + + ``kind`` is the wire discriminator (``"file"`` — the ``00_phase.md`` + JSON shape is the contract). ``path`` is SOURCE-RELATIVE (the RAG + view prefixes the source in its breadcrumb); ``title`` / + ``chunks`` (content + ``is_summary`` chunks — the same count + ``GET /api/docs`` returns) / ``indexed_at`` (ISO-8601) are verbatim + from the catalogue row the endpoint reads. + """ + + kind: Literal["file"] = "file" + path: str + title: str + chunks: int = Field(ge=0) + indexed_at: str + + +class KbTreeFolder(BaseModel): + """One folder node of the KB drill-down tree (phase 97, task 02). + + ``path`` is the source-relative folder (never ``""`` — the SOURCE + node IS the root); ``documents`` is the recursive subtree count + (the phase-94 ``ls`` count rule: every path equal to the folder or + starting with ``folder + "/"`` — the file sharing a folder's name + counts); ``summary`` is the stored ``folder_summaries`` row (AI or + manual — any row) or null; ``children`` are the direct subfolders + (path order) followed by the direct files (catalog order) — the + recursive union (Pydantic v2 resolves it with + ``from __future__ import annotations``). + """ + + kind: Literal["folder"] = "folder" + path: str + documents: int = Field(ge=0) + summary: str | None = None + children: list[KbTreeFolder | KbTreeFile] = Field(default_factory=list) + + +class KbTreeSource(BaseModel): + """One source node of the KB drill-down tree (phase 97, task 02). + + Sources list the REGISTERED names first (registry order — each + always present, a registered 0-document source lists with + ``documents: 0`` and no children), then the indexed-only sources + (alphabetical) — the phase-97 superset rule. ``documents`` is the + source's whole recursive count; ``summary`` is the stored + ``(source, "")`` source-root row or null; ``children`` are the + source's direct subfolders + direct files (same shape as a folder + node's). + """ + + name: str + documents: int = Field(ge=0) + summary: str | None = None + children: list[KbTreeFolder | KbTreeFile] = Field(default_factory=list) + + +class KbTree(BaseModel): + """Response of ``GET /api/docs/tree`` (phase 97, task 02). + + The FULL recursive tree in ONE fetch — the RAG view (admin) drills + client-side, zero per-level fetches (the ``00_phase.md`` "The tree + endpoint" contract). + """ + + sources: list[KbTreeSource] + + class DocContent(BaseModel): """One indexed document's full content (feeds the viewer page, phase 10).""" @@ -294,6 +363,44 @@ class SummaryResult(BaseModel): chunks: int +class FolderSummaryUpdate(BaseModel): + """``PATCH /api/folders/summary`` body (phase 97, task 03). + + ``source`` / ``folder_path`` name the folder whose stored + description is edited — ``folder_path = ""`` is the SOURCE ROOT + (the phase-94 ``folder_summaries`` convention). ``summary`` is the + raw new text: the API strips it before storing, and an + empty/whitespace-only value is the *clear* operation (row deleted + — the reset path, the phase-57 analog), not a 422. Unconstrained on + purpose: an unknown source or a folder with no indexed descendant + must 404 (``source not found`` / ``folder not found`` — + registry/row-lookup semantics), and traversal strings such as + ``../../etc`` are simply not prefixes of any indexed path (the + DB-only rule of the ``/documents/content`` lookup — no filesystem + access). + """ + + source: str + folder_path: str + summary: str + + +class FolderSummaryResult(BaseModel): + """``PATCH /api/folders/summary`` response (phase 97, task 03). + + ``summary`` is the stored text after the change — ``null`` after a + clear (the RAG view hides the level block / empties the description + cell on null). Every non-empty save was stored with + ``manually_edited = true`` (the flag itself is not echoed — the + response mirrors the phase-57 ``SummaryResult`` shape minus the + chunk count, which a folder description has no role in). + """ + + source: str + folder_path: str + summary: str | None + + class SteeringNoteIn(BaseModel): """``POST /api/steering`` body: one tuning instruction (phase 15). diff --git a/frontend/assets/sources.js b/frontend/assets/sources.js index 7d69220..b1f2005 100644 --- a/frontend/assets/sources.js +++ b/frontend/assets/sources.js @@ -39,14 +39,108 @@ * RE-SHOWS an already-mounted view (a switch back onto it, a re-click * of the RAG nav link, or back/forward) — the first show (mount) and * boot never (the mount's own load is the first fetch). This module - * listens on root and re-runs `loadDocs()`, which is now re-entrant: - * a re-load drops the tbody's rows BEFORE the fetch, so a refresh - * from a populated list into an empty result replaces the list - * (no ghost rows) — the History pattern (task 01). #sources-empty - * lives OUTSIDE the tbody (a .empty-state div), so the top clear is - * a bare replaceChildren(). The listener is armed only in the ADMIN + * listens on root and re-runs the catalog load, which is re-entrant: + * a re-load clears the row containers' rows BEFORE filling them (the + * History pattern from task 01 — phase 97 moved the clear from the + * fetch top into the render, and from the one tbody to both), so a + * refresh from a populated level into a sparser one replaces the rows + * (no ghost rows). #sources-empty lives OUTSIDE the tbody (a + * .empty-state div, not a row), so the clear is a bare + * replaceChildren(). The listener is armed only in the ADMIN * branch, after the whoami gate passes: anonymous shows the gate and - * never fetches /api/docs (the phase-16 soft rule). + * never fetches the catalog (the phase-16 soft rule). + * + * Phase 97 (task 04) — the catalog becomes the DRILL-DOWN TREE the + * agent's `ls` sees (the phase-94 concept, one end to end). The load + * is now `loadTree()`: ONE fetch of GET /api/docs/tree returns the + * FULL recursive tree (sources → folders → files, each folder carrying + * its STORED description — the rows the agent reads), and the view + * drills CLIENT-SIDE from that single fetch — zero per-level fetches, + * no URL change (the navigation is a re-render of the already-fetched + * tree). + * + * • the drill state is module-scoped: `current` — + * { source: null, folder: null } = the top level (the rows ARE the + * sources themselves — the ls() equivalence: name, recursive count, + * the (source, "") description), `folder: ""` = the source root, + * `folder: "one/two"` = the nested folder. The last fetched tree + * lives in the module-scoped `kbTree`. + * • `renderLevel()` from `current` + `kbTree`: #kb-crumb (hidden at + * the top — one link per ancestor: the top level, the source, then + * the folder chain, the last segment an aria-current span), #kb-level + * (the current level's STORED description — title = the full + * source-relative path; HIDDEN when none is stored — the ls rule: + * count only, no placeholder), #folders-table = the level's direct + * subfolders (top level: the sources), #docs-table = the level's + * DIRECT files only — makeRow UNCHANGED (the path link still opens + * the same-page modal; the no-JS href escape hatch intact), hidden + * when the level has none (at the top level it is ALWAYS hidden — + * files are seen per source, as with ls(source)). + * • the stat cards walk the WHOLE tree (document count, chunks sum, + * max indexed_at) — the values identical to the former flat walk. + * • the empty state (#sources-empty) is now the "zero SOURCES" + * semantic (nothing registered, nothing indexed) — the deliberate + * phase-97 change: a registered 0-document source renders its row + * (`0` documents) instead (the ls invariant — the agent lists it + * too). A failed tree fetch renders that same no-data state (the + * former flat-load failure behavior, unchanged in kind). + * • Phase 77/79 carry over: the monotonic loadSeq race token (only + * the newest load may touch the DOM after its await) and the + * re-entrant render — renderLevel() clears BOTH row containers + * BEFORE filling them, so a refresh from a populated level into a + * sparser one leaves no ghost rows. NEVER-STALE (PLAN §7.4): after + * a re-fetch (re-show, sync success, upload success), if the current + * location no longer exists in the NEW tree (source unregistered/ + * pruned, folder vanished), `current` RESETS to the top level BEFORE + * rendering — no stale breadcrumb, no stale block. + * • the refresh wirings move with the rename: `loadTree()` at the + * boot load, the bor:view-refresh listener, applySyncSuccess, and + * the upload-success branch of startSyncPolling — the anonymous + * branch still NEVER fetches (no /api/docs/tree request at all). + * • the sync button / poll / label / banner / error-modal machinery + * and the document-modal wiring are UNTOUCHED (the sync section + * above is byte-identical save its two catalog-refresh call + * sites, which now call loadTree() — the phase-97 rename). + * + * Phase 97 (task 05) — the folder-description EDITOR (the phase-57 + * affordance, mirrored). The owner edits (or clears) any directory's + * stored description with the EXACT file-summary interaction: Edit → + * inline textarea (prefilled via .value) → Save / Cancel → a + * role=status live-region status. + * + * • ONE shared function, wireDescriptionEdit(), drives BOTH + * surfaces: the static #kb-level Edit button (the current level's + * description — the block ships the

+ a .kb-level-body holding + * the

+ the Edit button) and the ALWAYS-present Edit button in + * every source/folder row's Description cell (makeDescCell builds + * it inside makeSourceRow / makeFolderRow — a description can be + * CREATED where none is stored: a < 2-document folder, the + * generator's fail-soft miss — the editor opens prefilled with the + * empty string). NO whoami gate in the view: the RAG view is + * admin-only already (the phase-16 gate) and the endpoint's + * require_admin is the API-level gate. + * • Save → PATCH /api/folders/summary with { source, folder_path, + * summary } — folder_path "" for the source root, the + * source-relative folder path otherwise (both known from the + * target: the row's node, or `current` for the level). 200 → + * re-render the description text (textContent ONLY — the XSS + * contract) in the surface where the edit happened + status + * "Description updated."; an empty save (the server echoes + * summary null) → the text goes away (level block hidden / row + * cell emptied) + "Description cleared."; the in-memory kbTree + * node's summary is updated IN PLACE (no re-fetch — the tree state + * stays coherent; the re-fetch is the safety net). Failure + * (non-2xx / network) → neutral retry copy (the phase-55 + * convention), the editor stays open with the user's text, the + * stored text untouched. Cancel → restore the text node. + * • the level block is PERSISTENT (reused across levels as you + * drill), so wireDescriptionEdit takes a getTarget() getter (a + * row's is a constant) and returns a handle whose reset() tears + * down an open editor before every re-render (PLAN §7.4 — a + * navigate-away or refresh never leaves a stale open editor). + * • every editor part is static createElement; the description text + * is a text node (textContent / .value) — this module never builds + * HTML from document-derived data (the house rule, unchanged). */ import { fetchIsAdmin } from "./header.js"; @@ -61,6 +155,163 @@ export function documentUrl(source, path) { return "/document.html?source=" + encodeURIComponent(source) + "&path=" + encodeURIComponent(path); } +/* ---------- folder-description editing (phase 97, task 05) ---------- + * The phase-57 edit affordance, mirrored for the RAG view's folder + * descriptions (the owner edits/clears any directory's stored + * description exactly like a file summary): Edit → inline textarea + * (prefilled via .value — the XSS contract) → Save / Cancel → a + * role=status live-region status, wired to PATCH /api/folders/summary. + * Every part is static createElement; the description text is a text + * node (textContent / .value) — this module never builds HTML from + * document-derived data (the house rule). */ +function mkBtn(cls, label) { + const b = document.createElement("button"); + b.type = "button"; + b.className = cls; + b.textContent = label; + return b; +} + +/* The shared description editor. `container` holds ONLY the + * description UI (the row's , or the level block's .kb-level-body) + * — the swap is a bare replaceChildren on it, so any persistent + * sibling (the level's

) is untouched. `getTarget()` returns the + * current { node, source, folder } at open/save time: a row's is a + * constant (the row's node), the level's reads `current` + + * currentLevelNode() (the block is reused across levels). `node` is + * the in-memory kbTree node: its .summary is read on open (prefill) + * and updated IN PLACE on success (no re-fetch — the tree state stays + * coherent; the re-fetch is the safety net). `folder` is "" for the + * source root, the source-relative folder path otherwise. `onCleared` + * (optional) runs after a successful clear — the level block hides + * itself (the ls rule); a row cell just goes empty (the always-present + * button stays). Returns a handle whose reset() tears down an OPEN + * editor before a re-render (PLAN §7.4 — never a stale open editor). */ +function wireDescriptionEdit({ editBtn, textEl, container, getTarget, onCleared }) { + const actions = document.createElement("div"); + actions.className = "kb-summary-actions"; + const saveBtn = mkBtn("kb-summary-save", "Save"); + const cancelBtn = mkBtn("kb-summary-cancel", "Cancel"); + actions.append(saveBtn, cancelBtn); + const status = document.createElement("p"); + status.className = "kb-summary-status"; + status.setAttribute("role", "status"); + status.setAttribute("aria-live", "polite"); + let editor = null; + let isOpen = false; + + /* Back to the display state: the text element re-rendered from the + * node (the CURRENT stored description, textContent only), the Edit + * button available again. A cleared node empties the text and, for + * the level block, hides the whole block via onCleared. */ + function closeEditor(message) { + const target = getTarget(); + const node = target ? target.node : null; + const value = node && typeof node.summary === "string" ? node.summary : ""; + textEl.textContent = value; // text node — the CURRENT stored description + editBtn.hidden = false; + status.textContent = message; + container.replaceChildren(textEl, editBtn, status); + isOpen = false; + if (value.trim() === "" && onCleared) onCleared(); // level: hide the block + else editBtn.focus(); // return focus to the opener + } + + function openEditor() { + if (isOpen) return; + const target = getTarget(); + if (!target) return; + editor = document.createElement("textarea"); + editor.className = "kb-summary-editor"; + editor.value = typeof target.node.summary === "string" ? target.node.summary : ""; // .value, never innerHTML + status.textContent = ""; + editBtn.hidden = true; + container.replaceChildren(editor, actions, status); + editor.focus(); + isOpen = true; + } + + async function saveDescription() { + const target = getTarget(); + if (!target) return; + const { node, source, folder } = target; + const value = editor.value; + saveBtn.disabled = true; // one PATCH at a time (never stale) + status.textContent = ""; + try { + const res = await fetch("/api/folders/summary", { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ source, folder_path: folder, summary: value }), + }); + if (!res.ok) { + // Neutral retry copy (phase-55) — the editor stays OPEN with the + // user's text (no swap back, the stored text is untouched). + status.textContent = "Couldn't save the description — try again."; + return; + } + const data = await res.json(); + node.summary = data.summary; // in-place kbTree update (no re-fetch) + closeEditor(data.summary === null ? "Description cleared." : "Description updated."); + } catch { + // Network failure: the reachable? copy; the editor stays open. + status.textContent = "Couldn't save the description — is the app reachable?"; + } finally { + saveBtn.disabled = false; + } + } + + editBtn.addEventListener("click", openEditor); + saveBtn.addEventListener("click", () => { + void saveDescription(); + }); + cancelBtn.addEventListener("click", () => closeEditor("")); + + return { + /* Tear down an OPEN editor with NO message and WITHOUT re-rendering + * the text from the (possibly stale) node — the surface is about to + * re-render for a DIFFERENT level (navigation / refresh): a stale + * open editor must never survive a re-render (PLAN §7.4). A closed + * editor is a no-op. */ + reset() { + if (!isOpen) return; + editor = null; + saveBtn.disabled = false; + status.textContent = ""; + editBtn.hidden = false; + container.replaceChildren(textEl, editBtn); + isOpen = false; + }, + }; +} + +/* A source/folder row's Description cell (task 05): the stored + * description text (a text node, textContent only) + the ALWAYS-present + * Edit button (`.kb-summary-edit`) — a description can be CREATED where + * none is stored (a < 2-document folder, the generator's fail-soft + * miss), so the button is added unconditionally (the view is + * admin-only already — the endpoint's require_admin is the API gate). + * The shared editor is wired with a CONSTANT target (the row's node). + * `label` is the human name for the button's aria-label. */ +function makeDescCell(node, source, folder, label) { + const td = document.createElement("td"); + const text = document.createElement("span"); + text.textContent = (node && node.summary) || ""; // text node — never innerHTML + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "kb-summary-edit"; + btn.textContent = "Edit"; + btn.setAttribute("aria-label", `Edit description: ${label}`); + td.append(text, btn); + wireDescriptionEdit({ + editBtn: btn, + textEl: text, + container: td, + getTarget: () => ({ node, source, folder }), + }); + return td; +} + export async function mount(root) { /* ---------- Sync sources button (Sources page only) ---------- * @@ -87,7 +338,7 @@ export async function mount(root) { * 3. sync success → the phase-32 settle (counts + catalog refresh); * 4. sync failed → the phase-32 failure (banner + modal); * 5. upload success → settle "Sync sources" + catalog refresh - * (loadDocs — phase 90: an upload no longer + * (loadTree — phase 90: an upload no longer * changes the KB, the re-read is a no-op safety * net); the upload's result line lives on the * Sources page, never in #sync-result (A3); @@ -99,7 +350,7 @@ export async function mount(root) { * rides the button title (hover) and #sync-result (the aria-live * announcer — screen readers hear it). The load-time re-attach * (initSyncButton) re-enters a RUNNING upload the same way; a terminal - * upload is a no-op there (the boot-time loadDocs() already shows the + * upload is a no-op there (the boot-time loadTree() already shows the * current catalog). * * Elements: #sync-btn (the button), #sync-label (the text), @@ -265,7 +516,7 @@ export async function mount(root) { hideSyncError(); emitSyncStatus(status); // Refresh the catalog live — the KB just changed. - loadDocs(); + loadTree(); } function applySyncFailure(status) { @@ -356,7 +607,7 @@ export async function mount(root) { if (syncResult) syncResult.textContent = ""; hideSyncError(); emitSyncStatus({ state: "idle" }); - loadDocs(); + loadTree(); return; } // 6. upload failed: settle only — the failure is the Sources page's @@ -416,7 +667,7 @@ export async function mount(root) { * the sync IDLE, an in-flight background upload RUN (phase 90: unpack * + register — the bare "Importing…" label) adopts the button the * same way — the "user clicked upload, then opened sources" case; a - * terminal upload is a no-op (the boot-time loadDocs() already shows + * terminal upload is a no-op (the boot-time loadTree() already shows * the current catalog). */ async function initSyncButton() { if (!syncBtn) return; @@ -470,12 +721,28 @@ export async function mount(root) { const tbody = root.querySelector("#docs-tbody"); const emptyEl = root.querySelector("#sources-empty"); - const tableWrap = root.querySelector(".table-wrap"); + /* Phase 97 (task 04): the new #folders-wrap ALSO carries the shared + * .table-wrap card class — the FILE table's wrap is therefore looked + * up through its own table, not the class (a class lookup would hit + * #folders-wrap first in document order). */ + const tableWrap = root.querySelector("#docs-table").parentElement; const statCards = root.querySelector("#stat-cards"); const gateEl = root.querySelector("#sources-gate"); const statDocs = root.querySelector("#stat-docs"); const statChunks = root.querySelector("#stat-chunks"); const statLast = root.querySelector("#stat-last"); + const crumbEl = root.querySelector("#kb-crumb"); + const levelEl = root.querySelector("#kb-level"); + const levelTitleEl = root.querySelector("#kb-level-title"); + const levelSummaryEl = root.querySelector("#kb-level-summary"); + const foldersWrap = root.querySelector("#folders-wrap"); + const foldersTbody = root.querySelector("#folders-tbody"); + /* Phase 97 (task 05): the level block's static Edit button + the + * .kb-level-body that holds the description UI (the

+ the + * button) — the shared editor swaps INSIDE the body, leaving the + * block's

untouched. */ + const levelEditBtn = root.querySelector("#kb-level-edit"); + const levelBody = levelEl ? levelEl.querySelector(".kb-level-body") : null; /* Phase 16: whoami BEFORE the docs fetch. Anonymous visitors get the * sign-in gate (stat cards + table hidden) and NO /api/docs call — the @@ -493,52 +760,336 @@ export async function mount(root) { } - /* Phase 77 (task 02): re-entrant — the top clear (the History - pattern from task 01) drops the tbody's rows BEFORE the fetch, - so a re-show refresh from a populated list into an empty result - replaces the list instead of leaving ghost rows. #sources-empty - lives OUTSIDE the tbody (a .empty-state div, not a row), so the - clear is a bare replaceChildren(). Phase 79 (task 04): the clear - alone is NOT enough when two loads interleave — the boot re-attach - (applySyncSuccess → loadDocs) and the boot-time loadDocs both clear - first, then the SLOWER fetch appends after the newer load's clear, - duplicating every row (2×). The monotonic seq token invalidates an - in-flight load the moment a newer one starts: only the newest load - may touch the DOM after its await. */ + /* Phase 97 (task 04): the drill-down tree (the module docstring + carries the full contract). `kbTree` = the last fetched tree; + `current` = the drill state ({ null, null } = top, `folder: ""` + = the source root). Phase 79 carries over: the monotonic loadSeq + race token invalidates an in-flight load the moment a newer one + starts — only the newest load may touch the DOM after its await + (the boot re-attach (applySyncSuccess → loadTree) and the boot-time + loadTree interleave exactly as the flat load once did). Re-entrancy + moves from the fetch top into the render: renderLevel() clears + BOTH row containers BEFORE filling them (the phase-77 History + pattern, extended to the second container), so a refresh from a + populated level into a sparser one leaves no ghost rows. */ + let kbTree = { sources: [] }; + let current = { source: null, folder: null }; let loadSeq = 0; - async function loadDocs() { + let levelEditor = null; // phase 97 (task 05): the level editor handle + + async function loadTree() { const my = ++loadSeq; - tbody.replaceChildren(); let r; try { - r = await fetch("/api/docs"); + r = await fetch("/api/docs/tree"); } catch { - if (my === loadSeq) showEmpty(); + if (my === loadSeq) renderEmpty(); return; } if (!r.ok) { - if (my === loadSeq) showEmpty(); + if (my === loadSeq) renderEmpty(); return; } - const { documents } = await r.json(); - if (my !== loadSeq) return; // a newer load owns the tbody now - if (!documents.length) { - showEmpty(); + const tree = await r.json(); + if (my !== loadSeq) return; // a newer load owns the DOM now + kbTree = tree && Array.isArray(tree.sources) ? tree : { sources: [] }; + resetVanishedLocation(); + renderLevel(); + } + + /* Never-stale (PLAN §7.4, phase 97): after a re-fetch, the drilled + * location may no longer exist in the NEW tree (the source was + * unregistered/pruned, the folder vanished) — reset to the top level + * BEFORE rendering: no stale breadcrumb, no stale block. */ + function resetVanishedLocation() { + if (current.source === null) return; + const src = kbTree.sources.find((s) => s.name === current.source); + if (!src) { + current = { source: null, folder: null }; + return; + } + if (current.folder !== null && !folderExistsIn(src, current.folder)) { + current = { source: null, folder: null }; + } + } + + /* The phase-94 existence rule, mirrored client-side over the tree's + * file paths: a folder exists under the source iff some of the + * source's indexed paths starts with `folder + "/"` (the source root + * — `""` — always exists). */ + function folderExistsIn(src, folderPath) { + if (folderPath === "") return true; + const paths = []; + const collect = (node) => { + for (const child of node.children || []) { + if (child.kind === "file") paths.push(child.path); + else collect(child); + } + }; + collect(src); + return paths.some((p) => p.startsWith(folderPath + "/")); + } + + /* The node for `current`: the source node at the root, or the folder + * node found by walking the tree with the folder's cumulative + * source-relative path (the builder's folder paths are + * source-relative, so `one/two` resolves one level at a time). */ + function currentLevelNode() { + const src = kbTree.sources.find((s) => s.name === current.source); + if (!src) return null; + if (current.folder === null || current.folder === "") return src; + let node = src; + let acc = ""; + for (const part of current.folder.split("/")) { + acc = acc ? acc + "/" + part : part; + const next = (node.children || []).find( + (c) => c.kind === "folder" && c.path === acc + ); + if (!next) return null; + node = next; + } + return node; + } + + /* The KB-wide stat cards (phase 97): walk the WHOLE tree — document + * count, chunks sum, max indexed_at — the values identical to the + * former flat /api/docs walk (the same documents, one level deeper). + * fmtDate reuse for the max. */ + function treeStats() { + let docs = 0; + let totalChunks = 0; + let last = ""; + const walk = (node) => { + for (const child of node.children || []) { + if (child.kind === "file") { + docs += 1; + totalChunks += child.chunks; + if (child.indexed_at > last) last = child.indexed_at; + } else { + walk(child); + } + } + }; + for (const s of kbTree.sources) walk(s); + return { docs, totalChunks, last }; + } + + /* The breadcrumb (phase 97): hidden at the top level; when drilled + * in, one link per ancestor — the top level (back to the sources + * list), the source, then the folder chain — the LAST segment a + * span with aria-current="page". Client-side only: no fetch, no URL + * change (the navigation is a re-render of the fetched tree). */ + function renderCrumb() { + if (!crumbEl) return; + crumbEl.replaceChildren(); + if (current.source === null) { + crumbEl.hidden = true; + return; + } + crumbEl.hidden = false; + let appended = false; + const append = (el) => { + if (appended) { + const sep = document.createElement("span"); + sep.className = "kb-crumb-sep"; + sep.setAttribute("aria-hidden", "true"); + sep.textContent = "/"; + crumbEl.appendChild(sep); + } + crumbEl.appendChild(el); + appended = true; + }; + append(crumbSegment("Knowledge base", { source: null, folder: null })); + if (current.folder === null || current.folder === "") { + append(crumbCurrent(current.source)); + } else { + append(crumbSegment(current.source, { source: current.source, folder: "" })); + let acc = ""; + const parts = current.folder.split("/"); + parts.forEach((part, i) => { + acc = acc ? acc + "/" + part : part; + if (i === parts.length - 1) append(crumbCurrent(part)); + else append(crumbSegment(part, { source: current.source, folder: acc })); + }); + } + } + + function crumbSegment(label, target) { + const a = document.createElement("a"); + a.className = "kb-crumb-link"; + a.href = "#"; // client-side navigation only — no URL change + a.textContent = label; // document-derived text — never innerHTML + a.addEventListener("click", (e) => { + e.preventDefault(); + goTo(target); + }); + return a; + } + + function crumbCurrent(label) { + const span = document.createElement("span"); + span.className = "kb-crumb-current"; + span.setAttribute("aria-current", "page"); + span.textContent = label; // document-derived text — never innerHTML + return span; + } + + /* The drill navigation (client-side, no fetch, no URL change). */ + function goTo(target) { + current = { source: target.source, folder: target.folder }; + renderLevel(); + } + + /* ONE table for every level (phase 97): at the TOP level the rows + * are the SOURCES themselves (the ls() equivalence — name, recursive + * count, the stored (source, "") description). textContent only. */ + function makeSourceRow(s) { + const tr = document.createElement("tr"); + const nameTd = document.createElement("td"); + const link = document.createElement("a"); + link.className = "folder-link"; + link.href = "#"; // client-side drill — no URL change + link.title = s.name; // hover name (the cell may ellipsize) + link.textContent = s.name; // document-derived text — never innerHTML + link.addEventListener("click", (e) => { + e.preventDefault(); + goTo({ source: s.name, folder: "" }); + }); + nameTd.appendChild(link); + tr.appendChild(nameTd); + const countTd = document.createElement("td"); + countTd.textContent = String(s.documents); + tr.appendChild(countTd); + tr.appendChild(makeDescCell(s, s.name, "", s.name)); // Description + ALWAYS-present Edit (task 05) + return tr; + } + + /* A level's subfolder row: the folder's LAST path segment as the + * label (the full source-relative path rides the title — the cell + * ellipsizes), the recursive count, the stored description (AI or + * manual — any row) or an empty cell. textContent only. */ + function makeFolderRow(f) { + const tr = document.createElement("tr"); + const nameTd = document.createElement("td"); + const link = document.createElement("a"); + link.className = "folder-link"; + link.href = "#"; // client-side drill — no URL change + link.title = current.source + "/" + f.path; // full path on hover + link.textContent = f.path.split("/").pop(); // never innerHTML + link.addEventListener("click", (e) => { + e.preventDefault(); + goTo({ source: current.source, folder: f.path }); + }); + nameTd.appendChild(link); + tr.appendChild(nameTd); + const countTd = document.createElement("td"); + countTd.textContent = String(f.documents); + tr.appendChild(countTd); + tr.appendChild( + makeDescCell(f, current.source, f.path, current.source + "/" + f.path) + ); // Description + ALWAYS-present Edit (task 05) + return tr; + } + + /* Render the CURRENT level from `current` + `kbTree` (phase 97). + * Clears BOTH row containers FIRST (re-entrancy — no ghost rows), + * then: zero sources → the no-data state; top level → the source + * rows (level block + file table hidden); inside a source/folder → + * the breadcrumb, the level block (the level's stored description — + * hidden when none is stored, the ls rule), the direct subfolders, + * and the direct files (makeRow, unchanged — the file node carries + * no source, the row object restores the flat shape makeRow reads). */ + function renderLevel() { + if (foldersTbody) foldersTbody.replaceChildren(); + if (tbody) tbody.replaceChildren(); + if (levelEditor) levelEditor.reset(); // a re-render never keeps a stale open editor (§7.4) + + if (!kbTree.sources.length) { + renderEmpty(); + return; + } + emptyEl.hidden = true; + + const st = treeStats(); + statDocs.textContent = String(st.docs); + statChunks.textContent = String(st.totalChunks); + statLast.textContent = st.last ? fmtDate(st.last) : "–"; + + renderCrumb(); + + if (current.source === null) { + // Top level: the level block is hidden (nothing above the + // sources); the folders table lists the SOURCES themselves and + // the file table is ALWAYS hidden (files are seen per source, + // as with ls(source)). + levelEl.hidden = true; + for (const s of kbTree.sources) foldersTbody.appendChild(makeSourceRow(s)); + foldersWrap.hidden = false; + if (tableWrap) tableWrap.hidden = true; return; } - let totalChunks = 0; - let last = ""; - for (const d of documents) { - totalChunks += d.chunks; - if (d.indexed_at > last) last = d.indexed_at; - tbody.appendChild(makeRow(d)); + const node = currentLevelNode(); + if (node === null) { + // Defensive: the reset above guarantees the level exists in the + // tree this render reads — a null means the tree changed under + // us, in which case the top level is the honest view. + current = { source: null, folder: null }; + renderLevel(); + return; } - statDocs.textContent = String(documents.length); - statChunks.textContent = String(totalChunks); - statLast.textContent = last ? fmtDate(last) : "–"; - emptyEl.hidden = true; - tableWrap.hidden = false; + + // The level block: the current level's STORED description (source: + // the (source, "") row; folder: its row) — hidden when none is + // stored (the ls rule: count only, no placeholder). Title = the + // full source-relative path (e.g. `alpha/two`). + if (node.summary) { + levelTitleEl.textContent = current.folder + ? current.source + "/" + current.folder + : current.source; + levelSummaryEl.textContent = node.summary; + levelEl.hidden = false; + } else { + levelEl.hidden = true; + } + + const children = node.children || []; + const subfolders = children.filter((c) => c.kind === "folder"); + const files = children.filter((c) => c.kind === "file"); + for (const f of subfolders) foldersTbody.appendChild(makeFolderRow(f)); + foldersWrap.hidden = subfolders.length === 0; + for (const f of files) { + tbody.appendChild( + makeRow({ + source: current.source, + path: f.path, + title: f.title, + chunks: f.chunks, + indexed_at: f.indexed_at, + }) + ); + } + if (tableWrap) tableWrap.hidden = files.length === 0; + } + + /* The no-data state (phase 97): zero sources (nothing registered, + * nothing indexed) OR a failed tree fetch (the former showEmpty + * failure behavior, unchanged in kind) — every catalog surface + * hidden, the stat cards read zero. A registered 0-document source + * does NOT land here: it renders its `0 documents` row (the ls + * invariant — the deliberate semantic change, module docstring). */ + function renderEmpty() { + if (levelEditor) levelEditor.reset(); // a re-render never keeps a stale open editor (§7.4) + statDocs.textContent = "0"; + statChunks.textContent = "0"; + statLast.textContent = "–"; + if (crumbEl) crumbEl.hidden = true; + levelEl.hidden = true; + if (foldersTbody) foldersTbody.replaceChildren(); + if (tbody) tbody.replaceChildren(); + if (foldersWrap) foldersWrap.hidden = true; + if (tableWrap) tableWrap.hidden = true; + emptyEl.hidden = false; } function makeRow(d) { @@ -575,40 +1126,68 @@ export async function mount(root) { return tr; } - function showEmpty() { - statDocs.textContent = "0"; - statChunks.textContent = "0"; - statLast.textContent = "–"; - emptyEl.hidden = false; - if (tableWrap) tableWrap.hidden = true; - } + /* Phase 97 (task 05): wire the LEVEL block's editor (the static + * #kb-level-edit button). The level block is PERSISTENT — reused for + * whichever level is current — so the target is a GETTER (a row's is + * a constant): it reads `current` + currentLevelNode() at open/save + * time. onCleared hides the whole block (the ls rule: no description + * → no block). The handle's reset() runs on every re-render so a + * navigate-away / refresh never leaves a stale open editor (§7.4). */ + if (levelEditBtn && levelBody) { + levelEditor = wireDescriptionEdit({ + editBtn: levelEditBtn, + textEl: levelSummaryEl, + container: levelBody, + getTarget: () => { + const node = currentLevelNode(); + return node ? { node, source: current.source, folder: current.folder } : null; + }, + onCleared: () => { + // The phase-57 announcement beat: keep the block visible a short + // beat so the role=status "Description cleared." is still + // readable, THEN hide it (the ls rule: no description → no + // block) — GUARDED: if a newer level is current by then (a + // navigate-away first) or its node has a description again (a + // re-create within the beat), the hide is a no-op. + setTimeout(() => { + const n = currentLevelNode(); + if (n && !n.summary) levelEl.hidden = true; + }, 2000); + }, + }); + } + /* ---------- view boot (phase 76 task 02) ---------- * The shared header is NOT booted here — in the shell it runs * exactly once, via the chat module (app.js) at shell boot. The * admin gate reads fetchIsAdmin() — the SAME cached whoami promise * header.js exports (zero extra requests): the sync button joins * the admin reveal on that one whoami (no extra fetch), and the - * anonymous branch gates the catalog in / out with NO /api/docs - * request at all (the Sources-page soft rule, unchanged). */ + * anonymous branch gates the catalog in / out with NO + * /api/docs/tree request at all (the phase-16 soft rule — the + * tree IS the catalog; phase 97). */ const admin = await fetchIsAdmin(); if (syncBtn) syncBtn.hidden = !admin; // admin-only: ship-hidden, revealed on the same whoami if (!admin) { - // Anonymous: gate in, catalog out, and no /api/docs request at all. + // Anonymous: gate in, catalog out, and no /api/docs/tree request + // at all — the tree surfaces ship hidden and never fill. if (statCards) statCards.hidden = true; + if (foldersWrap) foldersWrap.hidden = true; if (tableWrap) tableWrap.hidden = true; if (emptyEl) emptyEl.hidden = true; if (gateEl) gateEl.hidden = false; return; } if (gateEl) gateEl.hidden = true; - /* Phase 77 (task 02): a user-initiated re-show of this already- - mounted view makes the router dispatch bor:view-refresh on the - section — re-load the catalog then (loadDocs is re-entrant). - Armed ONLY here, after the whoami gate passed: anonymous shows - the gate and must never fetch /api/docs (the phase-16 soft - rule the story E2E pins). */ - root.addEventListener("bor:view-refresh", () => loadDocs()); - loadDocs(); + /* Phase 77 (task 02) + phase 97 (task 04): a user-initiated + re-show of this already-mounted view makes the router dispatch + bor:view-refresh on the section — re-load the catalog tree then + (loadTree is race-tokened: only the newest load touches the + DOM). Armed ONLY here, after the whoami gate passed: anonymous + shows the gate and must never fetch /api/docs/tree (the + phase-16 soft rule the story E2E pins). */ + root.addEventListener("bor:view-refresh", () => loadTree()); + loadTree(); } diff --git a/frontend/assets/styles.css b/frontend/assets/styles.css index 47993df..fee6359 100644 --- a/frontend/assets/styles.css +++ b/frontend/assets/styles.css @@ -1899,6 +1899,161 @@ details.thinking .thinking-text ul { margin: 0 0 0.5rem; } .docs-table tbody tr:hover { background: var(--bg); } .docs-table tbody tr:last-child td { border-bottom: 0; } +/* ---------- KB drill-down tree (phase 97, task 04) ---------- + The RAG view lists the catalog the way the agent's `ls` sees it + (the phase-94 concept, one end to end): #kb-crumb (the location + breadcrumb — hidden at the top level), #kb-level (the current + directory's STORED description — the surface-panel language of the + phase-93 page heads), and #folders-table (the ONE folders/sources + table — the .docs-table language; .kb-folders-table only re-styles + the 2nd/3rd column cells). Phase-08 tokens only — NO new hue + (the phase-92 monochrome invariant): brand-ink on surface 9.0:1, + ink 13.8:1, ink-soft 5.1:1 (every pair AA). :focus-visible via the + global 3px outline rule; row targets keep the .docs-table cell + padding (>= 44px effective — the phase-08 touch floor). No CDN, + system fonts. */ +.kb-crumb { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.1rem 0.3rem; + margin: 0.9rem 0 0; + font-size: 0.92rem; +} +.kb-crumb-link { + color: var(--brand-ink); /* 10.4:1 on --bg, 9.5:1 on the --brand-soft hover */ + text-decoration: none; + padding: 0.25rem 0.35rem; + border-radius: var(--radius-sm); +} +.kb-crumb-link:hover { background: var(--brand-soft); text-decoration: underline; } +.kb-crumb-current { + color: var(--ink); /* 16.7:1 on --bg */ + font-weight: 600; + padding: 0.25rem 0.35rem; +} +.kb-crumb-sep { color: var(--ink-soft); } +.kb-level { + background: var(--surface); + border: 1px solid var(--line); + border-radius: var(--radius); + padding: 1rem 1.25rem; + margin: 0.9rem 0 0.75rem; +} +.kb-level h2 { + margin: 0 0 0.35rem; + font-family: var(--mono); + font-size: 1rem; + color: var(--brand-ink); /* 9.0:1 on --surface — the path, in the path voice */ + word-break: break-all; +} +.kb-level p { margin: 0; color: var(--ink-soft); } /* 5.1:1 on --surface */ +.kb-folders-table { min-width: 480px; } +.kb-folders-table td:nth-child(2) { font-family: var(--font); font-size: inherit; max-width: none; } +.kb-folders-table td:nth-child(3) { + white-space: normal; + min-width: 18rem; + max-width: 44rem; + font-size: 0.88rem; +} +.folder-link { + color: var(--brand-ink); /* 9.0:1 on --surface (the .doc-link pair, row variant) */ + text-decoration: none; + border-radius: var(--radius-sm); +} +.folder-link:hover, .folder-link:focus-visible { + background: var(--brand-soft); + text-decoration: underline; +} + +/* ---------- KB folder-description editor (phase 97, task 05) ---------- + The phase-57 edit affordance on the RAG view's folder descriptions + (the .kb-summary-* family, mirroring the viewer's .doc-summary-*): + the Edit button (a 24px+ target; the [hidden] override beats its + display rule while the editor is open), the inline editor (the + 8rem-min textarea + Save/Cancel + the role=status live region). + House dark-tech palette (phase-08 tokens), system fonts, no CDN; + :focus-visible via the global 3px outline rule. No new hue (the + phase-92 monochrome invariant) — every color is a var(). */ +.kb-summary-edit { + flex: 0 0 auto; + display: inline-flex; + align-items: center; + min-height: 24px; + padding: 0.15rem 0.7rem; + border: 1px solid var(--line); + border-radius: 999px; + background: transparent; + color: var(--ink-soft); /* 5.1:1 on --surface (AA) */ + font: inherit; + font-weight: 600; + font-size: 0.78rem; + letter-spacing: 0.02em; + cursor: pointer; +} +.kb-summary-edit:hover { background: var(--brand-soft); color: var(--brand-ink); border-color: var(--brand); } +.kb-summary-edit[hidden] { display: none; } /* the hidden attr must beat the display above */ +/* Context spacing: the row cell's button sits after the description + text; the level block's sits below the

. */ +.kb-folders-table .kb-summary-edit { margin-left: 0.4rem; } +.kb-level-body .kb-summary-edit { margin-top: 0.5rem; } +.kb-summary-editor { + display: block; + width: 100%; + min-height: 8rem; /* the phase-57 spec */ + padding: 0.6rem 0.8rem; + border: 1px solid var(--line); + border-radius: var(--radius-sm); + background: var(--bg); /* inset against the --surface panel / row */ + color: var(--ink); /* 16.7:1 on --bg (AA) */ + font: inherit; + line-height: 1.5; + resize: vertical; +} +.kb-summary-actions { + display: flex; + align-items: center; + gap: 0.5rem; + margin-top: 0.75rem; +} +.kb-summary-save { + display: inline-flex; + align-items: center; + min-height: 32px; + padding: 0.35rem 0.95rem; + border: 0; + border-radius: 999px; + background: var(--brand); + color: var(--bg); /* --bg on --brand = 5.2:1 (AA) */ + font: inherit; + font-weight: 600; + font-size: 0.85rem; + cursor: pointer; +} +.kb-summary-save:hover { background: var(--brand-hover); } /* the house hover lightening */ +.kb-summary-save:disabled { opacity: 0.6; cursor: default; } /* one PATCH at a time */ +.kb-summary-cancel { + display: inline-flex; + align-items: center; + min-height: 32px; + padding: 0.35rem 0.95rem; + border: 1px solid var(--line); + border-radius: 999px; + background: transparent; + color: var(--ink-soft); /* 5.1:1 on --surface (AA) */ + font: inherit; + font-weight: 600; + font-size: 0.85rem; + cursor: pointer; +} +.kb-summary-cancel:hover { background: var(--err-bg); color: var(--err-ink); border-color: var(--err-line); } +.kb-summary-status { + margin: 0.6rem 0 0; + font-size: 0.85rem; + color: var(--ink-soft); /* 5.1:1 on --surface (AA) */ +} +.kb-summary-status:empty { margin-top: 0; } + /* ---------- Git sources page (phase 35) ---------- /git-sources.html: the admin-only manager for the stored git source list (add / remove, git-sources table). Same full-width table diff --git a/frontend/index.html b/frontend/index.html index 74e9ffd..28ae6ec 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -424,6 +424,48 @@ + +

+ + +
diff --git a/tests/e2e/test_admin_auth.py b/tests/e2e/test_admin_auth.py index 4489d50..ffdde17 100644 --- a/tests/e2e/test_admin_auth.py +++ b/tests/e2e/test_admin_auth.py @@ -23,6 +23,13 @@ Phase 79 (API tokens): the anonymous pins moved to the gated contract — phase-16 "the viewer stays open" soft rule is SUPERSEDED, shared chats are the only open surface). The password sign-in / sign-out / wrong-password assertions are UNCHANGED. + +Phase 97 adaptation: the admin catalog pin (test 4) is re-pointed at +the DRILL-DOWN TREE — the top level lists the fixture's single +indexed-only source (``docs``), the flat 13-row tbody no longer +renders, so the total is re-asserted PER LEVEL (drill → count; the +stat cards carry the KB total). The asserted document behavior is +unchanged; navigation only. """ from __future__ import annotations @@ -81,6 +88,28 @@ def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None: return _run_in_thread(_import_fixtures(mock_port)) +# --------------------------------------------------------------------------- +# Phase 97: the catalog is the drill-down tree the agent's `ls` walks — +# top level = the sources (the fixture's single indexed-only source, +# `docs`), then one folder link per path segment. Small per-suite +# drill helpers (the test_kb_tree house pattern). +# --------------------------------------------------------------------------- + + +def _drill(page: Page, *names: str) -> None: + """Drill one level at a time: each name is the EXACT text of the + source/folder link at the current level (client-side — no fetch, + no URL change).""" + for name in names: + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + + +def _go_top(page: Page) -> None: + """Back to the top level: the breadcrumb's top-level link (call + between drills only — the breadcrumb is hidden at the top).""" + page.locator("#kb-crumb a.kb-crumb-link").first.click() + + def _ask(page: Page, question: str) -> None: """Send one turn and wait until the grounded answer has fully landed.""" page.fill("#message-input", question) @@ -247,8 +276,32 @@ def test_admin_login_unlocks_sources_and_tuning( expect(page.locator("#sources-gate")).to_be_hidden() expect(page.locator("#stat-docs")).to_have_text("13") # phase 47: +quadlet/j2 expect(page.locator("#stat-chunks")).not_to_have_text("–") + # Phase 97: the catalog is the drill-down tree — the top level + # lists the source (the catalog-rendered signal), the file table + # is hidden there; the flat 13-row total is re-asserted PER LEVEL + # (drill → count; the sum is 3 + 1 + 2 + 1 + 3 + 1 + 1 + 1 = 13). + page.locator("#folders-tbody .folder-link").first.wait_for(state="visible") + expect(page.locator("#docs-table")).to_be_hidden() + _drill(page, "docs") + expect(page.locator("#folders-tbody tr")).to_have_count(2) + expect(page.locator("#docs-tbody tr")).to_have_count(0) # no root-level files + _drill(page, "homelab") expect(page.locator("#docs-table")).to_be_visible() - expect(page.locator("#docs-tbody tr")).to_have_count(13) + expect(page.locator("#docs-tbody tr")).to_have_count(3) + _go_top(page) + _drill(page, "docs", "deployments") + expect(page.locator("#docs-tbody tr")).to_have_count(1) + for folder, count in ( + ("container_gitlab", 2), + ("networking", 1), + ("quadlet", 3), + ("scripts", 1), + ("ssh", 1), + ("templates", 1), + ): + _go_top(page) + _drill(page, "docs", "homelab", folder) + expect(page.locator("#docs-tbody tr")).to_have_count(count) # Chat: the tuning UI is back — Sign out instead of Sign in, Tune # under the answer. The header toggle is NOT back: removed from the diff --git a/tests/e2e/test_archive_upload_sources.py b/tests/e2e/test_archive_upload_sources.py index 74704b5..b5285ee 100644 --- a/tests/e2e/test_archive_upload_sources.py +++ b/tests/e2e/test_archive_upload_sources.py @@ -524,7 +524,13 @@ def test_upload_registers_without_indexing( # Phase 90 A1: the upload indexes NOTHING — the KB is empty… assert _docs(page, app_url) == [] # …and so is the RAG catalog (the scan is the Sync button's job). + # Phase 97: the registered source renders its 0-document row + # (the load-settled signal; #sources-empty is the zero-SOURCES + # state only) and the file table stays empty. page.goto(app_url + SOURCES_URL) + row = page.locator("#folders-tbody tr", has_text=SOURCE_NAME) + expect(row).to_have_count(1, timeout=30_000) + expect(row.locator("td:nth-child(2)")).to_have_text("0") expect(page.locator("#docs-tbody tr")).to_have_count(0) diff --git a/tests/e2e/test_chat_persistence.py b/tests/e2e/test_chat_persistence.py index ec8f58b..2290aab 100644 --- a/tests/e2e/test_chat_persistence.py +++ b/tests/e2e/test_chat_persistence.py @@ -280,7 +280,10 @@ def test_persists_across_page_navigation( # bar at owner request, 2026-08-28 — pinned in # tests/e2e/test_shared_header.py). page.goto(f"{app_url}/sources.html") - expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=15_000) + # Phase 97: the top level lists the sources (the file table is + # per-level, hidden at the top) — the source row is the + # catalog-rendered signal. + expect(page.locator("#folders-tbody tr").first).to_be_visible(timeout=15_000) expect(page.locator("#new-chat-btn")).to_be_hidden() # Back to the chat: the conversation is exactly as left — both turns, diff --git a/tests/e2e/test_document_back_navigation.py b/tests/e2e/test_document_back_navigation.py index a5a33dd..77d558b 100644 --- a/tests/e2e/test_document_back_navigation.py +++ b/tests/e2e/test_document_back_navigation.py @@ -150,7 +150,10 @@ def test_back_from_sources_returns_to_sources( ) -> None: _reset_db(mock_llm, seed=True) login(page, app_url) # phase 16: the Sources table is admin-only - + # Phase 97: the catalog is the drill-down tree — the kubernetes.md + # row lives at the homelab level (the drill is the only change). + for name in ("docs", "homelab"): + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') row = page.locator("#docs-tbody tr", has_text="kubernetes.md") expect(row).to_have_count(1) link = row.locator("td:nth-child(2) a.doc-link") @@ -174,7 +177,10 @@ def test_back_from_sources_returns_to_sources( back.click() expect(page).to_have_url(f"{app_url}/sources.html") - expect(page.locator("#docs-table")).to_be_visible() + # Phase 97: the re-mount lands on the tree's top level (the sources + # list — the file table is per-level, hidden at the top); the + # source row is the catalog-rendered signal. + expect(page.locator("#folders-tbody tr").first).to_be_visible(timeout=15_000) # --------------------------------------------------------------------------- diff --git a/tests/e2e/test_document_viewer.py b/tests/e2e/test_document_viewer.py index a8bc453..0979cbe 100644 --- a/tests/e2e/test_document_viewer.py +++ b/tests/e2e/test_document_viewer.py @@ -107,6 +107,16 @@ def _assert_closed(page: Page) -> None: expect(page.locator(".doc-modal")).not_to_be_visible() +def _drill(page: Page, *names: str) -> None: + """Phase 97: the catalog is the drill-down tree the agent's `ls` + sees — click through the source/folder rows (exact name match, + one per name) to the level that holds the asserted file. The drill + is the only change from the flat-table era; the row itself is + unchanged.""" + for name in names: + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + + # --------------------------------------------------------------------------- # 1. Chat source chip → SAME-PAGE modal (no new tab) # --------------------------------------------------------------------------- @@ -172,7 +182,9 @@ def test_sources_row_opens_modal( ) -> None: _reset_db(mock_llm, seed=True) login(page, app_url) # phase 16: the Sources catalog is admin-only - + # Phase 97: the row lives at its folder level (docs → homelab → + # container_gitlab) — the drill is the only change. + _drill(page, "docs", "homelab", "container_gitlab") row = page.locator("#docs-tbody tr", has_text="gitlab-compose.yaml") expect(row).to_have_count(1) link = row.locator("td:nth-child(2) a.doc-link") @@ -320,6 +332,9 @@ def test_modal_xss_safe( # The Sources table lists every indexed document — the admin entry # point into the modal for a doc the chat never cited. login(page, app_url) + # Phase 97: the seeded doc's row lives at the notes/ level — the + # drill is the only change. + _drill(page, "docs", "notes") row = page.locator("#docs-tbody tr", has_text="xss-fixture.md") expect(row).to_have_count(1) row.locator("td:nth-child(2) a.doc-link").click() diff --git a/tests/e2e/test_edit_summaries.py b/tests/e2e/test_edit_summaries.py index 6c7c48d..4a85ee2 100644 --- a/tests/e2e/test_edit_summaries.py +++ b/tests/e2e/test_edit_summaries.py @@ -387,6 +387,11 @@ def test_admin_modal_surface_edit(page: Page, app_url: str) -> None: assert before["summary"] == expected login(page, app_url) # lands on /sources.html (the catalog is admin-only) + # Phase 97: the catalog is the drill-down tree — the row lives at + # the quadlet level (the drill is the only change; the phase-57 + # summary-edit flow on the viewer itself is untouched). + for name in (SOURCE, "quadlet"): + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') row = page.locator("#docs-tbody tr", has_text=DOC_PATH) expect(row).to_have_count(1) before_tabs = len(page.context.pages) diff --git a/tests/e2e/test_global_tuning.py b/tests/e2e/test_global_tuning.py index 66a996e..2928042 100644 --- a/tests/e2e/test_global_tuning.py +++ b/tests/e2e/test_global_tuning.py @@ -379,7 +379,13 @@ def test_tuning_page_a11y_and_no_cdn( # Landmarks (PLAN §7.2). assert page.locator("header.app-header").count() == 1, "header missing" - assert page.locator("nav[aria-label]").count() == 1, "labeled nav missing" + # Phase 97: the shell carries a SECOND nav in the RAG view + # (#kb-crumb, the catalog breadcrumb — hidden at the top level, + # pinned in test_kb_tree), so the primary-nav landmark is pinned by + # its label. + assert page.locator("nav[aria-label='Primary']").count() == 1, ( + "labeled primary nav missing" + ) assert page.locator("main#main").count() == 1, "main#main missing" assert page.locator("footer.app-footer").count() == 1, "footer missing" diff --git a/tests/e2e/test_history_page_width.py b/tests/e2e/test_history_page_width.py index e67b658..2eb821d 100644 --- a/tests/e2e/test_history_page_width.py +++ b/tests/e2e/test_history_page_width.py @@ -177,13 +177,25 @@ def _open_menu(page: Page) -> None: expect(page.locator("#nav-toggle")).to_have_attribute("aria-expanded", "true") +def _drill_to_files(page: Page) -> None: + """Phase 97: the catalog is the drill-down tree — the top level + lists the sources (the file table is per-level, hidden at the + top). The 640px ``min-width`` pin targets the FILE table, so both + RAG-view tests drill to a level that has files (the fixture + source + its ``homelab`` folder) before pinning.""" + for name in ("docs", "homelab"): + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + + def _wrap_handle(page: Page, view: str) -> JSHandle: """The table card (``.table-wrap``) of ``view``: the id'd cards for - history/tokens, the RAG card found through its ``.docs-table`` - (that card is the only one without an id).""" + history/tokens, the RAG card found through its FILE table (phase 97: + the RAG view carries TWO cards — ``#folders-wrap`` at the top level + and the per-level file table's card; the 640px pin is the file + table's, and the tests drill there first).""" if view == "sources": return page.evaluate_handle( - "() => document.querySelector('.docs-table').closest('.table-wrap')" + "() => document.querySelector('#docs-table').closest('.table-wrap')" ) return page.evaluate_handle(f"() => document.querySelector('#{view}-table-wrap')") @@ -381,14 +393,25 @@ def test_rag_view_regression( regress it — AND its table is still full-width inside the card: the 640px ``min-width`` still engages, so the card keeps its in-card scroll (the shared rule added a containing block, not a - width).""" + width). + + Phase 97: the catalog is the drill-down tree — the top level lists + the sources, so the FILE table (the 640px pin's subject) is reached + by drilling ``docs`` → ``homelab`` first; the drill is the only + change to this pin.""" summary = _seed_kb(mock_llm) assert summary is not None and summary.added == 13 # A9 formats (phase 47 added quadlet+j2) page = _mobile_page(browser) try: login(page, app_url, next="/sources.html") _wait_settled_admin(page) - expect(page.locator(".docs-table")).to_be_visible(timeout=15_000) + # Phase 97: the top level lists the sources (the source row is + # the catalog-rendered signal); the 640px in-card-scroll pin + # targets the FILE table, so drill to a level that has files + # first (the drill is the only change). + page.locator("#folders-tbody tr").first.wait_for(state="visible", timeout=15_000) + _drill_to_files(page) + expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=15_000) _assert_viewport_width(page, "/sources.html (direct)") _assert_in_card_scroll(page, _wrap_handle(page, "sources"), "/sources.html") @@ -411,7 +434,10 @@ def test_desktop_unchanged( than the mobile 346px card. The zero-offset ``position: relative`` changed no layout, so desktop is byte-identical in behavior. The KB is seeded the house way for - the RAG view's table (it hides itself on an empty KB). + the RAG view's table (it hides itself on an empty KB). Phase 97: + the RAG view's table is the FILE table — the top level lists the + sources, so the sources pass drills ``docs`` → ``homelab`` first + (the drill is the only change). Data state: the desktop pin is enforced on the EMPTY-table state the module docstring declares ("the History/Tokens contract holds @@ -443,8 +469,15 @@ def test_desktop_unchanged( if i: # the first view is the login landing page.goto(app_url + path) _wait_settled_admin(page) - marker = ".docs-table" if view == "sources" else f"#{view}-table-wrap" - expect(page.locator(marker)).to_be_visible(timeout=15_000) + if view == "sources": + # Phase 97: the top level lists the sources — the + # container-width pin is the FILE table's (the 640px + # min-width's desktop counterpart), so drill to a level + # that has files first. + page.locator("#folders-tbody tr").first.wait_for(state="visible", timeout=15_000) + _drill_to_files(page) + marker = ("#docs-tbody tr" if view == "sources" else f"#{view}-table-wrap") + expect(page.locator(marker).first).to_be_visible(timeout=15_000) _assert_viewport_width(page, f"{path} (desktop)") report = _wrap_handle(page, view).evaluate( diff --git a/tests/e2e/test_import_documents.py b/tests/e2e/test_import_documents.py index 9bd9cd8..68315de 100644 --- a/tests/e2e/test_import_documents.py +++ b/tests/e2e/test_import_documents.py @@ -11,14 +11,33 @@ fixture, not the subject of the tests. Phase 16 adaptation: the Sources catalog is admin-only — every test performs the real form login (``e2e.auth_helpers.login``) first. + +Phase 97 adaptation: the catalog is the DRILL-DOWN TREE the agent's +``ls`` walks — the top level lists the sources (this fixture's single +indexed-only source, ``docs``), then one folder link per path segment; +the flat all-documents table no longer renders. The asserted rows, +links, and stat cards are UNCHANGED in intent — the drill is the only +change (every row here is nested under a folder). The empty-state test +now pins the zero-SOURCES state (a registered 0-document source renders +its row instead) — for that state to be reachable at all, this suite +runs its OWN module app (the conftest leak-guard pattern) with +``BOR_GIT_SOURCES`` forced empty: the session app would inherit an +operator's local ``.env`` fallback source, which would leak a +0-document top-level row into the tree. """ from __future__ import annotations import asyncio +import json +import os +import subprocess +import sys +from collections.abc import Iterator from pathlib import Path from threading import Thread from typing import Any +import pytest from playwright.sync_api import Browser, Page, expect from sqlalchemy import text @@ -27,10 +46,86 @@ from app.db import SessionLocal from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from e2e.auth_helpers import login +from e2e.conftest import ( + ADMIN_PASSWORD, + SESSION_SECRET, + USE_REAL_LLM, + _wait_http, +) REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" +# Phase 79 (task 04, full inventory): the conftest session app owns its +# port in a combined run — this module app binds its own port instead +# (a same-port second uvicorn dies on bind and would drive the wrong +# server). Env-overridable. +APP_PORT = int(os.environ.get("E2E_APP_PORT_IMPORTDOCS", "8140")) +APP_URL = f"http://127.0.0.1:{APP_PORT}" + + +@pytest.fixture(scope="module") +def app_server(mock_llm: int) -> Iterator[str]: + """The real app under test — per-module env (the conftest pattern): + the leak guards force the code defaults, and ``BOR_GIT_SOURCES`` is + forced EMPTY (phase 97 — the registered sources now render as + top-level rows: the operator's ``.env`` fallback source would leak + a 0-document source into the tree and break the empty-state test's + zero-SOURCES state). The session app is never started in this + isolated run, so no port clash.""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = ( + "https://aipi.reeseapps.com/v1" + if USE_REAL_LLM + else f"http://127.0.0.1:{mock_llm}/v1" + ) + # Mock-calibrated threshold (conftest pattern). + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + # Phase 67: instant retry waits + the code-default budget. + env["BOR_LLM_RETRY_DELAY"] = "0" + env["BOR_LLM_RETRIES"] = str(Settings.model_fields["llm_retries"].default) + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + # Phase 16: admin auth must be set or create_app() refuses to boot. + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + # Phase 97: the registry is this suite's own concern (see the + # fixture docstring) — the env fallback is git-only, empty here. + env["BOR_GIT_SOURCES"] = "" + # Leak guards (conftest pattern): an operator's local (gitignored) + # .env cannot leak corpus-specific settings into the app under test. + env["BOR_DOCS_REPO"] = "" + env["BOR_SUGGESTIONS"] = json.dumps( + Settings.model_fields["suggestions"].default + ) + env["BOR_INPUT_PLACEHOLDER"] = Settings.model_fields["input_placeholder"].default + env["BOR_FOOTER_TEXT"] = Settings.model_fields["footer_text"].default + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], + cwd=REPO, + env=env, + ) + try: + _wait_http(f"{APP_URL}/api/health") + yield APP_URL + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + EXPECTED_ROWS = ( "homelab/kubernetes.md", "homelab/backups.md", @@ -78,15 +173,57 @@ def _run_in_thread(coro: Any) -> Any: def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None: - """Truncate the KB (and query log), then optionally re-import fixtures.""" + """Truncate the KB (and query log), then optionally re-import fixtures. + + Phase 97: the registry (``git_sources``) and the stored folder + descriptions (``folder_summaries``) are truncated too — they now + RENDER in the RAG view (top-level source rows + descriptions), so a + leftover row from another suite would show up as a 0-document + source and break the empty-state test's zero-SOURCES state.""" with SessionLocal() as db: - db.execute(text("TRUNCATE chunks, documents, query_log")) + db.execute( + text("TRUNCATE chunks, documents, query_log, git_sources, folder_summaries") + ) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) +# --------------------------------------------------------------------------- +# Phase 97: the catalog is the drill-down tree the agent's `ls` walks — +# the top level lists the SOURCES (this fixture's single indexed-only +# source: `docs`, the fixtures dir's basename — import_sources seeded it +# with no registry row), then one folder link per path segment. The +# flat all-documents table is gone; the drill is the only change (the +# asserted rows/links are the same). +# --------------------------------------------------------------------------- + + +SOURCE_NAME = "docs" # the fixtures dir's basename (the indexed-only source) + + +def _drill(page: Page, *names: str) -> None: + """Drill one level at a time (client-side — no fetch, no URL + change): each name is the EXACT text of the source/folder link at + the current level (the test_kb_tree house pattern).""" + for name in names: + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + + +def _go_top(page: Page) -> None: + """Back to the top level (the sources list): the breadcrumb's + top-level link (hidden AT the top — call between drills only).""" + page.locator("#kb-crumb a.kb-crumb-link").first.click() + + +def _wait_top_level(page: Page) -> None: + """The tree's single fetch settled: the source row is rendered + (the catalog-rendered signal — the top-level file table is always + hidden, so it is no longer a usable one).""" + page.locator("#folders-tbody .folder-link").first.wait_for(state="visible") + + def test_sources_page_lists_indexed_docs( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: @@ -111,11 +248,25 @@ def test_sources_page_lists_indexed_docs( expect(page.locator("#stat-last")).not_to_have_text("–") expect(page.locator("#sources-empty")).to_be_hidden() - for row_path in EXPECTED_ROWS: + _wait_top_level(page) + # Phase 97: every row is nested under a folder — drill source → + # folder(s) per path before asserting (the flat 13-row tbody no + # longer exists; the stat cards above hold the KB total). + for i, row_path in enumerate(EXPECTED_ROWS): + if i > 0: + _go_top(page) + _drill(page, SOURCE_NAME, *row_path.rsplit("/", 1)[0].split("/")) expect(page.locator("#docs-tbody tr", has_text=row_path)).to_have_count(1) - # The hidden junk was never indexed (A9 scope). + # The hidden junk was never indexed (A9 scope) — scoped to the + # level where it WOULD appear: no `.hidden` file row at the source + # level AND no `.hidden` folder row there either (the stat cards + # pin the 13 total, so the junk is counted nowhere). + _go_top(page) + _drill(page, SOURCE_NAME) expect(page.locator("#docs-tbody tr", has_text=".hidden")).to_have_count(0) + expect(page.locator("#folders-tbody .folder-link", has_text=".hidden")).to_have_count(0) # The path column carries the full path for hover (ellipsis is visual only). + _drill(page, "homelab") expect(page.locator("#docs-tbody tr", has_text="homelab/kubernetes.md") .get_by_role("cell").nth(1)).to_have_attribute("title", "homelab/kubernetes.md") @@ -125,11 +276,18 @@ def test_sources_table_layout( ) -> None: _reset_db(mock_llm, seed=True) login(page, app_url) # phase 16: the catalog is admin-only + # Phase 97: the top level lists the sources (the file table is + # hidden there) — wait for the source row, then drill to a file + # level (docs → homelab) for the file table's layout asserts. + _wait_top_level(page) + _drill(page, SOURCE_NAME, "homelab") page.locator("#docs-tbody tr").first.wait_for(state="visible") # Phase 76 (task 02): the shell carries BOTH views' .table-wrap — - # scope to the RAG view. - wrap = page.locator("#view-rag .table-wrap") + # scope to the RAG view. Phase 97: the RAG view carries TWO + # .table-wraps (#folders-wrap + the file table's) — :has() targets + # the file table's (the one these layout pins were written for). + wrap = page.locator("#view-rag .table-wrap:has(#docs-table)") expect(wrap).to_be_visible() expect(wrap).to_have_attribute("role", "region") expect(wrap).to_have_attribute("tabindex", "0") @@ -146,9 +304,13 @@ def test_sources_table_layout( mobile = browser.new_page(viewport={"width": 375, "height": 812}) try: login(mobile, app_url) # phase 16: the catalog is admin-only + _wait_top_level(mobile) + _drill(mobile, SOURCE_NAME, "homelab") mobile.locator("#docs-tbody tr").first.wait_for(state="visible") + # Phase 97: target the FILE table's wrap explicitly (a bare + # .table-wrap query would hit #folders-wrap first now). scroll_width, client_width = mobile.evaluate( - "() => { const el = document.querySelector('#view-rag .table-wrap');" + "() => { const el = document.querySelector('#docs-table').parentElement;" " return [el.scrollWidth, el.clientWidth]; }" ) assert scroll_width > client_width @@ -165,6 +327,9 @@ def test_empty_state_when_no_docs(page: Page, app_url: str, mock_llm: int, db_re expect(page.locator("#sources-empty code")).to_have_text( "uv run python -m scripts.import_docs" ) - expect(page.locator("#view-rag .table-wrap")).to_be_hidden() + # Phase 97: BOTH catalog tables ship in the RAG view — the empty + # state hides both (the original single-wrap pin, extended). + expect(page.locator("#view-rag .table-wrap:has(#docs-table)")).to_be_hidden() + expect(page.locator("#folders-wrap")).to_be_hidden() expect(page.locator("#stat-docs")).to_have_text("0") expect(page.locator("#stat-chunks")).to_have_text("0") diff --git a/tests/e2e/test_import_extensions_env.py b/tests/e2e/test_import_extensions_env.py index 87284c5..6c2edfc 100644 --- a/tests/e2e/test_import_extensions_env.py +++ b/tests/e2e/test_import_extensions_env.py @@ -16,6 +16,11 @@ DB isolation: the fixture's source name (``extension_kb``) is distinctive — the suite never asserts on absolute row counts and deletes the rows it creates in a ``finally`` (other suites' documents stay untouched in the shared E2E database). + +Phase 97 adaptation: the catalog is the DRILL-DOWN TREE — the rows +live at their folder levels (``extension_kb`` → ``homelab`` → +``scripts`` / ``notes``); the drill is the only change, the asserted +rows/links/modal are unchanged. """ from __future__ import annotations @@ -86,6 +91,20 @@ def _delete_source_rows() -> None: db.commit() +def _drill(page: Page, *names: str) -> None: + """Drill one level at a time (phase 97 — client-side, no fetch, + no URL change): each name is the EXACT text of the source/folder + link at the current level.""" + for name in names: + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + + +def _go_top(page: Page) -> None: + """Back to the top level: the breadcrumb's top-level link (call + between drills only — the breadcrumb is hidden at the top).""" + page.locator("#kb-crumb a.kb-crumb-link").first.click() + + @pytest.fixture(autouse=True) def extension_kb(mock_llm: int, db_ready: None) -> Iterator[ImportSummary]: """Seed the fixture with the NOVEL scope (``md,sh``) for one test @@ -110,14 +129,27 @@ def test_admin_sources_lists_the_novel_extension( login(page, app_url) # phase 16: the catalog is admin-only # The novel .sh document is listed; the path cell carries the full # path (the column is ellipsized — the title attribute is the pin). + # Phase 97: the rows live at their folder levels — drill source → + # homelab → scripts first. + _drill(page, SOURCE, "homelab", "scripts") row = page.locator("#docs-tbody tr", has_text=SH_REL) expect(row).to_have_count(1) link = row.locator("td:nth-child(2) a.doc-link") expect(link).to_have_count(1) expect(link).to_have_attribute("title", SH_REL) - # The markdown control doc is listed too (never asserted by count — - # other suites' documents may share the shared E2E database). + # The markdown control doc is listed too (its OWN level — never + # asserted by count: other suites' documents may share the shared + # E2E database). + _go_top(page) + _drill(page, SOURCE, "homelab", "notes") expect(page.locator("#docs-tbody tr", has_text=MD_REL)).to_have_count(1) + # Back to the .sh row's level — the modal asserts below click its + # path link. + _go_top(page) + _drill(page, SOURCE, "homelab", "scripts") + link = page.locator("#docs-tbody tr", has_text=SH_REL).locator( + "td:nth-child(2) a.doc-link" + ) # Format badge: the row's path link opens the same-page modal and # its meta row shows the .sh format (house assertion style — diff --git a/tests/e2e/test_kb_tree.py b/tests/e2e/test_kb_tree.py new file mode 100644 index 0000000..a40bfc3 --- /dev/null +++ b/tests/e2e/test_kb_tree.py @@ -0,0 +1,941 @@ +"""Phase 97 task 06 E2E (Playwright, mock-only): the RAG view's drill-down +catalog tree + the editable folder descriptions. + +The dedicated story suite for ``97_kb_tree_catalog`` (owner request, +2026-09-11): the Knowledge base view lists the KB the way the agent's +``ls`` sees it (the phase-94 concept, one end to end) — sources at the +top, then per level the subfolders with their stored descriptions and +the level's files — and the owner edits (or clears) any directory's +description with the phase-57 inline affordance. Everything is pinned +against the real app + the deterministic mock. + +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_kb_tree.py -v --no-cov + +KB fixture — a host temp-dir tree (``tmp_path_factory``; the app runs on +the same host) with TWO local sources registered through the +authenticated API, then the real in-process ``POST /api/sync`` pipeline +(the ``test_local_directory_sources.py`` / phase-94 registration + +real-Sync pattern; no git anywhere): + +* ``alpha/`` — ``root-note.md`` at the source root, ``one/`` (2 docs), + ``two/`` (2 docs); +* ``beta/`` — ``gamma/`` (2 docs). + +Total: 7 documents; alpha counts 5 (root + 2 + 2), beta counts 2. Every +stored description is deterministic: the mock's EXISTING +``FOLDER_SUMMARY_MODE`` branch (phase 94 — no mock changes needed) +stores, per ≥ 2-doc folder, the canned one-liner naming the folder, +``Fixture folder summary for [/].`` — the ``synced_kb`` +module fixture pins those exact rows (all with +``manually_edited = false``) after the sync, and the tests assert on +that exact text. + +MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the real +``lite`` does whatever it does with the folder prompts, while this +story's assertions key on the CANNED summaries (the phase-94 +convention): the stored rows are a pure function of the request, so the +drill, the edit/clear, and the keep-through-sync observables are +byte-exact only against the mock. + +The view is admin-only (phase 16): the admin flows drive the REAL form +login (``e2e.auth_helpers.login`` → ``/sources.html``); the anonymous +test uses a fresh context and asserts the sign-in gate + zero catalog +fetches. + +Test → observable mapping (Playwright Mapping Rule): +1. ``test_top_level_lists_sources_with_descriptions`` — the source rows: + ``alpha`` (5) + ``beta`` (2) in registry order with the canned + source-root descriptions (the top-level rows ARE the sources — the + ``ls()`` equivalence); the top-level file table is HIDDEN (files are + per-source); the stat cards read 7 documents + the computed chunks + total (the tree walk = the former flat walk). +2. ``test_drill_into_source`` — click the ``alpha`` row → breadcrumb + ``alpha`` (the ``aria-current`` segment); the level block shows + alpha's root description (the canned text, title = the source name); + the folder rows ``one`` / ``two`` each with count 2 + their canned + summaries; the file row ``root-note.md`` (Source column, title, + chunks column) in the level's file table. +3. ``test_drill_into_folder`` — drill to ``alpha`` → ``two`` → + breadcrumb ``alpha`` → ``two``; the level block shows ``two``'s + description (title = the full source-relative path); the file rows + ``two-a`` / ``two-b`` (titles, the Source column ``alpha``); + ``#folders-wrap`` hidden (no subfolders); the breadcrumb link on + ``alpha`` goes back up to the source level. +4. ``test_edit_folder_description`` — on the ``alpha`` level, Edit on + the ``one/`` ROW (pinned: the row surface — the level-block surface + is test 6's) → the textarea prefilled with the canned text → set the + new text → Save → the new text renders in the row + status + "Description updated."; a SQL assert (the ``SessionLocal`` house + pattern) on ``folder_summaries``: the row's ``summary`` is the new + text AND ``manually_edited`` is true. +5. ``test_clear_folder_description`` — an empty save → the text is gone + (the row cell emptied — the always-present Edit button stays) + + status "Description cleared."; the SQL assert: no row for the folder + (the next KB-changing sync regenerates an AI description — the + reset path). +6. ``test_manual_description_survives_a_changed_sync`` — the + LEVEL-BLOCK edit on the ``beta`` source root (the static + ``#kb-level-edit`` button, ``folder_path ""``); a new file is added + to the fixture dir; the real in-process sync (the mock regenerates + the OTHER folders' summaries) → the catalog re-fetch renders the + edited description UNCHANGED (the SQL row keeps the manual text + + the flag — the ``kept_manual`` path, E2E-pinned) while the untouched + folders show the canned regenerated text (flag back to false) and + the new document lands in the tree (the ``one`` folder counts 3). +7. ``test_reload_falls_back_to_top_level`` — drill to ``alpha/two``; + delete ``two/``'s documents + chunks directly (the house DB + pattern); re-show the RAG view (the nav link re-click — the + ``bor:view-refresh`` trigger) → the breadcrumb is hidden and the top + level renders (the never-stale contract, PLAN §7.4). +8. ``test_anonymous_sees_the_gate`` — the RAG view for an anonymous + visitor: the sign-in gate visible, no stat cards, no folders/file + table, no Edit affordance, and NO ``/api/docs*`` request (the + phase-16 soft rule — asserted via the request log, the + ``test_admin_auth.py`` pattern). +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import httpx +import pytest +from playwright.sync_api import Page, expect +from sqlalchemy import select, text + +from app.config import Settings as _Settings +from app.db import SessionLocal +from app.models import FolderSummary +from e2e.auth_helpers import login +from e2e.conftest import ( + ADMIN_PASSWORD, + SESSION_SECRET, + USE_REAL_LLM, + _wait_http, +) + +REPO = Path(__file__).resolve().parents[2] + +# Phase 79 (task 04, full inventory): the conftest session app owns its +# port in a combined run — this module app binds its own port instead +# (a same-port second uvicorn dies on bind and would drive the wrong +# server). Env-overridable. +APP_PORT = int(os.environ.get("E2E_APP_PORT_KBTREE", "8139")) +APP_URL = f"http://127.0.0.1:{APP_PORT}" + +# -------------------------------------------------------------------------- +# Fixture constants (deterministic, token-controlled) +# -------------------------------------------------------------------------- + +ALPHA = "alpha" +BETA = "beta" + +ROOT_NOTE = "root-note.md" +ONE_A = "one/one-a.md" +ONE_B = "one/one-b.md" +ONE_C = "one/one-c.md" # test 6's KB-changing addition +TWO_A = "two/two-a.md" +TWO_B = "two/two-b.md" +GAMMA_A = "gamma/gamma-a.md" +GAMMA_B = "gamma/gamma-b.md" + +ALPHA_COUNT = 5 # 1 root note + 2 one/ + 2 two/ +BETA_COUNT = 2 +TOTAL_DOCS = ALPHA_COUNT + BETA_COUNT + +#: The sync-time folder descriptions the mock's canned +#: ``FOLDER_SUMMARY_MODE`` branch stores (the phase-94 byte-stable +#: template — the one-liner names the folder), in ``(source, +#: folder_path)`` order: one row per ≥ 2-doc folder (the +#: recursive-subtree rule) — the ``""`` rows are the source roots. +SUMMARY_FOR = "Fixture folder summary for {}." +ALPHA_ROOT_SUM = SUMMARY_FOR.format(ALPHA) +ONE_SUM = SUMMARY_FOR.format(f"{ALPHA}/one") +TWO_SUM = SUMMARY_FOR.format(f"{ALPHA}/two") +BETA_ROOT_SUM = SUMMARY_FOR.format(BETA) +GAMMA_SUM = SUMMARY_FOR.format(f"{BETA}/gamma") + +EXPECTED_SUMMARIES: list[tuple[str, str, str]] = [ + (ALPHA, "", ALPHA_ROOT_SUM), + (ALPHA, "one", ONE_SUM), + (ALPHA, "two", TWO_SUM), + (BETA, "", BETA_ROOT_SUM), + (BETA, "gamma", GAMMA_SUM), +] +assert [ + (source, folder) for source, folder, _s in EXPECTED_SUMMARIES +] == sorted((source, folder) for source, folder, _s in EXPECTED_SUMMARIES) + +#: The hand-edited descriptions (tests 4 and 6) — distinctive sentences +#: no part of the fixture or the canned template contains, so the +#: round-trip assertions can never pass against the old text. +NEW_ALPHA_ONE = ( + "Owner override: alpha/one holds the one-a and one-b fixture notes — " + "the drill catalog's first hand-written description. (RESE-KBTREE-01)" +) +NEW_BETA_ROOT = ( + "Owner override: beta is the second fixture source — gamma only. " + "(RESE-KBTREE-02)" +) + + +def _md(title: str, body: str) -> str: + return f"# {title}\n\n{body}\n" + + +# -------------------------------------------------------------------------- +# Fixtures +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def kb_tree_dirs(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]: + """The two-source temp tree (see the module docstring): the app + server runs on the same host, so the paths are visible to it. The + directory NAMES are the source names (``kind=local`` → the + directory's basename, phase 38).""" + root = tmp_path_factory.mktemp("bor_kb_tree") + alpha = root / ALPHA + beta = root / BETA + (alpha / "one").mkdir(parents=True) + (alpha / "two").mkdir(parents=True) + (beta / "gamma").mkdir(parents=True) + + (alpha / ROOT_NOTE).write_text( + _md( + "Alpha Root Note", + "This file sits directly under the alpha source, not in any " + "folder.", + ), + encoding="utf-8", + ) + (alpha / ONE_A).write_text( + _md( + "Alpha One A", + "Alpha one fixture note A: covers topic A of the alpha " + "source tree.", + ), + encoding="utf-8", + ) + (alpha / ONE_B).write_text( + _md( + "Alpha One B", + "Alpha one fixture note B: covers topic B of the alpha " + "source tree.", + ), + encoding="utf-8", + ) + (alpha / TWO_A).write_text( + _md( + "Alpha Two A", + "Alpha two fixture note A: covers topic A of the alpha " + "source tree.", + ), + encoding="utf-8", + ) + (alpha / TWO_B).write_text( + _md( + "Alpha Two B", + "Alpha two fixture note B: covers topic B of the alpha " + "source tree.", + ), + encoding="utf-8", + ) + (beta / GAMMA_A).write_text( + _md( + "Beta Gamma A", + "Beta gamma fixture note A: covers topic A of the beta " + "source tree.", + ), + encoding="utf-8", + ) + (beta / GAMMA_B).write_text( + _md( + "Beta Gamma B", + "Beta gamma fixture note B: covers topic B of the beta " + "source tree.", + ), + encoding="utf-8", + ) + assert (alpha / TWO_A).is_file() and (beta / GAMMA_B).is_file() + return alpha, beta + + +@pytest.fixture(scope="module") +def app_server(mock_llm: int, kb_tree_dirs: tuple[Path, Path]) -> Iterator[str]: + """The real app under test — per-module app (the conftest pattern, + cf. ``test_local_directory_sources.py`` / ``test_ls_tree_drilldown. + py``): NO ``BOR_GIT_SOURCES`` (the env fallback is git-only — the + sources here are DB-registered local directories), the mock LLM, the + mock-calibrated threshold, and the leak-guarded code defaults. The + session app is never started in this isolated run, so no port clash. + ``kb_tree_dirs`` is a dependency only for the fixture ordering (the + temp tree exists before the app boots — the sync reads it).""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = ( + "https://aipi.reeseapps.com/v1" + if USE_REAL_LLM + else f"http://127.0.0.1:{mock_llm}/v1" + ) + # Mock-calibrated threshold (conftest pattern): this suite never + # asks the chat model anything — the gate is never on a path. + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + # Phase 67: instant retry waits + the code-default budget (the + # conftest leak-guard pattern). + env["BOR_LLM_RETRY_DELAY"] = "0" + env["BOR_LLM_RETRIES"] = str(_Settings.model_fields["llm_retries"].default) + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + # Phase 16: admin auth must be set or create_app() refuses to boot. + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + # The repo's .env file carries the owner's BOR_GIT_SOURCES (the app + # reads it from cwd) — override it with an EMPTY value (the env var + # beats the .env file): the registry must hold EXACTLY the two + # local directories this suite registers (a leftover env git list + # would pollute the top-level rows the whole story asserts on). + env["BOR_GIT_SOURCES"] = "" + # Leak guards (conftest pattern): an operator's local (gitignored) + # .env cannot leak corpus-specific settings into the app under test. + env["BOR_DOCS_REPO"] = "" + env["BOR_SUGGESTIONS"] = json.dumps( + _Settings.model_fields["suggestions"].default + ) + env["BOR_INPUT_PLACEHOLDER"] = _Settings.model_fields["input_placeholder"].default + env["BOR_FOOTER_TEXT"] = _Settings.model_fields["footer_text"].default + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], + cwd=REPO, + env=env, + ) + try: + _wait_http(f"{APP_URL}/api/health") + yield APP_URL + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +def _truncate_all() -> None: + """Fresh registry + KB (the E2E isolation pattern): the E2E suites + share one Postgres, so a leftover git_sources row would pollute the + top-level rows and a leftover document would show up in the level + file tables and the stat cards the suite asserts on exactly.""" + with SessionLocal() as db: + db.execute( + text( + "TRUNCATE chunks, documents, query_log, steering_notes, " + "kb_overview, git_sources, folder_summaries" + ) + ) + db.commit() # without the commit the TRUNCATE rolls back (the house pattern) + + +def _wait_sync_done_http(client: httpx.Client, timeout_s: float = 180.0) -> dict[str, Any]: + """Poll the (cookie-authenticated) status endpoint until the run + reaches a terminal state (the test_local_directory_sources pattern, + over plain httpx).""" + deadline = time.monotonic() + timeout_s + body: dict[str, Any] = {} + while time.monotonic() < deadline: + r = client.get("/api/sync/status") + assert r.status_code == 200, r.text + body = r.json() + if body["state"] in ("success", "failed"): + return body + time.sleep(0.5) + raise AssertionError(f"sync did not reach a terminal state: {body}") + + +@pytest.fixture(scope="module") +def synced_kb(app_server: str, kb_tree_dirs: tuple[Path, Path]) -> None: + """The story's precondition: the folder-structured KB synced under + the deterministic mock. + + Registers the two temp directories through the authenticated API + (the ``test_local_directory_sources.py`` pattern — ``alpha`` FIRST, + committed separately, so the registry order — ``(added_at, id)`` — + lists alpha before beta, the top-level row order the suite + asserts), runs the REAL in-process sync (``POST /api/sync`` — + walk → chunk → embed → overview → folder summaries → version + bump), and pins the stored folder descriptions: the mock's canned + ``FOLDER_SUMMARY_MODE`` branch (phase 94) makes the sync store one + deterministic row per ≥ 2-doc folder — the tests assert on that + exact text (and on the phase-97 ``manually_edited`` flag: every + stored row starts out AI-written). + """ + alpha, beta = kb_tree_dirs + _truncate_all() + with httpx.Client(base_url=app_server, timeout=30.0) as client: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, r.text + r = client.post( + "/api/git-sources", json={"kind": "local", "path": str(alpha)} + ) + assert r.status_code == 201, r.text + time.sleep(0.05) # distinct added_at: alpha before beta (registry order) + r = client.post( + "/api/git-sources", json={"kind": "local", "path": str(beta)} + ) + assert r.status_code == 201, r.text + r = client.post("/api/sync") + assert r.status_code == 202, r.text + body = _wait_sync_done_http(client) + assert body["state"] == "success", body + detail = body["detail"] + assert detail["added"] == TOTAL_DOCS, detail + assert detail["pruned"] == 0, detail + assert detail["overview"] is True, detail + # The change-gated folder descriptions (phase 94) landed: one row + # per ≥ 2-doc folder, the mock's byte-stable text — and every row + # AI-written (the ``manually_edited`` flag starts false). + with SessionLocal() as db: + rows = db.execute( + select( + FolderSummary.source, + FolderSummary.folder_path, + FolderSummary.summary, + FolderSummary.manually_edited, + ).order_by(FolderSummary.source, FolderSummary.folder_path) + ).all() + assert [ + (source, folder, summary, False) + for source, folder, summary in EXPECTED_SUMMARIES + ] == [(s, f, t, m) for s, f, t, m in rows], rows + + +# -------------------------------------------------------------------------- +# Page + DB helpers +# -------------------------------------------------------------------------- + + +def _kb_totals() -> tuple[int, int]: + """The KB-wide ``(documents, chunks)`` totals for the two sources, + straight from the DB — the values the stat cards' tree walk must + render (the former flat /api/docs walk, one level deeper).""" + with SessionLocal() as db: + docs = db.scalar( + text("SELECT count(*) FROM documents WHERE source IN (:a, :b)"), + {"a": ALPHA, "b": BETA}, + ) + chunks = db.scalar( + text( + "SELECT count(*) FROM chunks c JOIN documents d " + "ON c.document_id = d.id WHERE d.source IN (:a, :b)" + ), + {"a": ALPHA, "b": BETA}, + ) + return int(docs), int(chunks) + + +def _doc_chunks(source: str, path: str) -> int: + """One document's chunk count (the Chunks column value the file + row must render — the count ``GET /api/docs`` returns for it).""" + with SessionLocal() as db: + n = db.scalar( + text( + "SELECT count(*) FROM chunks c JOIN documents d " + "ON c.document_id = d.id WHERE d.source = :s AND d.path = :p" + ), + {"s": source, "p": path}, + ) + return int(n) + + +def _folder_row(source: str, folder: str) -> tuple[str, bool] | None: + """The stored ``(source, folder)`` description as + ``(summary, manually_edited)`` (``None`` when no row exists) — + read back through a fresh session (the ``SessionLocal`` house + pattern).""" + with SessionLocal() as db: + row = db.scalar( + select(FolderSummary).where( + FolderSummary.source == source, + FolderSummary.folder_path == folder, + ) + ) + return (row.summary, row.manually_edited) if row is not None else None + + +def _run_sync(app_url: str) -> dict[str, Any]: + """One admin sync through the API — the cookie-authenticated + ``POST /api/sync`` + the status poll (the ``synced_kb`` pattern, + for test 6's mid-suite KB-changing sync).""" + with httpx.Client(base_url=app_url, timeout=30.0) as client: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, r.text + r = client.post("/api/sync") + assert r.status_code == 202, r.text + return _wait_sync_done_http(client) + + +def _wait_top_level(page: Page) -> None: + """The admin boot has rendered the top level: the two source rows + are in the folders table (the tree's single fetch settled) and the + top-level file table is empty (files are per-source — always hidden + at the top).""" + expect(page.locator("#folders-tbody tr")).to_have_count(2, timeout=30_000) + expect(page.locator("#docs-tbody tr")).to_have_count(0) + + +def _drill(page: Page, *names: str) -> None: + """Drill one level at a time (client-side — no fetch, no URL + change): each name is the EXACT text of the source/folder link at + the current level (the row builders' link text: the source name, + or the folder's last path segment).""" + for name in names: + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + + +def _expect_level(page: Page, title: str, summary: str) -> None: + """The level block shows the current directory's stored + description: the full source-relative path as the title, the + description as the text.""" + expect(page.locator("#kb-level")).to_be_visible() + expect(page.locator("#kb-level-title")).to_have_text(title) + expect(page.locator("#kb-level-summary")).to_have_text(summary) + + +# -------------------------------------------------------------------------- +# 1. The top level: the source rows (the ls() equivalence) + stat cards +# -------------------------------------------------------------------------- + + +def test_top_level_lists_sources_with_descriptions( + page: Page, app_url: str, synced_kb: None, db_ready: None +) -> None: + page.set_default_timeout(30_000) + docs, chunks = _kb_totals() + assert docs == TOTAL_DOCS # the fixture's precondition (7 documents) + + login(page, app_url) # → /sources.html (the RAG view) + _wait_top_level(page) + + # The source rows: registry order (alpha registered first), the + # recursive count, the stored (source, "") description — the + # top-level rows ARE the sources (the ls() equivalence: name, + # count, description). + rows = page.locator("#folders-tbody tr") + expect(rows.nth(0).locator("a.folder-link")).to_have_text(ALPHA) + expect(rows.nth(0).locator("td:nth-child(2)")).to_have_text(str(ALPHA_COUNT)) + expect(rows.nth(0).locator("td:nth-child(3) span")).to_have_text(ALPHA_ROOT_SUM) + expect(rows.nth(1).locator("a.folder-link")).to_have_text(BETA) + expect(rows.nth(1).locator("td:nth-child(2)")).to_have_text(str(BETA_COUNT)) + expect(rows.nth(1).locator("td:nth-child(3) span")).to_have_text(BETA_ROOT_SUM) + # The edit affordance is ALWAYS present (a description can be + # CREATED where none is stored) — the row surface, one per row. + expect(page.locator("#folders-tbody .kb-summary-edit")).to_have_count(2) + + # The top level carries no breadcrumb and no level block; the + # empty state is off (a source IS listed). + expect(page.locator("#kb-crumb")).to_be_hidden() + expect(page.locator("#kb-level")).to_be_hidden() + expect(page.locator("#sources-empty")).to_be_hidden() + + # The file table is HIDDEN at the top level (files are seen per + # source, as with ls(source) — the ls() equivalence). + expect(page.locator("#docs-table")).to_be_hidden() + + # The stat cards: the KB-wide walk of the WHOLE tree — identical + # values to the former flat /api/docs walk. + expect(page.locator("#stat-docs")).to_have_text(str(TOTAL_DOCS)) + expect(page.locator("#stat-chunks")).to_have_text(str(chunks)) + expect(page.locator("#stat-last")).not_to_have_text("–") + + +# -------------------------------------------------------------------------- +# 2. Drill into a source: breadcrumb, level block, folder rows, files +# -------------------------------------------------------------------------- + + +def test_drill_into_source( + page: Page, app_url: str, synced_kb: None, db_ready: None +) -> None: + page.set_default_timeout(30_000) + root_chunks = _doc_chunks(ALPHA, ROOT_NOTE) + + login(page, app_url) + _wait_top_level(page) + _drill(page, ALPHA) + + # Breadcrumb: the top-level link + the current source segment. + expect(page.locator("#kb-crumb")).to_be_visible() + links = page.locator("#kb-crumb a.kb-crumb-link") + expect(links).to_have_count(1) + expect(links.nth(0)).to_have_text("Knowledge base") + expect(page.locator("#kb-crumb .kb-crumb-current")).to_have_text(ALPHA) + expect(page.locator("#kb-crumb .kb-crumb-current")).to_have_attribute( + "aria-current", "page" + ) + + # The level block: the source's stored root description (title = + # the source name — the source root is folder "") + the level's + # own Edit button (task 05). + _expect_level(page, ALPHA, ALPHA_ROOT_SUM) + expect(page.locator("#kb-level-edit")).to_be_visible() + + # The subfolder rows: path order, the recursive count, the stored + # (AI) description. + rows = page.locator("#folders-tbody tr") + expect(rows).to_have_count(2) + expect(rows.nth(0).locator("a.folder-link")).to_have_text("one") + expect(rows.nth(0).locator("td:nth-child(2)")).to_have_text("2") + expect(rows.nth(0).locator("td:nth-child(3) span")).to_have_text(ONE_SUM) + expect(rows.nth(1).locator("a.folder-link")).to_have_text("two") + expect(rows.nth(1).locator("td:nth-child(2)")).to_have_text("2") + expect(rows.nth(1).locator("td:nth-child(3) span")).to_have_text(TWO_SUM) + + # The level's direct files: root-note.md — the UNCHANGED 5-column + # contract (makeRow): Source | Path | Title | Chunks | Indexed. + expect(page.locator("#docs-table")).to_be_visible() + frows = page.locator("#docs-tbody tr") + expect(frows).to_have_count(1) + expect(frows.nth(0).locator("td:nth-child(1)")).to_have_text(ALPHA) + expect(frows.nth(0).locator("a.doc-link")).to_have_text(ROOT_NOTE) + expect(frows.nth(0).locator("td:nth-child(3)")).to_have_text("Alpha Root Note") + expect(frows.nth(0).locator("td:nth-child(4)")).to_have_text(str(root_chunks)) + expect(frows.nth(0).locator("td:nth-child(5)")).not_to_have_text("") + + +# -------------------------------------------------------------------------- +# 3. Drill into a folder: the file level + the breadcrumb going back up +# -------------------------------------------------------------------------- + + +def test_drill_into_folder( + page: Page, app_url: str, synced_kb: None, db_ready: None +) -> None: + page.set_default_timeout(30_000) + + login(page, app_url) + _wait_top_level(page) + _drill(page, ALPHA, "two") + + # Breadcrumb: the top-level link + the source link + the current + # folder segment. + links = page.locator("#kb-crumb a.kb-crumb-link") + expect(links).to_have_count(2) + expect(links.nth(0)).to_have_text("Knowledge base") + expect(links.nth(1)).to_have_text(ALPHA) + expect(page.locator("#kb-crumb .kb-crumb-current")).to_have_text("two") + + # The level block: two's stored description, the FULL + # source-relative path as the title. + _expect_level(page, f"{ALPHA}/two", TWO_SUM) + + # The file rows: two-a / two-b (path order), the Source column + # carries the source (the file nodes are source-scoped in the tree + # — the row object restores the flat shape makeRow reads). + frows = page.locator("#docs-tbody tr") + expect(frows).to_have_count(2) + expect(frows.nth(0).locator("td:nth-child(1)")).to_have_text(ALPHA) + expect(frows.nth(0).locator("a.doc-link")).to_have_text(TWO_A) + expect(frows.nth(0).locator("td:nth-child(3)")).to_have_text("Alpha Two A") + expect(frows.nth(1).locator("td:nth-child(1)")).to_have_text(ALPHA) + expect(frows.nth(1).locator("a.doc-link")).to_have_text(TWO_B) + expect(frows.nth(1).locator("td:nth-child(3)")).to_have_text("Alpha Two B") + + # No subfolders under two: the folders table is hidden. + expect(page.locator("#folders-wrap")).to_be_hidden() + + # The breadcrumb link on the source goes back UP to the source + # level (client-side — no fetch, no URL change). + page.click(f'#kb-crumb a.kb-crumb-link:text-is("{ALPHA}")') + expect(page.locator("#kb-crumb .kb-crumb-current")).to_have_text(ALPHA) + expect(page.locator("#kb-crumb a.kb-crumb-link")).to_have_count(1) + rows = page.locator("#folders-tbody tr") + expect(rows).to_have_count(2) + expect(rows.nth(0).locator("a.folder-link")).to_have_text("one") + expect(rows.nth(1).locator("a.folder-link")).to_have_text("two") + + +# -------------------------------------------------------------------------- +# 4. Edit a folder description (the row surface) → manually_edited +# -------------------------------------------------------------------------- + + +def test_edit_folder_description( + page: Page, app_url: str, synced_kb: None, db_ready: None +) -> None: + page.set_default_timeout(30_000) + + login(page, app_url) + _wait_top_level(page) + _drill(page, ALPHA) + + rows = page.locator("#folders-tbody tr") + expect(rows).to_have_count(2) + row = rows.nth(0) + expect(row.locator("a.folder-link")).to_have_text("one") + + # Edit → the inline editor: textarea PREFILLED with the stored + # (canned) text, Save / Cancel, and the role=status live region. + row.locator(".kb-summary-edit").click() + editor = page.locator(".kb-summary-editor") + expect(editor).to_have_count(1) + expect(editor).to_be_visible() + expect(editor).to_have_value(ONE_SUM) + expect(page.locator(".kb-summary-save")).to_be_visible() + expect(page.locator(".kb-summary-cancel")).to_be_visible() + status = row.locator(".kb-summary-status") + expect(status).to_have_attribute("role", "status") + expect(status).to_have_attribute("aria-live", "polite") + + # Replace the text with the distinctive hand-edit and Save. + page.fill(".kb-summary-editor", NEW_ALPHA_ONE) + page.click(".kb-summary-save") + + # The live-region confirmation; the new text renders in the row + # (the textContent re-render — the canned text is gone); the + # always-present Edit button is back. + expect(status).to_have_text("Description updated.") + expect(row.locator("td:nth-child(3) span")).to_have_text(NEW_ALPHA_ONE) + expect(row.locator(".kb-summary-edit")).to_be_visible() + + # The server row: the new text AND the manually_edited flag (the + # task-01 keep/keep-out rules apply from this save on — the + # sync-time generator must never rewrite or prune the row). + stored = _folder_row(ALPHA, "one") + assert stored == (NEW_ALPHA_ONE, True), stored + + +# -------------------------------------------------------------------------- +# 5. An empty save CLEARS — the row is deleted (the reset path) +# -------------------------------------------------------------------------- + + +def test_clear_folder_description( + page: Page, app_url: str, synced_kb: None, db_ready: None +) -> None: + page.set_default_timeout(30_000) + + login(page, app_url) + _wait_top_level(page) + _drill(page, ALPHA) + + rows = page.locator("#folders-tbody tr") + expect(rows).to_have_count(2) + row = rows.nth(0) + expect(row.locator("a.folder-link")).to_have_text("one") + # The row holds a description (the module's canned row — or test + # 4's manual edit when the tests run in order); the editor opens + # either way (the button is always present). + row.locator(".kb-summary-edit").click() + expect(page.locator(".kb-summary-editor")).to_be_visible() + + # Select-all + delete — clear the prefilled editor — then Save. + page.fill(".kb-summary-editor", "") + page.click(".kb-summary-save") + + status = row.locator(".kb-summary-status") + expect(status).to_have_text("Description cleared.") + # The text is gone: the cell empties (the always-present Edit + # button stays — a description can be re-created from the empty + # cell). + expect(row.locator("td:nth-child(3) span")).to_have_text("") + expect(row.locator(".kb-summary-edit")).to_be_visible() + + # The SQL assert: no row for the folder (the next KB-changing + # sync regenerates an AI description — the reset path). + assert _folder_row(ALPHA, "one") is None + + +# -------------------------------------------------------------------------- +# 6. The manual description survives a KB-changing sync (kept_manual) +# -------------------------------------------------------------------------- + + +def test_manual_description_survives_a_changed_sync( + page: Page, + app_url: str, + synced_kb: None, + kb_tree_dirs: tuple[Path, Path], + db_ready: None, +) -> None: + page.set_default_timeout(30_000) + alpha, _beta = kb_tree_dirs + + login(page, app_url) + _wait_top_level(page) + _drill(page, BETA) + + # The beta level: the stored root description + the gamma row + # (both canned — AI-written at the module sync). + _expect_level(page, BETA, BETA_ROOT_SUM) + rows = page.locator("#folders-tbody tr") + expect(rows).to_have_count(1) + expect(rows.nth(0).locator("a.folder-link")).to_have_text("gamma") + expect(rows.nth(0).locator("td:nth-child(3) span")).to_have_text(GAMMA_SUM) + + # The LEVEL-BLOCK Edit (the level surface — test 4 pins the row + # surface): the static #kb-level-edit button, the source root + # (folder_path ""). The textarea is prefilled with the stored text. + page.click("#kb-level-edit") + editor = page.locator(".kb-summary-editor") + expect(editor).to_have_count(1) + expect(editor).to_have_value(BETA_ROOT_SUM) + page.fill(".kb-summary-editor", NEW_BETA_ROOT) + page.click(".kb-summary-save") + expect(page.locator("#kb-level .kb-summary-status")).to_have_text( + "Description updated." + ) + expect(page.locator("#kb-level-summary")).to_have_text(NEW_BETA_ROOT) + + # The KB-changing sync: a new file lands in alpha/one (the fixture + # dir — the app runs on the same host), then the real in-process + # sync (the mock's FOLDER_SUMMARY_MODE branch regenerates the + # non-manual folders' summaries). + (alpha / ONE_C).write_text( + _md( + "Alpha One C", + "Alpha one fixture note C: added for the changed-sync test.", + ), + encoding="utf-8", + ) + body = _run_sync(app_url) + assert body["state"] == "success", body + assert body["detail"]["added"] == 1, body["detail"] + + # The catalog re-fetch (the nav re-click — the bor:view-refresh + # trigger, no document load): the current position (the beta root) + # still exists, so the view stays put and re-renders from the NEW + # tree — the EDITED description is UNCHANGED (kept_manual: the + # generator skipped the owner's row, no lite burn on it)… + page.click("#nav-sources") + expect(page.locator("#kb-level-summary")).to_have_text(NEW_BETA_ROOT) + # …while the untouched folder shows the canned REGENERATED text. + expect( + page.locator("#folders-tbody tr") + .nth(0) + .locator("td:nth-child(3) span") + ).to_have_text(GAMMA_SUM) + + # The new document landed in the tree: back to the top (the + # breadcrumb's top-level link), then into alpha — the one folder + # now counts 3 with its regenerated canned description. + page.locator("#kb-crumb a.kb-crumb-link").nth(0).click() + _drill(page, ALPHA) + one_row = page.locator("#folders-tbody tr").nth(0) + expect(one_row.locator("a.folder-link")).to_have_text("one") + expect(one_row.locator("td:nth-child(2)")).to_have_text("3") + expect(one_row.locator("td:nth-child(3) span")).to_have_text(ONE_SUM) + + # The SQL asserts: the manual row survived (text + flag), the + # untouched rows were regenerated (the canned text, the flag back + # to false — the rows are AI-written again). + assert _folder_row(BETA, "") == (NEW_BETA_ROOT, True) + assert _folder_row(BETA, "gamma") == (GAMMA_SUM, False) + assert _folder_row(ALPHA, "one") == (ONE_SUM, False) + + +# -------------------------------------------------------------------------- +# 7. A vanished location resets the view to the top level (never stale) +# -------------------------------------------------------------------------- + + +def test_reload_falls_back_to_top_level( + page: Page, app_url: str, synced_kb: None, db_ready: None +) -> None: + page.set_default_timeout(30_000) + + login(page, app_url) + _wait_top_level(page) + _drill(page, ALPHA, "two") + expect(page.locator("#kb-crumb .kb-crumb-current")).to_have_text("two") + + # The folder vanishes: its documents + chunks deleted directly (the + # house DB pattern — no sync, no API). + with SessionLocal() as db: + db.execute( + text( + "DELETE FROM chunks WHERE document_id IN " + "(SELECT id FROM documents WHERE source = :s AND path LIKE 'two/%')" + ), + {"s": ALPHA}, + ) + db.execute( + text("DELETE FROM documents WHERE source = :s AND path LIKE 'two/%'"), + {"s": ALPHA}, + ) + db.commit() + + # Re-show the RAG view: the nav-link RE-CLICK is the + # bor:view-refresh trigger (the router's re-fetch on the mounted + # view — no document load). + page.click("#nav-sources") + + # The vanished location resets the view to the top level BEFORE + # rendering (PLAN §7.4 — no stale breadcrumb, no stale block): the + # breadcrumb and the level block are hidden, the source rows + # re-render. + expect(page.locator("#kb-crumb")).to_be_hidden() + expect(page.locator("#kb-level")).to_be_hidden() + rows = page.locator("#folders-tbody tr") + expect(rows).to_have_count(2) + expect(rows.nth(0).locator("a.folder-link")).to_have_text(ALPHA) + expect(rows.nth(1).locator("a.folder-link")).to_have_text(BETA) + + +# -------------------------------------------------------------------------- +# 8. Anonymous: the sign-in gate — and zero catalog fetches +# -------------------------------------------------------------------------- + + +def test_anonymous_sees_the_gate( + page: Page, app_url: str, synced_kb: None, db_ready: None +) -> None: + page.set_default_timeout(30_000) + + docs_calls: list[str] = [] + page.on( + "request", + lambda r: docs_calls.append(r.url) if "/api/docs" in r.url else None, + ) + + # Fresh context (the function-scoped page fixture — no login). + page.goto(app_url + "/sources.html") + gate = page.locator("#sources-gate") + expect(gate).to_be_visible(timeout=30_000) + expect(gate).to_contain_text("Sign in to view the full catalog") + + # The catalog surfaces stay hidden (the tree ships hidden and never + # fills for anonymous)… + expect(page.locator("#stat-cards")).to_be_hidden() + expect(page.locator("#folders-wrap")).to_be_hidden() + expect(page.locator("#docs-table")).to_be_hidden() + expect(page.locator("#kb-crumb")).to_be_hidden() + expect(page.locator("#kb-level")).to_be_hidden() + expect(page.locator("#sources-empty")).to_be_hidden() + expect(page.locator("#folders-tbody tr")).to_have_count(0) + expect(page.locator("#docs-tbody tr")).to_have_count(0) + # …and there is no Edit affordance anywhere (the row buttons are + # never built; the static level button sits in the hidden block). + expect(page.locator(".kb-summary-edit:visible")).to_have_count(0) + expect(page.locator("#kb-level-edit")).to_be_hidden() + + # The soft rule's wire-level proof (the phase-16 pattern, the + # tree edition): not one catalog request — the anonymous gate + # branch never fetches /api/docs/tree. + assert docs_calls == [], f"anonymous RAG view fetched the catalog: {docs_calls}" diff --git a/tests/e2e/test_ls_tree_drilldown.py b/tests/e2e/test_ls_tree_drilldown.py index 148a0b1..3d4a4b1 100644 --- a/tests/e2e/test_ls_tree_drilldown.py +++ b/tests/e2e/test_ls_tree_drilldown.py @@ -393,6 +393,7 @@ def _truncate_all() -> None: "kb_overview, git_sources, folder_summaries" ) ) + db.commit() # without the commit the TRUNCATE rolls back (the house pattern) db.commit() diff --git a/tests/e2e/test_markdown_tables.py b/tests/e2e/test_markdown_tables.py index 998dc84..bb1da62 100644 --- a/tests/e2e/test_markdown_tables.py +++ b/tests/e2e/test_markdown_tables.py @@ -133,8 +133,14 @@ def _ask_table_answer(page: Page, app_url: str) -> Any: def _open_tables_doc_modal(page: Page, app_url: str) -> None: - """Admin → Sources → the tables.md row → same-page document modal.""" + """Admin → Sources → the tables.md row → same-page document modal. + + Phase 97: the catalog is the drill-down tree — the row lives at + its folder level (``docs`` → ``homelab``); the drill is the only + change.""" login(page, app_url) # phase 16: the Sources catalog is admin-only + for name in ("docs", "homelab"): # the fixture's indexed-only source + folder + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') row = page.locator("#docs-tbody tr", has_text=TABLES_PATH) expect(row).to_have_count(1) row.locator("td:nth-child(2) a.doc-link").click() diff --git a/tests/e2e/test_nav_consistency.py b/tests/e2e/test_nav_consistency.py index c440c9e..1d2f5d2 100644 --- a/tests/e2e/test_nav_consistency.py +++ b/tests/e2e/test_nav_consistency.py @@ -608,15 +608,16 @@ def test_viewer_nav_click_full_loads_the_shell_rag_view( # The arrival is the SHELL at the RAG view's URL: the document loaded # for real (the sentinel is gone), the RAG view is rendered (first - # table row visible), the Chat view is hidden AND inert in the same - # document, and the RAG link carries the router's single-writer - # active stamp. + # catalog row visible — phase 97: the top level lists the sources; + # the file table is per-level and hidden at the top), the Chat view + # is hidden AND inert in the same document, and the RAG link carries + # the router's single-writer active stamp. expect(page).to_have_url(app_url + SOURCES_URL, timeout=30_000) assert page.evaluate("() => window.__phase76_viewer") is None, ( "a surviving document's nav click must be a real departure " "(a fresh document load wipes window globals)" ) - expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=15_000) + expect(page.locator("#folders-tbody tr").first).to_be_visible(timeout=15_000) expect(page.locator("#view-chat")).to_be_hidden() assert page.evaluate("() => document.getElementById('view-chat').inert") is True, ( "the chat view ships hidden AND inert in the shell" diff --git a/tests/e2e/test_nav_switch_keeps_stream.py b/tests/e2e/test_nav_switch_keeps_stream.py index ee148d9..19a8d78 100644 --- a/tests/e2e/test_nav_switch_keeps_stream.py +++ b/tests/e2e/test_nav_switch_keeps_stream.py @@ -106,7 +106,9 @@ ANSWER = ".msg.brain .bubble:not(.typing)" #: The other three navbar views (test 2): the nav link, the view's #: URL (pushState target), and an admin-visible content marker inside #: the view (proof the view actually showed — the RAG view gets the -#: same treatment with ``#docs-tbody tr`` in test 1). +#: same treatment with ``#folders-tbody tr`` in test 1: phase 97's +#: top level lists the sources, the file table is per-level and +#: hidden at the top). OTHER_VIEWS: tuple[tuple[str, str, str], ...] = ( ("#nav-git-sources", "/git-sources.html", "#git-sources-content"), ("#nav-tuning", "/tuning.html", "#tune-save"), @@ -329,10 +331,11 @@ def test_rag_switch_mid_stream_completes( "a real navigation would have wiped the window sentinel — " "the switch must be same-document" ) - # The RAG view actually showed (the fixture docs' rows are listed) - # and the chat view is hidden (the stream fills it in the - # background — that persistence IS the fix). - expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=15_000) + # The RAG view actually showed (the source rows are listed — phase + # 97: the top level lists the sources; the file table is per-level + # and hidden at the top) and the chat view is hidden (the stream + # fills it in the background — that persistence IS the fix). + expect(page.locator("#folders-tbody tr").first).to_be_visible(timeout=15_000) expect(page.locator("#view-chat")).to_be_hidden() # Stay on the RAG view while the stream keeps running (the switch diff --git a/tests/e2e/test_navbar_refresh.py b/tests/e2e/test_navbar_refresh.py index aaff12c..d4544bc 100644 --- a/tests/e2e/test_navbar_refresh.py +++ b/tests/e2e/test_navbar_refresh.py @@ -109,7 +109,10 @@ TITLE_E = "Phase77 popstate E" # pollers hit OTHER paths and only run while a job is in flight). DATA_VIEWS: tuple[tuple[str, str, str], ...] = ( ("#nav-history", "/history.html", "/api/chats"), - ("#nav-sources", "/sources.html", "/api/docs"), + # Phase 97: the RAG view's list endpoint is the drill-down tree + # (GET /api/docs/tree — the flat /api/docs is the unchanged API + # surface, no longer the view's fetch). + ("#nav-sources", "/sources.html", "/api/docs/tree"), ("#nav-git-sources", "/git-sources.html", "/api/git-sources"), ("#nav-tuning", "/tuning.html", "/api/steering"), ) @@ -568,10 +571,11 @@ def test_stream_survival_control(page: Page, app_url: str, mock_llm: int, db_rea assert page.evaluate("() => window.__shell_boot") == "phase77", ( "a real navigation would have wiped the window sentinel" ) - # The RAG view actually showed (the fixture docs' rows are listed) - # and the chat view is hidden (the stream fills it in the - # background — that persistence IS the phase-76 fix). - expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=15_000) + # The RAG view actually showed (the source rows are listed — phase + # 97: the top level lists the sources; the file table is per-level + # and hidden at the top) and the chat view is hidden (the stream + # fills it in the background — that persistence IS the phase-76 fix). + expect(page.locator("#folders-tbody tr").first).to_be_visible(timeout=15_000) expect(page.locator("#view-chat")).to_be_hidden() # Stay on RAG while the stream keeps running, then return to Chat. diff --git a/tests/e2e/test_quadlet_jinja_import.py b/tests/e2e/test_quadlet_jinja_import.py index 7c4f2dc..29d2270 100644 --- a/tests/e2e/test_quadlet_jinja_import.py +++ b/tests/e2e/test_quadlet_jinja_import.py @@ -26,6 +26,11 @@ Test → story mapping (Playwright Mapping Rule): below threshold **and** zero FTS hits) → honest-positive: the answer bubble is not ``.is-deflected`` and a source chip names ``templates/deploy.j2``. + +Phase 97 adaptation: the Sources table is the DRILL-DOWN TREE — the +rows live at their folder levels (``docs`` → ``homelab`` → ``quadlet`` +/ ``templates``); the drill is the only change, the asserted +rows/links are unchanged. """ from __future__ import annotations @@ -99,6 +104,20 @@ def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None: return _run_in_thread(_import_fixtures(mock_port)) +def _drill(page: Page, *names: str) -> None: + """Drill one level at a time (phase 97 — client-side, no fetch, + no URL change): each name is the EXACT text of the source/folder + link at the current level.""" + for name in names: + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + + +def _go_top(page: Page) -> None: + """Back to the top level: the breadcrumb's top-level link (call + between drills only — the breadcrumb is hidden at the top).""" + page.locator("#kb-crumb a.kb-crumb-link").first.click() + + # --------------------------------------------------------------------------- # 1. Default-extensions import indexes the new formats (API view) # --------------------------------------------------------------------------- @@ -133,7 +152,12 @@ def test_quadlet_and_jinja_indexed(page: Page, app_url: str, mock_llm: int, db_r def test_sources_table_shows_them(page: Page, app_url: str, mock_llm: int, db_ready: None) -> None: _reset_db(mock_llm, seed=True) login(page, app_url) # phase 16: the Sources catalog is admin-only - for path in NEW_FORMAT_DOCS: + for i, path in enumerate(NEW_FORMAT_DOCS): + # Phase 97: the rows live at their folder levels — drill + # source → folder(s) per path (the drill is the only change). + if i > 0: + _go_top(page) + _drill(page, "docs", *path.rsplit("/", 1)[0].split("/")) row = page.locator("#docs-tbody tr", has_text=path) expect(row).to_have_count(1) link = row.locator("td:nth-child(2) a.doc-link") @@ -154,6 +178,8 @@ def test_container_content_viewable( _reset_db(mock_llm, seed=True) login(page, app_url) # phase 16: the Sources catalog is admin-only + # Phase 97: the row lives at its folder level — drill first. + _drill(page, "docs", "homelab", "quadlet") row = page.locator("#docs-tbody tr", has_text="homelab/quadlet/compose.container") expect(row).to_have_count(1) link = row.locator("td:nth-child(2) a.doc-link") diff --git a/tests/e2e/test_responsive_polish.py b/tests/e2e/test_responsive_polish.py index d55306f..0114669 100644 --- a/tests/e2e/test_responsive_polish.py +++ b/tests/e2e/test_responsive_polish.py @@ -213,7 +213,10 @@ def test_no_horizontal_overflow_at_viewports( _assert_no_doc_overflow(page, f"chat @ {width}px") page.goto(f"{app_url}/sources.html") # phase 16: admin-only - page.locator("#docs-tbody tr").first.wait_for(state="visible", timeout=10_000) + # Phase 97: the top level lists the sources (the file table + # is per-level, always hidden at the top) — the source row + # is the catalog-rendered signal. + page.locator("#folders-tbody tr").first.wait_for(state="visible", timeout=10_000) _assert_no_doc_overflow(page, f"sources @ {width}px") finally: page.close() @@ -276,10 +279,14 @@ def test_sources_table_full_width( page = browser.new_page(viewport={"width": 1280, "height": 800}) try: login(page, app_url, next="/sources.html") # phase 16: admin-only - page.locator("#docs-tbody tr").first.wait_for(state="visible", timeout=10_000) - # Phase 76 (task 02): the shell carries BOTH views' .table-wrap - # (the git-sources one ships hidden) — scope to the RAG view. - wrap_box = page.locator("#view-rag .table-wrap").bounding_box() + # Phase 97: the top level lists the sources (the file table is + # per-level, hidden at the top) — the source row is the + # catalog-rendered signal, and the visible catalog card is + # #folders-wrap (the ONE .table-wrap without an id in the RAG + # view is no longer unique — the shell carries the git-sources + # one too, so scope by the id). + page.locator("#folders-tbody tr").first.wait_for(state="visible", timeout=10_000) + wrap_box = page.locator("#folders-wrap").bounding_box() shell_box = page.locator(".sources-shell").bounding_box() assert wrap_box is not None and shell_box is not None assert wrap_box["width"] >= 0.80 * shell_box["width"], ( @@ -292,9 +299,11 @@ def test_sources_table_full_width( mobile = browser.new_page(viewport={"width": 375, "height": 812}) try: login(mobile, app_url, next="/sources.html") # phase 16: admin-only - mobile.locator("#docs-tbody tr").first.wait_for(state="visible", timeout=10_000) + # Phase 97: the top-level catalog card is #folders-wrap (the + # .kb-folders-table keeps its min-width and scrolls in-card). + mobile.locator("#folders-tbody tr").first.wait_for(state="visible", timeout=10_000) scroll, client = mobile.evaluate( - "() => { const el = document.querySelector('#view-rag .table-wrap');" + "() => { const el = document.querySelector('#folders-wrap');" " return [el.scrollWidth, el.clientWidth]; }" ) assert scroll > client, ( @@ -366,9 +375,14 @@ def test_a11y_landmarks_and_labels( page.wait_for_load_state("networkidle") expect(page.locator("#auth-gate")).to_be_hidden() - # Landmarks (PLAN §7.2). + # Landmarks (PLAN §7.2). Phase 97: the shell carries a SECOND + # nav in the RAG view (#kb-crumb, the catalog breadcrumb — + # hidden at the top level, pinned in test_kb_tree), so the + # primary-nav landmark is pinned by its label. assert page.locator("header.app-header").count() == 1, f"header missing on {path}" - assert page.locator("nav[aria-label]").count() == 1, f"labeled nav missing on {path}" + assert page.locator("nav[aria-label='Primary']").count() == 1, ( + f"labeled primary nav missing on {path}" + ) assert page.locator("main#main").count() == 1, f"main#main missing on {path}" assert page.locator("footer.app-footer").count() == 1, f"footer missing on {path}" @@ -577,6 +591,11 @@ def test_long_content_wraps_without_overflow( phone = browser.new_page(viewport={"width": 360, "height": 740}) try: login(phone, app_url, next="/sources.html") # phase 16: admin-only + # Phase 97: the catalog is the drill-down tree — the row lives + # at the deep level of the longkb source (the drill is the only + # change). + for name in ("longkb", "deep"): + phone.click(f'#folders-tbody a.folder-link:text-is("{name}")') row = phone.locator("#docs-tbody tr", has_text="backup_rotation").first row.wait_for(state="visible", timeout=10_000) cell = row.get_by_role("cell").nth(1) diff --git a/tests/e2e/test_retrieval_quality.py b/tests/e2e/test_retrieval_quality.py index 47bc3dc..e982f88 100644 --- a/tests/e2e/test_retrieval_quality.py +++ b/tests/e2e/test_retrieval_quality.py @@ -118,8 +118,15 @@ def test_multi_format_import_hidden_doc_excluded( } # The Sources page (we're already on it, signed in) reflects the set. + # Phase 97: the file table is hidden at the top level (the source + # row is the tree's top) — drill into the source, where a + # `.hidden` junk row WOULD appear, and scope the absence there + # (the stat card pins the 13 total: the junk is counted nowhere). expect(page.locator("#stat-docs")).to_have_text("13") + page.locator("#folders-tbody .folder-link").first.wait_for(state="visible") + page.click('#folders-tbody a.folder-link:text-is("docs")') expect(page.locator("#docs-tbody tr", has_text=".hidden")).to_have_count(0) + expect(page.locator("#folders-tbody .folder-link", has_text=".hidden")).to_have_count(0) def test_gitlab_question_is_grounded_with_gitlab_chip( diff --git a/tests/e2e/test_shared_header.py b/tests/e2e/test_shared_header.py index a016d4c..2bcda97 100644 --- a/tests/e2e/test_shared_header.py +++ b/tests/e2e/test_shared_header.py @@ -266,7 +266,10 @@ def test_admin_bar_on_all_pages( page.goto(app_url + SOURCES_URL) expect(page.locator("#sources-gate")).to_be_hidden() - expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=15_000) + # Phase 97: the top level lists the sources (the file table is + # per-level, hidden at the top) — the source row is the + # catalog-rendered signal. + expect(page.locator("#folders-tbody tr").first).to_be_visible(timeout=15_000) assert_shared_bar(page, admin=True, page_kind="sources") page.goto(app_url + VIEWER_URL) diff --git a/tests/e2e/test_sources_midstream_bug.py b/tests/e2e/test_sources_midstream_bug.py index eb64c59..cd7fa92 100644 --- a/tests/e2e/test_sources_midstream_bug.py +++ b/tests/e2e/test_sources_midstream_bug.py @@ -305,8 +305,10 @@ def test_full_answer_completes_after_rag_nav_midstream( page.click("#nav-sources") expect(page).to_have_url(app_url + "/sources.html") assert page.evaluate("() => window.__shell_boot") == "phase76" - # The RAG view actually mounted (the fixture docs' rows are listed). - expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=15_000) + # The RAG view actually mounted (the source rows are listed — phase + # 97: the top level lists the sources; the file table is per-level + # and hidden at the top). + expect(page.locator("#folders-tbody tr").first).to_be_visible(timeout=15_000) # Stay on the RAG view while the stream keeps running in the # background (the switch is ~t+2s; the full answer needs ~9s). @@ -381,7 +383,9 @@ def test_nav_switch_before_first_token_completes( page.click("#nav-sources") expect(page).to_have_url(app_url + "/sources.html") assert page.evaluate("() => window.__shell_boot") == "phase76" - expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=15_000) + # Phase 97: the source rows are the top-level catalog-rendered + # signal (the file table is per-level, hidden at the top). + expect(page.locator("#folders-tbody tr").first).to_be_visible(timeout=15_000) # Let the 4s pre-token pause elapse WHILE the RAG view is up — the # first content frames land while the chat view is still hidden — diff --git a/tests/e2e/test_sticky_navbar.py b/tests/e2e/test_sticky_navbar.py index e7d65a3..75d78a2 100644 --- a/tests/e2e/test_sticky_navbar.py +++ b/tests/e2e/test_sticky_navbar.py @@ -214,7 +214,14 @@ def test_app_header_stuck_at_top_on_sources( ) -> None: page.set_default_timeout(30_000) login(page, app_url) # → /sources.html (the table is admin-only) - expect(page.locator("#docs-tbody tr")).to_have_count(TOTAL_DOCS) + # Phase 97: the catalog is the drill-down tree — the top level + # lists the two sources (the KB total rides the stat cards; the + # flat 55-row tbody no longer exists). The LONG page the story + # needs is a file level: drill into `gen` (41 direct docs). + page.locator("#folders-tbody .folder-link").first.wait_for(state="visible") + expect(page.locator("#stat-docs")).to_have_text(str(TOTAL_DOCS)) + page.click('#folders-tbody a.folder-link:text-is("gen")') + expect(page.locator("#docs-tbody tr")).to_have_count(TOTAL_DOCS - 13) # The precondition the story needs: the page must actually scroll — # fail loudly if the table ever stops being long enough. diff --git a/tests/e2e/test_summary_in_viewer.py b/tests/e2e/test_summary_in_viewer.py index 1930489..b9a1ceb 100644 --- a/tests/e2e/test_summary_in_viewer.py +++ b/tests/e2e/test_summary_in_viewer.py @@ -222,6 +222,10 @@ def test_modal_shows_panel_and_full_page_agrees( login(page, app_url) # phase 16: the Sources catalog is admin-only digest_line, pointer_line = _summary_lines(SOURCE, YAML_PATH) + # Phase 97: the catalog is the drill-down tree — the row lives at + # the quadlet level (the drill is the only change). + for name in (SOURCE, "quadlet"): + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') row = page.locator("#docs-tbody tr", has_text=YAML_PATH) expect(row).to_have_count(1) before = len(page.context.pages) @@ -301,6 +305,11 @@ def test_markdown_doc_has_no_summary_panel( # Modal: same story — no panel, .doc-md is the sole content child. # (the session is already signed in — the form login above) page.goto(f"{app_url}/sources.html") + # Phase 97: the re-mount lands on the tree's top level — drill to + # the notes level where the row lives (the drill is the only + # change). + for name in (SOURCE, "notes"): + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') row = page.locator("#docs-tbody tr", has_text=MD_PATH) expect(row).to_have_count(1) row.locator("td:nth-child(2) a.doc-link").click() diff --git a/tests/e2e/test_sync_button.py b/tests/e2e/test_sync_button.py index 1fdfd85..0525328 100644 --- a/tests/e2e/test_sync_button.py +++ b/tests/e2e/test_sync_button.py @@ -185,6 +185,29 @@ def _overview_row() -> KbOverview | None: return db.get(KbOverview, 1) +# The git fixture repo's directory name — the source name the import +# records for it (the repo_name basename rule, phase 28). +GIT_SOURCE_NAME = "homelab-notes" + + +def _go_top(page: Page) -> None: + """Back to the top level (no-op when already there — the + breadcrumb is hidden at the top).""" + if not page.locator("#kb-crumb").is_hidden(): + page.locator("#kb-crumb a.kb-crumb-link").first.click() + + +def _expect_fixture_row(page: Page) -> None: + """The REAL clone was imported — the fixture path is listed. Phase + 97: the catalog is the drill-down tree, so drill source → folder + ``notes`` first (the flat all-docs row is the same row, one level + deeper).""" + _go_top(page) + page.click(f'#folders-tbody a.folder-link:text-is("{GIT_SOURCE_NAME}")') + page.click('#folders-tbody a.folder-link:text-is("notes")') + expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1) + + def _wait_sync_done(page: Page, app_url: str, timeout_s: float = 60.0) -> dict[str, Any]: """Poll the (cookie-authenticated) status endpoint until the run reaches a terminal state — exactly what the UI's 2 s poll loop @@ -244,8 +267,9 @@ def test_admin_sync_lifecycle(page: Page, app_url: str, db_ready: None) -> None: expect(page.locator("#sync-result")).to_have_text("1 added") # The REAL clone was imported: the fixture path is in the Sources - # table (the sentinel lives inside it). - expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1) + # table (the sentinel lives inside it) — via the drill-down tree + # (phase 97). + _expect_fixture_row(page) # Phase-31 regeneration ran (the import changed the KB): the single # kb_overview row is fresh and non-empty (DB check — truncated @@ -264,8 +288,9 @@ def test_admin_sync_lifecycle(page: Page, app_url: str, db_ready: None) -> None: # Nothing re-embedded (sha256 delta) — the no-op run announces the # unchanged count instead of an empty live region. expect(page.locator("#sync-result")).to_have_text("0 added · 1 unchanged") - # The doc survived the prune re-import (its file is still in the repo). - expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1) + # The doc survived the prune re-import (its file is still in the + # repo) — via the drill-down tree (phase 97). + _expect_fixture_row(page) # --- 3. Concurrency: one sync at a time ------------------------------------ diff --git a/tests/e2e/test_sync_model_down.py b/tests/e2e/test_sync_model_down.py index b8a3aa3..9ee962c 100644 --- a/tests/e2e/test_sync_model_down.py +++ b/tests/e2e/test_sync_model_down.py @@ -267,6 +267,29 @@ def _overview_row() -> KbOverview | None: return db.get(KbOverview, 1) +# The git fixture repo's directory name — the source name the import +# records for it (the repo_name basename rule, phase 28). +GIT_SOURCE_NAME = "homelab-notes" + + +def _go_top(page: Page) -> None: + """Back to the top level (no-op when already there — the + breadcrumb is hidden at the top).""" + if not page.locator("#kb-crumb").is_hidden(): + page.locator("#kb-crumb a.kb-crumb-link").first.click() + + +def _expect_fixture_row(page: Page) -> None: + """The REAL clone was imported — the fixture path is listed. Phase + 97: the catalog is the drill-down tree, so drill source → folder + ``notes`` first (the flat all-docs row is the same row, one level + deeper).""" + _go_top(page) + page.click(f'#folders-tbody a.folder-link:text-is("{GIT_SOURCE_NAME}")') + page.click('#folders-tbody a.folder-link:text-is("notes")') + expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1) + + def _dismiss_reattached_modal(page: Page) -> None: """Close the modal the page load RE-ATTACHED to (phase 32/41 behavior): the dead app's status is still ``failed`` from an @@ -433,8 +456,9 @@ def test_healthy_sync_still_succeeds( expect(page.locator("#sync-result")).to_have_text("1 added") # The REAL clone was imported: the fixture path is in the Sources - # table (the sentinel lives inside it). - expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1) + # table (the sentinel lives inside it) — via the drill-down tree + # (phase 97). + _expect_fixture_row(page) # Phase-31 regeneration ran (the import changed the KB): the single # kb_overview row is fresh and non-empty (DB check — truncated @@ -451,7 +475,9 @@ def test_healthy_sync_still_succeeds( expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) expect(btn).to_be_enabled() expect(page.locator("#sync-result")).to_have_text("0 added · 1 unchanged") - expect(page.locator("#docs-tbody tr", has_text=FIXTURE_DOC)).to_have_count(1) + # The doc survived the prune re-import — via the drill-down tree + # (phase 97). + _expect_fixture_row(page) # And no modal ever opened (no failure on the healthy app). expect(page.locator(".sync-modal")).to_have_count(0) diff --git a/tests/e2e/test_sync_upload_progress.py b/tests/e2e/test_sync_upload_progress.py index af5480f..02e6f3f 100644 --- a/tests/e2e/test_sync_upload_progress.py +++ b/tests/e2e/test_sync_upload_progress.py @@ -568,7 +568,12 @@ def test_upload_toast_then_navigate_away( # page). page.goto(app_url + SOURCES_URL) - # Phase 90 A1: the upload indexed NOTHING — the catalog is empty… + # Phase 90 A1: the upload indexed NOTHING — the catalog shows the + # registered source with 0 documents (phase 97: the source row IS + # the load-settled signal; the file table stays hidden/empty)… + row = page.locator("#folders-tbody tr", has_text=UPLOAD_NAME) + expect(row).to_have_count(1, timeout=30_000) + expect(row.locator("td:nth-child(2)")).to_have_text("0") expect(page.locator("#docs-tbody tr")).to_have_count(0) # …and the sync button settles idle with no error UI (the # sub-second upload run is over by the time this page's 2 s poll @@ -623,6 +628,11 @@ def test_upload_toast_then_navigate_away( expect(btn).to_be_enabled() expect(btn).not_to_have_attribute("aria-busy") expect(page.locator("#sync-result")).to_have_text(f"{N_FILES} added") + # Phase 97: the catalog is the drill-down tree — drill source → + # folder `docs` before the per-level row asserts (the flat total + # no longer exists in one tbody). + page.click(f'#folders-tbody a.folder-link:text-is("{UPLOAD_NAME}")') + page.click('#folders-tbody a.folder-link:text-is("docs")') expect(page.locator("#docs-tbody tr")).to_have_count(N_FILES, timeout=30_000) expect(page.locator("#docs-tbody tr", has_text="docs/00.md")).to_have_count(1) expect(page.locator("#docs-tbody tr", has_text="docs/24.md")).to_have_count(1) @@ -738,7 +748,17 @@ def test_sync_live_file_label( btn = page.locator("#sync-btn") expect(btn).to_be_visible(timeout=30_000) - expect(page.locator("#sync-label")).to_have_text("Sync sources") + # The button is retry-ready at boot: idle ("Sync sources") — or the + # RE-ATTACHED terminal from a prior test's sync (the module app is + # shared, its in-memory sync state survives the test boundary, and + # the phase-32 boot re-attach renders the last result — "Synced + # HH:MM" after test 1's successful run). Either way the button is + # enabled and a click starts THIS test's run (the assertions below + # are the run's own). + expect(page.locator("#sync-label")).to_have_text( + re.compile(r"^(Sync sources|Synced \d{1,2}:\d{2})$") + ) + expect(btn).to_be_enabled() recorder = _TickRecorder(app_url, "/api/sync/status") recorder.start() @@ -778,6 +798,11 @@ def test_sync_live_file_label( expect(btn).to_be_enabled() expect(btn).not_to_have_attribute("aria-busy") expect(page.locator("#sync-result")).to_have_text(f"{N_FILES} added") + # Phase 97: the catalog is the drill-down tree — drill source → + # folder `notes` before the per-level row asserts (the flat total + # no longer exists in one tbody). + page.click(f'#folders-tbody a.folder-link:text-is("{SYNC_SOURCE_DIR}")') + page.click('#folders-tbody a.folder-link:text-is("notes")') expect(page.locator("#docs-tbody tr")).to_have_count(N_FILES, timeout=30_000) expect(page.locator("#docs-tbody tr", has_text="notes/00.md")).to_have_count(1) expect(page.locator("#docs-tbody tr", has_text=SYNC_SOURCE_DIR)).to_have_count(N_FILES) diff --git a/tests/e2e/test_theme_semantic_completion.py b/tests/e2e/test_theme_semantic_completion.py index 6d77c47..0bfa7fd 100644 --- a/tests/e2e/test_theme_semantic_completion.py +++ b/tests/e2e/test_theme_semantic_completion.py @@ -122,7 +122,7 @@ from sqlalchemy.orm import Session from app.config import Settings from app.core.theming import COLOR_FIELDS from app.db import SessionLocal -from app.models import Chunk, Document, SavedChat, UiSettings +from app.models import Chunk, Document, GitSource, SavedChat, UiSettings from app.rag.sources_meta import bump_sources_version from e2e.auth_helpers import login from e2e.conftest import ( @@ -823,7 +823,22 @@ def _seed_tool_docs(db: Session) -> None: """The two-document pair (see the constants above): the retrievable grounding document (one chunk carrying the mock's own bag-of-words embedding) + the catalog-only read target (no chunks — retrieval - never puts it in context, so the agent's read tool accepts it).""" + never puts it in context, so the agent's read tool accepts it). + + Phase 94: the drill-down ``ls`` top level reads the registry — the + seed registers BOTH sources (TRUNCATEd in + ``_reset_kb_with_tool_docs``), ``Checklist`` FIRST: registry order + is ``(added_at, id)``, so the mock's drill (first source of the + listing) — and therefore the read — lands on ``read-me.md`` + deterministically, independent of the shared dev DB's leftover + registry rows.""" + # COMMIT between the inserts (not flush): ``added_at`` is + # ``server_default now()`` — the transaction timestamp — and the + # tie-break is the random uuid ``id``, so one-transaction rows order + # nondeterministically. + db.add(GitSource(url=READ_SOURCE, kind="local")) + db.commit() + db.add(GitSource(url=TOOL_DOC_SOURCE, kind="local")) db.add( Document( source=TOOL_DOC_SOURCE, @@ -866,11 +881,18 @@ def _seed_tool_docs(db: Session) -> None: def _reset_kb_with_tool_docs() -> None: """Truncate the KB tables (the house reset) and seed the pair — a direct DB seed, so the turn is genuinely grounded and the mock's - single-read flow runs to completion: ls → read on the first catalog - line (the catalog-only document) → the quoted answer.""" + single-read flow runs to completion: ls → drill ls (phase 94 — the + top level lists sources only) → read on the first file line (the + catalog-only document) → the quoted answer. Phase 94: ``git_sources`` + joins the TRUNCATE — the top-level listing reads the registry, so + the leftover rows of other suites would change what the drill + targets.""" with SessionLocal() as db: db.execute( - text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview") + text( + "TRUNCATE chunks, documents, query_log, steering_notes, " + "kb_overview, git_sources" + ) ) db.commit() _seed_tool_docs(db) @@ -904,20 +926,25 @@ def test_tool_call_lines_gray(page: Page, app_url: str, db_ready: None) -> None: # Screenshot 1: the yellow "Listing documents" / "Reading" lines — # now gray accent-ink text with the gray accent-line left border, - # both lines' text intact. + # the lines' text intact. Phase 94: the drill-down ls adds a THIRD + # line between them — the drill ls scoped to the first source of + # the top level (registry order: Checklist — the read target's + # source). lines = page.locator(".msg.brain .tool-call") - expect(lines).to_have_count(2) + expect(lines).to_have_count(3) expect(lines.nth(0)).to_contain_text("Listing documents") - expect(lines.nth(1)).to_contain_text("Reading") - expect(lines.nth(1)).to_contain_text(READ_SP) + expect(lines.nth(1)).to_contain_text("Listing documents in") + expect(lines.nth(1)).to_contain_text(READ_SOURCE) + expect(lines.nth(2)).to_contain_text("Reading") + expect(lines.nth(2)).to_contain_text(READ_SP) _assert_gray(lines.nth(0), "color", GRAY["accent_ink"], label="ls line text") _assert_gray( lines.nth(0), "borderLeftColor", GRAY["accent_line"], label="ls line border" ) - _assert_gray(lines.nth(1), "color", GRAY["accent_ink"], label="read line text") + _assert_gray(lines.nth(2), "color", GRAY["accent_ink"], label="read line text") # The path chip on the Reading line: gray brand-soft background + # gray ink (the screenshot's code chip — still gray under the ramp). - code = lines.nth(1).locator("code") + code = lines.nth(2).locator("code") _assert_gray(code, "backgroundColor", GRAY["brand_soft"], label="read chip bg") _assert_gray(code, "color", GRAY["ink"], label="read chip text") diff --git a/tests/e2e/test_upload_no_scan.py b/tests/e2e/test_upload_no_scan.py index f2ca684..120bf24 100644 --- a/tests/e2e/test_upload_no_scan.py +++ b/tests/e2e/test_upload_no_scan.py @@ -60,7 +60,9 @@ Contract under test: control) and its folder exists on the host under ``BOR_UPLOAD_DIR`` with all three files — but **zero documents are indexed**: ``GET /api/docs`` is empty and the RAG catalog (``/sources.html``) - settles on its empty state; + settles on the source's 0-document row (phase 97: ``#sources-empty`` + is the zero-SOURCES state only — a registered 0-document source + renders its ``0`` count instead); * **test 2 — ignore list, then Sync scans**: fresh state → upload → success line → the row's phase-89 "Ignore paths" editor: type ``notes``, Save → the row shows the "1 ignored" count tag (the A5 @@ -465,12 +467,17 @@ def test_upload_does_not_scan( assert _folder_files(folder) == {"alpha.md", "beta.md", "notes/skipme.md"}, ( f"unexpected unpacked files: {_folder_files(folder)}" ) - # …and so is the RAG catalog: it settles on its empty state (the - # visible #sources-empty is the deterministic "the load finished - # with zero documents" signal — a count-0 check alone would race - # the boot loadDocs fetch). + # …and so is the RAG catalog: the registered source renders its + # 0-document row (phase 97: the "zero documents" signal moved from + # #sources-empty — the zero-SOURCES state only — to the source + # row's own `0` count, which is ALSO the deterministic load-settled + # signal; a count-0 check alone would race the boot loadTree + # fetch). page.goto(app_url + SOURCES_URL) - expect(page.locator("#sources-empty")).to_be_visible(timeout=30_000) + row = page.locator("#folders-tbody tr", has_text=SOURCE_NAME) + expect(row).to_have_count(1, timeout=30_000) + expect(row.locator("td:nth-child(2)")).to_have_text("0") + expect(page.locator("#sources-empty")).to_be_hidden() expect(page.locator("#docs-tbody tr")).to_have_count(0) expect(page.locator("#stat-docs")).to_have_text("0") @@ -533,10 +540,13 @@ def test_ignore_list_then_sync_scans( ("local", str(upload_dir / SOURCE_NAME), [IGNORE_ENTRY]) ] - # The RAG page — the catalog is still empty before the scan… + # The RAG page — the catalog is still empty before the scan (the + # source row reads 0 — phase 97's load-settled zero signal)… page.goto(app_url + SOURCES_URL) expect(page.locator("#sync-btn")).to_be_visible(timeout=30_000) - expect(page.locator("#sources-empty")).to_be_visible(timeout=30_000) + row = page.locator("#folders-tbody tr", has_text=SOURCE_NAME) + expect(row).to_have_count(1, timeout=30_000) + expect(row.locator("td:nth-child(2)")).to_have_text("0") expect(page.locator("#sync-label")).to_have_text("Sync sources") expect(page.locator("#sync-error-banner")).to_be_hidden() @@ -566,7 +576,12 @@ def test_ignore_list_then_sync_scans( assert status["files_done"] == 2 and status["files_total"] == 2 # The catalog: exactly the two non-ignored docs, for the source — - # and the ignored one is NOT there. + # and the ignored one is NOT there. Phase 97: files are seen per + # source — drill into the source row first (the two docs sit at + # its root; the ignored notes/ folder is not even listed — the + # existence rule runs over INDEXED paths). + page.click(f'#folders-tbody a.folder-link:text-is("{SOURCE_NAME}")') + expect(page.locator("#folders-wrap")).to_be_hidden() # no subfolders expect(page.locator("#docs-tbody tr")).to_have_count(2, timeout=30_000) expect(page.locator("#docs-tbody tr", has_text=SOURCE_NAME)).to_have_count(2) expect(page.locator("#docs-tbody tr", has_text="alpha.md")).to_have_count(1) diff --git a/tests/integration/test_docs_api.py b/tests/integration/test_docs_api.py index 7a7a2d7..c79ab96 100644 --- a/tests/integration/test_docs_api.py +++ b/tests/integration/test_docs_api.py @@ -1,15 +1,68 @@ -"""Integration tests: GET /api/docs — empty shape + populated shape. +"""Integration tests: GET /api/docs — empty shape + populated shape — +and GET /api/docs/tree (phase 97, task 02 — the full recursive KB tree +the RAG view renders: shape, ordering, counts, summaries, the 403 gate, +the superset rule, and the stat-walk equivalence with ``GET /api/docs``). Uses the real compose Postgres (``db`` fixture) and FastAPI's TestClient. """ from __future__ import annotations +import inspect +import itertools import uuid -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta -from sqlalchemy import text +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import func, select, text -from app.models import Chunk, Document +import app.api.docs as docs_api +from app.config import Settings +from app.core import tokens as token_service +from app.main import app as fastapi_app +from app.models import Chunk, Document, FolderSummary, GitSource +from app.rag import git_sources as rag_git_sources + +_TREE_TABLES = "chunks, documents, folder_summaries, git_sources" + + +def _truncate_tree_tables(db) -> None: + db.execute(text(f"TRUNCATE {_TREE_TABLES}")) + db.commit() + + +_counter = itertools.count() + + +def _seed_doc(db, source: str, path: str, title: str, n_chunks: int, indexed_at: datetime) -> None: + """One indexed document with *n_chunks* content chunks (unique hash).""" + doc = Document( + source=source, + path=path, + full_path=f"/tmp/{source}/{path}", + title=title, + content=f"# {title}\n\nBody.", + content_hash=f"{next(_counter):064d}", + indexed_at=indexed_at, + ) + db.add(doc) + db.flush() + db.add_all( + Chunk(document_id=doc.id, position=i, content=f"chunk {i}", embedding=[0.01] * 768) + for i in range(n_chunks) + ) + + +def _tree_file_nodes(sources) -> list[dict]: + """Every file node of a tree response, walked recursively.""" + files: list[dict] = [] + for source in sources: + for child in source["children"]: + if child["kind"] == "file": + files.append(child) + else: + files.extend(_tree_file_nodes([child])) + return files def test_docs_empty_shape(admin_client, db) -> None: @@ -77,3 +130,521 @@ def test_docs_response_matches_schema_shape(admin_client, db) -> None: for d in body["documents"]: assert set(d) == {"id", "source", "path", "title", "chunks", "indexed_at"} assert isinstance(d["chunks"], int) and d["chunks"] >= 0 + + +# -------------------------------------------------------------------- +# GET /api/docs/tree (phase 97, task 02). +# -------------------------------------------------------------------- + + +def test_docs_tree_populated_shape_order_counts_summaries(admin_client, db) -> None: + _truncate_tree_tables(db) + base = datetime.now(UTC) + # Registry order (added_at) is Homelab → Deployments — deliberately + # NOT alphabetical (the registry order leads, the phase-97 rule). + db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) + db.add( + GitSource( + url="https://github.com/reese/Deployments.git", + kind="git", + added_at=base + timedelta(hours=1), + ) + ) + _seed_doc(db, "Homelab", "k8s/talos.md", "Talos", 3, base) + _seed_doc(db, "Homelab", "k8s/cluster.md", "Cluster", 2, base) + _seed_doc(db, "Homelab", "k8s/helm/charts.md", "Charts", 1, base) + _seed_doc(db, "Homelab", "root-note.md", "Root note", 4, base) + _seed_doc(db, "Deployments", "deploy-a.md", "Deploy A", 1, base) + _seed_doc(db, "Deployments", "deploy-b.md", "Deploy B", 0, base) + db.add_all( + [ + FolderSummary(source="Homelab", folder_path="", summary="Homelab docs."), + FolderSummary(source="Homelab", folder_path="k8s", summary="K8s stuff."), + # A stored row for a folder with NO indexed descendants — + # e.g. a manual row surviving the prune below the 2-doc + # minimum (phase 97, task 01): the tree's folders come from + # INDEXED paths (the existence rule), not from summary rows. + FolderSummary(source="Deployments", folder_path="orphan", summary="Ghost."), + ] + ) + db.commit() + + r = admin_client.get("/api/docs/tree") + assert r.status_code == 200 + body = r.json() + assert set(body) == {"sources"} + sources = body["sources"] + assert [s["name"] for s in sources] == ["Homelab", "Deployments"] + + homelab, deployments = sources + assert set(homelab) == {"name", "documents", "summary", "children"} + assert homelab["documents"] == 4 # the whole recursive count + assert homelab["summary"] == "Homelab docs." # the (source, "") row + assert deployments["summary"] is None # no stored root row + assert deployments["documents"] == 2 + + # Homelab: subfolder first (path order among folders), then the + # root files; the nested folder k8s/helm recurses one level deeper. + k8s, root_note = homelab["children"] + assert k8s["kind"] == "folder" + assert k8s["path"] == "k8s" + assert k8s["documents"] == 3 # talos + cluster + charts (subtree) + assert k8s["summary"] == "K8s stuff." + helm, cluster, talos = k8s["children"] + assert (helm["kind"], helm["path"], helm["documents"], helm["summary"]) == ( + "folder", + "k8s/helm", + 1, + None, + ) + assert [ + (c["kind"], c["path"], c["title"], c["chunks"]) + for c in (cluster, talos) + ] == [("file", "k8s/cluster.md", "Cluster", 2), ("file", "k8s/talos.md", "Talos", 3)] + chart = helm["children"][0] + assert (chart["kind"], chart["path"], chart["title"], chart["chunks"]) == ( + "file", + "k8s/helm/charts.md", + "Charts", + 1, + ) + datetime.fromisoformat(chart["indexed_at"]) # valid ISO-8601 + assert (root_note["kind"], root_note["path"], root_note["title"], root_note["chunks"]) == ( + "file", + "root-note.md", + "Root note", + 4, + ) + + # Deployments: flat — direct files in catalog order, and the + # ``orphan`` summary row does NOT create a folder node. + assert [(c["kind"], c["path"]) for c in deployments["children"]] == [ + ("file", "deploy-a.md"), + ("file", "deploy-b.md"), + ] + + _truncate_tree_tables(db) + + +def test_docs_tree_403_anonymous(client, db) -> None: + """Admin-only, like ``GET /api/docs``: anonymous → 403 + ``admin only`` (the RAG view's anonymous gate never fetches the + tree).""" + _truncate_tree_tables(db) + r = client.get("/api/docs/tree") + assert r.status_code == 403 + assert r.json() == {"detail": "admin only"} + + +def test_docs_tree_empty_registry_and_catalog( + admin_client, db, monkeypatch: pytest.MonkeyPatch +) -> None: + """Nothing registered, nothing indexed → ``{"sources": []}``. + + "Empty registry" means the ``git_sources`` table AND the + ``BOR_GIT_SOURCES`` env fallback are both empty — the dev ``.env`` + names a source, so the resolver's ``get_settings`` is patched with + a fresh ``Settings(_env_file=None, git_sources="")`` (the + ``test_sync_api`` / ``test_git_sources_api`` pattern: the dev + ``.env`` never leaks in).""" + monkeypatch.setattr( + rag_git_sources, + "get_settings", + lambda: Settings(_env_file=None, git_sources=""), # pyright: ignore[reportCallIssue] + ) + _truncate_tree_tables(db) + r = admin_client.get("/api/docs/tree") + assert r.status_code == 200 + assert r.json() == {"sources": []} + + +def test_docs_tree_indexed_only_source_after_registered(admin_client, db) -> None: + """The superset rule: a registered 0-document source still lists + (first — the registry leads), and indexed-only sources (documents + whose source is not in ``git_sources`` — ad-hoc imports, removed + but not-yet-pruned sources) trail in alphabetical order.""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Alpha.git", kind="git", added_at=base)) + _seed_doc(db, "Zeta", "z1.md", "Z1", 1, base) + _seed_doc(db, "Midx", "m1.md", "M1", 2, base) + db.commit() + + r = admin_client.get("/api/docs/tree") + assert r.status_code == 200 + sources = r.json()["sources"] + assert [s["name"] for s in sources] == ["Alpha", "Midx", "Zeta"] + alpha, midx, zeta = sources + assert (alpha["documents"], alpha["children"], alpha["summary"]) == (0, [], None) + assert midx["documents"] == 1 + assert zeta["documents"] == 1 + assert [c["path"] for c in midx["children"]] == ["m1.md"] + + _truncate_tree_tables(db) + + +# -------------------------------------------------------------------- +# PATCH /api/folders/summary (phase 97, task 03) — the admin +# folder-description editor: update / create / source-root / clear / +# double-clear, the 404s, the 403s, and the round-trip through +# ``GET /api/docs/tree`` (task 02). +# -------------------------------------------------------------------- + +#: A deliberately old stamp: a save must ADVANCE ``updated_at`` past it. +OLDER_STAMP = datetime(2020, 1, 1, tzinfo=UTC) + + +def _seed_folder_pair(db, source: str, folder: str, base: datetime) -> None: + """Two documents under ``source/folder`` — the ≥ 2-document + minimum a folder must hold to be summarizable.""" + _seed_doc(db, source, f"{folder}/one.md", f"{folder} one", 1, base) + _seed_doc(db, source, f"{folder}/two.md", f"{folder} two", 2, base) + + +def _get_folder_row(db, source: str, folder: str) -> FolderSummary | None: + return db.scalar( + select(FolderSummary).where( + FolderSummary.source == source, FolderSummary.folder_path == folder + ) + ) + + +def test_folder_summary_update_ai_row_flips_manual_and_advances_stamp( + admin_client: TestClient, db +) -> None: + """Update an AI-written row: the text is replaced, ``manually_edited`` + flips to true, ``updated_at`` advances past the seeded stamp — and a + second save with different text updates IN PLACE (same PK, one + row). The round-trip: ``GET /api/docs/tree`` shows the new text on + the folder node.""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) + _seed_folder_pair(db, "Homelab", "k8s", base) + db.add( + FolderSummary( + source="Homelab", + folder_path="k8s", + summary="AI text.", + manually_edited=False, + updated_at=OLDER_STAMP, + ) + ) + db.commit() + db.expire_all() + + r = admin_client.patch( + "/api/folders/summary", + json={"source": "Homelab", "folder_path": "k8s", "summary": " Owner words. "}, + ) + assert r.status_code == 200, r.text + assert set(r.json()) == {"source", "folder_path", "summary"} + assert r.json() == {"source": "Homelab", "folder_path": "k8s", "summary": "Owner words."} + + db.expire_all() + row = _get_folder_row(db, "Homelab", "k8s") + assert row is not None + assert row.summary == "Owner words." # stripped before storing + assert row.manually_edited is True # a manual row from this save on + assert row.updated_at > OLDER_STAMP # fresh stamp, not the seeded one + + # The task-02 tree shows the owner's text on the folder node. + tree = admin_client.get("/api/docs/tree").json() + k8s = next(c for c in tree["sources"][0]["children"] if c["kind"] == "folder") + assert (k8s["path"], k8s["summary"]) == ("k8s", "Owner words.") + + # A second save updates in place — same row, no second PK. + r2 = admin_client.patch( + "/api/folders/summary", + json={"source": "Homelab", "folder_path": "k8s", "summary": "Owner words v2."}, + ) + assert r2.status_code == 200, r2.text + assert r2.json()["summary"] == "Owner words v2." + db.expire_all() + assert db.scalar(select(func.count()).select_from(FolderSummary)) == 1 # one row + row = _get_folder_row(db, "Homelab", "k8s") + assert row is not None + assert row.summary == "Owner words v2." + assert row.manually_edited is True + + _truncate_tree_tables(db) + + +def test_folder_summary_create_where_no_row_exists( + admin_client: TestClient, db +) -> None: + """A manual description can be CREATED where no row exists — a + < 2-document folder the generator never wrote (or its fail-soft + miss): insert, not update, with ``manually_edited = true``.""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) + _seed_doc(db, "Homelab", "solo/only.md", "Only", 1, base) # 1-doc folder + db.commit() + db.expire_all() + assert _get_folder_row(db, "Homelab", "solo") is None # no AI row for < 2 docs + + r = admin_client.patch( + "/api/folders/summary", + json={"source": "Homelab", "folder_path": "solo", "summary": "One-off scripts."}, + ) + assert r.status_code == 200, r.text + assert r.json() == { + "source": "Homelab", + "folder_path": "solo", + "summary": "One-off scripts.", + } + db.expire_all() + row = _get_folder_row(db, "Homelab", "solo") + assert row is not None + assert row.summary == "One-off scripts." + assert row.manually_edited is True + + _truncate_tree_tables(db) + + +def test_folder_summary_root_save_on_registered_zero_doc_source( + admin_client: TestClient, db +) -> None: + """``folder_path = ""`` is valid for ANY allowed source — a registered + 0-document source (nothing indexed yet) still accepts a root + description: the source check is registered-OR-indexed.""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Empty.git", kind="git", added_at=base)) + db.commit() + + r = admin_client.patch( + "/api/folders/summary", + json={"source": "Empty", "folder_path": "", "summary": "Docs incoming."}, + ) + assert r.status_code == 200, r.text + assert r.json() == {"source": "Empty", "folder_path": "", "summary": "Docs incoming."} + db.expire_all() + row = _get_folder_row(db, "Empty", "") + assert row is not None + assert row.manually_edited is True + + _truncate_tree_tables(db) + + +def test_folder_summary_source_root_round_trips_through_tree( + admin_client: TestClient, db +) -> None: + """The source root (``folder_path: ""``) round-trips through the + task-02 tree endpoint: save → the source node carries the text; + clear → it is null again.""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) + _seed_doc(db, "Homelab", "a/b.md", "B", 1, base) + _seed_doc(db, "Homelab", "c.md", "C", 1, base) + db.commit() + + r = admin_client.patch( + "/api/folders/summary", + json={"source": "Homelab", "folder_path": "", "summary": "All the homelab docs."}, + ) + assert r.status_code == 200, r.text + assert r.json() == { + "source": "Homelab", + "folder_path": "", + "summary": "All the homelab docs.", + } + tree = admin_client.get("/api/docs/tree").json() + assert tree["sources"][0]["summary"] == "All the homelab docs." + db.expire_all() + assert _get_folder_row(db, "Homelab", "") is not None + + r = admin_client.patch( + "/api/folders/summary", + json={"source": "Homelab", "folder_path": "", "summary": " "}, + ) + assert r.status_code == 200, r.text + assert r.json() == {"source": "Homelab", "folder_path": "", "summary": None} + tree = admin_client.get("/api/docs/tree").json() + assert tree["sources"][0]["summary"] is None + + _truncate_tree_tables(db) + + +def test_folder_summary_clear_deletes_row_and_double_clear_is_noop( + admin_client: TestClient, db +) -> None: + """Empty/whitespace clears: the row is deleted — AI-written OR + manual, either way it is gone (the next KB-changing sync regenerates + an AI row: the reset path) — and the response ``summary`` is null. + A second clear with no row is a 200 no-op. The task-02 tree shows + null on both folder nodes after the clears.""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) + _seed_folder_pair(db, "Homelab", "k8s", base) + _seed_folder_pair(db, "Homelab", "manual", base) + db.add( + FolderSummary( + source="Homelab", + folder_path="k8s", + summary="AI text.", + manually_edited=False, + updated_at=OLDER_STAMP, + ) + ) + db.add( + FolderSummary( + source="Homelab", + folder_path="manual", + summary="Owner text.", + manually_edited=True, + updated_at=OLDER_STAMP, + ) + ) + db.commit() + db.expire_all() + + for folder, blank in (("k8s", " "), ("manual", "")): # whitespace, then empty + r = admin_client.patch( + "/api/folders/summary", + json={"source": "Homelab", "folder_path": folder, "summary": blank}, + ) + assert r.status_code == 200, r.text + assert r.json() == {"source": "Homelab", "folder_path": folder, "summary": None} + db.expire_all() + assert db.scalar(select(func.count()).select_from(FolderSummary)) == 0 # both rows gone + + # A second clear (no row) is a 200 no-op. + r = admin_client.patch( + "/api/folders/summary", + json={"source": "Homelab", "folder_path": "k8s", "summary": ""}, + ) + assert r.status_code == 200, r.text + assert r.json() == {"source": "Homelab", "folder_path": "k8s", "summary": None} + db.expire_all() + assert db.scalar(select(func.count()).select_from(FolderSummary)) == 0 # still nothing + + # The task-02 tree shows null on both folder nodes after the clears. + tree = admin_client.get("/api/docs/tree").json() + nodes = { + c["path"]: c for c in tree["sources"][0]["children"] if c["kind"] == "folder" + } + assert nodes["k8s"]["summary"] is None + assert nodes["manual"]["summary"] is None + + _truncate_tree_tables(db) + + +def test_folder_summary_404_unknown_source_folder_and_traversal( + admin_client: TestClient, db +) -> None: + """404s, in check order: an unknown source (neither registered nor + indexed) wins over the folder check — including for a folder that + EXISTS under another source; an unknown folder, a traversal folder + path, and a file merely sharing a folder's name (no indexed + descendant) are all ``folder not found``. Nothing is written.""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) + _seed_folder_pair(db, "Homelab", "real", base) + _seed_doc(db, "Homelab", "note", "A file named like a folder", 1, base) + db.commit() + + cases = ( + # (source, folder_path, detail) + ("Ghost", "", "source not found"), # unknown source, root + ("Ghost", "real", "source not found"), # folder exists ELSEWHERE — source wins + ("Homelab", "nope", "folder not found"), # unknown folder + ("Homelab", "../../etc", "folder not found"), # traversal: no prefix match + ("Homelab", "note", "folder not found"), # file named like a folder, no descendants + ("Homelab", "real/", "folder not found"), # trailing slash: strict prefix rule + ) + for source, folder, detail in cases: + r = admin_client.patch( + "/api/folders/summary", + json={"source": source, "folder_path": folder, "summary": "whatever"}, + ) + assert r.status_code == 404, (source, folder, r.status_code) + assert r.json() == {"detail": detail}, (source, folder) + + db.expire_all() + assert db.scalar(select(func.count()).select_from(FolderSummary)) == 0 # nothing written + + _truncate_tree_tables(db) + + +def test_folder_summary_403_anonymous_and_token_user( + client: TestClient, db +) -> None: + """The ``require_admin`` gate: an anonymous caller AND a live + access-token user who is not the admin (phase 79 token users exist) + both get 403 ``admin only`` — the endpoint gate is the API-level + defense in depth, not the RAG view's render gate.""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) + _seed_folder_pair(db, "Homelab", "k8s", base) + db.execute(text("TRUNCATE api_tokens")) + db.commit() + body = {"source": "Homelab", "folder_path": "k8s", "summary": "x"} + try: + # Anonymous (the shared ``client`` is unsigned in this module). + r = client.patch("/api/folders/summary", json=body) + assert r.status_code == 403 + assert r.json() == {"detail": "admin only"} + + # A live access-token user who is not the admin (phase 79): + # signed in via the public token login, the row is active. + _row, plaintext = token_service.create_token(db, "pin-holder") + db.commit() + holder = TestClient(fastapi_app) + s = holder.post("/api/token-auth", json={"token": plaintext}) + assert s.status_code == 204, s.text + r = holder.patch("/api/folders/summary", json=body) + assert r.status_code == 403 + assert r.json() == {"detail": "admin only"} + finally: + db.execute(text("TRUNCATE api_tokens")) + db.commit() + _truncate_tree_tables(db) + + +def test_folder_summary_patch_never_constructs_an_llm_client() -> None: + """Source pin (the house pattern): a folder description is NEVER + embedded — no chunk, no retrieval role beyond the ``ls`` line — so + the PATCH handler must never touch the LLM client (the deliberate + contrast with the phase-57 document-summary re-embed).""" + src = inspect.getsource(docs_api.update_folder_summary) + assert "LLMClient" not in src + + +def test_docs_tree_stat_walk_equivalence_with_flat_list(admin_client, db) -> None: + """The RAG view computes its KB-wide stat cards by walking the + in-memory tree — the walk must yield EXACTLY what + ``GET /api/docs`` reports for the same data (the stat-card values + are unchanged by the redesign).""" + _truncate_tree_tables(db) + base = datetime.now(UTC) + db.add(GitSource(url="https://github.com/reese/Homelab.git", kind="git", added_at=base)) + _seed_doc(db, "Homelab", "a/b/c.md", "C", 3, base) + _seed_doc(db, "Homelab", "a/d.md", "D", 2, base) + _seed_doc(db, "Homelab", "top.md", "Top", 5, base) + _seed_doc(db, "Homelab", "solo.md", "Solo", 0, base) + db.commit() + + tree = admin_client.get("/api/docs/tree").json() + flat = admin_client.get("/api/docs").json() + + files = _tree_file_nodes(tree["sources"]) + assert len(files) == len(flat["documents"]) # document count + assert sum(f["chunks"] for f in files) == sum(d["chunks"] for d in flat["documents"]) + # The tree's per-source counts match the flat list's per-source counts. + for source in tree["sources"]: + flat_count = sum( + 1 for d in flat["documents"] if d["source"] == source["name"] + ) + assert source["documents"] == flat_count + source_files = _tree_file_nodes([source]) + assert sum(f["chunks"] for f in source_files) == sum( + d["chunks"] for d in flat["documents"] if d["source"] == source["name"] + ) + + _truncate_tree_tables(db) diff --git a/tests/integration/test_migration_0018.py b/tests/integration/test_migration_0018.py new file mode 100644 index 0000000..46a3b0e --- /dev/null +++ b/tests/integration/test_migration_0018.py @@ -0,0 +1,223 @@ +"""Integration: migration 0018 (folder_summaries.manually_edited) +schema contract (phase 97, task 01). + +Drives the **real Alembic engine** against the live dev database +(``podman compose up -d db``), mirroring the house pattern of +``test_migration_0017.py`` (information_schema assertions on the state +the migration must leave). The tests target the 0017 → 0018 step +explicitly so later migrations cannot break them: + +* upgrade 0017 → 0018 → the ``manually_edited`` column exists with the + full contract — BOOLEAN NOT NULL, server default ``false`` — while + the 0017 ``folder_summaries`` schema (PK, summary, updated_at) + survives; +* pre-0018 rows backfill ``false`` (an AI-written row stays an AI row) + and a row written without the column takes the default; +* a row written with ``manually_edited = true`` round-trips the flag; +* downgrade to 0017 → the column is GONE (A13 — reversible) while the + rows + their summaries survive; +* upgrade back to 0018 → the column is back (round-trip). + +The ``alembic`` fixture guarantees the DB ends at head even if a test +fails or the process is interrupted. +""" +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +import pytest +from alembic.config import Config +from sqlalchemy import text +from sqlalchemy.orm import Session + +from alembic import command +from app.db import db_available + + +@pytest.fixture() +def alembic(db: Session) -> Iterator[Config]: + """Real Alembic config bound to the dev DB (URL from app settings). + + Starts at head (repairs an interrupted earlier run); teardown upgrades + to head no matter what happened, so the dev DB is never left below + head. + """ + if not db_available(): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + cfg = Config() # no alembic.ini file — env.py gets the URL from app config + cfg.set_main_option("script_location", "alembic") + command.upgrade(cfg, "head") + try: + yield cfg + finally: + # Release the test session's open transaction BEFORE the repair + # DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock + # on ``folder_summaries``, which would deadlock the repair's + # ``ALTER TABLE`` (0018) forever. + db.rollback() + command.upgrade(cfg, "head") + + +def _version(db: Session) -> str | None: + return db.execute(text("SELECT version_num FROM alembic_version")).scalar() + + +def _column(db: Session, table: str, column: str) -> tuple[Any, ...] | None: + """(data_type, is_nullable, column_default, character_maximum_length) + for one table column.""" + row = db.execute( + text( + "SELECT data_type, is_nullable, column_default, character_maximum_length" + " FROM information_schema.columns" + " WHERE table_name = :t AND column_name = :c" + ), + {"t": table, "c": column}, + ).fetchone() + return tuple(row) if row is not None else None + + +def _flag(db: Session, source: str, folder_path: str) -> Any: + return db.execute( + text( + "SELECT manually_edited FROM folder_summaries" + " WHERE source = :s AND folder_path = :f" + ), + {"s": source, "f": folder_path}, + ).scalar_one() + + +def _clear_rows(db: Session) -> None: + db.execute(text("DELETE FROM folder_summaries")) + db.commit() + + +def test_upgrade_to_0018_adds_manually_edited(db: Session, alembic: Config) -> None: + """Upgrade 0017 → 0018: the column exists with the full contract + (BOOLEAN NOT NULL, server default ``false``), is ABSENT at 0017, + pre-0018 rows backfill ``false`` (an AI row stays an AI row), a new + row without the column takes the default, and an explicit ``true`` + round-trips — while the 0017 table contract survives.""" + command.downgrade(alembic, "0017") # start from the pre-0018 state + assert _version(db) == "0017" + assert _column(db, "folder_summaries", "manually_edited") is None, ( + "the flag must be absent at 0017" + ) + + try: + # A pre-0018 AI-written row — must backfill ``false``. + db.execute( + text( + "INSERT INTO folder_summaries (source, folder_path, summary)" + " VALUES ('OldSource', 'old/folder', 'pre-0018 summary')" + ) + ) + db.commit() + + command.upgrade(alembic, "0018") + assert _version(db) == "0018", "alembic_version must be at 0018" + + flag = _column(db, "folder_summaries", "manually_edited") + assert flag is not None, "folder_summaries.manually_edited is missing" + assert flag[0] == "boolean", "manually_edited must be BOOLEAN" + assert flag[1] == "NO", "manually_edited must be NOT NULL" + assert flag[2] is not None and "false" in str(flag[2]), ( + "manually_edited must carry the `false` server default" + ) + + # The pre-0018 row backfilled ``false`` — an AI row stays an AI row. + assert _flag(db, "OldSource", "old/folder") is False + + # A row written without the column takes the server default. + db.execute( + text( + "INSERT INTO folder_summaries (source, folder_path, summary)" + " VALUES ('NewSource', '', 'root summary')" + ) + ) + db.commit() + assert _flag(db, "NewSource", "") is False, ( + "an omitted flag takes the `false` server default" + ) + + # The flag round-trips through an explicit ``true``. + db.execute( + text( + "UPDATE folder_summaries SET manually_edited = true" + " WHERE source = 'NewSource'" + ) + ) + db.commit() + assert _flag(db, "NewSource", "") is True, ( + "manually_edited = true must round-trip" + ) + + # The 0017 schema survives the additive upgrade. + summary = _column(db, "folder_summaries", "summary") + assert summary is not None and summary[0] == "text" and summary[1] == "NO", ( + "folder_summaries.summary (0017) must survive the upgrade" + ) + folder = _column(db, "folder_summaries", "folder_path") + assert folder is not None and folder[3] == 1000, ( + "folder_summaries.folder_path (0017) must survive the upgrade" + ) + finally: + _clear_rows(db) + + +def test_downgrade_to_0017_drops_the_column(db: Session, alembic: Config) -> None: + """Downgrade 0018 → 0017: the column is gone (A13 — fully + reversible) while the rows + their summaries survive, and the rest + of the schema (the 0017 table contract, ``documents``) is intact.""" + command.upgrade(alembic, "head") + try: + db.execute( + text( + "INSERT INTO folder_summaries" + " (source, folder_path, summary, manually_edited)" + " VALUES ('ManualSrc', '', 'owner text', true)" + ) + ) + db.commit() + command.downgrade(alembic, "0017") + assert _version(db) == "0017" + assert _column(db, "folder_summaries", "manually_edited") is None, ( + "the flag must be dropped" + ) + row = db.execute( + text( + "SELECT source, folder_path, summary FROM folder_summaries" + " WHERE source = 'ManualSrc'" + ) + ).fetchone() + assert row is not None and row[2] == "owner text", ( + "the row and its summary must survive the column drop" + ) + + summary = _column(db, "folder_summaries", "summary") + assert summary is not None and summary[0] == "text", ( + "the 0017 table contract must survive the downgrade" + ) + doc_path = _column(db, "documents", "path") + assert doc_path is not None and doc_path[3] == 1000, ( + "documents.path must survive the downgrade" + ) + finally: + _clear_rows(db) + # Repair: the fixture teardown re-upgrades to head. + + +def test_upgrade_round_trip_restores_the_flag(db: Session, alembic: Config) -> None: + """Downgrade to 0017, then upgrade back to 0018: the column is back + with the full contract (BOOLEAN NOT NULL, the `false` default).""" + command.downgrade(alembic, "0017") + command.upgrade(alembic, "0018") + assert _version(db) == "0018", "round-trip upgrade must land at 0018" + + flag = _column(db, "folder_summaries", "manually_edited") + assert flag is not None, "folder_summaries.manually_edited must be back" + assert flag[0] == "boolean", "manually_edited must be BOOLEAN after the round-trip" + assert flag[1] == "NO", "manually_edited must be NOT NULL after the round-trip" + assert flag[2] is not None and "false" in str(flag[2]), ( + "the `false` server default must survive the round-trip" + ) diff --git a/tests/integration/test_sync_folder_summaries.py b/tests/integration/test_sync_folder_summaries.py index 30c8596..36e419e 100644 --- a/tests/integration/test_sync_folder_summaries.py +++ b/tests/integration/test_sync_folder_summaries.py @@ -275,7 +275,8 @@ def test_changed_import_generates_folder_rows( assert _updated_at(db, "MyDocs", "a") is not None # The stats log line (PLAN §9 ample logging). assert any( - "folder_summaries: generated=2 failed=0 pruned=0" in r.getMessage() + "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=0" + in r.getMessage() for r in records ) @@ -470,6 +471,96 @@ def test_folder_lite_failure_keeps_previous_row_and_stays_green( assert root_stamp_after is not None and root_stamp_after > root_stamp_before +def test_changed_import_never_overwrites_a_manual_row( + db: Session, + src: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Phase 97 (task 01): a manually-edited folder description survives + a KB-changing sync — the generator SKIPS it (zero + ``FOLDER_SUMMARY_MODE`` calls for it), the summary-line token + STAYS 3 fields (``folder_summaries=//`` + — ``kept_manual`` is a stat, not a token), the manual row's text, + stamp, and flag are untouched, and the other folders regenerate + (the ``kept_manual`` stat lands on the generator's log line).""" + # First sync: full generation — root + a/ (b/ holds 1 doc: none). + llm1 = FakeEmbedder() + rc, out = _run_main(monkeypatch, llm1, ["--source", str(src)], capsys) + assert rc == 0 + assert out.rstrip().endswith( + "overview=updated sources_version=1 folder_summaries=2/0/0" + ) + assert set(_rows(db)) == {("MyDocs", ""), ("MyDocs", "a")} + + # The owner edits the source-root description (task 03's PATCH is + # the writer; task 01 pins the generator's behavior, so the row is + # inserted directly — the ``test_import_docs_overview.py`` pattern). + manual_text = "Owner's own words about MyDocs." + db.execute( + text( + "UPDATE folder_summaries SET summary = :s, manually_edited = true" + " WHERE source = 'MyDocs' AND folder_path = ''" + ), + {"s": manual_text}, + ) + db.commit() + root_stamp_before = _updated_at(db, "MyDocs", "") + assert root_stamp_before is not None + + records: list[logging.LogRecord] = [] + + class _Sink(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + records.append(record) + + fs_logger = logging.getLogger("app.rag.folder_summaries") + sink = _Sink() + fs_logger.addHandler(sink) + fs_logger.setLevel(logging.INFO) + try: + # A KB-changing re-sync (a new doc under a/) — the gate fires a + # full regeneration ... + (src / "a" / "three.md").write_text( + "# A Three\nAnother folder document.\n", encoding="utf-8" + ) + llm2 = FakeEmbedder() + rc, out = _run_main(monkeypatch, llm2, ["--source", str(src)], capsys) + finally: + fs_logger.removeHandler(sink) + + assert rc == 0 + assert "added=1" in out + # The token STAYS 3 fields — kept_manual is a stat, not a token. + assert out.rstrip().endswith( + "overview=updated sources_version=2 folder_summaries=1/0/0" + ) + # Zero folder calls for the owner's folder — only a/ (now 3 docs). + calls = _folder_calls(llm2) + assert [c[1]["content"].splitlines()[0] for c in calls] == ["Folder: MyDocs/a"] + # The owner's text, stamp, and flag are untouched ... + rows_after = _rows(db) + assert rows_after[("MyDocs", "")] == manual_text + assert _updated_at(db, "MyDocs", "") == root_stamp_before, ( + "the manual row is never re-stamped" + ) + flag = db.execute( + text( + "SELECT manually_edited FROM folder_summaries" + " WHERE source = 'MyDocs' AND folder_path = ''" + ) + ).scalar_one() + assert flag is True, "the generator never clears the flag" + # ... while the other folder regenerates. + assert rows_after[("MyDocs", "a")] is not None + # The 4-field stats line carries the skip (PLAN §9 ample logging). + assert any( + "folder_summaries: generated=1 failed=0 pruned=0 kept_manual=1" + in r.getMessage() + for r in records + ) + + def test_limit_run_skips_folder_generation( db: Session, src: Path, diff --git a/tests/unit/test_folder_summaries.py b/tests/unit/test_folder_summaries.py index 6992d3f..ee7b0b4 100644 --- a/tests/unit/test_folder_summaries.py +++ b/tests/unit/test_folder_summaries.py @@ -441,7 +441,7 @@ def test_generate_happy_path_upserts_every_candidate_folder( llm = _FakeLLM() with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"): stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 3, "failed": 0, "pruned": 0} + assert stats == {"generated": 3, "failed": 0, "pruned": 0, "kept_manual": 0} assert llm.calls == 3, "one lite call per candidate folder (the solo folder: none)" stored = _rows(db) @@ -472,7 +472,8 @@ def test_generate_happy_path_upserts_every_candidate_folder( assert "root.md — Root" not in a_prompt assert ( - "folder_summaries: generated=3 failed=0 pruned=0" in caplog.text + "folder_summaries: generated=3 failed=0 pruned=0 kept_manual=0" + in caplog.text ), "the stats line must be greppable (PLAN §9 ample logging)" @@ -488,7 +489,7 @@ def test_generate_per_folder_fail_soft_keeps_previous_and_lands_others( llm = _FakeLLM(fail_folders=("FSU/a/b",)) with caplog.at_level(logging.ERROR, logger="app.rag.folder_summaries"): stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 2, "failed": 1, "pruned": 0} + assert stats == {"generated": 2, "failed": 1, "pruned": 0, "kept_manual": 0} assert llm.calls == 3 # the failing folder was attempted too stored = _rows(db) @@ -547,7 +548,7 @@ def test_generate_skip_is_a_full_noop(db: Session, clean_tables) -> None: db.commit() llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm, skip=True)) - assert stats == {"generated": 0, "failed": 0, "pruned": 0} + assert stats == {"generated": 0, "failed": 0, "pruned": 0, "kept_manual": 0} assert llm.calls == 0 assert _rows(db) == {("FSU", ""): "existing"} @@ -560,7 +561,7 @@ def test_generate_empty_kb_prunes_every_row(db: Session, clean_tables) -> None: db.commit() llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm)) - assert stats == {"generated": 0, "failed": 0, "pruned": 2} + assert stats == {"generated": 0, "failed": 0, "pruned": 2, "kept_manual": 0} assert llm.calls == 0 assert _rows(db) == {} @@ -613,6 +614,113 @@ def test_generate_only_flushes_caller_commits(db: Session, clean_tables) -> None assert MIN_DOCS_PER_FOLDER == 2 # the ≥ 2 scope rule, pinned by name +# ---------- manually_edited (phase 97, task 01) ---------- + + +def test_manual_row_survives_regeneration( + db: Session, clean_tables, caplog: pytest.LogCaptureFixture +) -> None: + """An owner-edited row is SKIPPED on regeneration (phase 97, task + 01): the fake LLM is never called for it (no ``lite`` burn on owner + text — not even a prompt is built), its text AND ``updated_at`` + stay byte-identical, ``kept_manual`` counts it, the flag is never + cleared, and the 4-field log line carries it (PLAN §9).""" + _seed_catalogue(db) + manual_text = "Owner's own words about a/." + db.add( + FolderSummary( + source="FSU", folder_path="a", summary=manual_text, + manually_edited=True, + ) + ) + db.commit() + stamp_before = _updated_at(db, "FSU", "a") + assert stamp_before is not None + + llm = _FakeLLM() + with caplog.at_level(logging.INFO, logger="app.rag.folder_summaries"): + stats = asyncio.run(generate_folder_summaries(db, llm)) + + assert stats == {"generated": 2, "failed": 0, "pruned": 0, "kept_manual": 1} + assert llm.calls == 2, "the manual folder burns zero lite calls" + assert [user.splitlines()[0] for _s, user in llm.requests] == [ + "Folder: FSU", + "Folder: FSU/a/b", + ], "no prompt is ever built for the owner's folder" + + stored = _rows(db) + assert stored[("FSU", "a")] == manual_text, "the owner's text survives" + assert _updated_at(db, "FSU", "a") == stamp_before, ("never re-stamped") + assert stored[("FSU", "")] == REPLY and stored[("FSU", "a/b")] == REPLY, ( + "the non-manual candidates still regenerate (the flag is the difference)" + ) + row = db.get(FolderSummary, ("FSU", "a")) + assert row is not None and row.manually_edited is True, ( + "the generator never clears the flag" + ) + assert ( + "folder_summaries: generated=2 failed=0 pruned=0 kept_manual=1" + in caplog.text + ), "the 4-field stats line must be greppable (PLAN §9 ample logging)" + + +def test_manual_row_survives_the_prune(db: Session, clean_tables) -> None: + """A manual row is NEVER pruned (phase 97, task 01): two folders + drop below 2 documents — the MANUAL one keeps its row (owner + content persists until cleared — the clear deletes it, so the next + KB-changing sync regenerates an AI description) while the + NON-manual twin loses its now-stale row; the flag is the only + difference. A vanished folder's manual row is kept too, and its + non-manual twin is pruned.""" + _add_doc(db, "FSU", "a/one.md", "One") + _add_doc(db, "FSU", "a/two.md", "Two") + _add_doc(db, "FSU", "b/one.md", "B One") + _add_doc(db, "FSU", "b/two.md", "B Two") + manual_text = "Owner's words about a/." + db.add( + FolderSummary( + source="FSU", folder_path="a", summary=manual_text, + manually_edited=True, + ) + ) + db.add(FolderSummary(source="FSU", folder_path="b", summary="ai words")) + db.add( + FolderSummary( + source="FSU", folder_path="gone/manual", summary="owner kept", + manually_edited=True, + ) + ) + db.add(FolderSummary(source="FSU", folder_path="gone/ai", summary="stale ai")) + db.commit() + + # a/ and b/ each drop below the minimum (2 -> 1 recursive doc). + db.execute( + text( + "DELETE FROM documents WHERE source = 'FSU'" + " AND path IN ('a/two.md', 'b/two.md')" + ) + ) + db.commit() + + llm = _FakeLLM() + stats = asyncio.run(generate_folder_summaries(db, llm)) + + assert stats == {"generated": 1, "failed": 0, "pruned": 2, "kept_manual": 0} + assert llm.calls == 1, "only the surviving candidate (the root) regenerates" + stored = _rows(db) + assert stored[("FSU", "a")] == manual_text, ( + "the manual row survives its folder dropping below the minimum" + ) + assert ("FSU", "b") not in stored, ( + "the non-manual twin loses its stale row (the flag is the difference)" + ) + assert stored[("FSU", "gone/manual")] == "owner kept", ( + "a vanished folder's manual row is kept — owner content until cleared" + ) + assert ("FSU", "gone/ai") not in stored, ("the non-manual twin is pruned") + assert stored[("FSU", "")] == REPLY # the root (2 docs) still regenerates + + # ---------- missing_folder_summaries (phase 96, task 02) ---------- @@ -715,7 +823,7 @@ def test_only_missing_fills_exactly_the_missing_keys( llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True)) - assert stats == {"generated": 2, "failed": 0, "pruned": 0} + assert stats == {"generated": 2, "failed": 0, "pruned": 0, "kept_manual": 0} assert llm.calls == 2, "one call per MISSING key — zero for present rows" assert [user.splitlines()[0] for _s, user in llm.requests] == [ "Folder: FSU", @@ -738,7 +846,7 @@ def test_only_missing_no_gap_burns_zero_calls(db: Session, clean_tables) -> None stamps = {f: _updated_at(db, "FSU", f) for f in ("", "a", "a/b")} llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True)) - assert stats == {"generated": 0, "failed": 0, "pruned": 0} + assert stats == {"generated": 0, "failed": 0, "pruned": 0, "kept_manual": 0} assert llm.calls == 0, "zero-burn: no gap, no lite call" assert _rows(db) == before for folder, stamp in stamps.items(): @@ -758,7 +866,7 @@ def test_only_missing_still_prunes_stale_rows(db: Session, clean_tables) -> None llm = _FakeLLM() stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True)) - assert stats == {"generated": 2, "failed": 0, "pruned": 1} + assert stats == {"generated": 2, "failed": 0, "pruned": 1, "kept_manual": 0} assert llm.calls == 2 stored = _rows(db) @@ -781,7 +889,7 @@ def test_only_missing_fail_soft_keeps_prior_and_lands_others( db.commit() llm = _FakeLLM(fail_folders=("FSU/a/b",)) stats = asyncio.run(generate_folder_summaries(db, llm, only_missing=True)) - assert stats == {"generated": 1, "failed": 1, "pruned": 0} + assert stats == {"generated": 1, "failed": 1, "pruned": 0, "kept_manual": 0} assert llm.calls == 2 # both missing folders were attempted stored = _rows(db) diff --git a/tests/unit/test_frontend_router.py b/tests/unit/test_frontend_router.py index dc75a7d..c6dc669 100644 --- a/tests/unit/test_frontend_router.py +++ b/tests/unit/test_frontend_router.py @@ -51,9 +51,11 @@ Phase 77 task 02 (the other data views join the refresh): RAG (``tuning.js``) each listen for ``bor:view-refresh`` on their root and re-run their existing load (armed only in the admin branch, after the whoami gate — the same gate guard as History). ``sources.js``'s -``loadDocs`` clears the tbody's rows at the TOP (before the fetch — -the History pattern), so a refresh from a populated list into an empty -result leaves no ghost rows. The Chat view (``app.js``) does NOT +catalog load is re-entrant: the phase-97 ``loadTree`` (one fetch of +``GET /api/docs/tree``) renders through ``renderLevel``, which clears +BOTH row containers at the top before filling them, so a refresh from +a populated level into a sparser (or empty) one leaves no ghost rows. The Chat +view (``app.js``) does NOT listen — the negative pin: the in-flight SSE stream and the local conversation must survive every switch (the phase-76 contract), so the exclusion is a contract, not an oversight. @@ -559,25 +561,34 @@ def _pin_refresh_listener(js: str, gate: str, listener_call: str, name: str) -> def test_rag_view_refetches_on_reshow() -> None: - """Phase 77 task 02: the RAG (knowledge base) view re-fetches on a - user-initiated re-show — sources.js listens and re-runs - ``loadDocs()``. ``loadDocs`` is now re-entrant: the tbody's rows - are cleared at the TOP, before the fetch (the History pattern from - task 01), so a refresh from a populated list into an empty result - replaces the list instead of leaving ghost rows.""" + """Phase 77 task 02 (+ phase 97 task 04): the RAG (knowledge base) + view re-fetches on a user-initiated re-show — sources.js listens + and re-runs ``loadTree()`` (the phase-97 catalog load: ONE fetch of + ``GET /api/docs/tree``). The load is race-tokened (phase 79) and + the RE-ENTRANT render — ``renderLevel`` clears BOTH row containers + at the TOP before filling them (the History pattern from task 01, + extended to the folders table) — so a refresh from a populated + level into a sparser (or empty) result replaces the rows instead + of leaving ghost rows.""" js = _asset("sources.js") _pin_refresh_listener( - js, "const admin = await fetchIsAdmin();", "() => loadDocs()", "sources.js" + js, "const admin = await fetchIsAdmin();", "() => loadTree()", "sources.js" ) - load = js.find("async function loadDocs()") - assert load != -1, "loadDocs must exist" + load = js.find("async function loadTree()") + assert load != -1, "loadTree must exist" body = js[load : js.find("\n }", load)] - clear_i = body.find("tbody.replaceChildren()") - fetch_i = body.find('fetch("/api/docs")') - assert 0 <= clear_i < fetch_i, ( - "the row clearing must precede the fetch (a populated → empty refresh " - "must not leave ghost rows)" - ) + assert 'fetch("/api/docs/tree")' in body, "the load must fetch the tree endpoint" + assert "++loadSeq" in body, "the race token stays (phase 79)" + render = js.find("function renderLevel()") + assert render != -1, "renderLevel must exist" + render_body = js[render : js.find("\n }", render)] + for container in ("foldersTbody", "tbody"): + clear_i = render_body.find(f"{container}.replaceChildren()") + append_i = render_body.find(f"{container}.appendChild") + assert 0 <= clear_i < append_i, ( + f"{container}: the clear must precede the fill (a populated → " + "sparser refresh must not leave ghost rows)" + ) def test_git_sources_view_refetches_on_reshow() -> None: diff --git a/tests/unit/test_frontend_sync_upload.py b/tests/unit/test_frontend_sync_upload.py index ca10edd..6777f71 100644 --- a/tests/unit/test_frontend_sync_upload.py +++ b/tests/unit/test_frontend_sync_upload.py @@ -285,7 +285,7 @@ def test_tick_decision_tree_order_and_branches() -> None: 'syncResult.textContent = ""', "hideSyncError()", 'emitSyncStatus({ state: "idle" })', - "loadDocs()", + "loadTree()", # phase 97: the catalog load is the tree fetch ): assert line in up_ok, f"the upload-success settle must carry {line!r}" assert "fmtSyncResult" not in up_ok, "no upload counts in #sync-result (A3)" @@ -300,7 +300,7 @@ def test_tick_decision_tree_order_and_branches() -> None: 'emitSyncStatus({ state: "idle" })', ): assert line in up_fail, f"the upload-failed settle must carry {line!r}" - assert "loadDocs()" not in up_fail, "no KB change on a failed upload" + assert "loadTree()" not in up_fail, "no KB change on a failed upload" assert "showSyncError" not in up_fail, "no banner on this page (A3)" assert "applySyncFailure" not in up_fail and "showSyncModal" not in up_fail # 7. both idle: the unchanged idle settle. @@ -333,7 +333,7 @@ def test_reattach_adopts_a_running_upload_only() -> None: upload run re-attaches (running state, upload kind — phase 90: no live file, the label stays bare "Importing…" — the synthetic running frame, the poll starts); a terminal upload is a no-op — - the fall-through is the plain idle settle (the boot-time loadDocs() + the fall-through is the plain idle settle (the boot-time loadTree() already shows the current catalog).""" body = _fn(_js(), "initSyncButton") assert "await fetchIsAdmin()" in body, "admin-only (no extra fetch)" @@ -370,7 +370,7 @@ def test_section_header_documents_the_two_job_contract() -> None: header = js[marker : js.find("const syncBtn")] assert "Phase 64 (task 04)" in header assert "/api/git-sources/upload/status" in header, "the second job's endpoint" - assert "loadDocs" in header, "the A3 catalog refresh" + assert "loadTree" in header, "the A3 catalog refresh (phase 97 rename)" assert "A3" in header and "A4" in header diff --git a/tests/unit/test_kb_tree_builder.py b/tests/unit/test_kb_tree_builder.py new file mode 100644 index 0000000..8af061d --- /dev/null +++ b/tests/unit/test_kb_tree_builder.py @@ -0,0 +1,285 @@ +"""Unit tests: ``app.api.docs.build_kb_tree`` (phase 97, task 02). + +The PURE tree builder behind ``GET /api/docs/tree`` — the RAG view's +drill-down tree, the same tree the agent's ``ls`` walks plus file +metadata. Driven without a database (module-level function, plain +inputs): multi-source ordering (the superset rule), the 0-document +registered source, the nested recursive counts, the phase-94 existence +rule, the file/folder name collision, ordering, summaries, verbatim +file metadata, and the ``group_folder_listing`` cross-check property +("the UI shows what the agent sees") at the root and one nested level. +""" +from __future__ import annotations + +from app.api.docs import TreeDocRow, build_kb_tree +from app.rag.agent import group_folder_listing + +T0 = "2026-09-01T08:00:00+00:00" +T1 = "2026-09-02T08:00:00+00:00" +T2 = "2026-09-03T08:00:00+00:00" + + +def _folder_nodes(node) -> list: + """The folder-kind children of a source/folder node, in order.""" + return [child for child in node.children if child.kind == "folder"] + + +def _file_nodes(node) -> list: + """The file-kind children of a source/folder node, in order.""" + return [child for child in node.children if child.kind == "file"] + + +def test_multi_source_registry_order_leads_and_indexed_only_appended() -> None: + """Sources = the registry names in order, then the distinct + indexed-only sources in alphabetical order (the superset rule).""" + names = ["beta", "alpha", "empty"] # registry order — NOT alphabetical + doc_rows: list[TreeDocRow] = [ + ("beta", "b.md", "B", 1, T0), + ("alpha", "a.md", "A", 1, T0), + ("gamma", "g.md", "G", 1, T0), # indexed-only → appended + ("delta", "d.md", "D", 1, T0), # indexed-only → appended + ] + tree = build_kb_tree(names, doc_rows, {}) + assert [s.name for s in tree] == ["beta", "alpha", "empty", "delta", "gamma"] + # The registry sources keep their registry order even though it is + # not alphabetical; the indexed-only ones trail, sorted. + assert tree[0].name == "beta" + assert tree[1].name == "alpha" + assert tree[3].name == "delta" + assert tree[4].name == "gamma" + + +def test_registered_zero_document_source_lists_empty() -> None: + """A registered source with no indexed documents lists (the + phase-70/72 invariant): ``documents: 0``, no children, summary null + when nothing is stored.""" + tree = build_kb_tree(["solo"], [], {}) + assert len(tree) == 1 + assert tree[0].name == "solo" + assert tree[0].documents == 0 + assert tree[0].children == [] + assert tree[0].summary is None + + +def test_nested_document_counts_into_source_ancestors_and_own_folder() -> None: + """A document under ``a/b/c/`` contributes to the source, to ``a``, + to ``a/b``, and to ``a/b/c`` (the recursive subtree, the phase-94 + ``ls`` count rule).""" + rows = [ + ("S", "a/b/c/deep.md", "Deep", 1, T0), + ("S", "a/b/shallow.md", "Shallow", 1, T0), + ("S", "a/top.md", "Top", 1, T0), + ("S", "root.md", "Root", 1, T0), + ] + (source,) = build_kb_tree(["S"], rows, {}) + assert source.documents == 4 + a = _folder_nodes(source)[0] + assert a.path == "a" + assert a.documents == 3 # deep + shallow + top + a_b = _folder_nodes(a)[0] + assert a_b.path == "a/b" + assert a_b.documents == 2 # deep + shallow + a_b_c = _folder_nodes(a_b)[0] + assert a_b_c.path == "a/b/c" + assert a_b_c.documents == 1 # deep only + assert [f.path for f in _file_nodes(a_b_c)] == ["a/b/c/deep.md"] + assert [f.path for f in _file_nodes(a)] == ["a/top.md"] + + +def test_existence_rule_a_file_path_is_never_a_folder() -> None: + """A folder node appears only with a true descendant (some path + starts with ``folder + "/"``); a document's own path — even one + with dots — never creates a folder.""" + rows = [ + ("S", "x.md", "X", 1, T0), + ("S", "x.y/z.md", "Z", 1, T0), + ] + (source,) = build_kb_tree(["S"], rows, {}) + folders = _folder_nodes(source) + # Only ``x.y`` exists (``x.y/z.md`` starts with ``x.y/``); ``x`` and + # ``x.md`` are file names, not folders. + assert [f.path for f in folders] == ["x.y"] + assert [f.path for f in _file_nodes(source)] == ["x.md"] + assert [f.path for f in _file_nodes(folders[0])] == ["x.y/z.md"] + + +def test_file_folder_name_collision_both_appear() -> None: + """A document sharing a directory's name: BOTH appear — the folder + node (via its descendants) and the file node (its own row); the + colliding file counts into the folder's subtree (the ``ls`` count + rule's ``path == folder`` arm).""" + rows = [ + ("S", "a", "File A", 1, T0), # a file wearing the folder's name + ("S", "a/b.md", "B", 1, T0), # makes ``a`` a folder + ] + (source,) = build_kb_tree(["S"], rows, {}) + folders = _folder_nodes(source) + files = _file_nodes(source) + assert [f.path for f in folders] == ["a"] + assert folders[0].documents == 2 # "a" itself + "a/b.md" + assert [f.path for f in files] == ["a"] + assert files[0].title == "File A" + assert [f.path for f in _file_nodes(folders[0])] == ["a/b.md"] + + +def test_subfolder_path_order_and_file_catalog_order() -> None: + """Direct subfolders list in path (sorted) order; direct files keep + the input (catalog — ``GET /api/docs``) order, independent of the + subfolder ordering.""" + rows = [ + ("S", "zeta/z1.md", "Z1", 1, T0), + ("S", "alpha/a1.md", "A1", 1, T0), + ("S", "mike/m1.md", "M1", 1, T0), + ("S", "beta/b1.md", "B1", 1, T0), + ("S", "z-file.md", "Z", 1, T0), # file AFTER the folders in input + ("S", "a-file.md", "A", 1, T0), # file before it in input + ] + (source,) = build_kb_tree(["S"], rows, {}) + assert [f.path for f in _folder_nodes(source)] == ["alpha", "beta", "mike", "zeta"] + # Input order is preserved for the files (z-file.md precedes + # a-file.md in the input, so it does too here — catalog order is + # the INPUT order, not a re-sort). + assert [f.path for f in _file_nodes(source)] == ["z-file.md", "a-file.md"] + + +def test_summaries_present_and_absent() -> None: + """``summary`` is the stored row (source root ``""`` or folder + path) or null when absent — any row (AI or manual is indistinguishable + here; the builder carries whatever is stored).""" + rows = [ + ("S", "one/a.md", "A", 1, T0), + ("S", "one/b.md", "B", 1, T0), + ("S", "two/c.md", "C", 1, T0), + ] + summaries = {("S", ""): "Source desc.", ("S", "one"): "One desc."} + # ("S", "two") is NOT stored → null. + (source,) = build_kb_tree(["S"], rows, summaries) + assert source.summary == "Source desc." + one, two = _folder_nodes(source) + assert one.summary == "One desc." + assert two.summary is None + # File nodes carry no summary key at all (the 00_phase.md shape). + file = _file_nodes(one)[0] + assert set(file.model_dump()) == {"kind", "path", "title", "chunks", "indexed_at"} + assert "summary" not in file.__class__.model_fields + + +def test_file_metadata_unchanged_in_tree() -> None: + """File ``title`` / ``chunks`` / ``indexed_at`` ride into the tree + verbatim from the catalogue row (no reformatting).""" + rows = [("S", "deep/x/y.md", "The Title", 7, T2)] + (source,) = build_kb_tree(["S"], rows, {}) + file = _file_nodes(_folder_nodes(_folder_nodes(source)[0])[0])[0] + assert file.path == "deep/x/y.md" + assert file.title == "The Title" + assert file.chunks == 7 + assert file.indexed_at == T2 + + +def test_empty_inputs_empty_tree() -> None: + """No registry sources and no indexed documents → an empty tree + (the RAG view's ``{"sources": []}`` case).""" + assert build_kb_tree([], [], {}) == [] + + +def test_indexed_document_under_unlisted_source_is_impossible() -> None: + """By construction (the superset rule) every doc source is listed — + the registry names lead and every other doc source follows; there + is no input where a document is dropped.""" + names = ["reg-b", "reg-a"] + rows: list[TreeDocRow] = [ + ("reg-b", "b.md", "B", 1, T0), + ("zzz", "z.md", "Z", 1, T0), + ("aaa", "a.md", "A", 1, T0), + ("reg-a", "a.md", "A2", 1, T0), + ] + tree = build_kb_tree(names, rows, {}) + listed = [s.name for s in tree] + assert listed == ["reg-b", "reg-a", "aaa", "zzz"] + doc_paths = { + f.path for s in tree for f in s.children if f.kind == "file" + } + # Every document from the input is present exactly once. + assert doc_paths == {"b.md", "z.md", "a.md"} + assert sum(s.documents for s in tree) == len(rows) + + +# -------------------------------------------------------------------- +# The cross-check property — "the UI shows what the agent sees": +# for a single-source dataset the builder's level equals +# ``app.rag.agent.group_folder_listing``'s output. +# -------------------------------------------------------------------- + +#: The shared single-source dataset: nested folders, root files, a +#: summary-stored folder and an unstored one. Paths are in (source, +#: path) catalog order; titles map 1:1 to paths. +CROSS_ROWS: list[TreeDocRow] = [ + ("S", "note", "Note", 1, T0), + ("S", "one/a.md", "A", 2, T1), + ("S", "one/b.md", "B", 0, T1), + ("S", "one/two/c.md", "C", 3, T1), + ("S", "one/two/d.md", "D", 1, T1), + ("S", "root.md", "Root", 4, T0), + ("S", "zz/e.md", "E", 2, T2), + ("S", "zz/f.md", "F", 2, T2), +] + +CROSS_SUMMARIES = { + ("S", ""): "Source desc.", + ("S", "one"): "One desc.", + ("S", "zz"): "Zz desc.", + # ("S", "one/two") deliberately unstored → null at that level. +} + + +def _cross_check(folder: str, builder_node) -> None: + """Assert the builder's level *folder* equals + ``group_folder_listing("S", folder, ...)`` — same subfolder + ``(path, count, summary)`` triples in order AND same file + ``(path, title)`` pairs in order (uncapped — the dataset is well + under the ``ls`` 50-line cap, so the cap is inert).""" + rows = [(path, title) for _source, path, title, _chunks, _stamp in CROSS_ROWS] + source_summaries = { + folder_path: summary + for (source, folder_path), summary in CROSS_SUMMARIES.items() + if source == "S" + } + subs, files, _total = group_folder_listing("S", folder, rows, source_summaries) + assert [(f.path, f.documents, f.summary) for f in _folder_nodes(builder_node)] == subs + assert [(f.path, f.title) for f in _file_nodes(builder_node)] == [ + (path, title) for _source, path, title in files + ] + + +def test_cross_check_root_level_matches_group_folder_listing() -> None: + """At the source root the builder's direct subfolders / direct + files equal the agent's root level, element for element.""" + (source,) = build_kb_tree(["S"], CROSS_ROWS, CROSS_SUMMARIES) + _cross_check("", source) + # And the explicit expectations (the test is readable without the + # helper): one → 4 docs (its whole subtree), zz → 2. + assert source.documents == 8 + assert [(f.path, f.documents, f.summary) for f in _folder_nodes(source)] == [ + ("one", 4, "One desc."), + ("zz", 2, "Zz desc."), + ] + assert [(f.path, f.title) for f in _file_nodes(source)] == [ + ("note", "Note"), + ("root.md", "Root"), + ] + + +def test_cross_check_nested_level_matches_group_folder_listing() -> None: + """One nested level (``one``): subfolder ``one/two`` (2 docs, no + stored summary) + the direct files ``one/a.md`` / ``one/b.md`` — + equal to the agent's drill into the same folder.""" + (source,) = build_kb_tree(["S"], CROSS_ROWS, CROSS_SUMMARIES) + one = next(f for f in _folder_nodes(source) if f.path == "one") + _cross_check("one", one) + assert [(f.path, f.documents, f.summary) for f in _folder_nodes(one)] == [ + ("one/two", 2, None), + ] + assert [(f.path, f.title) for f in _file_nodes(one)] == [ + ("one/a.md", "A"), + ("one/b.md", "B"), + ] diff --git a/tests/unit/test_kb_tree_ui.py b/tests/unit/test_kb_tree_ui.py new file mode 100644 index 0000000..20ab950 --- /dev/null +++ b/tests/unit/test_kb_tree_ui.py @@ -0,0 +1,1014 @@ +"""Unit: the RAG view's drill-down catalog tree (phase 97, task 04). + +The RAG (Knowledge base) view lists the catalog the way the agent's +``ls`` sees it (the phase-94 concept, ONE end to end): sources at the +top, then per level the subfolders (recursive count + the STORED +description) and the level's files (the existing 5-column +``#docs-table`` — unchanged). The view's catalog load is now +``loadTree()``: ONE fetch of ``GET /api/docs/tree`` (the full recursive +tree in a single payload — task 02), then client-side drill navigation +(no per-level fetch, no URL change). + +This module pins the source-level contract (the house pattern of +``tests/unit/test_sync_button.py`` — read the frontend files as text, +no browser; the behavior is E2E-covered by the phase's dedicated +suite, task 06): + +* the shell's RAG view ships the static tree skeleton (``#kb-crumb``, + ``#kb-level`` + ``#kb-level-title``/``#kb-level-summary``, + ``#folders-wrap`` + ``#folders-table`` — ONE table for every level, + the Folder | Documents | Description head) in order after + ``#stat-cards`` and BEFORE the unchanged file table; +* the exact ``/api/docs/tree`` fetch (and the flat ``/api/docs`` fetch + is gone from the view module); +* the drill state (``current`` / ``kbTree``), the state transitions + (source row click, folder row click, breadcrumb up, top reset), and + the aria-current last segment; +* the textContent contract (no ``innerHTML`` with document-derived + data — the module's standing rule); +* the top-level semantics (the rows ARE the sources — the ``ls()`` + equivalence — and the file table is hidden at the top); +* the level block's ls rule (hidden when no description is stored); +* the stat cards' whole-tree walk; +* the empty-state semantic (``#sources-empty`` ONLY on zero sources — + the deliberate phase-97 change: a registered 0-document source + renders its row instead); +* the never-stale reset-to-top fallback (PLAN §7.4 — a vanished + location resets ``current`` to the top BEFORE rendering); +* ``loadTree`` wired at exactly the three refresh points (view-refresh, + sync success, upload success) plus the boot load — and the + anonymous branch still never fetches the catalog. + +Phase 97 (task 05) adds the FOLDER-DESCRIPTION EDITOR (the phase-57 +affordance, mirrored) — the source pins this module gains for it: + +* the static ``#kb-level-edit`` button in the level block (type=button, + the .kb-summary-edit class, the "Edit" label, after the description + ``

`` — in the .kb-level-body the editor swaps inside); +* the Description cell of EVERY source/folder row (``makeDescCell``, + called from both row builders) — the stored description as a text + node + the ALWAYS-present Edit button (a description can be CREATED + where none is stored: no gate on the stored value); +* the shared editor (``wireDescriptionEdit``): the swap builds a + textarea prefilled via ``.value`` (never innerHTML — the XSS + contract), Save / Cancel, and the ``role="status"`` + ``aria-live="polite"`` live region; +* the exact PATCH — ``/api/folders/summary``, method PATCH, body + ``{ source, folder_path, summary }`` with ``folder_path ""`` for the + source root (exactly one call site in the module); +* the outcomes — success updates the in-memory ``kbTree`` node IN + PLACE (no re-fetch) + re-renders the text via textContent + " + Description updated."; a cleared echo (summary null) empties the + text (row cell) / hides the level block (the phase-57 announcement + beat, guarded) + "Description cleared."; a failure (non-ok OR + network) keeps the editor open with neutral retry copy (phase-55); + Cancel restores the text node without a fetch; +* the double-click guard (disabled before the fetch, re-enabled in the + ``finally``); the level editor's ``getTarget()`` getter + the + ``reset()`` handle called on every re-render (PLAN §7.4); +* the five ``.kb-summary-*`` classes in styles.css (house palette, no + new hue in the phase-97 block). +""" +from __future__ import annotations + +import re +from pathlib import Path + +FRONTEND = Path(__file__).resolve().parents[2] / "frontend" +ASSETS = FRONTEND / "assets" +SOURCES_JS = ASSETS / "sources.js" +STYLES_CSS = ASSETS / "styles.css" +SHELL_HTML = FRONTEND / "index.html" + + +def _text(path: Path) -> str: + assert path.is_file(), f"missing frontend file: {path}" + return path.read_text(encoding="utf-8") + + +def _js() -> str: + return _text(SOURCES_JS) + + +def _rag_view(html: str) -> str: + """The RAG view section of the shell (view-scoped scope — the shell + carries many views, so whole-file matches hit the wrong view).""" + i = html.find('

str: + """The CSS between two phase markers (comments included — the no-new + hue check wants to see that none was added in prose either).""" + i = css_text.find(start_marker) + assert i != -1, f"missing CSS marker: {start_marker!r}" + j = css_text.find(end_marker, i) + assert j != -1, f"missing CSS end marker: {end_marker!r}" + return css_text[i:j] + + +def _fn(js: str, name: str) -> str: + """The source of a (possibly async, possibly nested) function via + balanced-brace counting (module-level and mount-scoped alike). The + brace count starts AFTER the parameter list — a destructured + parameter (wireDescriptionEdit's target object) may carry braces of + its own.""" + for prefix in ("async function ", "function "): + start = js.find(f"{prefix}{name}(") + if start != -1: + # Skip the parameter list (balanced parens). + depth = 0 + i = js.find("(", start) + close = i + while i < len(js): + if js[i] == "(": + depth += 1 + elif js[i] == ")": + depth -= 1 + if depth == 0: + close = i + break + i += 1 + # Then brace-count the body. + brace = js.find("{", close) + depth = 0 + for j in range(brace, len(js)): + if js[j] == "{": + depth += 1 + elif js[j] == "}": + depth -= 1 + if depth == 0: + return js[start : j + 1] + raise AssertionError(f"unbalanced braces in {name}()") + raise AssertionError(f"{name}() must exist in sources.js") + + +# ---------- the shell's RAG view: the static tree skeleton ---------- + + +def test_rag_view_ships_the_tree_skeleton_in_order() -> None: + """The house no-JS-safe skeleton convention: the tree surfaces ship + in the static HTML (after the stat cards, before the file table) + and ship HIDDEN — assets/sources.js fills them with createElement. + ``#kb-crumb`` is the location nav; ``#kb-level`` is the level block + (title + description); ``#folders-wrap`` hosts the ONE + folders/sources table (``.table-wrap`` card, Folder | Documents | + Description, visually-hidden caption, the ``.docs-table`` + language + ``.kb-folders-table``).""" + view = _rag_view(_text(SHELL_HTML)) + for fragment in ( + '', + '
', + '', + '', + ): + assert fragment in view, f"the RAG view must ship {fragment!r}" + head = re.search(r"(.*?)", view[view.find('id="folders-table"'):], re.S) + assert head, "#folders-table must keep a static thead" + for column in ( + "", + "", + "", + ): + assert column in head.group(1), f"#folders-table head must carry {column!r}" + # The deliberate placement: after the stat cards, before the file + # table (the file table keeps its own wrap + head, unchanged). + assert ( + view.find('id="stat-cards"') + < view.find('id="kb-crumb"') + < view.find('id="kb-level"') + < view.find('id="folders-wrap"') + < view.find('id="docs-table"') + ), "the skeleton must sit between the stat cards and the file table" + # The file table is UNCHANGED (the 5-column contract, makeRow's home). + doc_head = re.search( + r'
FolderDocumentsDescription
.*?(.*?)', view, re.S + ) + assert doc_head, "the file table must keep its static thead" + for column in ("Source", "Path", "Title", "Chunks", "Indexed"): + assert f">{column}" in doc_head.group(1), f"#docs-table head must keep {column!r}" + + +# ---------- the tree load: the ONE fetch + the race token ---------- + + +def test_sources_js_fetches_the_tree_endpoint_only() -> None: + """The catalog load is `loadTree()`: ONE fetch of the tree endpoint. + The flat `GET /api/docs` fetch is GONE from the view module (the + endpoint itself stays — the API surface is untouched, task 02).""" + js = _js() + assert 'fetch("/api/docs/tree")' in js, "the load must fetch /api/docs/tree" + assert js.count('fetch("/api/docs/tree")') == 1, ( + "the tree is the view's SINGLE catalog fetch (zero per-level fetches)" + ) + assert 'fetch("/api/docs")' not in js, "the flat /api/docs fetch must be gone" + + +def test_load_tree_is_race_tokened_and_resets_before_rendering() -> None: + """Phase 79 carries over: the monotonic loadSeq token — only the + newest load may touch the DOM after its await. On success the load + stores the tree in the module-scoped `kbTree`, runs the never-stale + reset, and renders (in that order — the reset must see the NEW + tree and happen BEFORE the render).""" + js = _js() + assert "async function loadTree()" in js, "loadTree must exist" + body = js[js.find("async function loadTree()") :] + body = body[: body.find("\n }")] + assert "const my = ++loadSeq;" in body, "the race token stays (phase 79)" + assert "if (my !== loadSeq) return;" in body, "stale loads must not touch the DOM" + store_i = body.find("kbTree = tree && Array.isArray(tree.sources) ? tree : { sources: [] };") + reset_i = body.find("resetVanishedLocation();") + render_i = body.find("renderLevel();") + assert -1 < store_i < reset_i < render_i, ( + "store the tree → reset a vanished location → render (in order)" + ) + + +# ---------- the drill state + the navigation transitions ---------- + + +def test_drill_state_and_module_tree_are_scoped_to_the_mount() -> None: + """`current` ({ source, folder } — null/null = the top level, + folder "" = the source root) and `kbTree` are mount-scoped module + state (the router mounts a view ONCE — the state survives every + switch, as the sync state machine does).""" + js = _js() + assert "let kbTree = { sources: [] };" in js + assert "let current = { source: null, folder: null };" in js, ( + "the drill state starts at the top level" + ) + + +def test_source_row_click_drills_into_the_source_root() -> None: + """Top-level row = a SOURCE (the ls() equivalence): the name cell + is a `.folder-link` whose click sets `{ source, "" }` (the source + ROOT) and re-renders — client-side only (no fetch, no URL change). + The row carries the recursive count and the stored (source, "") + description — textContent only.""" + js = _js() + body = js[js.find("function makeSourceRow(") : js.find("function makeSourceRow(") + 1200] + assert 'link.className = "folder-link"' in body, "the source row uses the row drill link" + assert "link.textContent = s.name" in body + assert "goTo({ source: s.name, folder: \"\" })" in body, ( + "a source row drills to the source ROOT (folder \"\")" + ) + assert "countTd.textContent = String(s.documents)" in body, "the recursive count" + assert 'makeDescCell(s, s.name, "", s.name)' in body, ( + "the Description cell (task 05): the stored (source, \"\") " + "description + the ALWAYS-present Edit button — folder_path \"\" " + "for the source root" + ) + assert "link.href = \"#\"" in body, "client-side drill — no URL change" + + +def test_folder_row_click_drills_into_the_folder() -> None: + """A level's subfolder row: the folder's LAST path segment as the + label (the full source-relative path rides the title), the + recursive count, the stored description — and the click sets + `{ source, folder: f.path }` (the source-relative folder path).""" + js = _js() + start = js.find("function makeFolderRow(") + body = js[start : start + 1400] + assert 'link.className = "folder-link"' in body + assert "link.textContent = f.path.split(\"/\").pop()" in body, ( + "the label is the last segment (the breadcrumb language)" + ) + assert "link.title = current.source + \"/\" + f.path" in body, "full path on hover" + assert "goTo({ source: current.source, folder: f.path })" in body, ( + "a folder row drills to that folder" + ) + assert "countTd.textContent = String(f.documents)" in body + assert ( + 'makeDescCell(f, current.source, f.path, current.source + "/" + f.path)' + in body + ), "the Description cell (task 05): the stored row + the ALWAYS-present Edit" + + +def test_breadcrumb_links_go_up_and_the_top_reset() -> None: + """The breadcrumb: hidden at the top level; when drilled in, one + link per ancestor — the TOP level (→ `{ null, null }` — the reset), + the source (→ `{ source, "" }`), then the folder chain (→ each + ancestor folder) — the LAST segment a span with + aria-current=\"page\" (not a link).""" + js = _js() + start = js.find("function renderCrumb(") + body = js[start : js.find("function crumbSegment(")] + assert "if (current.source === null) {" in body + assert "crumbEl.hidden = true" in body, "the breadcrumb is hidden at the top level" + assert "crumbSegment(\"Knowledge base\", { source: null, folder: null })" in body, ( + "the top-level segment goes back to the sources list (the reset)" + ) + assert "crumbSegment(current.source, { source: current.source, folder: \"\" })" in body, ( + "the source segment goes up to the source root" + ) + assert "crumbSegment(part, { source: current.source, folder: acc })" in body, ( + "each folder-ancestor segment goes up to that folder" + ) + assert "crumbCurrent(part)" in body, "the last segment is the current span" + seg = js[js.find("function crumbSegment(") : js.find("function crumbCurrent(")] + assert 'a.className = "kb-crumb-link"' in seg + assert "a.textContent = label" in seg, "segment labels are textContent" + cur = js[js.find("function crumbCurrent(") : js.find("function goTo(")] + assert 'span.className = "kb-crumb-current"' in cur + assert 'span.setAttribute("aria-current", "page")' in cur, ( + "the last segment is aria-current (a span, never a link)" + ) + assert "span.textContent = label" in cur + + +def test_navigation_is_client_side_only() -> None: + """goTo() is the ONLY navigation: set `current`, re-render. No + fetch (the tree is already in memory), no URL change (history is + untouched — the deep-link contract of the shell is unchanged).""" + js = _js() + body = js[js.find("function goTo(") : js.find("function makeSourceRow(")] + assert "current = { source: target.source, folder: target.folder };" in body + assert "renderLevel();" in body + assert "fetch(" not in body, "the drill never fetches (client-side only)" + assert "pushState" not in body and "replaceState" not in body, "no URL change" + + +# ---------- the level rendering ---------- + + +def test_top_level_lists_sources_and_hides_the_file_table() -> None: + """Top level: the breadcrumb + level block are hidden; the folders + table lists the SOURCES themselves (makeSourceRow for every source) + and the file table is ALWAYS hidden (files are seen per source — + `ls()` shows none at the top).""" + js = _js() + start = js.find("function renderLevel(") + body = js[start : js.find("function renderEmpty(")] + top = body[body.find("if (current.source === null) {") :] + assert "levelEl.hidden = true;" in top, "the level block is hidden at the top level" + assert "for (const s of kbTree.sources) foldersTbody.appendChild(makeSourceRow(s));" in top, ( + "ONE table for every level: at the top the rows are the sources" + ) + assert "foldersWrap.hidden = false;" in top + assert "if (tableWrap) tableWrap.hidden = true;" in top, ( + "the file table is always hidden at the top level" + ) + + +def test_level_block_shows_the_stored_description_only() -> None: + """Inside a source/folder: the level block shows the CURRENT + level's stored description — the title is the full source-relative + path (e.g. `alpha/two`; the source root shows the source name) — + and is HIDDEN when none is stored (the ls rule: count only, no + placeholder).""" + js = _js() + start = js.find("function renderLevel(") + body = js[start : js.find("function renderEmpty(")] + assert "if (node.summary) {" in body, "the ls rule: no description → no block" + summary_if = body.find("if (node.summary) {") + shown = body[summary_if : body.find("} else {", summary_if)] + assert "levelTitleEl.textContent = current.folder" in shown + assert "current.source + \"/\" + current.folder" in shown, ( + "the folder title is the full source-relative path" + ) + assert ": current.source;" in shown, "the source root title is the source name" + assert "levelSummaryEl.textContent = node.summary;" in shown + assert "levelEl.hidden = false;" in shown + hidden = body[body.find("} else {", body.find("if (node.summary) {")) :] + assert "levelEl.hidden = true;" in hidden[:200], "no stored row → the block is hidden" + + +def test_level_lists_subfolders_and_direct_files_only() -> None: + """A level's folders table = its DIRECT subfolders (hidden when + none); its file table = its DIRECT files (hidden when none) — + makeRow UNCHANGED (the file node carries no source: the row object + restores the flat /api/docs shape makeRow reads, so the phase-10/26 + pins on makeRow stay green).""" + js = _js() + start = js.find("function renderLevel(") + body = js[start : js.find("function renderEmpty(")] + assert 'children.filter((c) => c.kind === "folder")' in body + assert 'children.filter((c) => c.kind === "file")' in body + assert "for (const f of subfolders) foldersTbody.appendChild(makeFolderRow(f));" in body + assert "foldersWrap.hidden = subfolders.length === 0;" in body, ( + "#folders-wrap hidden when the level has no subfolders" + ) + assert "for (const f of files) {" in body + assert "source: current.source," in body, "the row object restores the source" + assert "makeRow({" in body, "makeRow is unchanged (the phase-10/26 pins)" + assert "if (tableWrap) tableWrap.hidden = files.length === 0;" in body, ( + "the file table hides when the level has no direct files" + ) + # makeRow itself is untouched — the standing pins on its body. + assert "link.href = documentUrl(d.source, d.path)" in js + assert "openDocumentModal(d.source, d.path, link)" in js + + +def test_render_clears_both_row_containers_before_filling() -> None: + """Re-entrancy (the phase-77 pattern, extended to the second + container): the render clears BOTH tbodies at the top — a refresh + from a populated level into a sparser one leaves no ghost rows.""" + js = _js() + start = js.find("function renderLevel(") + body = js[start : js.find("\n }", start)] + clear_folders = body.find("if (foldersTbody) foldersTbody.replaceChildren();") + clear_docs = body.find("if (tbody) tbody.replaceChildren();") + assert 0 <= clear_folders < clear_docs, "both containers clear at the top" + for append in ("foldersTbody.appendChild", "tbody.appendChild"): + assert clear_docs < body.find(append), f"{append} must come after the clears" + + +# ---------- the stat cards: the whole-tree walk ---------- + + +def test_stat_cards_walk_the_whole_tree() -> None: + """The KB-wide stat cards walk the WHOLE tree (every file node at + any depth): document count, chunks sum, max indexed_at (fmtDate + reuse) — the values identical to the former flat walk. A + 0-document registered source contributes nothing (and still lists + its row — the ls invariant).""" + js = _js() + body = js[js.find("function treeStats()") : js.find("function renderCrumb(")] + assert "docs += 1;" in body + assert "totalChunks += child.chunks;" in body + assert "if (child.indexed_at > last) last = child.indexed_at;" in body + assert "walk(child);" in body, "the walk recurses into subfolders" + assert "for (const s of kbTree.sources) walk(s);" in body, "every source is walked" + render = js[js.find("function renderLevel(") : js.find("function renderEmpty(")] + assert "const st = treeStats();" in render + assert "statDocs.textContent = String(st.docs);" in render + assert "statChunks.textContent = String(st.totalChunks);" in render + assert 'st.last ? fmtDate(st.last) : "–"' in render, "the fmtDate reuse" + + +# ---------- the empty state: the zero-sources semantic ---------- + + +def test_empty_state_is_zero_sources_only() -> None: + """The deliberate phase-97 semantic change (module docstring): + #sources-empty shows ONLY when the tree has zero sources — a + registered 0-document source renders its row instead (the ls + invariant). A failed tree fetch renders the same no-data state + (the former loadDocs failure behavior, unchanged in kind).""" + js = _js() + render = js[js.find("function renderLevel(") : js.find("function renderEmpty(")] + assert "if (!kbTree.sources.length) {" in render, ( + "the empty state fires on ZERO SOURCES (not zero documents)" + ) + assert "renderEmpty();" in render[render.find("if (!kbTree.sources.length) {"):] + empty_start = js.find("function renderEmpty(") + empty = js[empty_start : js.find("\n }", empty_start)] + assert 'statDocs.textContent = "0";' in empty + assert 'statChunks.textContent = "0";' in empty + assert 'statLast.textContent = "–";' in empty + assert "if (foldersWrap) foldersWrap.hidden = true;" in empty, "both wraps hidden" + assert "if (tableWrap) tableWrap.hidden = true;" in empty + assert "emptyEl.hidden = false;" in empty, "#sources-empty shows" + # The failure paths of the load land in the same no-data state. + load = js[js.find("async function loadTree()") : js.find("async function loadTree()") + 900] + assert load.count("renderEmpty();") >= 2, "a failed fetch renders the no-data state" + + +def test_module_docstring_documents_the_semantic_change() -> None: + """The house module-docstring convention: the phase-97 section + records the tree load, the drill state, and the deliberate + 0-document-source semantic change.""" + js = _js() + header = js[: js.find("import { fetchIsAdmin }")] + assert "Phase 97 (task 04)" in header + assert "zero SOURCES" in header, "the empty-state semantic change is documented" + assert "0-document source renders its row" in header, ( + "the registered 0-document source exception is documented" + ) + assert "loadTree()" in header + + +# ---------- the never-stale reset-to-top fallback ---------- + + +def test_reset_to_top_when_the_location_vanished() -> None: + """PLAN §7.4: after a re-fetch, if the current source is no longer + in the tree (unregistered/pruned) OR the current folder no longer + exists under it, `current` RESETS to the top level BEFORE rendering + — no stale breadcrumb, no stale block. The folder existence check + mirrors the phase-94 rule: some indexed path starts with + `folder + "/"` (the source root — \"\" — always exists).""" + js = _js() + body = js[js.find("function resetVanishedLocation()") : js.find("function folderExistsIn(")] + assert "if (current.source === null) return;" in body, "the top level never vanishes" + assert "kbTree.sources.find((s) => s.name === current.source)" in body + assert body.count("current = { source: null, folder: null };") == 2, ( + "both vanishing arms reset to the top level" + ) + exists = js[js.find("function folderExistsIn(") : js.find("function currentLevelNode(")] + assert 'if (folderPath === "") return true;' in exists, "the source root always exists" + assert "p.startsWith(folderPath + \"/\")" in exists, "the phase-94 existence rule" + # The reset runs on the success path of the load, before the render + # (pinned in the load-order test — here: it is defined and called). + assert "resetVanishedLocation();" in js[js.find("async function loadTree()"):] + + +# ---------- the refresh wirings: loadTree at the three points ---------- + + +def test_load_tree_wired_at_exactly_the_three_refresh_points_plus_boot() -> None: + """loadDocs's call sites become loadTree's (same points, same + semantics): the boot load, the bor:view-refresh listener (admin + branch), applySyncSuccess (the KB just changed), and the + upload-success branch of startSyncPolling — and the old name is + gone. Exactly 5 occurrences of `loadTree()`: the definition + + those 4 calls (no other call site exists).""" + js = _js() + assert "loadDocs" not in js, "the old load name is gone from the module" + assert _code(js).count("loadTree()") == 5, ( + "the definition + the 4 call sites, no others (code, not comments)" + ) + # 1. the view-refresh listener, armed in the admin branch. + listener = js.find('addEventListener("bor:view-refresh"') + assert listener != -1 + gate = js.find("const admin = await fetchIsAdmin();") + assert -1 < gate < listener, "armed after the whoami gate (the phase-77 pin)" + assert "() => loadTree()" in js[listener : listener + 120] + # 2. the boot load — the last statement of mount, right after the + # listener is armed (the mount's own load is the first fetch). + assert 'root.addEventListener("bor:view-refresh", () => loadTree());\n loadTree();' in js, ( + "the boot load follows the listener" + ) + # 3. applySyncSuccess (the sync's terminal — the KB just changed). + success = js[js.find("function applySyncSuccess(") : js.find("function applySyncFailure(")] + assert "loadTree();" in success + # 4. the upload-success branch of the tick (settle + refresh). + tick = js[js.find("function startSyncPolling(") : js.find("function startSync()")] + up_ok = tick.find('uploadStatus && uploadStatus.state === "success"') + up_fail = tick.find('uploadStatus && uploadStatus.state === "failed"') + assert -1 < up_ok < up_fail + assert "loadTree();" in tick[up_ok:up_fail], "the upload success refreshes the tree" + + +def test_anonymous_branch_never_fetches_the_tree() -> None: + """The phase-16 soft rule (unchanged in kind): the anonymous branch + gates the catalog in / out with NO catalog fetch — no /api/docs/ + tree request at all, and the tree surfaces stay hidden (they ship + hidden and never fill).""" + js = _js() + start = js.find("if (!admin) {") + end = js.find("if (gateEl) gateEl.hidden = true;") + assert -1 < start < end, "the anonymous branch must exist" + branch = js[start:end] + assert "fetch(" not in branch, "anonymous never fetches the catalog" + assert "loadTree" not in branch, "anonymous never starts a tree load" + assert "if (foldersWrap) foldersWrap.hidden = true;" in branch, ( + "the folders wrap is part of the anonymous hide" + ) + assert "if (tableWrap) tableWrap.hidden = true;" in branch + assert "if (gateEl) gateEl.hidden = false;" in branch + + +# ---------- the textContent contract ---------- + + +def _code(js: str) -> str: + """sources.js with comments stripped (a comment may legally carry + any string — the count pins below must only see code).""" + no_block = re.sub(r"/\*.*?\*/", "", js, flags=re.S) + return re.sub(r"//.*", "", no_block) + + +def test_tree_cells_never_use_innerhtml_with_derived_data() -> None: + """The module's standing rule: document-derived text goes through + textContent only. The ONE innerHTML left in the module CODE is the + static (non-derived) sync-error modal skeleton — unchanged since + phase 41.""" + js = _js() + code = _code(js) + assert code.count("innerHTML") == 1, ( + "only the static sync-modal skeleton may use innerHTML (code, not comments)" + ) + modal = code.find("function createSyncModal()") + assert modal != -1 + assert code.find("innerHTML") > modal, "the single innerHTML is the sync-modal creation" + # The tree's derived-text sinks are all textContent (spot the + # contract across the builders + the level block). + for sink in ( + "link.textContent = s.name", + "link.textContent = f.path.split(\"/\").pop()", + 'text.textContent = (node && node.summary) || ""', + "levelSummaryEl.textContent = node.summary;", + "a.textContent = label", + "span.textContent = label", + "countTd.textContent = String(s.documents)", + "countTd.textContent = String(f.documents)", + ): + assert sink in js, f"the derived-text contract: {sink!r}" + + +# ---------- the file-table wrap lookup (the new class collision) ---------- + + +def test_file_wrap_lookup_avoids_the_new_folders_wrap() -> None: + """#folders-wrap ALSO carries the shared .table-wrap class (the + card language) — the FILE table's wrap is looked up through its + own table, not the class (a class lookup would hit #folders-wrap + first in document order and hide/show the wrong card).""" + js = _js() + assert 'root.querySelector("#docs-table").parentElement' in js + assert 'root.querySelector(".table-wrap")' not in js, ( + "the bare class lookup must not survive (it would hit #folders-wrap)" + ) + + +# ---------- the styles: the tree's classes, no new hue ---------- + + +def test_styles_carry_the_tree_classes_with_no_new_hue() -> None: + """The new classes exist (.kb-crumb + link/current/separator, + .kb-level, .kb-folders-table, .folder-link) and the phase-97 block + introduces NO new hue (the phase-92 monochrome invariant — every + color via a var(); no hex/rgb literal).""" + css = _text(STYLES_CSS) + for selector in ( + ".kb-crumb", + ".kb-crumb-link", + ".kb-crumb-current", + ".kb-crumb-sep", + ".kb-level", + ".kb-folders-table", + ".folder-link", + ): + assert re.search(r"(? None: + """The level block ships the phase-57 Edit button in the static + HTML: a real type=button with the .kb-summary-edit class and the + "Edit" label, INSIDE #kb-level after p#kb-level-summary — in the + .kb-level-body (the editor swaps inside it, the

stays put).""" + view = _rag_view(_text(SHELL_HTML)) + assert '
' in view, ( + "the level's description UI ships in the .kb-level-body" + ) + assert ( + '' + in view + ), "the static level Edit button (phase-57 label, real button)" + body = view[view.find('id="kb-level"') : view.find('id="folders-wrap"')] + assert ( + body.find('

') + < body.find('

') + < body.find('id="kb-level-edit"') + ), "title → description → Edit button, in order" + + +def test_row_description_cell_builds_text_and_always_present_edit() -> None: + """makeDescCell (the shared Description-cell builder, called from + BOTH row builders): the stored description as a text node + (textContent — never innerHTML) + the Edit button added + UNCONDITIONALLY — a description can be CREATED where none is + stored (a < 2-document folder, the generator's fail-soft miss), so + there is NO gate on the stored value. The button is a real + type=button with a human aria-label, and the shared editor is + wired with a CONSTANT target { node, source, folder }.""" + body = _fn(_js(), "makeDescCell") + assert 'text.textContent = (node && node.summary) || ""' in body, ( + "the stored description is a text node (or an empty cell)" + ) + assert 'btn.type = "button"' in body + assert 'btn.className = "kb-summary-edit"' in body + assert 'btn.textContent = "Edit"' in body + assert 'btn.setAttribute("aria-label", `Edit description: ${label}`)' in body + assert "td.append(text, btn)" in body + assert "if (node" not in body, ( + "always present: no gate on the stored description" + ) + assert "getTarget: () => ({ node, source, folder })" in body, ( + "a row's target is a constant (its own node)" + ) + + +def test_editor_swap_builds_textarea_save_cancel_and_live_region() -> None: + """Edit swaps the description UI for the inline editor: a +