From 065304b52f5f9f2a31343ce88c0fee18fa453b91 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Thu, 15 Oct 2026 23:19:27 -0400 Subject: [PATCH] phase: 184_sync_bounded_session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All gates green; no code changes were needed — the prior failure was a missing final report, not a test failure. **Verified / implemented (phase 184 final pass)** - All 4 tasks present in `.agents/phases/complete/184_sync_bounded_session/`; confirmed their output in the worktree: `sync_session_reset_files=200` / `sync_prune_batch=500` on `Settings` (+ non-negative validators), both `SettingMeta` rows under `"Import & sources"`, `.env.example` + `README.md` entries. - `import_sources` expunges at **exactly two** sites (per-file loop top L972, source boundary L1012) — grep proves no third `expunge`/`expire_all` anywhere in `app/`; `app/db.py:30` untouched. - `_prune` uses `load_only(id, source, path, is_image, image_path, page_count).order_by(Document.id)` + per-batch commits, with the phase-122/126 guards and `changed_paths` intact. **Tests / lint / coverage (all 100% green)** - `uv run pytest --cov=app --cov-report=term-missing` → **4506 passed**, `app/` TOTAL **99%** (>90%). - `uv run pytest tests/integration/test_import_session_retention.py -v --no-cov` → **12 passed**. - `uv run pytest tests/e2e/test_sync_session_reset.py -v --no-cov` → **4 passed in isolation**. - Regression sweep of neighbouring sync suites (each isolated): `test_sync_changed_only` 3, `test_sync_summary_visibility` 3, `test_prune_cascade_cleanup` 3, `test_sync_git_freeze` 2 — all passed. - `uv run ruff check .` → All checks passed. `uv run pyright` → 0 errors, 0 warnings. **Completion criteria** 1. Settings + CATALOG + `.env.example` + README; `test_settings_catalog.py` → ✅ green 2. Retention test green with the post-fix bound and both measured numbers in the docstring (pre-fix 12-row monotone staircase / ≈265 MB extrapolated; post-fix ends at 0, add-path peak 14 vs 21→49) → ✅ 3. Expunge only at the two documented-safe points; light-row batched prune with guards intact → ✅ 4. Every existing importer/sync/removal test green with unchanged expectations → ✅ 5. Full suite green, coverage >90%, ruff + pyright clean → ✅ 6. Phase E2E green in isolation → ✅ 7. README + AGENTS accurate/consistent → ✅ 8. Atomic commit — left to the harness (I ran no git write commands). **Documentation** — Full README (1886 lines) + AGENTS.md (189 lines) review: **verified consistent — no changes needed.** Programmatic cross-checks: every `Settings` field appears in `.env.example` and in the README config reference; every `.env.example` var is documented in that reference; every `tests/…` and `app/…` path cited in README/AGENTS exists; all 10 README page URLs are registered in `app/main.py`; documented E2E port vars (`E2E_APP_PORT_SYNCRESET[_ZERO]`) exist in the test. The only reference-var without a `Settings` field is `BOR_THEME`, which the README explicitly documents as retired (correct as written). **Notable decisions/deviations** — None. No code, test, or doc edits were made this pass; changes remain entirely in the working tree for the harness to stage. `00_phase.md` still sits in `todo/` while tasks 01–04 are in `complete/` — harness-owned, so I left it untouched. **Next pending phase:** `185_sync_defer_content_fast_path`. --- .../184_sync_bounded_session/00_phase.md | 0 .../01_identity_map_probe.md | 0 .../02_session_reset_cadence.md | 0 .../03_batched_prune.md | 0 .../04_e2e_bounded_sync.md | 0 .../184_sync_bounded_session__00_phase.a1.err | 0 .../184_sync_bounded_session__00_phase.a1.md | 1 + ...ync_bounded_session__00_phase.a1.md.status | 1 + ...sync_bounded_session__00_phase.a1.validate | 143 ++ .../184_sync_bounded_session__00_phase.a2.err | 0 .../184_sync_bounded_session__00_phase.a2.md | 29 + ...ync_bounded_session__00_phase.a2.md.status | 1 + ...sync_bounded_session__00_phase.a2.validate | 143 ++ ...nded_session__01_identity_map_probe.a1.err | 0 ...unded_session__01_identity_map_probe.a1.md | 19 + ...ession__01_identity_map_probe.a1.md.status | 1 + ...session__01_identity_map_probe.a1.validate | 143 ++ ...d_session__02_session_reset_cadence.a1.err | 0 ...ed_session__02_session_reset_cadence.a1.md | 17 + ...ion__02_session_reset_cadence.a1.md.status | 1 + ...sion__02_session_reset_cadence.a1.validate | 143 ++ ...c_bounded_session__03_batched_prune.a1.err | 0 ...nc_bounded_session__03_batched_prune.a1.md | 20 + ...ded_session__03_batched_prune.a1.md.status | 1 + ...nded_session__03_batched_prune.a1.validate | 143 ++ ...ounded_session__04_e2e_bounded_sync.a1.err | 0 ...bounded_session__04_e2e_bounded_sync.a1.md | 10 + ..._session__04_e2e_bounded_sync.a1.md.status | 1 + ...d_session__04_e2e_bounded_sync.a1.validate | 143 ++ .env.example | 25 + AGENTS.md | 39 + README.md | 36 +- app/config.py | 51 + app/core/settings_catalog.py | 6 + app/rag/importer.py | 136 +- tests/e2e/test_sync_session_reset.py | 919 ++++++++++++ .../test_import_session_retention.py | 1247 +++++++++++++++++ tests/integration/test_importer.py | 16 +- tests/unit/test_config.py | 55 + tests/unit/test_settings_catalog.py | 35 + 40 files changed, 3512 insertions(+), 13 deletions(-) rename .agents/phases/{todo => complete}/184_sync_bounded_session/00_phase.md (100%) rename .agents/phases/{todo => complete}/184_sync_bounded_session/01_identity_map_probe.md (100%) rename .agents/phases/{todo => complete}/184_sync_bounded_session/02_session_reset_cadence.md (100%) rename .agents/phases/{todo => complete}/184_sync_bounded_session/03_batched_prune.md (100%) rename .agents/phases/{todo => complete}/184_sync_bounded_session/04_e2e_bounded_sync.md (100%) create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.err create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.md create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.md.status create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.validate create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.err create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.md create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.md.status create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.validate create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.err create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.md create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.md.status create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.validate create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.err create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.md create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.md.status create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.validate create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.err create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.md create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.md.status create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.validate create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.err create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.md create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.md.status create mode 100644 .agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.validate create mode 100644 tests/e2e/test_sync_session_reset.py create mode 100644 tests/integration/test_import_session_retention.py diff --git a/.agents/phases/todo/184_sync_bounded_session/00_phase.md b/.agents/phases/complete/184_sync_bounded_session/00_phase.md similarity index 100% rename from .agents/phases/todo/184_sync_bounded_session/00_phase.md rename to .agents/phases/complete/184_sync_bounded_session/00_phase.md diff --git a/.agents/phases/todo/184_sync_bounded_session/01_identity_map_probe.md b/.agents/phases/complete/184_sync_bounded_session/01_identity_map_probe.md similarity index 100% rename from .agents/phases/todo/184_sync_bounded_session/01_identity_map_probe.md rename to .agents/phases/complete/184_sync_bounded_session/01_identity_map_probe.md diff --git a/.agents/phases/todo/184_sync_bounded_session/02_session_reset_cadence.md b/.agents/phases/complete/184_sync_bounded_session/02_session_reset_cadence.md similarity index 100% rename from .agents/phases/todo/184_sync_bounded_session/02_session_reset_cadence.md rename to .agents/phases/complete/184_sync_bounded_session/02_session_reset_cadence.md diff --git a/.agents/phases/todo/184_sync_bounded_session/03_batched_prune.md b/.agents/phases/complete/184_sync_bounded_session/03_batched_prune.md similarity index 100% rename from .agents/phases/todo/184_sync_bounded_session/03_batched_prune.md rename to .agents/phases/complete/184_sync_bounded_session/03_batched_prune.md diff --git a/.agents/phases/todo/184_sync_bounded_session/04_e2e_bounded_sync.md b/.agents/phases/complete/184_sync_bounded_session/04_e2e_bounded_sync.md similarity index 100% rename from .agents/phases/todo/184_sync_bounded_session/04_e2e_bounded_sync.md rename to .agents/phases/complete/184_sync_bounded_session/04_e2e_bounded_sync.md diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.err b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.md b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.md new file mode 100644 index 0000000..428828c --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.md @@ -0,0 +1 @@ +(no final assistant message — see the .err log) diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.md.status b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.md.status new file mode 100644 index 0000000..65a791b --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.md.status @@ -0,0 +1 @@ +error: no final assistant text diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.validate b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.validate new file mode 100644 index 0000000..02ee690 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a1.validate @@ -0,0 +1,143 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 11%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 99%] +.......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 318 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 862 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 252 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6870 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.err b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.md b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.md new file mode 100644 index 0000000..ecc1a6d --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.md @@ -0,0 +1,29 @@ +All gates green; no code changes were needed — the prior failure was a missing final report, not a test failure. + +**Verified / implemented (phase 184 final pass)** +- All 4 tasks present in `.agents/phases/complete/184_sync_bounded_session/`; confirmed their output in the worktree: `sync_session_reset_files=200` / `sync_prune_batch=500` on `Settings` (+ non-negative validators), both `SettingMeta` rows under `"Import & sources"`, `.env.example` + `README.md` entries. +- `import_sources` expunges at **exactly two** sites (per-file loop top L972, source boundary L1012) — grep proves no third `expunge`/`expire_all` anywhere in `app/`; `app/db.py:30` untouched. +- `_prune` uses `load_only(id, source, path, is_image, image_path, page_count).order_by(Document.id)` + per-batch commits, with the phase-122/126 guards and `changed_paths` intact. + +**Tests / lint / coverage (all 100% green)** +- `uv run pytest --cov=app --cov-report=term-missing` → **4506 passed**, `app/` TOTAL **99%** (>90%). +- `uv run pytest tests/integration/test_import_session_retention.py -v --no-cov` → **12 passed**. +- `uv run pytest tests/e2e/test_sync_session_reset.py -v --no-cov` → **4 passed in isolation**. +- Regression sweep of neighbouring sync suites (each isolated): `test_sync_changed_only` 3, `test_sync_summary_visibility` 3, `test_prune_cascade_cleanup` 3, `test_sync_git_freeze` 2 — all passed. +- `uv run ruff check .` → All checks passed. `uv run pyright` → 0 errors, 0 warnings. + +**Completion criteria** +1. Settings + CATALOG + `.env.example` + README; `test_settings_catalog.py` → ✅ green +2. Retention test green with the post-fix bound and both measured numbers in the docstring (pre-fix 12-row monotone staircase / ≈265 MB extrapolated; post-fix ends at 0, add-path peak 14 vs 21→49) → ✅ +3. Expunge only at the two documented-safe points; light-row batched prune with guards intact → ✅ +4. Every existing importer/sync/removal test green with unchanged expectations → ✅ +5. Full suite green, coverage >90%, ruff + pyright clean → ✅ +6. Phase E2E green in isolation → ✅ +7. README + AGENTS accurate/consistent → ✅ +8. Atomic commit — left to the harness (I ran no git write commands). + +**Documentation** — Full README (1886 lines) + AGENTS.md (189 lines) review: **verified consistent — no changes needed.** Programmatic cross-checks: every `Settings` field appears in `.env.example` and in the README config reference; every `.env.example` var is documented in that reference; every `tests/…` and `app/…` path cited in README/AGENTS exists; all 10 README page URLs are registered in `app/main.py`; documented E2E port vars (`E2E_APP_PORT_SYNCRESET[_ZERO]`) exist in the test. The only reference-var without a `Settings` field is `BOR_THEME`, which the README explicitly documents as retired (correct as written). + +**Notable decisions/deviations** — None. No code, test, or doc edits were made this pass; changes remain entirely in the working tree for the harness to stage. `00_phase.md` still sits in `todo/` while tasks 01–04 are in `complete/` — harness-owned, so I left it untouched. + +**Next pending phase:** `185_sync_defer_content_fast_path`. diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.md.status b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.validate b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.validate new file mode 100644 index 0000000..02ee690 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__00_phase.a2.validate @@ -0,0 +1,143 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 11%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 99%] +.......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 318 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 862 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 252 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6870 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.err b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.md b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.md new file mode 100644 index 0000000..f773d5f --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.md @@ -0,0 +1,19 @@ +All gates green. Report: + +**Implemented** +- `app/config.py`: `sync_session_reset_files: int = 200` + `sync_prune_batch: int = 500` (Import & sources section, `BOR_`-prefixed, each with a non-negative `field_validator` following the `agent_max_rounds`/`compress_min_bytes` precedent; `0` = exact pre-phase-184 behaviour, documented). +- `app/core/settings_catalog.py`: `CATALOG` rows `"Sync session reset files"` / `"Sync prune batch"` in `"Import & sources"` — plain, live, overridable. +- `.env.example`: new `--- Sync session bounds (phase 184) ---` block (defaults, `0` semantics, the batched-prune atomicity note, "inert until tasks 02/03"). +- NEW `tests/integration/test_import_session_retention.py` — the retention probe with the `_map_size()` door: added-run release, the unchanged-run staircase pin (`0,1,…,11` → exactly 12 `Document`s resident, content included, **not** collectable, `expunge_all()` releases all), the `0`-means-pre-phase rollback pin, and the weak-clean/dirty-strong mechanism test. +- Unit pins: 4 in `tests/unit/test_config.py` (defaults, env override, `0` accepted, negative refuses startup), 2 in `tests/unit/test_settings_catalog.py` (group/label/flags + field types/defaults). + +**Gates** — `uv run pytest --cov=app --cov-report=term-missing` → **4498 passed**, `TOTAL 99%` (`app/config.py` 100%, `settings_catalog.py` 100%); `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors; `tests/integration/test_import_session_retention.py tests/unit/test_settings_catalog.py -v` → 19 passed (run 4× for flake-stability); sanity E2E in isolation: `test_admin_settings_panel.py` 12 passed, `test_sync_changed_only.py` 3 passed. + +**Docs** — `README.md`: two rows in the Configuration reference "Import & sources" table (flagged *inert until task 02/03 wires it*, since nothing reads them yet). No `AGENTS.md` change (it lists no env knobs). + +**Notable findings / deviations (important for task 02)** +- D1's premise is **CONFIRMED in effect but wrong in mechanism**: the identity map is *weak*, so the ADD run does **not** accumulate (84/84 present only until one `gc.collect()`, which drops it to 0) — the task's `len(identity_map) >= 12*2` pin on a fresh add run is **not true**. The real, monotone, KB-proportional retention is the **unchanged** path: `_unchanged_doc`'s `doc.file_hash = file_digest` stamp marks every row `modified` while `session.is_modified()` is false (no UPDATE, commit skipped), so the row lands in `Session.dirty` — a *strong* session-side set — and survives every collection with its full `content` (extrapolated ≈265 MB for the 17,407-doc / 15.2 KB KB). `expire_on_commit=False` is not the retainer; the phase still must not change it. +- Hence `expunge_all()` **is** the right lever, but it discards unflushed changes: safe only at D3's point (per-file commit inside `_index_file`, `importer.py:1575`) — written into the module docstring as a hazard note for task 02. Task 02's post-fix bound against this fixture is "≤ reset + slack" starting from `MEASURED_UNCHANGED_MAP_SIZE = 12`. +- RSS is recorded, never asserted (inherited A4 rule): ≈35 KB/file, unreturned after `gc.collect()` while live-object count stays flat — allocator, not session; `expunge_all` cannot fix that part. + +**Next pending task:** `.agents/phases/todo/184_sync_bounded_session/02_session_reset_cadence.md`. diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.md.status b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.validate b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.validate new file mode 100644 index 0000000..e0d8e4c --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__01_identity_map_probe.a1.validate @@ -0,0 +1,143 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 9%] +........................................................................ [ 11%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 97%] +........................................................................ [ 99%] +.................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 318 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 855 4 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 252 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6863 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.err b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.md b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.md new file mode 100644 index 0000000..28b9d81 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.md @@ -0,0 +1,17 @@ +**Implemented (phase 184 task 02)** +- `app/rag/importer.py` `import_sources`: `reset_every = llm.settings.sync_session_reset_files` read ONCE before the source loop; `session.expunge_all()` at exactly two guarded points — top of the per-file loop (`summary.files % reset_every == 0`) and once per source before the two post-passes — with the full safety argument (commits already done, `expunge_all` discards only the fast path's no-op stamp, hooks/`dates_map`/`seen`/`total` plain, rollback path touches nothing) plus a docstring paragraph. No other call sites. +- `tests/integration/test_import_session_retention.py`: added `test_reset_cadence_bounds_the_map` (cadence 3 vs the cadence-0 reference: identical `ImportSummary` via `asdict`, identical DB per path/content/summary/`file_hash`/chunk-count; end map ≤ `MAP_BOUND`=17, mid-run ≤ 2×7 objects), `test_reset_cadence_bounds_the_unchanged_walk` (sawtooth `[0,1,2]×4` vs the pre-fix ramp, end map `0` → pins the source boundary), `test_reset_cadence_survives_an_embedding_error` (stub refuses file #4 with cadence 2: `errors=1`, 11 survivors × 6 chunks, `changed_paths` correct); kept both cadence-`0` guarantee tests; docstring flipped to `CONFIRMED → FIXED` with both measured map sizes. +- Fixed one stale existing test: `tests/integration/test_importer.py::test_backfill_never_touches_standalone_and_copyless_rows` called `db.expire()` on objects a run-supplied session may now detach → re-reads by key. Counts/log expectations unchanged (only affected test in 4501). + +**Gates (all green)** +- `uv run pytest tests/integration/test_import_session_retention.py --no-cov` → 7 passed +- importer/importer_e2e/importer_dates/sync_api/unit-importer + retention → 129 passed +- `uv run pytest --cov=app --cov-report=term-missing` → **4501 passed**, `app/rag/importer.py` 99%, TOTAL 99% (>90%) +- `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors +- E2E spot-checks in isolation: `test_sync_changed_only.py` 3 passed; `test_prune_cascade_cleanup.py` + `test_sync_summary_visibility.py` 6 passed + +**Docs**: `README.md` (reset knob no longer "inert", notes the per-source release), `.env.example` (status line: walk reads `BOR_SYNC_SESSION_RESET_FILES`; prune knob still inert until task 03), `AGENTS.md` (new rule 13: release only at the proven-safe points). + +**Decisions/deviations**: added the unchanged-walk test beyond the task's list because task 01 proved the retention lives on the unchanged path (the ADD run's rows are GC-collectable, so its bound alone pins nothing); `MAP_BOUND = RESET_FILES + (RESET_FILES-1)×7 = 17` documented as a ceiling, measured end-of-run 0. No live-infra changes. + +**Next pending task**: `.agents/phases/todo/184_sync_bounded_session/03_batched_prune.md`. diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.md.status b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.validate b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.validate new file mode 100644 index 0000000..9520119 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__02_session_reset_cadence.a1.validate @@ -0,0 +1,143 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 11%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 99%] +..................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 318 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 860 4 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 252 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6868 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.err b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.md b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.md new file mode 100644 index 0000000..8221905 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.md @@ -0,0 +1,20 @@ +All green. Final report: + +**Implemented (task 03 — `184_sync_bounded_session/03_batched_prune.md`)** +- `app/rag/importer.py` `_prune`: the candidate walk is now `select(Document).options(load_only(id, source, path, is_image, image_path, page_count)).where(source.in_(…)).order_by(Document.id))` — `content` (avg 15.2 KB) never loaded; `from sqlalchemy.orm import load_only` added. +- `_prune(..., prune_batch: int = 500)` keyword; commits every Nth delete, and the final commit is skipped only when the last batch boundary already committed (so `10` → exactly 3 commits for 30 docs; `0` → exactly the pre-phase 1 commit). Caller (`import_sources`, L1034) passes `prune_batch=llm.settings.sync_prune_batch` (read once per run). +- Docstring: the light-row contract (per-column rationale for the six, why `content` is out), the batch-commit contract, and the LOCKED A2 atomicity note (crash leaves a prefix pruned; `seen` recomputed from disk → next sync converges). +- Tests (`tests/integration/test_import_session_retention.py`, +5 prune tests + a `before_cursor_execute` capture): no `documents.content` in the prune SELECT and projection == the six columns; **exactly 1** `FROM documents` SELECT per prune (no per-row lazy load); page-copy cleanup (`id`+`page_count`) proven off light rows; 0 orphan chunks; commit counts 3 vs 1; the `Settings → _prune` wiring via a forwarding spy; both prune guards (phase-122 images-off, phase-126 parent-PDF) on light rows. Mutation-checked: dropping `page_count`/`image_path`, loading full rows, the unconditional trailing commit and dropping the caller kwarg each FAIL a test. + +**Gates (all 100% green)** +- `uv run pytest tests/integration/test_import_session_retention.py -q --no-cov` → 12 passed +- `uv run pytest tests/integration/test_importer.py tests/integration/test_importer_e2e.py tests/integration/test_source_removal_api.py tests/unit/test_importer.py tests/unit/test_importer_ignore.py` → 120 passed +- `uv run pytest` → **4506 passed**; `--cov=app` TOTAL **99%** (`app/rag/importer.py` 99%) +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors +- E2E spot-check (unchanged phases): `test_prune_cascade_cleanup.py` 3 passed, `test_sync_changed_only.py`+`test_sync_summary_visibility.py` 6 passed + +**Docs**: `README.md` — new "A sync's memory is bounded per RUN, not per knowledge base" operations bullet + `BOR_SYNC_PRUNE_BATCH` row no longer "inert" + CLI `--prune` knob note; `AGENTS.md` — new rule 14 (six-column prune, batch commits, no bulk `delete()`/`raiseload`); `.env.example` — both knobs now live. + +**Decisions**: default `prune_batch=500` (task-specified); `scripts/import_docs.py` has no direct `_prune` call (it goes through `import_sources`, so its counts/log lines are unchanged and it follows the setting). Deviation from the task's literal snippet: the end-of-loop commit is guarded so a batch that divides the count emits no redundant empty transaction — this is what makes the pinned count exactly 3. + +**Next pending task**: `184_sync_bounded_session/04_e2e_bounded_sync.md`. diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.md.status b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.validate b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.validate new file mode 100644 index 0000000..02ee690 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__03_batched_prune.a1.validate @@ -0,0 +1,143 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 11%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 99%] +.......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 318 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 862 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 252 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6870 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.err b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.md b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.md new file mode 100644 index 0000000..27e7183 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.md @@ -0,0 +1,10 @@ +All gates are green and the working tree is final. Report: + +**Task 04 complete — phase 184's dedicated Playwright suite + docs pass.** + +- **New** `tests/e2e/test_sync_session_reset.py` (919 lines, mock-only, `E2E_REAL_LLM=1` skipped): two module apps on their own ports (`E2E_APP_PORT_SYNCRESET` 8200 = reset 3 / batch 2, `E2E_APP_PORT_SYNCRESET_ZERO` 8201 = both `0`), one 10-file/two-folder fixture source with a sentinel, shared E2E Postgres truncated before **and** after every test, admin login via `auth_helpers`, `/api/` 4xx/5xx + request-failure watcher. +- Tests → observables: ① cadence-ON walk lands all 10 docs **exactly once** (`10 added`, `errors=0`, stat cards == `select(count())`, both folders' ls/tree rows, no wire errors); ② unchanged re-sync `0 added · 10 unchanged`, per-path chunk counts byte-still (no silent re-embed); ③ five-file prune at `BOR_SYNC_PRUNE_BATCH=2` (3 commits inside the UI run) → `0 added · 5 unchanged · 5 pruned`, rows gone from both folders, survivors' chunks intact, **0 orphan chunk rows**; ④ the byte-identical cross-check — cadence-`0` leg vs the recorded cadence-`3` leg: same paths/counts, `content`, `summary`, `content_hash`, `file_hash`, chunk contents in position order, same counts line (non-vacuous: it pins its own baseline shape first). +- Gates: `pytest tests/e2e/test_sync_session_reset.py -v --no-cov` → **4 passed in isolation**; `test_sync_changed_only.py` → 3 passed; `test_sync_git_freeze.py` → 2 passed; `pytest --cov=app` → **4506 passed, app/ TOTAL 99%**; `ruff check .` → All checks passed; `pyright` → **0 errors, 0 warnings**. +- Docs: `README.md` (phase-184 operations paragraph, both knobs in the env table with defaults/`0` semantics, CLI-import note, E2E pointer), `AGENTS.md` rules 13/14 (now cite the E2E suite; corrected the prune pin to "exactly one **document-wide** SELECT"), `.env.example` (dropped the task-progress wording; added the two fields it omitted project-wide — `BOR_APP_VERSION`, `BOR_SESSION_COOKIE` — so every `Settings` field is exemplified). Audited programmatically: every `Settings` field documented in README and `.env.example`. +- No defects found in app code; no locked decision touched. RSS is asserted nowhere (phase 183 A4 inherited) — the memory teeth stay in `tests/integration/test_import_session_retention.py`. +- **Left to the harness:** the atomic `--no-gpg-sign` commit and the `todo/ → complete/` move (I ran no git write commands, per the pipeline rules). diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.md.status b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.validate b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.validate new file mode 100644 index 0000000..02ee690 --- /dev/null +++ b/.agents/reports/184_sync_bounded_session/184_sync_bounded_session__04_e2e_bounded_sync.a1.validate @@ -0,0 +1,143 @@ +........................................................................ [ 1%] +........................................................................ [ 3%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 11%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 99%] +.......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 292 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 318 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 111 0 100% +app/core/theming.py 40 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 105 0 100% +app/models.py 141 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 494 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 862 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 78 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 162 0 100% +app/rag/retriever.py 252 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 549 0 100% +-------------------------------------------------- +TOTAL 6870 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.env.example b/.env.example index ab8186f..28e9e22 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,7 @@ # --- App --- BOR_ENVIRONMENT=development # BOR_APP_NAME=Brain of Reese # display name on all pages — titles, header brand, status labels, aria text (phase 39) +# BOR_APP_VERSION=0.1.0 # version the app reports (GET /api/health, the OpenAPI doc) # BOR_INPUT_PLACEHOLDER=Ask me anything… # composer placeholder, chat page (phase 62) # BOR_FOOTER_TEXT=Powered by self-hosted models # footer line on every page (phase 62) # (Phase 91: the retired CSS-file theme env var is gone — the colors @@ -157,6 +158,28 @@ BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py,container,network,volume, # BOR_UPLOAD_MAX_MB=512 # caps BOTH the compressed upload and the total # extracted bytes (zip-bomb guard); must be > 0 +# --- Sync session bounds (phase 184) --- +# The two knobs that bound ONE sync run, so its cost stops scaling with +# the size of the whole knowledge base. Both read per run (the importer +# reads its settings per sync), so a value saved in the admin Settings +# panel applies to the NEXT sync - no restart. 0 = the exact +# pre-phase-184 behaviour on both (the no-redeploy rollback); a negative +# value is a typo and refuses startup, naming the field. +# The walk reads BOR_SYNC_SESSION_RESET_FILES (released every Nth file plus +# once per source, before the two post-passes) and the prune reads +# BOR_SYNC_PRUNE_BATCH (one COMMIT every Nth delete, over a row that loads +# six columns and never the document's content). +# BOR_SYNC_SESSION_RESET_FILES=200 # files between session releases + # (expunge_all) inside the import walk; + # 0 = never release anything (pre-phase) +# BOR_SYNC_PRUNE_BATCH=500 # pruned documents per COMMIT inside the + # prune instead of one commit at the end + # of the whole import; 0 = one commit at + # the end (pre-phase). A crash mid-prune + # then leaves a PREFIX pruned - the next + # sync converges (seen is recomputed + # from disk every run) + # --- Image documents (phase 122) --- # Master switch for indexing standalone image files (direct uploads, # uploaded archives, git/local source walks) as first-class documents. @@ -257,6 +280,8 @@ BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py,container,network,volume, BOR_ADMIN_PASSWORD= BOR_SESSION_SECRET= # BOR_SESSION_MAX_AGE=43200 # signed-cookie lifetime, seconds (default 12 h, sliding) +# BOR_SESSION_COOKIE=bor_session # cookie NAME; boot-wired, so a change here + # needs a restart (the panel marks it the same) # --- Debugging (0/1 — 1 enables attach-on-demand debugpy on port 5678) --- DEBUGPY=0 diff --git a/AGENTS.md b/AGENTS.md index 29c3105..e6113e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -133,6 +133,45 @@ `doc.chunks = [...]` **re-embed** path is the deliberate, documented exception (collection replacement legitimately needs the loaded collection; changed files only). +13. **Release the sync session at the proven-safe points only:** an import + run holds one session and `expire_on_commit=False` (`app/db.py`) means a + commit releases nothing, so `_unchanged_doc`'s `file_hash` stamp keeps + every walked `Document` — content included — resident in the session's + own strong `dirty` set (not collectable): memory grew with the KB. The + cadence is therefore `session.expunge_all()` every + `BOR_SYNC_SESSION_RESET_FILES` files at the TOP of the per-file loop and + once per source before the two post-passes (phase 184; `0` = never, the + pre-phase rollback). `expunge_all` DISCARDS unflushed changes, so never + move it into `_index_file`, `_unchanged_doc`, `_store_summary` or a + post-pass — their locals are live across statements — and never add a + reset before a file's per-file commit. Pinned by + `tests/integration/test_import_session_retention.py` (bounded with the + cadence on, unbounded with it `0`, byte-identical summaries and DB) and + through the UI by `tests/e2e/test_sync_session_reset.py` (two module apps, + one per cadence: every document exactly once, the all-`unchanged` re-sync, + and the `0`-cadence run byte-identical to the `3`-cadence one). +14. **The prune reads six columns and commits in batches:** `_prune` + selects its candidates through + `load_only(id, source, path, is_image, image_path, page_count)` + `order_by(Document.id)` — the exact set the phase-122 images-off guard, + the phase-126 parent-PDF rule and the image/page copy cleanup read, and + NOT `content` (252 MB of markdown per full KB walk) — and commits every + `BOR_SYNC_PRUNE_BATCH` deletes (`0` = the pre-phase single commit; + phase 184). Never read another `Document` attribute inside that loop (a + deferred column costs one lazy SELECT per document and unbounds the + walk — no `raiseload` paper-over, add the column instead), and never + swap the per-document `session.delete(doc)` for a bulk `delete()` (the + guards, `changed_paths` and the log line depend on the per-document + shape). A non-zero batch deliberately trades the all-or-nothing prune + for a prefix (LOCKED A2): `seen` is recomputed from disk, so the next + sync converges. Pinned by + `tests/integration/test_import_session_retention.py` (the captured + `SELECT`'s column list, exactly one document-wide `SELECT` per prune + (a second one is a per-row lazy load), the commit counts at `10` and + `0`, both guards on light rows) and + `tests/e2e/test_sync_session_reset.py` (the five-document prune at batch + `2` through the Sources page: both folders cleared, the survivors' chunks + intact, zero orphan chunk rows). ## Quick reference ```bash diff --git a/README.md b/README.md index 0a46e1d..f613f3e 100644 --- a/README.md +++ b/README.md @@ -457,7 +457,11 @@ reflect the new files. its `chunks` rows (the embeddings) go together, the latter through the database's own `ON DELETE CASCADE`, so the cost of a mass prune is bounded by documents, not chunks (see *A mass prune costs documents, - not chunks* under [Sync from the UI](#sync-from-the-ui)). + not chunks* under [Sync from the UI](#sync-from-the-ui)). The CLI run + follows the same two phase-184 knobs as the UI sync + (`BOR_SYNC_SESSION_RESET_FILES`, `BOR_SYNC_PRUNE_BATCH`): it loads only + the prune's six columns and commits every Nth delete (`0` on either knob + restores the pre-phase-184 run). - The import prints one line per file and ends with a greppable summary, so it is safe to run from cron or after every commit. - Indexed formats: **md, markdown, txt, yaml, yml, json, py**, the Podman @@ -651,6 +655,34 @@ only) runs the whole source refresh in one click: that fails the suite if the cascade clause ever disappears; the owner flow itself (delete upstream → re-sync → exactly those documents and chunks gone) is pinned by `tests/e2e/test_prune_cascade_cleanup.py`. +- **A sync's memory is bounded per RUN, not per knowledge base** (phase + 184). One import run holds one session, and `expire_on_commit=False` + (`app/db.py`) means a commit releases nothing — so before this phase the + run kept every row it touched (each unchanged document with its full + ~15 KB `content`) until it finished, and the prune read every document of + every pruned source whole, deletes held in ONE transaction. Now the walk + releases its session every `BOR_SYNC_SESSION_RESET_FILES` files (plus once + per source), and the prune (`_prune`) reads only the SIX columns its + guards and its copy cleanup need (`id`, `source`, `path`, `is_image`, + `image_path`, `page_count` — never `content`) ordered by id, committing + every `BOR_SYNC_PRUNE_BATCH` deletes. **Where the operator sets the + ceiling:** those two `BOR_`-prefixed settings (defaults `200` / + `500`, live on the next sync without a restart, `0` on either one restores + the exact pre-phase-184 run as the no-redeploy rollback). Counts, log + lines, `changed_paths`, the phase-122/126 prune guards and the database + are byte-identical either way — pinned by + `tests/integration/test_import_session_retention.py` (the identity-map + bound, the no-`content` prune `SELECT` with no per-row lazy load, the + 3-commits-vs-1 cadence, both prune guards on light rows) and end to end + through the Sources page by `tests/e2e/test_sync_session_reset.py` — two + module apps on their own ports (`E2E_APP_PORT_SYNCRESET` / + `E2E_APP_PORT_SYNCRESET_ZERO`), one per cadence: the ten-file walk lands + every document exactly once with the totals the stat cards show, the + re-sync is all-`unchanged` with an unchanged chunk total, a five-document + prune at batch `2` clears both folders with zero orphan chunk rows, and a + `0`-cadence run of the same fixture is byte-identical to the `3`-cadence + one (per-path chunk counts, `content`, `summary`, both hashes, the counts + line). - **Cancel**: while a run is in flight, a **Cancel** button appears next to the sync button; clicking it posts `POST /api/sync/cancel` (admin only) and registers a cooperative cancel (`202`) — the run @@ -1548,6 +1580,8 @@ startup (the DB rows shadow the env, `.env` is never rewritten). | `BOR_SOURCES_DIR` | `~/bor-sources` | Where git repos are cloned/pulled | | `BOR_UPLOAD_DIR` | `~/bor-sources/uploads` | Where uploaded source archives are unpacked | | `BOR_UPLOAD_MAX_MB` | `512` | Cap (MiB) for uploaded source archives (zip-bomb guard) | +| `BOR_SYNC_SESSION_RESET_FILES` | `200` | Files between releases of the import run's SQLAlchemy session (phase 184): every Nth file the walk hands back everything the session still holds (`expunge_all`), so an unchanged-file sync stops keeping one full document row per file walked. `0` = never release — the exact pre-phase-184 behaviour; a negative value refuses startup naming the field. The release also runs once per source, before the two post-passes | +| `BOR_SYNC_PRUNE_BATCH` | `500` | Pruned documents per COMMIT inside the prune (phase 184) instead of one commit at the end of the whole import, so a mass prune releases its deletes as it goes instead of holding one enormous transaction. `0` = one commit at the end — the exact pre-phase-184 behaviour; a negative value refuses startup naming the field. With a non-zero batch a crash mid-prune leaves a **prefix** pruned, and the next sync converges (`seen` is recomputed from disk every run) | | `BOR_SUGGESTIONS` | built-in list | JSON seed for onboarding chips (shown before the first saved question) | ### Image support (phases 122/123/124/181) diff --git a/app/config.py b/app/config.py index a4aed47..03ee129 100644 --- a/app/config.py +++ b/app/config.py @@ -446,6 +446,29 @@ class Settings(BaseSettings): #: the validator fails loudly at startup (the ``agent_max_rounds`` #: pattern). upload_max_mb: int = 512 + #: Files between identity-map releases inside + #: ``app.rag.importer.import_sources`` (phase 184, + #: ``BOR_SYNC_SESSION_RESET_FILES``): every Nth file of the walk — at + #: the one point where no object from a finished file is reachable — + #: the run's session calls ``expunge_all()``, so whatever the walk + #: still holds goes before the next file is read. ``0`` = NEVER reset + #: — the exact pre-phase-184 behaviour (the operator's no-redeploy + #: rollback, the ``BOR_LOG_LARGE_RESPONSE_BYTES`` precedent). A + #: negative value is a typo — the validator fails startup loudly (the + #: ``agent_max_rounds`` pattern). Read per run (the importer reads + #: ``llm.settings``, so a saved override applies to the NEXT sync). + sync_session_reset_files: int = 200 + #: Pruned documents per commit inside ``app.rag.importer._prune`` + #: (phase 184, ``BOR_SYNC_PRUNE_BATCH``): the delete loop commits + #: every Nth document instead of once at the end of the whole import, + #: so a mass prune releases its rows as it goes. ``0`` = ONE commit at + #: the end — the exact pre-phase-184 behaviour (the same rollback + #: knob). A negative value is a typo — the validator fails startup + #: loudly (the ``agent_max_rounds`` pattern). Note the deliberate + #: atomicity change of a non-zero batch (phase 184 A2): a crash + #: mid-prune leaves a PREFIX pruned, and the next sync converges + #: because ``seen`` is recomputed from the filesystem every run. + sync_prune_batch: int = 500 # --- Image documents (phase 122: standalone images as documents) --- #: Master switch for image-document indexing (phase 122, @@ -809,6 +832,34 @@ class Settings(BaseSettings): raise ValueError("upload_max_mb must be > 0 (MiB)") return v + @field_validator("sync_session_reset_files") + @classmethod + def _sync_session_reset_files_non_negative(cls, v: int) -> int: + """``0`` is the pre-phase-184 rollback (never reset) — a negative + value is a typo that would make the cadence comparison constant, + so it fails startup naming the field (the + ``agent_max_rounds`` pattern, phase 184).""" + if v < 0: + raise ValueError( + "sync_session_reset_files must be >= 0 (0 = never reset the" + " import session)" + ) + return v + + @field_validator("sync_prune_batch") + @classmethod + def _sync_prune_batch_non_negative(cls, v: int) -> int: + """``0`` is the pre-phase-184 rollback (one commit at the end of + the prune) — a negative value is a typo that would make the batch + comparison constant, so it fails startup naming the field (the + ``agent_max_rounds`` pattern, phase 184).""" + if v < 0: + raise ValueError( + "sync_prune_batch must be >= 0 (0 = one commit at the end of" + " the prune)" + ) + return v + @field_validator("chat_image_max_mb") @classmethod def _chat_image_max_mb_positive(cls, v: int) -> int: diff --git a/app/core/settings_catalog.py b/app/core/settings_catalog.py index e798a1a..076af84 100644 --- a/app/core/settings_catalog.py +++ b/app/core/settings_catalog.py @@ -288,6 +288,12 @@ CATALOG: dict[str, SettingMeta] = { "upload_max_mb": SettingMeta( "upload_max_mb", "Import & sources", "Upload max MB" ), + "sync_session_reset_files": SettingMeta( + "sync_session_reset_files", "Import & sources", "Sync session reset files" + ), + "sync_prune_batch": SettingMeta( + "sync_prune_batch", "Import & sources", "Sync prune batch" + ), # --- Images --- "images": SettingMeta("images", "Images", "Images"), "image_extensions": SettingMeta( diff --git a/app/rag/importer.py b/app/rag/importer.py index 59675c4..2ed4363 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -346,7 +346,7 @@ from typing import Any, Protocol import pymupdf as fitz # PyMuPDF (phase 126, locked A1); `fitz` = the stable API name from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, load_only from app.config import Settings from app.db import SessionLocal @@ -867,6 +867,14 @@ async def import_sources( failure leaves the row untouched — the next sync retries it). Driven by committed doc rows, so it is safe under ``limit`` too (it deletes nothing). + + Bounded session (phase 184, task 02): the walk releases its SQLAlchemy + identity map every ``llm.settings.sync_session_reset_files`` files and + once per source before the two post-passes, so the run's resident set + scales with the CADENCE rather than with the size of the knowledge base + (``0`` = never — the exact pre-phase behaviour, the operator's + no-redeploy rollback). The cadence changes no count, no log line and no + committed byte: see the safety argument at the reset site. """ if limit is not None and limit <= 0: raise ValueError("limit must be >= 1") @@ -875,6 +883,12 @@ async def import_sources( # toggle is on — the empty set admits nothing, so the toggle-off run # (walk, counts, prune) stays byte-identical to pre-phase. image_exts = llm.settings.image_extension_set if llm.settings.images else frozenset() + # Phase 184 (task 02): the identity-map release cadence, read ONCE here — + # before the source loop — so a settings overlay saved mid-run cannot + # change the cadence between two files of the same walk (the importer + # reads ``llm.settings`` per RUN, never per file). ``0`` disables the + # release entirely. + reset_every = llm.settings.sync_session_reset_files owns_session = session is None if session is None: session = SessionLocal() @@ -922,6 +936,40 @@ async def import_sources( ): if limit is not None and summary.files >= limit: break + # Phase 184 (task 02): the identity-map release. There is NO + # automatic one — ``expire_on_commit=False`` (``app/db.py:30``) + # means a commit drops nothing, and ``_unchanged_doc``'s + # ``doc.file_hash = file_digest`` stamp marks every + # already-current row DIRTY, which puts it in the session's + # own strong ``dirty`` set (cyclic GC cannot reach it, see + # ``tests/integration/test_import_session_retention.py``). + # Without this the run holds every Document — its full + # ``content`` included — it walked until the import ENDED: + # the 2026-10-15 incident's second driver, and the reason + # memory grew with the KB rather than with the work. + # + # This is the ONLY safe point, and WHY it is safe: + # ``expunge_all()`` discards UNFLUSHED changes, so it may run + # only where every real write of the finished file is already + # committed. Here the previous file's ``_index_file`` has + # RETURNED — each path commits per file (the changed path + # commits its chunks, ``_store_summary`` commits, and + # ``_unchanged_doc`` commits its date refresh, its stamp and + # whatever the phase-163 strip re-gate wrote) — and what + # ``expunge_all`` then drops is at most the fast path's stamp + # of a value the database ALREADY holds (no UPDATE was + # pending: ``session.is_modified`` is false for it). Nothing + # downstream reaches back into a finished file: ``_index_file`` + # returns ``None``, the progress/chunk/image hooks only assign + # fields on the sync's own status object, ``dates_map``/ + # ``ignore``/``seen`` are plain data, and ``total`` was + # computed before the loop. The ``EmbeddingError`` handler + # below calls ``session.rollback()`` and touches no object of + # its own, so detached state is irrelevant to it. + # ``reset_every == 0`` → never (the exact pre-phase + # behaviour, the operator's rollback knob). + if reset_every and summary.files and summary.files % reset_every == 0: + session.expunge_all() rel = path.relative_to(root).as_posix() seen.add((source, rel)) summary.files += 1 @@ -955,6 +1003,13 @@ async def import_sources( session.rollback() summary.errors += 1 logger.error("import: error source=%s path=%s — %s", source, rel, e) + # Phase 184 (task 02): the source-boundary release — one source's + # walk residue never crosses into the next source. Safe for the + # same reason as the per-file reset and additionally because BOTH + # post-passes below select the rows they need fresh (their own + # ``select(Document)``), so no walk object has to survive it. + if reset_every: + session.expunge_all() # Phase 125 (task 04): the post-index enrichment pass — the # markdown-FIRST case (a markdown indexed while its # referenced image had no document yet recorded the @@ -980,6 +1035,10 @@ async def import_sources( session, source_names, seen, images=llm.settings.images, summary=summary, page_dir=Path(llm.settings.pdf_page_dir).expanduser(), + # Phase 184 (task 03): the commit cadence, read ONCE + # here (the same per-RUN rule as the identity-map + # reset) — ``0`` = the pre-phase single commit. + prune_batch=llm.settings.sync_prune_batch, ) summary.embed_batches = llm.embed_batches summary.log() @@ -3864,6 +3923,7 @@ def _prune( images: bool = False, summary: ImportSummary | None = None, page_dir: Path | None = None, + prune_batch: int = 500, ) -> int: """Delete documents of *source_names* whose file is no longer in *seen*. @@ -3910,10 +3970,51 @@ def _prune( chunk row, never hydrates a 768-dim embedding, and a mass prune costs documents instead of chunks × 24.8 KiB (the 2026-10-15 incident: a 3,008-doc prune materialised ~720 MB to delete 72k rows and locked up - a 3,904 MB VM with no swap). The ``changed_paths`` feed and the - image/page copy cleanup above are unchanged, and they only ever read + a 3,904 MB VM with no swap). That fix removed the CHILD rows; a SECOND + driver of the same incident stood on this very walk, and phase 184 + (task 03) removed it with the two contracts below. + + Phase 184 (task 03) — the light-row contract. The walk below selects + through ``load_only`` and is ordered by ``Document.id``, so its resident + set is independent of the knowledge base: ``content`` averages 15.2 KB + per row (252 MB of markdown across the incident KB), and loading it for + every document of every pruned source — to decide a deletion and then + throw the string away — was that driver. The SIX columns loaded are + exactly what the guards and the cleanup read: ``is_image`` (the + phase-122 images-off toggle guard), ``path`` (that same guard's + ``seen`` tuple and the phase-126 ``_EMBEDDED_IMAGE_RE`` parent-PDF + rule), ``source`` (the ``seen`` tuple, ``changed_paths`` and the log + line), ``image_path`` (:func:`_delete_image_copy`), ``id`` + + ``page_count`` (:func:`_delete_page_copies`) — and ``content`` is NOT + among them (nor ``summary``, ``full_path``, ``title``, + ``image_context``, either hash, or any date). Reading anything else + inside the loop would emit one lazy SELECT per document and undo the + bound; ``tests/integration/test_import_session_retention.py`` pins both + halves (no ``documents.content`` in the SELECT, exactly ONE + ``FROM documents`` SELECT for the whole walk). Phase 185 does the same + job for the walk's fast path. + + Phase 184 (task 03) — the batch-commit contract. *prune_batch* is + ``Settings.sync_prune_batch`` (``BOR_SYNC_PRUNE_BATCH``, read once by + the caller from ``llm.settings``): the loop commits every Nth delete, + so a long prune no longer holds one enormous transaction and its rows + are released as it goes. ``prune_batch=0`` is the exact pre-phase + behaviour — ONE commit after the whole loop — the operator's + no-redeploy rollback (LOCKED A3). + + **Atomicity (LOCKED A2, deliberate).** With a non-zero batch a crash + mid-prune leaves a PREFIX pruned instead of rolling the whole prune + back. That is safe HERE, and only here: ``seen`` is recomputed from the + filesystem on every run, so the next sync converges on whatever this + one left, and the import already committed per file throughout the walk + — a run-level transaction never existed to lose. ``summary.pruned`` + stays the count of documents actually deleted, unchanged in meaning. + + What phase 183 left in place is unchanged: the ``changed_paths`` feed + and the image/page copy cleanup above, which only ever read ``doc.image_path`` / ``doc.page_count`` / ``doc.source`` / ``doc.path`` - — never ``doc.chunks``. The **re-embed** path is the deliberate + — never ``doc.chunks`` — and they are exactly the reads the six light + columns cover. The **re-embed** path is the deliberate exception (LOCKED A2): :func:`_replace_content_chunks` reassigns ``doc.chunks = [...]``, which is collection REPLACEMENT (the ORM must know which old rows to orphan) and legitimately needs the loaded @@ -3923,7 +4024,25 @@ def _prune( if not source_names: return 0 pruned = 0 - docs = session.scalars(select(Document).where(Document.source.in_(source_names))).all() + # Phase 184 (task 03): the light-row walk — see the docstring's + # light-row contract. SIX columns, ``content`` deliberately absent + # (252 MB of markdown per full KB walk), deterministic order so the + # batch prefix is reproducible. + docs = session.scalars( + select(Document) + .options( + load_only( + Document.id, + Document.source, + Document.path, + Document.is_image, + Document.image_path, + Document.page_count, + ) + ) + .where(Document.source.in_(source_names)) + .order_by(Document.id) + ).all() for doc in docs: if (doc.source, doc.path) not in seen: if doc.is_image and not images: @@ -3948,6 +4067,11 @@ def _prune( session.delete(doc) pruned += 1 logger.info("import: pruned source=%s path=%s", doc.source, doc.path) - if pruned: + # Phase 184 (task 03): a long prune no longer holds a huge + # transaction — commit the batch (``prune_batch=0`` → never + # here, the single commit below, i.e. the pre-phase run). + if prune_batch and pruned and pruned % prune_batch == 0: + session.commit() + if pruned and not (prune_batch and pruned % prune_batch == 0): session.commit() return pruned diff --git a/tests/e2e/test_sync_session_reset.py b/tests/e2e/test_sync_session_reset.py new file mode 100644 index 0000000..b3c2c74 --- /dev/null +++ b/tests/e2e/test_sync_session_reset.py @@ -0,0 +1,919 @@ +"""Phase 184 E2E (Playwright, mock-only): the BOUNDED sync still syncs all. + +Source: owner incident 2026-10-15 — the same run whose prune phase 183 fixed. +An import run holds ONE session and ``app/db.py`` builds it with +``expire_on_commit=False``, so a commit released NOTHING: the walk kept every +``Document`` it touched resident (each unchanged row holds its full ~15 KB +``content``, and ``_unchanged_doc``'s ``file_hash`` stamp puts the row in the +session's own strong ``dirty`` set, out of reach of the cycle collector), and +``_prune`` read every document of every pruned source WHOLE — ``content`` +included — holding all of its deletes in ONE transaction until the end. The +run's memory therefore grew with the SIZE OF THE KNOWLEDGE BASE instead of +with the work the run did. + +The fix is two operator knobs, both ``BOR_``-prefixed settings the importer +reads once per RUN: + +* ``BOR_SYNC_SESSION_RESET_FILES`` (default ``200``) — every Nth file of the + walk, at the one point where no object from a finished file is reachable, + the session calls ``expunge_all()``; it also runs once per source, before + the two post-passes. ``0`` = NEVER release = the exact pre-phase-184 run. +* ``BOR_SYNC_PRUNE_BATCH`` (default ``500``) — the prune commits every Nth + delete instead of once at the end of the whole import. ``0`` = the exact + pre-phase-184 single commit. A non-zero batch deliberately trades the + all-or-nothing prune for a PREFIX (LOCKED A2: ``seen`` is recomputed from + disk, so the next sync converges). + +Both knobs change the run's MEMORY PROFILE and nothing else — and that +"nothing else" is precisely what a browser can see. This module pins it: the +same counts line, every document exactly once, an all-``unchanged`` re-sync, +a multi-batch prune that behaves, and a Sources page that never stalls, never +errors and reports the same totals as the database. The retention itself (the +identity-map bound, the prune's six-column ``SELECT``, the commit counts) is +deterministic and pinned in +``tests/integration/test_import_session_retention.py``; RSS is asserted +nowhere in this phase (phase 183 LOCKED A4, inherited). + +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_sync_session_reset.py -v --no-cov + +MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is skipped — every number here (the +per-path chunk counts, the byte-identical cross-check) is a property of the +deterministic mock embedder/summarizer; against the real endpoint the +comparison would be a measurement, not a pin. + +TWO module apps on dedicated ports (the ``test_sync_summary_visibility.py`` +pattern — the conftest session app owns 8123 in a combined run, and a +same-port second uvicorn would drive the wrong server): + +* the CADENCE app — ``BOR_SYNC_SESSION_RESET_FILES=3`` and + ``BOR_SYNC_PRUNE_BATCH=2``, so a 10-file walk releases the session THREE + times mid-run (before files 4, 7 and 10) plus once at the source boundary, + and a 5-document prune commits THREE times (at 2, at 4, and the final one). + Tests 1–3 drive it; +* the ZERO app — both knobs ``0``, i.e. the exact pre-phase-184 run. Test 4 + drives it over the SAME fixture directory, which is what makes the two runs + comparable at all. + +Both boot against the mock LLM with ``BOR_GIT_SOURCES`` forced empty (the +owner's local gitignored ``.env`` must never leak a real source into a run), +their own ``BOR_SOURCES_DIR`` / ``BOR_PDF_PAGE_DIR`` under +``tmp_path_factory`` (nothing here clones or renders a PDF — the dirs exist so +no run writes into the operator's), and the mock-calibrated gate (the conftest +convention). The fixture KB is ONE plain (non-git) directory named +:data:`SOURCE_NAME` (``kind='local'`` rows index under ``Path(path).name``, so +the source label is stable whatever temp root pytest picked) holding TEN +``.md`` files across TWO folders — ``a/1..5.md`` and ``b/1..5.md`` — with +``a/1.md`` carrying the sentinel phrase. + +Isolation: the shared E2E Postgres is truncated BEFORE **and** AFTER every +test (``chunks, documents, query_log, kb_overview, git_sources, sources_meta, +folder_summaries``), after waiting for the app to hold no running sync (the +run state lives in the app's memory, and a leaked run would keep importing +into the next test's truncated KB). Each test registers its own source row. + +Test → observable mapping (Playwright Mapping Rule): +1. ``test_reset_cadence_produces_the_identical_run`` — the cadence ON is a + complete, healthy sync: the counts line reads ``10 added``, the terminal + reports ``added=10`` with ``errors=0``, every one of the ten paths exists + EXACTLY ONCE in the KB (the test's own SQL), the ls/tree rows of both + folders render all ten files, the stat cards equal ``select(count())`` over + ``documents`` (10) and ``chunks`` (> 10), the sentinel's phrase is in the + stored content, and THROUGH THE RUN no ``/api/`` request answered 4xx/5xx + and none failed — no error banner, no modal. +2. ``test_resync_is_unchanged_with_the_cadence_on`` — the second sync of the + same bytes settles ``0 added · 10 unchanged`` with ``pruned=0`` and + ``success``, and the chunk total and per-path chunk counts are byte-still: + the cadence loses no rows (a lost row would come back as ``added``) and + re-embeds nothing. +3. ``test_multi_batch_prune_from_the_ui`` — five of the ten files go and the + re-sync settles ``0 added · 5 unchanged · 5 pruned`` (5 deletes at batch + ``2`` = THREE commits inside the UI's own run); the five documents are gone + from the KB AND from the ls/tree rows of BOTH folders, the survivors' chunk + total is unchanged, and ZERO orphan chunk rows exist (the test's own SQL). +4. ``test_cadence_zero_matches_cadence_three`` — the byte-identical + cross-check the phase lives on: the SAME fixture walked with + ``BOR_SYNC_SESSION_RESET_FILES=0`` lands the same paths (each exactly + once), the same per-path chunk counts, the same ``content`` / ``summary`` / + ``content_hash`` / ``file_hash`` on every document, the same chunk contents + in position order and the same counts line as the cadence-``3`` run. The + reference is test 1's recorded run when the module runs in order; alone, + the test walks the cadence-``3`` app itself first — and either way it + asserts its own 10-documents / >10-chunks shape BEFORE comparing, so an + empty baseline can never make it pass vacuously. +""" +from __future__ import annotations + +import os +import re +import subprocess +import sys +import time +from collections.abc import Iterator +from datetime import datetime +from pathlib import Path +from typing import Any + +import httpx +import pytest +from playwright.sync_api import Page, expect +from sqlalchemy import func, select, text + +from app.db import SessionLocal +from app.models import Chunk, Document +from e2e.auth_helpers import login +from e2e.conftest import ( + ADMIN_PASSWORD, + MOCK_PORT, + SESSION_SECRET, + USE_REAL_LLM, + _wait_http, +) + +REPO = Path(__file__).resolve().parents[2] + +#: The CADENCE app (tests 1–3). Env-overridable, the house port pattern: the +#: conftest session app owns 8123 in a combined run. +CADENCE_PORT = int(os.environ.get("E2E_APP_PORT_SYNCRESET", "8200")) + +#: The ZERO app (test 4's reference leg): both knobs ``0`` — the exact +#: pre-phase-184 run (LOCKED A3's no-redeploy rollback). +ZERO_PORT = int(os.environ.get("E2E_APP_PORT_SYNCRESET_ZERO", "8201")) + +SOURCES_URL = "/sources.html" + +#: The fixture directory's basename — the ``source`` the importer records for +#: a ``kind='local'`` row (``Path(path).name``), so it is the same string in +#: the DB whatever temp root this run got. +SOURCE_NAME = "sync-session-reset" + +#: The fixture layout: two folders × five importable files. After the prune, +#: ``a`` keeps three survivors and ``b`` two — so BOTH folders still render +#: rows, and a prune that stopped after its first batch would show up. +ALL_PATHS = [*[f"a/{n}.md" for n in range(1, 6)], *[f"b/{n}.md" for n in range(1, 6)]] +KEEP_PATHS = ["a/1.md", "a/2.md", "a/3.md", "b/1.md", "b/2.md"] +GONE_PATHS = ["a/4.md", "a/5.md", "b/3.md", "b/4.md", "b/5.md"] +N_FILES = len(ALL_PATHS) + +#: The sentinel phrase, carried by :data:`SENTINEL_PATH` ONLY. Finding it in +#: the stored ``content`` proves the walk really indexed the fixture rather +#: than leaving a KB of merely the right SIZE. +SENTINEL = "RESE-SYNC-RESET-SENTINEL-4f27" +SENTINEL_PATH = "a/1.md" + +#: The cadences the two apps boot with — module constants so the narrative and +#: the app env cannot drift apart. 10 files / every 3rd = releases before +#: files 4, 7 and 10 (THREE mid-run resets, then the source boundary); a +#: 5-document prune at batch 2 = commits at 2, at 4 and the final one. +RESET_FILES = 3 +PRUNE_BATCH = 2 + +#: "Synced HH:MM" — the local-time last-result label (sources.js's +#: fmtSyncTime), any hour/minute (the test_sync_button.py pattern). +SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}$") + +#: Settle budgets: a ten-file mock sync (walk + embeds + overview + three +#: folder summaries) lands in seconds and the UI settles a terminal within one +#: 2 s poll tick, so this is an order of magnitude of head-room with no +#: client-side hard timeout (the test_prune_cascade_cleanup budget). +SETTLE_TIMEOUT_MS = 90_000 + +#: Test 1 records its numbers here for test 4's cross-check (the +#: ``test_sync_changed_only.py`` "state from the earlier test" idiom). Test 4 +#: runs its own cadence leg when this is empty, so it stays honest alone. +_CADENCE_RUN: dict[str, Any] = {} + + +# --------------------------------------------------------------------------- +# The fixture source (ONE directory BOTH apps walk) +# --------------------------------------------------------------------------- + + +def _doc_body(path: str) -> str: + """One small markdown file per fixture path, byte-stable across runs. + + The bodies differ from each other (so a per-path chunk count means + something) but never carry a timestamp or the absolute path, and only + :data:`SENTINEL_PATH` carries the sentinel — that is what lets test 4 + compare two runs of two different processes line for line. + """ + if path == SENTINEL_PATH: + return ( + "# Session reset sentinel\n" + "\n" + "The bounded sync keeps the sentinel phrase in the knowledge base.\n" + f"Marker: {SENTINEL}\n" + ) + folder, name = path.rsplit("/", 1) + stem = name.removesuffix(".md") + return ( + f"# Session reset note {folder} {stem}\n" + "\n" + f"One unrelated note from the {folder} folder of the {SOURCE_NAME} " + "fixture source, indexed so the cadence has something to walk.\n" + ) + + +def _seed_source(root: Path) -> Path: + """(Re-)write all ten fixture files under ``root / SOURCE_NAME``. + + Idempotent on purpose: test 3 deletes five files from the tree and test 4 + re-seeds the SAME directory, so the reference leg walks byte-identical + content through the other app. + """ + base = root / SOURCE_NAME + for rel in ALL_PATHS: + target = base / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(_doc_body(rel), encoding="utf-8") + assert not (base / ".git").exists() # a plain directory — no git anywhere + return base + + +@pytest.fixture(scope="module") +def source_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: + """The one fixture tree BOTH module apps walk (the app servers run on the + same host, so the path is visible to them).""" + return _seed_source(tmp_path_factory.mktemp("bor_sync_session_reset")) + + +# --------------------------------------------------------------------------- +# The two module apps +# --------------------------------------------------------------------------- + + +def _app_env(scratch: Path, reset_files: int, prune_batch: int) -> dict[str, str]: + """The per-module app env (the conftest pattern): the mock LLM, the + mock-calibrated gate, no git fallback, dedicated scratch dirs — and THIS + app's two phase-184 cadences, pinned explicitly so an operator's local + (gitignored) ``.env`` cannot retune them.""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = ( + "https://aipi.reeseapps.com/v1" + if USE_REAL_LLM + else f"http://127.0.0.1:{MOCK_PORT}/v1" + ) + # Mock-calibrated gate (the conftest pattern): the token-overlap mock + # embeddings score far below the production 0.62, and BOTH floors must + # stay <= the threshold or the app refuses to boot (app/config.py). + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15" + env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15" + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + # Phase 16: admin auth must be set or create_app() refuses to boot. + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + # No env-fallback rows: the registry each test writes through the + # authenticated API is the only source list, so every sync's counts are + # exactly this test's doing. + env["BOR_GIT_SOURCES"] = "" + env["BOR_SOURCES_DIR"] = str(scratch / "checkouts") + env["BOR_PDF_PAGE_DIR"] = str(scratch / "pdf-pages") + # THE SUBJECT OF THE PHASE. + env["BOR_SYNC_SESSION_RESET_FILES"] = str(reset_files) + env["BOR_SYNC_PRUNE_BATCH"] = str(prune_batch) + return env + + +def _serve(env: dict[str, str], port: int) -> Iterator[str]: + """Boot one uvicorn app under test on *port* and yield its URL.""" + proc = subprocess.Popen( + [ + sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning", + ], + cwd=REPO, + env=env, + ) + url = f"http://127.0.0.1:{port}" + try: + _wait_http(f"{url}/api/health") + yield url + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def app_server( + mock_llm: int, source_dir: Path, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[str]: + """The CADENCE app (tests 1–3): ``reset=3``, ``prune_batch=2``. + + ``mock_llm`` and ``source_dir`` are dependencies for the ORDERING only — + the mock answers before the app boots, and the fixture tree exists before + the first test registers it. + """ + if USE_REAL_LLM: + pytest.skip( + "mock-only suite — the byte-identical cross-check is a property " + "of the deterministic mock (E2E_REAL_LLM unsupported)" + ) + assert mock_llm and source_dir + scratch = tmp_path_factory.mktemp("bor_sync_session_reset_cadence") + yield from _serve(_app_env(scratch, RESET_FILES, PRUNE_BATCH), CADENCE_PORT) + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +@pytest.fixture(scope="module") +def zero_app_server( + mock_llm: int, source_dir: Path, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[str]: + """The ZERO app (test 4's reference leg): both knobs ``0``, the exact + pre-phase-184 run the bounded one must be byte-identical to.""" + if USE_REAL_LLM: + pytest.skip( + "mock-only suite — the byte-identical cross-check is a property " + "of the deterministic mock (E2E_REAL_LLM unsupported)" + ) + assert mock_llm and source_dir + scratch = tmp_path_factory.mktemp("bor_sync_session_reset_zero") + yield from _serve(_app_env(scratch, 0, 0), ZERO_PORT) + + +@pytest.fixture(scope="module") +def zero_app_url(zero_app_server: str) -> str: + return zero_app_server + + +# --------------------------------------------------------------------------- +# Isolation +# --------------------------------------------------------------------------- + + +def _truncate_all() -> None: + """Fresh registry + KB (the E2E isolation pattern). ``sources_meta`` (the + KB generation counter) and ``folder_summaries`` reset with the KB: the + sync's overview/bump is an upsert, and a leftover folder summary would + ride the very ``changed_paths`` feed the prune writes.""" + with SessionLocal() as db: + db.execute( + text( + "TRUNCATE chunks, documents, query_log, kb_overview, " + "git_sources, sources_meta, folder_summaries" + ) + ) + db.commit() # without the commit the TRUNCATE rolls back + + +def _wait_sync_settled(base_url: str, timeout_s: float = 180.0) -> None: + """No run may leak across a truncate: wait for a non-running status with + an own admin session (the endpoint is admin-only). Usually a no-op — the + tests settle before they end.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + with httpx.Client(base_url=base_url, timeout=5.0) as client: + client.post("/api/login", json={"password": ADMIN_PASSWORD}) + r = client.get("/api/sync/status") + assert r.status_code == 200, r.text + if r.json()["state"] != "running": + return + time.sleep(0.2) + raise AssertionError(f"a sync was still running at a test boundary: {base_url}") + + +@pytest.fixture(autouse=True) +def _clean(app_url: str, db_ready: None) -> Iterator[None]: + """Per-test KB isolation: the shared E2E Postgres is truncated before AND + after every test, the app holding no running sync at both boundaries. + + The ZERO app is deliberately NOT waited on: it is booting lazily for test 4 + only, and that test's own run is settled inside the test body, so the + post-truncate can never race it.""" + _wait_sync_settled(app_url) + _truncate_all() + yield + _wait_sync_settled(app_url) + _truncate_all() + + +# --------------------------------------------------------------------------- +# Helpers: the UI side +# --------------------------------------------------------------------------- + + +def _watch_network(page: Page) -> list[str]: + """Collect every ``/api/`` 4xx/5xx and every failed ``/api/`` request the + page sees — the owner's "the Sources page errored / stalled". + + Returns the (growing) list the caller asserts empty at the end of the + run: the page's own 2 s status poll lands in this list, so a run that made + the API answer 500 is caught here rather than at some later assertion. + """ + problems: list[str] = [] + + def _on_response(response: Any) -> None: + if "/api/" in response.url and response.status >= 400: + problems.append(f"HTTP {response.status} {response.url}") + + def _on_failed(request: Any) -> None: + if "/api/" in request.url: + problems.append(f"FAILED {request.url} ({request.failure})") + + page.on("response", _on_response) + page.on("requestfailed", _on_failed) + return problems + + +def _add_local_source(page: Page, base_url: str, path: Path) -> None: + """Register the fixture directory through the authenticated API (the + phase-49 shape — the page's own form for this is gone; the admin cookie + rides ``page.request``).""" + r = page.request.post( + f"{base_url}/api/git-sources", + data={"kind": "local", "path": str(path)}, + ) + assert r.status == 201, f"expected 201 for {path}: {r.status} {r.text}" + + +def _started_at(body: dict[str, Any]) -> float: + """The status body's ``started_at`` as a wall-clock epoch (the server + stamps it in UTC and this process runs on the same host).""" + started = body.get("started_at") + if not started: + return 0.0 + try: + return datetime.fromisoformat(str(started)).timestamp() + except ValueError: + return 0.0 + + +def _sync_from_sources_page(page: Page, base_url: str) -> dict[str, Any]: + """Click the Sources page **Sync sources** button and wait for THIS run's + terminal state (the module app is shared, so a terminal counts only after + a ``running`` tick or when its ``started_at`` post-dates the click). + + Settles the UI on success and proves the owner saw no error surface: the + button re-enabled, ``Synced HH:MM``, no error banner, no modal.""" + btn = page.locator("#sync-btn") + expect(btn).to_be_visible(timeout=30_000) + expect(btn).to_be_enabled() + clicked_at = time.time() + btn.click() + expect(btn).to_be_disabled() + expect(page.locator("#sync-label")).to_have_text("Syncing…") + + deadline = time.monotonic() + SETTLE_TIMEOUT_MS / 1000 + saw_running = False + body: dict[str, Any] = {} + while time.monotonic() < deadline: + r = page.request.get(f"{base_url}/api/sync/status") + assert r.status == 200, r.text + body = r.json() + state = body["state"] + if state == "running": + saw_running = True + elif state in ("success", "failed", "cancelled") and ( + saw_running or _started_at(body) >= clicked_at - 1.0 + ): + break + time.sleep(0.2) + else: + raise AssertionError(f"sync never reached a terminal state: {body}") + + assert body["state"] == "success", f"sync failed: {body}" + assert body["error"] is None, body + expect(btn).to_be_enabled(timeout=15_000) + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=15_000) + expect(page.locator("#sync-error-banner")).to_be_hidden() + assert page.locator(".sync-modal-backdrop").count() == 0 + return body + + +def _expect_counts_line(page: Page, expected: str) -> None: + """The counts line — the owner-visible sync report on the single sync text + surface (phase 150; sources.js fmtSyncResult).""" + expect(page.locator("#sync-progress-text")).to_have_text(expected, timeout=15_000) + + +def _counts_line(page: Page) -> str: + """The counts line as text — the string test 4 compares across the apps.""" + return str(page.locator("#sync-progress-text").inner_text()) + + +def _open_level(page: Page, *names: str) -> None: + """Open ONE level at a time from the catalogue top — each name the EXACT + link text of the source or folder row at the current level (the phase-97 + drill idiom, the ``test_kb_tree_lazy_load.py`` settle). + + The top is reached through the breadcrumb's first segment when the page is + drilled (the phase-99 drill state rides ``history.state``, so a nav + re-click would KEEP the level — the crumb is the control that actually + leaves it), and is a no-op when the page already sits there (the crumb + hidden). Every click after that awaits the level it produced: the phase-167 + node fetch is over once the breadcrumb names the level and no in-flight + ``.kb-level-loading`` row remains.""" + if not page.locator("#kb-crumb").is_hidden(): + page.click('#kb-crumb a.kb-crumb-link:text-is("Knowledge base")') + expect(page.locator("#kb-crumb")).to_be_hidden(timeout=30_000) + expect(page.locator("#folders-tbody tr")).to_have_count(1, timeout=30_000) + for name in names: + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + expect(page.locator("#kb-crumb .kb-crumb-current")).to_have_text( + name, timeout=30_000 + ) + expect(page.locator(".kb-level-loading")).to_have_count(0) + + +# --------------------------------------------------------------------------- +# Helpers: the DB side +# --------------------------------------------------------------------------- + + +def _paths_with_counts() -> dict[str, int]: + """``path`` → number of ``documents`` rows for that path — THE exactly-once + probe: a reset that dropped work and re-added it shows up here.""" + with SessionLocal() as db: + rows = db.execute( + select(Document.path, func.count(Document.id)) + .where(Document.source == SOURCE_NAME) + .group_by(Document.path) + ).all() + return {str(path): int(count) for path, count in rows} + + +def _chunk_count_by_path(paths: list[str] | None = None) -> dict[str, int]: + """``path`` → chunk rows per document (an absent path is not listed).""" + query = ( + select(Document.path, func.count(Chunk.id)) + .outerjoin(Chunk, Chunk.document_id == Document.id) + .where(Document.source == SOURCE_NAME) + .group_by(Document.path) + ) + if paths is not None: + query = query.where(Document.path.in_(paths)) + with SessionLocal() as db: + rows = db.execute(query).all() + return {str(path): int(count) for path, count in rows} + + +def _total_chunks() -> int: + with SessionLocal() as db: + return int(db.scalar(select(func.count()).select_from(Chunk)) or 0) + + +def _doc_ids(paths: list[str]) -> list[str]: + """The indexed document ids of ``(SOURCE_NAME, path)``, sorted. Test 3 + needs them BEFORE the prune — the orphan query runs on ids that no longer + exist afterwards.""" + with SessionLocal() as db: + return sorted( + str(doc_id) + for (doc_id,) in db.execute( + select(Document.id).where( + Document.source == SOURCE_NAME, Document.path.in_(paths) + ) + ).all() + ) + + +def _orphan_chunks() -> int: + """Chunk rows whose document is gone — phase 183's contract, asserted here + too so a BATCHED prune cannot leave any behind.""" + with SessionLocal() as db: + return int( + db.scalar( + select(func.count()) + .select_from(Chunk) + .where(Chunk.document_id.not_in(select(Document.id))) + ) + or 0 + ) + + +def _kb_snapshot() -> dict[str, tuple[str, str | None, str, str, list[str]]]: + """The byte-level shape of the fixture source: + + ``path`` → ``(content, summary, content_hash, file_hash, chunk contents in + position order)`` — every field the two runs must agree on. Timestamps and + ``full_path`` are deliberately absent: they belong to the run and to the + temp root, not to the imported bytes. + """ + with SessionLocal() as db: + docs = db.scalars( + select(Document) + .where(Document.source == SOURCE_NAME) + .order_by(Document.path) + ).all() + snapshot: dict[str, tuple[str, str | None, str, str, list[str]]] = {} + for doc in docs: + assert doc.content_hash and doc.file_hash, doc.path + contents = db.scalars( + select(Chunk.content) + .where(Chunk.document_id == doc.id) + .order_by(Chunk.position) + ).all() + snapshot[str(doc.path)] = ( + doc.content, + doc.summary, + str(doc.content_hash), + str(doc.file_hash), + [str(c) for c in contents], + ) + return snapshot + + +def _sentinel_in_kb() -> bool: + """Does the stored content of the sentinel document carry the sentinel? + (The walk really indexed the fixture; the KB is not merely the right size.)""" + with SessionLocal() as db: + content = db.scalar( + select(Document.content).where( + Document.source == SOURCE_NAME, Document.path == SENTINEL_PATH + ) + ) + return bool(content and SENTINEL in content) + + +def _file_rows(page: Page) -> list[str]: + """The current level's file rows, by path — the ls/tree rows the owner + reads (a pruned document must leave this list).""" + links = page.locator("#docs-tbody tr a.doc-link") + return [str(links.nth(i).inner_text()) for i in range(links.count())] + + +def _run_fresh_sync(page: Page, base_url: str, source: Path) -> dict[str, Any]: + """Register the fixture and run ONE fresh-KB sync through the Sources + page, returning the shape the cross-check compares.""" + _add_local_source(page, base_url, source) + body = _sync_from_sources_page(page, base_url) + return { + "detail": dict(body["detail"]), + "line": _counts_line(page), + "paths": _paths_with_counts(), + "chunks_by_path": _chunk_count_by_path(), + "total_chunks": _total_chunks(), + "snapshot": _kb_snapshot(), + } + + +# --------------------------------------------------------------------------- +# 1. The cadence ON syncs everything, cleanly, and the page agrees +# --------------------------------------------------------------------------- + + +def test_reset_cadence_produces_the_identical_run( + page: Page, app_url: str, source_dir: Path, db_ready: None +) -> None: + """``BOR_SYNC_SESSION_RESET_FILES=3`` over ten files: three mid-run + releases — and a sync whose every owner-visible number is the number the + unbounded run produces. + + The counts line reads ``10 added``, the terminal reports ``added=10`` with + ``errors=0``, each of the ten paths exists EXACTLY ONCE (a release that + discarded unflushed work would show up as a missing or a duplicated path), + the ls/tree rows of both folders render all ten files, the stat cards + equal the database's own ``select(count())`` over ``documents`` and + ``chunks``, the sentinel's phrase is in the stored content — and not one + ``/api/`` request answered 4xx/5xx or failed through the whole run.""" + page.set_default_timeout(30_000) + _seed_source(source_dir.parent) # idempotent — the tree is byte-stable + login(page, app_url, next=SOURCES_URL) + problems = _watch_network(page) + + leg = _run_fresh_sync(page, app_url, source_dir) + + # --- the owner's settle line + the terminal detail ------------------- + assert leg["line"] == f"{N_FILES} added", leg["line"] + detail = leg["detail"] + assert detail["added"] == N_FILES, detail + assert detail["updated"] == 0 and detail["unchanged"] == 0, detail + assert detail["pruned"] == 0 and detail["errors"] == 0, detail + assert detail["files"] == N_FILES, detail + + # --- every document landed EXACTLY once, fully embedded -------------- + assert leg["paths"] == dict.fromkeys(ALL_PATHS, 1), leg["paths"] + assert all(count > 0 for count in leg["chunks_by_path"].values()), ( + f"a document has no chunks — the release lost its embed work: " + f"{leg['chunks_by_path']}" + ) + assert leg["total_chunks"] > N_FILES, leg["total_chunks"] + assert _sentinel_in_kb(), "the sentinel document never reached the KB" + + # --- the page agrees with the database (the phase-167 totals) -------- + expect(page.locator("#stat-docs")).to_have_text(str(N_FILES), timeout=15_000) + expect(page.locator("#stat-chunks")).to_have_text( + str(leg["total_chunks"]), timeout=15_000 + ) + + # --- the ls/tree rows render the whole fixture ----------------------- + _open_level(page, SOURCE_NAME, "a") + assert _file_rows(page) == [f"a/{n}.md" for n in range(1, 6)] + _open_level(page, SOURCE_NAME, "b") + assert _file_rows(page) == [f"b/{n}.md" for n in range(1, 6)] + + # --- the run never errored on the wire or in the UI ------------------ + assert problems == [], f"the run put an error on the wire: {problems}" + + # Record for test 4's cross-check (the phase's byte-identical contract). + _CADENCE_RUN.clear() + _CADENCE_RUN.update(leg) + + +# --------------------------------------------------------------------------- +# 2. The unchanged re-sync with the cadence ON: nothing moves +# --------------------------------------------------------------------------- + + +def test_resync_is_unchanged_with_the_cadence_on( + page: Page, app_url: str, source_dir: Path, db_ready: None +) -> None: + """The run a production sync actually takes — every file unchanged — with + the release firing three times through it: the phase-147 fast path stays + the fast path. + + The second sync settles ``0 added · 10 unchanged`` with ``pruned=0``, the + terminal is ``success``, and the database's chunk total and per-path chunk + counts are byte-still: the cadence neither loses rows (which the next sync + would report as ``added``) nor silently re-embeds an unchanged document.""" + page.set_default_timeout(30_000) + _seed_source(source_dir.parent) + login(page, app_url, next=SOURCES_URL) + problems = _watch_network(page) + + first = _run_fresh_sync(page, app_url, source_dir) + assert first["detail"]["added"] == N_FILES, first["detail"] + _expect_counts_line(page, f"{N_FILES} added") + chunks_after_first = _chunk_count_by_path() + paths_after_first = _paths_with_counts() + total_after_first = _total_chunks() + assert total_after_first == sum(chunks_after_first.values()) + + # --- the unchanged re-sync ------------------------------------------- + second = _sync_from_sources_page(page, app_url) + detail = second["detail"] + assert detail["unchanged"] == N_FILES, detail + assert detail["added"] == 0 and detail["updated"] == 0, detail + assert detail["pruned"] == 0 and detail["errors"] == 0, detail + _expect_counts_line(page, f"0 added · {N_FILES} unchanged") + + # No silent re-embed: the same chunk total, per path, after the run. + assert _chunk_count_by_path() == chunks_after_first, ( + "an unchanged file changed chunk count — the cadence re-embedded a " + "document the fast path had left alone" + ) + assert _total_chunks() == total_after_first + assert _paths_with_counts() == paths_after_first + assert _sentinel_in_kb() + assert problems == [], f"the re-sync put an error on the wire: {problems}" + + +# --------------------------------------------------------------------------- +# 3. The prune through the UI at BOR_SYNC_PRUNE_BATCH=2 +# --------------------------------------------------------------------------- + + +def test_multi_batch_prune_from_the_ui( + page: Page, app_url: str, source_dir: Path, db_ready: None +) -> None: + """Five documents go, and the prune that takes them is a MULTI-BATCH run: + at ``BOR_SYNC_PRUNE_BATCH=2`` the five deletes commit at 2, at 4 and once + more at the end — THREE commits inside the UI's own run — while the owner + still sees exactly one prune report. + + The counts line settles ``0 added · 5 unchanged · 5 pruned``; the five + documents are gone from the KB AND from the ls/tree rows of BOTH folders + (three survivors in ``a``, two in ``b``); the survivors' chunk total is + unchanged; and ZERO orphan chunk rows exist. A batched commit changes WHEN + a delete becomes durable, never what the run removes.""" + page.set_default_timeout(30_000) + _seed_source(source_dir.parent) + login(page, app_url, next=SOURCES_URL) + problems = _watch_network(page) + + first = _run_fresh_sync(page, app_url, source_dir) + assert first["detail"]["added"] == N_FILES, first["detail"] + + # The subject of the prune, captured BEFORE it happens: the doomed rows' + # ids (the orphan query needs ids that no longer exist by the time it + # runs) and every survivor's chunk count. + doomed_ids = _doc_ids(GONE_PATHS) + keep_counts = _chunk_count_by_path(KEEP_PATHS) + assert len(doomed_ids) == len(GONE_PATHS), doomed_ids + assert set(keep_counts) == set(KEEP_PATHS), keep_counts + assert all(count > 0 for count in keep_counts.values()), keep_counts + + # --- the upstream deletion (this test owns the fixture tree) --------- + for rel in GONE_PATHS: + (source_dir / rel).unlink() + assert not any((source_dir / rel).exists() for rel in GONE_PATHS) + + # --- the prune -------------------------------------------------------- + body = _sync_from_sources_page(page, app_url) + detail = body["detail"] + assert detail["pruned"] == len(GONE_PATHS), detail + assert detail["added"] == 0 and detail["updated"] == 0, detail + assert detail["unchanged"] == len(KEEP_PATHS), detail + assert detail["errors"] == 0, detail + _expect_counts_line( + page, + f"0 added · {len(KEEP_PATHS)} unchanged · {len(GONE_PATHS)} pruned", + ) + + # The catalogue rows went WITH the documents — in both folders, which is + # where a prune that stopped after one batch would leave a stale row. + _open_level(page, SOURCE_NAME, "a") + assert _file_rows(page) == ["a/1.md", "a/2.md", "a/3.md"] + _open_level(page, SOURCE_NAME, "b") + assert _file_rows(page) == ["b/1.md", "b/2.md"] + + # The survivors are untouched and nothing was orphaned. + assert _chunk_count_by_path() == keep_counts, ( + "the prune took a survivor's chunks with it, or left a doomed row" + ) + assert _total_chunks() == sum(keep_counts.values()) + assert _orphan_chunks() == 0, ( + "a batched prune left orphan chunk rows — the deletes committed " + "without the FK cascade doing the child work" + ) + assert _sentinel_in_kb() + assert problems == [], f"the prune put an error on the wire: {problems}" + + +# --------------------------------------------------------------------------- +# 4. cadence 0 == cadence 3 (the byte-identical cross-check) +# --------------------------------------------------------------------------- + + +def test_cadence_zero_matches_cadence_three( + page: Page, + app_url: str, + zero_app_url: str, + source_dir: Path, + db_ready: None, +) -> None: + """THE PIN the phase lives on: the cadences change the run's memory + profile and NOTHING the owner or the database can see. + + The same fixture tree, walked fresh by the ZERO app + (``BOR_SYNC_SESSION_RESET_FILES=0``, ``BOR_SYNC_PRUNE_BATCH=0`` — the + pre-phase-184 run) against the CADENCE app's numbers (``3`` / ``2``): the + same paths (each exactly once), the same chunk count per path, the same + ``content`` / ``summary`` / ``content_hash`` / ``file_hash`` on every + document, the same chunk contents in position order, and the same counts + line. + + The reference is test 1's recorded run when the module runs in order; run + ALONE, the test walks the cadence-``3`` app itself first. Either way it + asserts the reference's own 10-documents / >10-chunks shape BEFORE + comparing, so an empty baseline can never make it pass vacuously.""" + page.set_default_timeout(30_000) + + # Re-seed: test 3 deleted five files from the fixture tree. + _seed_source(source_dir.parent) + on_disk = sorted(p.relative_to(source_dir).as_posix() for p in source_dir.rglob("*.md")) + assert on_disk == sorted(ALL_PATHS), on_disk + + # --- the reference leg: cadence 3 (test 1's numbers, or a fresh run) -- + if _CADENCE_RUN: + reference = dict(_CADENCE_RUN) + else: + login(page, app_url, next=SOURCES_URL) + reference = _run_fresh_sync(page, app_url, source_dir) + _truncate_all() + assert reference["paths"] == dict.fromkeys(ALL_PATHS, 1), reference["paths"] + assert reference["total_chunks"] > N_FILES, reference["total_chunks"] + + # --- the subject leg: cadence 0, the exact pre-phase run -------------- + login(page, zero_app_url, next=SOURCES_URL) + problems = _watch_network(page) + zero = _run_fresh_sync(page, zero_app_url, source_dir) + + assert zero["detail"]["added"] == N_FILES, zero["detail"] + assert zero["detail"]["errors"] == 0, zero["detail"] + assert zero["detail"]["pruned"] == 0, zero["detail"] + + # --- every number agrees --------------------------------------------- + assert zero["line"] == reference["line"], ( + f"the owner-visible counts line moved with the cadence: " + f"{zero['line']!r} != {reference['line']!r}" + ) + assert zero["paths"] == reference["paths"], ( + "the set of paths (or a path's row count) moved with the cadence" + ) + assert zero["chunks_by_path"] == reference["chunks_by_path"], ( + "the per-path chunk counts moved with the cadence: " + f"{zero['chunks_by_path']} != {reference['chunks_by_path']}" + ) + assert zero["total_chunks"] == reference["total_chunks"] + assert zero["snapshot"].keys() == reference["snapshot"].keys() + for path, row in zero["snapshot"].items(): + assert row == reference["snapshot"][path], ( + f"{path} is not byte-identical across the cadences" + ) + assert _orphan_chunks() == 0 + assert problems == [], f"the reference run put an error on the wire: {problems}" diff --git a/tests/integration/test_import_session_retention.py b/tests/integration/test_import_session_retention.py new file mode 100644 index 0000000..3772be2 --- /dev/null +++ b/tests/integration/test_import_session_retention.py @@ -0,0 +1,1247 @@ +"""Integration: phase 184 — what one import run RETAINS, and its bound. + +**VERDICT (task 01): CONFIRMED — the run kept every row it walked and +nothing released it — the accumulation lives on the UNCHANGED path, not the +added path.** The owner incident (2026-10-15, the same run phase 183 fixed +the prune of) is the reason: an import run of a big knowledge base grew in +memory with the SIZE OF THE KB rather than with the work the run actually +did. + +**→ FIXED (task 02):** :func:`app.rag.importer.import_sources` now calls +``session.expunge_all()`` every ``Settings.sync_session_reset_files`` files +(top of the per-file loop) plus once per source before the two post-passes, +so the map is bounded by the CADENCE instead of by the knowledge base — the +pre-fix and post-fix map sizes are both measured below, and the post-fix +ceiling is :data:`MAP_BOUND`. + +**The mechanism, exactly.** + +* ``app/db.py:30`` builds every session with + ``expire_on_commit=False`` — deliberately, and this phase does **NOT** + change it: every request path reads attributes after commit. What that + flag controls is attribute EXPIRY, not object retention; on its own it + keeps nothing alive. +* The SQLAlchemy identity map is therefore **weak** for a CLEAN row: a row + the run read, committed and no longer references is collected and leaves + the map by itself (pinned by + :func:`test_a_clean_row_leaves_the_map_and_a_dirty_one_never_does`). That + is why the **added** path is bounded instead of retained: nothing in the + session points at a finished file's cluster, so the next cyclic + collection frees it. Refcounting alone does not — a ``Document`` and its + ``Chunk`` rows form a cycle — which is why the added path's map size is a + question of GC timing, never of what the session holds. +* The **unchanged** path is the opposite, and it is the path a production + sync takes for nearly every file. ``_unchanged_doc`` + (``app/rag/importer.py``) stamps ``doc.file_hash = file_digest`` on EVERY + unchanged file. The value already equals the stored one, so + ``session.is_modified(doc)`` is ``False`` (no UPDATE, and the helper's + commit is skipped) — but the assignment still marks the state + ``modified``, which puts the row in ``Session.dirty``: a **strong** + reference held by the session itself. Cyclic GC cannot help (the session + IS the reference); ``commit()`` does not help + (``expire_on_commit=False``). The row — its full ``content`` string + included — is resident until the run ends. + +**Measured BY THIS FILE** (12 markdown files of ~4.4 KB each, the dev +Postgres, the deterministic in-process fake embedder): + +* objects the ADD run creates: 12 ``Document`` + 60 content ``Chunk`` + 12 + summary ``Chunk`` = **84**. Identity map at the end of that run: **84** + inside the pytest process (no cyclic collection had been triggered yet — + a document and its chunks form a reference cycle, so refcounting alone + cannot free them) and **0** after ONE collection point + (:func:`test_the_added_run_holds_its_objects_only_until_a_collection`). + Over a longer 400-file run the same map sawtoothed between 40 and 200 + entries with ``len(gc.get_objects())`` flat, i.e. the added rows are + released by the collector as the run goes: bounded by GC timing, NOT by + the session. +* objects the UNCHANGED run walks: **12** documents. Identity map at the end + of that run: **exactly 12** — every one a ``Document`` still holding its + full ``content`` (53 KB of content strings in this fixture). Sampled + through the ``progress`` hook it is a monotone staircase (before file 1: + 0 rows resident … before file 12: 11) — **one row gained per file, none + ever released**. ``gc.collect()`` releases **0** of them; + ``session.expunge_all()`` releases **all 12** without losing a row (task + 02's remedy, measured here so task 02 has a number to hit). +* extrapolated to the incident KB (17,407 documents, ``content`` averaging + **15.2 KB** — the phase-184 baseline): a full unchanged sync retains + ≈ **265 MB** of ``content`` strings plus row state on a 3,904 MB VM with + no swap that shares its RAM with a 3.1 GB Postgres. That is the "memory + grew with the knowledge base" the owner saw; it is proportional to the + files walked, so the phase-147 fast path (which makes the walk faster) + cannot reduce it. + +**The hazard the fix respects (task 02, implemented).** The retained rows +are retained BECAUSE they are dirty, and ``expunge_all()`` DISCARDS +unflushed changes. That is safe on the fast path — the stamp writes the +value the database already holds, so there is nothing to lose — but it is +only safe at a point where every real write of the finished file has been +committed (``_unchanged_doc`` commits its date refresh and its +``is_modified`` stamp, ``_store_summary`` and the index paths commit per +file). Expunging before such a commit would silently drop that file's +write, which is why the cadence sits at the top of the per-file loop +(after the previous file's ``_index_file`` returned) and at the source +boundary (before the two post-passes, which select their own rows fresh) +— and nowhere else: never inside ``_index_file``, ``_unchanged_doc``, +``_store_summary`` or a post-pass, whose locals are live across +statements. + +**Measured AFTER the fix (task 02, the same 12-file fixture).** With +``sync_session_reset_files=3`` the UNCHANGED walk is a sawtooth +``[0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2]`` sampled at the ``progress`` hook +— a release every third file where the cadence-``0`` run ramps +monotonically to 12 — and it ends at **0** rows, because the +source-boundary release is the last thing the walk does and the two +post-passes select no rows for a source with no image docs. The ADD run +(whose rows are only weakly held, so its mid-run number moves with GC +timing) peaks at **14** entries sampled mid-run — two files' worth of +seven objects, the widest gap the cadence allows — against the +``0``-cadence run's 21→49 climb, and it too ends at **0**. Both runs +report a :func:`dataclasses.asdict`-identical ``ImportSummary`` and leave +a byte-identical database next to the cadence-``0`` run (see +:func:`test_reset_cadence_bounds_the_map`): the release costs nothing but +residency. + +**Why RSS is never asserted here** (phase 183 LOCKED A4, inherited): the +allocator's behaviour is CI noise. ``VmRSS`` over a 400-file run of this +same fixture grew ≈ 35 KB/file and did NOT come back after +``gc.collect()`` — that is freed-but-unreturned allocator memory, NOT +session retention (``len(gc.get_objects())`` stayed flat across the whole +400-file run: 190k→210k sawtooth). ``len(session.identity_map)`` is the +exact, deterministic retention surface, and every assertion here measures +it through :func:`_map_size` — the one door tasks 02/03 assert through. + +**The prune side (task 03) — the OTHER half of the incident.** The walk +above is what a sync costs per file; ``_prune`` is what it costs per +DOCUMENT IN THE SOURCE, and on 2026-10-15 it was where the box actually +died. It used to ``select(Document)`` the FULL row — ``content`` included, +averaging **15.2 KB** — for every document of every pruned source (17,407 +rows ≈ **252 MB** of markdown) to decide a deletion it then threw away, and +to hold the whole delete in ONE transaction until the end. Post-task-03 it +selects six light columns (``id``, ``source``, ``path``, ``is_image``, +``image_path``, ``page_count`` — everything the guards and the copy cleanup +read) ``order_by(Document.id)`` and commits every +``Settings.sync_prune_batch`` deletes. Measured with the ``before_cursor_execute`` +capture below over a 30-doc × 5-chunk × 8 KB fixture: **1** ``SELECT … FROM +documents`` for the whole walk (no per-row follow-up: the six columns really +are enough), ``documents.content`` ABSENT from its column list, and +**3** commits at ``sync_prune_batch=10`` vs exactly **1** at ``0`` (the +pre-phase rollback) with every count, ``changed_paths`` entry and row total +identical. The guard pins +(:func:`test_prune_guards_unaffected_with_light_rows`) are what prove the +light row is COMPLETE — a column missing from ``load_only`` would emit one +lazy SELECT per row and fail that test, not get papered over. + +**What this file does NOT touch:** the walk's column set (phase 185), +``scripts/git_sync.py`` (phase 186), ``app/db.py:30``, and the +``doc.chunks = [...]`` re-embed path (phase 183 LOCKED A2). +""" +from __future__ import annotations + +import asyncio +import dataclasses +import gc +import re +from collections.abc import Callable, Iterator +from pathlib import Path +from uuid import UUID + +import pytest +from sqlalchemy import event, func, select, text +from sqlalchemy import inspect as sa_inspect +from sqlalchemy.orm import Session + +from app.config import Settings +from app.db import SessionLocal, engine +from app.models import Chunk, Document +from app.rag.importer import ImportSummary, _prune, import_sources +from app.rag.llm import EmbeddingError +from tests.fakes import FakeEmbedder + +# --------------------------------------------------------------------------- +# The fixture: 12 files of ~4.4 KB, each producing 5 content chunks +# --------------------------------------------------------------------------- + +#: The probe's own source name — nothing else in the KB uses it. +PROBE_SOURCE = "retention-probe" + +#: One import run's worth of files. Small on purpose: what the tests pin is +#: the SHAPE of the retention (an unbounded ramp with the knob ``0``, a +#: cadence-bounded sawtooth with it on), so any count works — 12 keeps the +#: run in seconds, divides by both cadences used here (2 and 3) and keeps +#: the staircase readable. Task 04 drives the cadence through the UI with a +#: bigger fixture. +FILE_COUNT = 12 + +#: One repeated sentence, so every file has the same shape (and therefore +#: the same chunk count) as every other. +_SENTENCE = ( + "The homelab runs podman quadlets on the hnb host with a static DNS " + "zone, nightly restic backups and a systemd timer that probes uptime. " +) + +#: The pinned output of :func:`_seed_files`: 5 content chunks per file (4 +#: sections of ~1.1 KB against ``chunk_target_chars=2000``) and 1 summary +#: chunk per document (phase 118 A2). +EXPECTED_CONTENT_CHUNKS = 60 +EXPECTED_SUMMARY_CHUNKS = 12 + +#: Every ORM object a full ADD run creates: the 12 documents, their content +#: chunks and their summary chunks. What the identity map would have to +#: hold if the ADDED path retained what the phase-184 baseline describes. +OBJECTS_CREATED_BY_ADD_RUN = ( + FILE_COUNT + EXPECTED_CONTENT_CHUNKS + EXPECTED_SUMMARY_CHUNKS +) + +#: The measured number of identity-map entries at the end of an UNCHANGED +#: run with the cadence OFF — one per file walked, each a full ``Document`` +#: (module docstring). The counterpart +#: :func:`test_reset_cadence_bounds_the_unchanged_walk` pins what the same +#: walk does with the cadence ON: a sawtooth capped by the cadence, ending +#: at 0. +MEASURED_UNCHANGED_MAP_SIZE = FILE_COUNT + +#: Task 02's cadence for the bounded tests: 12 files / every 3rd = FOUR +#: releases inside the walk, so a single missed reset cannot hide. +RESET_FILES = 3 + +#: Every ORM object ONE added file puts through the session: its +#: ``Document`` + its 5 content chunks + its 1 summary chunk. +OBJECTS_PER_ADDED_FILE = 1 + EXPECTED_CONTENT_CHUNKS // FILE_COUNT + 1 # 7 + +#: The post-fix ceiling (``sync_session_reset_files + slack``, task 02). +#: The arithmetic: between two cadence releases at most +#: ``RESET_FILES - 1`` files have been indexed, each contributing at most +#: :data:`OBJECTS_PER_ADDED_FILE` objects (the ``2 * 7 = 14`` the ADD run is +#: measured peaking at) — the last partial batch. ``RESET_FILES`` on top is +#: the slack for the file being indexed when the sample is taken. Measured: +#: the UNCHANGED run ends at 0 and the ADD run ends at 0, so 17 is a +#: ceiling, not an expectation — what matters is that it does not scale +#: with :data:`FILE_COUNT` (the pre-fix run measured +#: :data:`MEASURED_UNCHANGED_MAP_SIZE` = 12 grown from the KB, 49 on the +#: ADD path). +MAP_BOUND = RESET_FILES + (RESET_FILES - 1) * OBJECTS_PER_ADDED_FILE # 17 + +#: The content token :class:`_BoomEmbedder` refuses on — file #4 +#: (``probe-03.md`` carries ``# Probe document 3`` + ``of probe 3``). +BOOM_MARKER = "probe 3" + + +def _seed_files(tmp: Path) -> Path: + """``FILE_COUNT`` markdown files of ~4.4 KB under a fresh source root.""" + root = tmp / PROBE_SOURCE + root.mkdir() + for n in range(FILE_COUNT): + parts = [f"# Probe document {n}\n"] + for section in range(4): + parts.append(f"\n## Section {section} of probe {n}\n\n" + _SENTENCE * 8 + "\n") + body = "".join(parts) + assert 4_300 <= len(body) <= 4_700, len(body) + (root / f"probe-{n:02d}.md").write_text(body) + return root + + +def _probe_llm(reset_files: int = 0) -> FakeEmbedder: + """The fake embedder with BOTH phase-184 knobs explicit — ``0`` by + default, the exact pre-phase behaviour (LOCKED A3: ``0`` is the + operator's rollback, no re-deploy). The run is byte-identical to a run + predating the settings entirely; the explicitness is the point — these + tests stay true after tasks 02/03 ship, because they pin what ``0`` + guarantees. *reset_files* ``> 0`` asks for the shipped cadence.""" + llm = FakeEmbedder() + llm.settings = Settings( + _env_file=None, # pyright: ignore[reportCallIssue] + sync_session_reset_files=reset_files, + sync_prune_batch=0, + ) + return llm + + +class _BoomEmbedder(FakeEmbedder): + """A fake embedder that refuses every embed whose text carries + :data:`BOOM_MARKER` — the importer's ``EmbeddingError`` path (the + per-file ``session.rollback()``), driven here AFTER the identity map has + already been reset, to pin that detached state breaks nothing.""" + + async def embed( + self, + texts: list[str], + *, + on_progress: Callable[[int, int], None] | None = None, + ) -> list[list[float]]: + if any(BOOM_MARKER in t for t in texts): + raise EmbeddingError(f"simulated endpoint refusal for {BOOM_MARKER}") + return await super().embed(texts, on_progress=on_progress) + + +def _map_size(session: Session) -> int: + """The run's retention surface — the ONE door every assertion here (and + in tasks 02/03) measures through.""" + return len(session.identity_map) + + +def _map_classes(session: Session) -> dict[str, int]: + """How many identity-map entries belong to each mapped class.""" + counts: dict[str, int] = {} + for state in session.identity_map.all_states(): + name = state.class_.__name__ + counts[name] = counts.get(name, 0) + 1 + return counts + + +def _resident_content_chars(session: Session) -> int: + """Content characters held resident by the ``Document`` rows still in + the identity map — the per-run cost that scales with the KB.""" + total = 0 + for state in session.identity_map.all_states(): + if state.class_ is not Document: + continue + obj = state.obj() + assert isinstance(obj, Document) + total += len(obj.content) + return total + + +def _truncate(db: Session) -> None: + """Empty the KB mid-test AND drop every row *db* has cached, so the next + read is the database's truth rather than a stale identity-map entry.""" + db.execute(text("TRUNCATE chunks, documents")) + db.commit() + db.expunge_all() + + +def _kb_snapshot(db: Session) -> dict[str, tuple[str, str | None, str, int]]: + """path → ``(content, summary, file_hash, chunk count)`` — the DB-side + ground truth the cadence-bounded run must match the pre-phase run for, + byte for byte (task 02's identical-output contract).""" + counts: dict[UUID, int] = { + row[0]: row[1] + for row in db.execute( + select(Chunk.document_id, func.count(Chunk.id)).group_by(Chunk.document_id) + ) + } + snapshot: dict[str, tuple[str, str | None, str, int]] = {} + for doc in db.scalars( + select(Document) + .where(Document.source == PROBE_SOURCE) + .order_by(Document.path) + ): + assert doc.file_hash is not None # the phase-147 stamp is always set + snapshot[doc.path] = ( + doc.content, + doc.summary, + doc.file_hash, + counts.get(doc.id, 0), + ) + return snapshot + + +@pytest.fixture(autouse=True) +def clean_kb(db: Session) -> Iterator[None]: + """``documents`` / ``chunks`` are global state: reset around every test + (the ``test_sync_api.py`` truncate pattern).""" + db.execute(text("TRUNCATE chunks, documents")) + db.commit() + yield + db.execute(text("TRUNCATE chunks, documents")) + db.commit() + + +# --------------------------------------------------------------------------- +# The runs +# --------------------------------------------------------------------------- + + +def test_the_added_run_holds_its_objects_only_until_a_collection( + db: Session, tmp_path: Path +) -> None: + """The added path is NOT session-held — one collection point clears it. + + A fresh session indexes :data:`FILE_COUNT` new files + (``prune=False``), creating :data:`OBJECTS_CREATED_BY_ADD_RUN` ORM + objects. Measured: all 84 are still in the map at the end of the run + inside the pytest process, because a ``Document`` and its ``Chunk`` + rows form a reference cycle that refcounting cannot break — but every + one of them left the session at their commit (``session.is_modified`` + is false, nothing points at them), so a single ``gc.collect()`` drops + the map to **0**. Contrast that with + :func:`test_the_unchanged_run_keeps_every_document_it_walks`, where no + number of collections frees anything: THAT is the leak-shaped + retention, and it is the path a production sync takes for nearly every + file. + """ + root = _seed_files(tmp_path) + session = SessionLocal() + try: + summary = asyncio.run( + import_sources([root], _probe_llm(), prune=False, session=session) + ) + assert (summary.files, summary.added, summary.unchanged) == ( + FILE_COUNT, + FILE_COUNT, + 0, + ) + assert summary.chunks == EXPECTED_CONTENT_CHUNKS + assert summary.summaries == EXPECTED_SUMMARY_CHUNKS + assert summary.errors == 0 + + # The objects really were created — the DB is the ground truth for + # how many ORM objects the run put through the session. + n_docs = db.scalar(select(func.count()).select_from(Document)) + n_chunks = db.scalar(select(func.count()).select_from(Chunk)) + assert n_docs == FILE_COUNT + assert n_chunks == EXPECTED_CONTENT_CHUNKS + EXPECTED_SUMMARY_CHUNKS + assert n_docs + n_chunks == OBJECTS_CREATED_BY_ADD_RUN + + # Whatever the collector has not reached yet is NOT held by the + # session — it is only garbage the cycle collector has not visited. + gc.collect() + assert _map_size(session) == 0, ( + f"{_map_size(session)} entries of a finished file are still" + " reachable from the session after a collection point — the" + " weak-map mechanism of the module docstring no longer holds" + ) + finally: + session.close() + + +def test_the_unchanged_run_keeps_every_document_it_walks( + db: Session, tmp_path: Path +) -> None: + """THE PROBE: a repeat sync holds one full ``Document`` per file walked. + + The production-shaped case — the second sync of the same source, every + file unchanged, the phase-147 fast path for all of them. The identity + map grows by exactly one row per file and loses nothing: sampled + through the ``progress`` hook (which fires before each file's + ``_index_file``) it is the staircase ``0, 1, … FILE_COUNT-1``, and it + ends at :data:`MEASURED_UNCHANGED_MAP_SIZE`. + + This is the accumulation phase 184 exists for, and it is NOT + collectable: ``gc.collect()`` releases none of it — the session's own + ``dirty`` set is what holds the rows (the ``doc.file_hash`` stamp of + ``_unchanged_doc`` marks the state modified and never commits it) — + while ``session.expunge_all()`` releases all of it (see + :func:`test_a_clean_row_leaves_the_map_and_a_dirty_one_never_does`). + """ + root = _seed_files(tmp_path) + seed = SessionLocal() + try: + first = asyncio.run(import_sources([root], _probe_llm(), session=seed)) + assert first.added == FILE_COUNT + finally: + seed.close() + gc.collect() + + session = SessionLocal() + try: + staircase: list[int] = [] + + def progress(source: str, rel: str, done: int, total: int) -> None: + """The per-file sample — the hook fires BEFORE this file is + indexed, so the value is the residue of the files already + walked (``source``/``rel``/``done``/``total`` are not needed to + measure retention).""" + del source, rel, done, total + staircase.append(_map_size(session)) + + summary = asyncio.run( + import_sources( + [root], _probe_llm(), prune=False, session=session, progress=progress + ) + ) + assert (summary.files, summary.added, summary.unchanged) == ( + FILE_COUNT, + 0, + FILE_COUNT, + ) + + # One row resident per file already walked — a staircase, never a + # plateau and never a release. + assert staircase == list(range(FILE_COUNT)), ( + "expected exactly one document retained per file already walked," + f" got {staircase}" + ) + assert _map_size(session) == MEASURED_UNCHANGED_MAP_SIZE + + # …and every one of them is a full Document row, content included + # (the 15.2 KB-per-document production figure is what makes this + # scale with the knowledge base instead of with the work). + assert _map_classes(session) == {"Document": FILE_COUNT} + assert _resident_content_chars(session) >= FILE_COUNT * 4_000 + + # Not collectable: the session IS the reference. + gc.collect() + assert _map_size(session) == MEASURED_UNCHANGED_MAP_SIZE, ( + "the retained rows became collectable — the probe's premise (and" + " task 02's fix) needs re-measuring" + ) + finally: + session.close() + + +def test_reset_cadence_is_configurable_off(db: Session, tmp_path: Path) -> None: + """``sync_session_reset_files=0`` IS the pre-phase run — the rollback + guarantee (LOCKED A3), true before task 02 and after it ships. + + With both knobs explicitly ``0`` the unchanged-path run keeps exactly + what :func:`test_the_unchanged_run_keeps_every_document_it_walks` + measured (:data:`MEASURED_UNCHANGED_MAP_SIZE` rows, all ``Document``, + none collectable), and reports byte-identical counts. This is not a + pre-fix-only assertion: it is the operator's promise that + ``BOR_SYNC_SESSION_RESET_FILES=0`` restores today's behaviour exactly. + """ + llm = _probe_llm() + assert llm.settings.sync_session_reset_files == 0 + assert llm.settings.sync_prune_batch == 0 + + root = _seed_files(tmp_path) + seed = SessionLocal() + try: + first = asyncio.run(import_sources([root], _probe_llm(), session=seed)) + assert first.added == FILE_COUNT + finally: + seed.close() + gc.collect() + + session = SessionLocal() + try: + summary = asyncio.run(import_sources([root], llm, prune=False, session=session)) + assert (summary.files, summary.added, summary.unchanged, summary.pruned) == ( + FILE_COUNT, + 0, + FILE_COUNT, + 0, + ) + assert summary.chunks == 0 # the fast path re-embeds nothing + assert _map_size(session) == MEASURED_UNCHANGED_MAP_SIZE + assert _map_classes(session) == {"Document": FILE_COUNT} + gc.collect() + assert _map_size(session) == MEASURED_UNCHANGED_MAP_SIZE + finally: + session.close() + + +def test_reset_cadence_bounds_the_map(db: Session, tmp_path: Path) -> None: + """THE FIX: with ``sync_session_reset_files=3`` the map stays inside one + cadence window — and the run's OUTPUT is identical to the cadence-``0`` + run it replaces. + + The same :func:`_seed_files` fixture is imported twice into an empty KB: + once with the knob ``0`` (the pre-phase reference) and once with the + cadence at :data:`RESET_FILES`. The bounded run reports + ``added=12`` + :data:`EXPECTED_CONTENT_CHUNKS` chunks, its whole + ``ImportSummary`` compares equal to the reference's, and the database + matches it path by path (content, summary, ``file_hash`` and the per-path + chunk count) — the release drops residency, never a byte of work. + + The bound itself (:data:`MAP_BOUND`) is asserted twice: at the end of the + run (measured **0** — the source-boundary release is the last thing the + walk does, and the two post-passes select no rows for a source with no + image docs) and mid-run through the ``progress`` hook, where the ceiling + is the ``RESET_FILES - 1`` files indexed since the last release times + :data:`OBJECTS_PER_ADDED_FILE` (measured **14**). Neither number moves + when the fixture grows: that is the whole point of the phase. + """ + root = _seed_files(tmp_path) + + # 1 — the reference run: the exact pre-phase behaviour (both knobs 0). + reference = SessionLocal() + try: + ref_summary = asyncio.run( + import_sources([root], _probe_llm(), prune=False, session=reference) + ) + finally: + reference.close() + ref_kb = _kb_snapshot(db) + + # 2 — the same walk over an empty KB, now with the cadence on. + _truncate(db) + session = SessionLocal() + try: + staircase: list[int] = [] + + def progress(source: str, rel: str, done: int, total: int) -> None: + """Per-file sample — the hook fires BEFORE this file is indexed, + so it measures what the previous files left behind.""" + del source, rel, done, total + staircase.append(_map_size(session)) + + summary = asyncio.run( + import_sources( + [root], + _probe_llm(RESET_FILES), + prune=False, + session=session, + progress=progress, + ) + ) + assert _map_size(session) <= MAP_BOUND, ( + f"the finished run still holds {_map_size(session)} objects — the" + f" cadence window allows {MAP_BOUND}" + ) + assert staircase, "the progress hook never fired" + assert max(staircase) <= (RESET_FILES - 1) * OBJECTS_PER_ADDED_FILE, ( + f"mid-run the walk held {max(staircase)} objects — more than the" + f" {RESET_FILES - 1} files allowed between two releases" + ) + finally: + session.close() + + # 3 — the byte-identical contract (the phase's reason for existing). + assert summary.added == FILE_COUNT + assert summary.chunks == EXPECTED_CONTENT_CHUNKS + assert summary.summaries == EXPECTED_SUMMARY_CHUNKS + assert summary.errors == 0 + assert dataclasses.asdict(summary) == dataclasses.asdict(ref_summary) + assert _kb_snapshot(db) == ref_kb + + +def test_reset_cadence_bounds_the_unchanged_walk(db: Session, tmp_path: Path) -> None: + """The production path, bounded: a repeat sync sawtooths instead of + ramping. + + :func:`test_the_unchanged_run_keeps_every_document_it_walks` is the + PRE-fix picture (the staircase ``0, 1, … 11``). With the cadence at + :data:`RESET_FILES` the SAME second sync — every file unchanged, every + row dirty from the ``file_hash`` stamp, none of it collectable — samples + ``[0, 1, 2]`` repeated: the release lands before files 4, 7 and 10, and + the run ends at **0** rows resident, because the source-boundary release + runs after the last file and the post-passes (no image docs here) select + nothing. Both halves matter: the sawtooth pins the per-file reset, the + end-of-run ``0`` pins the source boundary. + """ + root = _seed_files(tmp_path) + seed = SessionLocal() + try: + first = asyncio.run(import_sources([root], _probe_llm(), session=seed)) + assert first.added == FILE_COUNT + finally: + seed.close() + gc.collect() + + session = SessionLocal() + try: + staircase: list[int] = [] + + def progress(source: str, rel: str, done: int, total: int) -> None: + del source, rel, done, total + staircase.append(_map_size(session)) + + summary = asyncio.run( + import_sources( + [root], + _probe_llm(RESET_FILES), + prune=False, + session=session, + progress=progress, + ) + ) + assert (summary.files, summary.added, summary.unchanged) == ( + FILE_COUNT, + 0, + FILE_COUNT, + ) + assert staircase == [n % RESET_FILES for n in range(FILE_COUNT)], ( + f"the bounded walk sampled {staircase} — expected a release every" + f" {RESET_FILES} files" + ) + assert max(staircase) < MEASURED_UNCHANGED_MAP_SIZE + assert _map_size(session) == 0, ( + "the source-boundary release did not run — one source's walk" + f" residue ({_map_size(session)} rows) is still resident" + ) + gc.collect() + assert _map_size(session) == 0 + finally: + session.close() + + +def test_reset_cadence_survives_an_embedding_error(db: Session, tmp_path: Path) -> None: + """The error funnel with a map already reset — nothing blows up, nothing + is lost. + + ``sync_session_reset_files=2`` means releases have already happened + before the stub embedder refuses file #4 (``probe-03.md``), so the + handler's ``session.rollback()`` runs on a session whose earlier objects + are DETACHED. The run records ``errors=1``, keeps walking, and indexes + the other 11 files completely: every survivor keeps all 6 of its chunks + and its content, the failed file leaves no row at all, and + ``changed_paths`` is the 11 survivors in walk order — the same shape the + pre-phase error pin produces + (``tests/unit/test_importer.py::test_embedding_failure_is_logged_and_import_continues`, + which runs with the cadence at its ``0`` default). + """ + root = _seed_files(tmp_path) + llm = _BoomEmbedder() + llm.settings = Settings( + _env_file=None, # pyright: ignore[reportCallIssue] + sync_session_reset_files=2, + sync_prune_batch=0, + ) + + session = SessionLocal() + try: + summary = asyncio.run( + import_sources([root], llm, prune=False, session=session) + ) + assert (summary.files, summary.errors) == (FILE_COUNT, 1) + assert summary.added == FILE_COUNT - 1 + assert summary.chunks == EXPECTED_CONTENT_CHUNKS - ( + EXPECTED_CONTENT_CHUNKS // FILE_COUNT + ) + assert summary.summaries == EXPECTED_SUMMARY_CHUNKS - 1 + assert _map_size(session) <= MAP_BOUND + finally: + session.close() + + failed = f"probe-{3:02d}.md" + assert summary.changed_paths == [ + (PROBE_SOURCE, f"probe-{n:02d}.md") for n in range(FILE_COUNT) if n != 3 + ] + snapshot = _kb_snapshot(db) + assert set(snapshot) == {f"probe-{n:02d}.md" for n in range(FILE_COUNT)} - {failed} + # Every file walked BEFORE the failure kept its full row and its chunks — + # detaching them cost the committed work nothing. + assert {path: row[3] for path, row in snapshot.items()} == { + path: OBJECTS_PER_ADDED_FILE - 1 for path in snapshot + } + assert all(row[0] and row[2] for row in snapshot.values()) + + +def test_a_clean_row_leaves_the_map_and_a_dirty_one_never_does(db: Session) -> None: + """The mechanism in one test — why ``expunge_all()`` is the lever. + + A committed, CLEAN row is held weakly: drop the last strong reference + and it leaves the identity map by itself (which is why the added run + above stays flat). Assign a value to it — even the value it already + has, which is exactly ``_unchanged_doc``'s ``doc.file_hash = + file_digest`` stamp — and the state becomes ``modified``, the session's + ``dirty`` set holds it STRONGLY, and the row survives every collection + until the session ends. ``expire_on_commit=False`` is not what retains + it, and ``commit()`` does not clear it; only ``expunge`` / ``close`` + does — and expunging drops no data, because the dirty value equals the + stored one. + """ + session = SessionLocal() + try: + doc = Document( + source=PROBE_SOURCE, + path="mechanism.md", + full_path=f"/srv/brain/{PROBE_SOURCE}/mechanism.md", + title="Mechanism", + content="x" * 64, + content_hash="a" * 64, + file_hash="a" * 64, + ) + session.add(doc) + session.commit() + assert _map_size(session) == 1 + + del doc + gc.collect() + assert _map_size(session) == 0, "a clean committed row must be weakly held" + + row = session.scalar( + select(Document).where( + Document.source == PROBE_SOURCE, Document.path == "mechanism.md" + ) + ) + assert row is not None + row.file_hash = row.file_hash # the stamp: same value, dirty state + assert sa_inspect(row).modified is True + assert session.is_modified(row) is False # no UPDATE will be emitted + del row + gc.collect() + assert _map_size(session) == 1, ( + "the stamped row is no longer held by the session — the unchanged" + " path's retention (and this phase's premise) changed" + ) + + # And the lever: expunging releases the row and loses nothing — the + # bytes in the database are already what the stamp wanted. + session.expunge_all() + gc.collect() + assert _map_size(session) == 0 + assert ( + db.scalar( + select(func.count()) + .select_from(Document) + .where(Document.source == PROBE_SOURCE) + ) + == 1 + ) + finally: + session.close() + + +# --------------------------------------------------------------------------- +# Task 03 — the prune: its ROW (the light select) and its TRANSACTIONS +# --------------------------------------------------------------------------- + +#: The prune probe's own source name — nothing else in the KB uses it. +PRUNE_SOURCE = "prune-probe" + +#: The task-03 fixture: 30 documents × 5 chunks. Small on purpose — what the +#: tests pin is the SHAPE of the prune (the columns it loads, the +#: transactions it runs), and 30 is more than the batch used here so one +#: missed commit boundary cannot hide. +PRUNE_DOC_COUNT = 30 +PRUNE_CHUNKS_PER_DOC = 5 + +#: ~8 KB of markdown per document (production ``content`` averages 15.2 KB). +#: A full row is FAR bigger than a light row, which is why "``content`` is +#: not in the SELECT" is the load-bearing pin of this task. +PRUNE_CONTENT = ("The homelab runs podman quadlets on the hnb host. " * 200)[:8_192] + +#: Task 03's batch cadence for the commit-count pin: 30 docs / every 10th = +#: THREE commits, so a single missed boundary cannot hide. +PRUNE_BATCH = 10 + +#: The SIX columns the prune loads — the ``load_only`` set of ``_prune`` +#: (docstring: the light-row contract). An eighth name here means the walk +#: got heavier; one missing means a guard lost the column it reads. +PRUNE_COLUMNS = frozenset( + {"id", "source", "path", "is_image", "image_path", "page_count"} +) + +#: ``… FROM documents`` — the prune's own walk AND (that is the pin) any +#: per-document follow-up lazy load of a deferred column. The word boundary +#: keeps ``DELETE FROM documents`` out of the count. +_DOC_SELECT_RE = re.compile(r"\bfrom\s+documents\b", re.IGNORECASE) + + +@pytest.fixture() +def statement_capture() -> Iterator[list[str]]: + """Every SQL the engine executes for the duration of ONE test. + + The ``before_cursor_execute`` technique of phase 183 task 01 + (``tests/integration/test_prune_cascade_statements.py``): the column + list a SELECT uses is invisible in the ORM's Python API and plain to + see in the statement it compiles, so a capture is the only + deterministic way to pin what the prune's row actually contains. + Installed on the engine and REMOVED in teardown, so no other test of + the session ever sees the listener. + """ + captured: list[str] = [] + + def _listener( + conn: object, # noqa: ARG001 + cursor: object, # noqa: ARG001 + statement: str, + parameters: object, # noqa: ARG001 + context: object, # noqa: ARG001 + executemany: bool, # noqa: ARG001 + ) -> None: + captured.append(statement) + + event.listen(engine, "before_cursor_execute", _listener) + try: + yield captured + finally: + event.remove(engine, "before_cursor_execute", _listener) + + +def _doc_selects(captured: list[str]) -> list[str]: + """The captured statements that SELECT the ``documents`` table.""" + return [ + stmt + for stmt in captured + if stmt.lstrip().lower().startswith("select") and _DOC_SELECT_RE.search(stmt) + ] + + +def _projected_columns(statement: str) -> frozenset[str]: + """The ``documents.`` names in ONE SELECT's PROJECTION. + + Only the part before ``FROM`` counts — ``WHERE documents.source IN`` and + ``ORDER BY documents.id`` are a predicate and a sort, not a column the + row holds. + """ + head = statement.split("\nFROM ", 1)[0] + return frozenset(re.findall(r"documents\.\w+", head)) + + +def _seed_prune_docs( + session: Session, *, page_dir: Path | None = None +) -> list[UUID]: + """``PRUNE_DOC_COUNT`` indexed docs of :data:`PRUNE_SOURCE`, each with + ``PRUNE_CHUNKS_PER_DOC`` embedded chunks and a full-fat ``content``. + + Committed and then EXPLICITLY expunged, so the prune starts from where a + real sync's prune starts: an empty identity map and a SELECT. The chunks + go in by ``document_id`` (never through ``doc.chunks``), the phase-183 + convention that keeps the child collection UNLOADED. + + With *page_dir* every doc becomes a page-QUALIFYING one (``page_count=2``, + phase 148) and its two ``-pN.png`` copies are written there — the + state in which the prune reads ``id`` + ``page_count``, i.e. the sixth + and seventh reads of the light row. + """ + ids: list[UUID] = [] + for n in range(PRUNE_DOC_COUNT): + doc = Document( + source=PRUNE_SOURCE, + path=f"docs/doc-{n:04d}.md", + full_path=f"/srv/brain/{PRUNE_SOURCE}/docs/doc-{n:04d}.md", + title=f"Prune probe doc {n}", + content=PRUNE_CONTENT, + content_hash=f"{n:064x}", + page_count=2 if page_dir is not None else None, + ) + session.add(doc) + session.flush() + ids.append(doc.id) + if page_dir is not None: + for page in (1, 2): + (page_dir / f"{doc.id}-p{page}.png").write_bytes(b"\x89PNG page") + rot = n % 768 + session.add_all( + [ + Chunk( + document_id=doc.id, + position=position, + content=PRUNE_CONTENT, + embedding=[((i + rot) % 768) / 768 for i in range(768)], + ) + for position in range(PRUNE_CHUNKS_PER_DOC) + ] + ) + session.commit() + session.expunge_all() + return ids + + +def _arm_commit_counter(session: Session) -> list[int]: + """A commit counter on *session* (task 03's transaction pin). + + ``after_commit`` fires once per ``Session.commit()`` that ends a root + transaction — including an empty one — so what the list counts is + exactly the prune's own transaction cadence and nothing else. + """ + commits: list[int] = [] + event.listen(session, "after_commit", lambda s: commits.append(1)) + return commits + + +def test_prune_holds_no_content( + db: Session, tmp_path: Path, statement_capture: list[str] +) -> None: + """THE PRUNE PROBE: the walk loads SIX columns and never ``content``. + + 30 documents × 5 chunks are pruned through the importer's own ``_prune`` + with an empty *seen* (every doc of the source goes), and the captured + SQL pins all three halves of the task-03 contract: + + * the walk is exactly **ONE** ``SELECT … FROM documents`` — no + per-document follow-up SELECT, i.e. nothing inside the loop reads a + deferred column. An eighth statement here means the light row is + INCOMPLETE, and the fix is to add that column to ``load_only`` (never + a ``raiseload``, never a paper-over); + * that SELECT's projection is EXACTLY :data:`PRUNE_COLUMNS` — no + ``documents.content`` (15.2 KB per row in production, ~252 MB across + the incident KB), no ``summary``, no ``full_path``, neither hash, no + date; + * the deletes still remove everything: 0 documents, **0 orphan chunk + rows** (phase 183's ``ON DELETE CASCADE`` doing the child work), the + pruned docs' PDF page copies gone from disk (the phase-148 cleanup, + which is what makes ``id`` + ``page_count`` real reads), plus the full + phase-127 ``changed_paths`` feed. + """ + page_dir = tmp_path / "pdf_pages" + page_dir.mkdir() + ids = _seed_prune_docs(db, page_dir=page_dir) + statement_capture.clear() # the seed's own SQL is not part of the probe + + summary = ImportSummary() + pruned = _prune( + db, + {PRUNE_SOURCE}, + set(), + images=False, + summary=summary, + page_dir=page_dir, + prune_batch=0, + ) + selects = _doc_selects(statement_capture) + + assert pruned == PRUNE_DOC_COUNT + assert len(selects) == 1, ( + f"{len(selects)} SELECTs over documents for a {PRUNE_DOC_COUNT}-doc " + "prune — a deferred column is being read per row, so the light row " + "is incomplete (add the column to load_only)" + ) + projected = { + col.removeprefix("documents.") + for col in _projected_columns(selects[0]) + } + assert projected == PRUNE_COLUMNS, ( + f"the prune's row is not the six-column light row: {sorted(projected)}" + ) + assert "content" not in projected and "summary" not in projected, ( + "the prune SELECTs content again — its resident set scales with the " + "knowledge base instead of with the deletes (the 2026-10-15 incident)" + ) + + # Everything really went, children included (no orphans — phase 183 D4, + # asserted here too so a light row cannot quietly change it). + assert db.scalar( + select(func.count()).select_from(Document).where(Document.source == PRUNE_SOURCE) + ) == 0 + assert db.scalar( + select(func.count()).select_from(Chunk).where(Chunk.document_id.in_(ids)) + ) == 0 + assert sorted(summary.changed_paths) == sorted( + (PRUNE_SOURCE, f"docs/doc-{n:04d}.md") for n in range(PRUNE_DOC_COUNT) + ) + # The page copies went too — read off the light row (id + page_count). + assert list(page_dir.iterdir()) == [] + # And the light rows leave nothing resident behind either. + gc.collect() + assert _map_classes(db) == {} + + +def test_prune_without_sources_touches_nothing( + db: Session, statement_capture: list[str] +) -> None: + """The empty-source guard is a real short-circuit, not a walk for naught. + + ``_prune`` with no source names returns ``0`` before the SELECT — so a + sync that imported nothing prunes nothing WITHOUT reading a single row + (the caller passes the set of sources it actually walked). + """ + _seed_prune_docs(db) + statement_capture.clear() + + summary = ImportSummary() + assert _prune(db, set(), set(), summary=summary, prune_batch=10) == 0 + assert statement_capture == [] + assert summary.changed_paths == [] + assert db.scalar( + select(func.count()).select_from(Document).where(Document.source == PRUNE_SOURCE) + ) == PRUNE_DOC_COUNT + + +def test_prune_batch_commits(db: Session) -> None: + """The transaction cadence: THREE commits at ``10``, exactly ONE at ``0``. + + The same 30-doc prune run twice against an empty KB — once with + ``prune_batch=10`` and once with ``prune_batch=0`` (the pre-phase + rollback, LOCKED A3). The counter sees the three boundaries (10, 20, 30) + and NOTHING after them, where the ``0`` run commits once at the end. + Every other number is identical, which is the point: the cadence changes + WHEN the deletes become durable, never how many there are — + + ``pruned=30``, the same 30 ``changed_paths``, 0 documents and 0 chunk + rows left. (A batch that divides the count does NOT get a fourth, empty + commit — the final commit is skipped exactly when the last boundary + already committed, so ``prune_batch=0`` stays the one-transaction run it + always was.) + """ + + def run(prune_batch: int) -> tuple[int, ImportSummary, int]: + ids = _seed_prune_docs(db) + session = SessionLocal() + summary = ImportSummary() + try: + commits = _arm_commit_counter(session) + pruned = _prune( + session, + {PRUNE_SOURCE}, + set(), + images=False, + summary=summary, + page_dir=None, + prune_batch=prune_batch, + ) + return pruned, summary, len(commits) + finally: + session.close() + # The FK cascade did the child work in EVERY transaction the + # prune opened, not just the last one. + assert db.scalar( + select(func.count()) + .select_from(Chunk) + .where(Chunk.document_id.in_(ids)) + ) == 0 + + pruned, summary, commits = run(PRUNE_BATCH) + assert pruned == PRUNE_DOC_COUNT + assert commits == PRUNE_DOC_COUNT // PRUNE_BATCH, ( + f"a {PRUNE_DOC_COUNT}-doc prune at batch {PRUNE_BATCH} ran " + f"{commits} commits — expected one per {PRUNE_BATCH} deletes" + ) + assert sorted(summary.changed_paths) == sorted( + (PRUNE_SOURCE, f"docs/doc-{n:04d}.md") for n in range(PRUNE_DOC_COUNT) + ) + assert db.scalar( + select(func.count()).select_from(Document).where(Document.source == PRUNE_SOURCE) + ) == 0 + + # …and the pre-phase run does the SAME work in ONE transaction. + _truncate(db) + pruned_0, summary_0, commits_0 = run(0) + assert pruned_0 == pruned == PRUNE_DOC_COUNT + assert commits_0 == 1, ( + f"prune_batch=0 ran {commits_0} commits — the rollback knob must " + "reproduce the pre-phase single transaction exactly" + ) + assert sorted(summary_0.changed_paths) == sorted(summary.changed_paths) + assert db.scalar( + select(func.count()).select_from(Document).where(Document.source == PRUNE_SOURCE) + ) == 0 + + +def test_prune_batch_setting_reaches_the_prune( + db: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The caller passes ``sync_prune_batch`` — the knob is not dead code. + + ``_prune`` takes the cadence as a keyword (it has no ``llm`` argument), + so this pins the ONE wire from ``Settings`` to the loop: a real sync of a + source with three files deleted, observed through a spy on ``_prune``. + The spy records the keyword and forwards to the real function, so the + run's numbers stay the real ones (``pruned=3``) — and the survivors stay + fully embedded, which is what proves the batched deletes were durable + rather than parked in a transaction that never ran. + """ + root = tmp_path / PRUNE_SOURCE + root.mkdir() + for n in range(4): + (root / f"keep-{n}.md").write_text( + f"# Keep {n}\n\nThe homelab runs podman quadlets on the hnb " + f"host with quadlet units, nightly restic backups and a static " + f"DNS zone for probe {n}.\n" + ) + + llm = _probe_llm() + llm.settings.sync_prune_batch = 3 + first = asyncio.run(import_sources([root], llm, session=db)) + assert (first.files, first.added, first.pruned) == (4, 4, 0) + + for n in range(3): + (root / f"keep-{n}.md").unlink() + + import app.rag.importer as importer + + seen_kwargs: list[object] = [] + real_prune = importer._prune + + def spy(*args: object, **kwargs: object) -> int: + assert "prune_batch" in kwargs, ( + "import_sources did not pass Settings.sync_prune_batch to _prune" + ) + seen_kwargs.append(kwargs["prune_batch"]) + return real_prune(*args, **kwargs) # type: ignore[operator] + + monkeypatch.setattr(importer, "_prune", spy) + second = asyncio.run(import_sources([root], llm, prune=True, session=db)) + + assert seen_kwargs == [3], ( + "import_sources did not pass Settings.sync_prune_batch to _prune" + ) + assert (second.files, second.pruned, second.errors) == (1, 3, 0) + assert sorted(path for (_src, path) in second.changed_paths) == [ + "keep-0.md", + "keep-1.md", + "keep-2.md", + ] + assert [path for (path,) in db.execute(select(Document.path))] == ["keep-3.md"] + # The survivor kept every chunk; the pruned docs left no orphans. + embedded = db.scalar( + select(func.count()).select_from(Chunk).where(Chunk.embedding.is_not(None)) + ) + assert embedded is not None and embedded > 0 + assert db.scalar( + select(func.count()) + .select_from(Chunk) + .where(Chunk.document_id.not_in(select(Document.id))) + ) == 0 + + +def test_prune_guards_unaffected_with_light_rows( + db: Session, tmp_path: Path +) -> None: + """The guards run off SIX columns — the light row is COMPLETE. + + A column missing from ``load_only`` does not announce itself: the guard + that needs it simply emits one lazy SELECT per row and carries on, so + the bug would show up as a walk that is quietly unbounded again. This + test drives BOTH prune guards over light rows and pins the survivors: + + * phase 122 (images OFF): an ``is_image`` doc whose file is gone + SURVIVES — ``is_image`` decides it; + * phase 126 (the embedded image's parent rule, toggle ON): a + ``#page=N#img=M`` doc SURVIVES while its parent PDF's rel is in + *seen* — ``path`` decides it — while a standalone image whose file IS + gone prunes and takes its ``image_dir`` copy with it, which reads + ``image_path`` (:func:`app.rag.importer._delete_image_copy`). + + Both toggles run over the same five rows, one run after the other (the + way two syncs of the same source actually happen): toggle OFF prunes ONE + document (the missing markdown — and leaves BOTH copies alone), toggle ON + then prunes the ONE row that is still there and still must go (the + standalone image, WITH its copy), the embedded image surviving in both. + The exact pre-phase guard semantics, on a row that never loaded + ``content``. + """ + image_dir = tmp_path / "image_dir" + image_dir.mkdir() + gone_copy = image_dir / "gone.png" + kept_copy = image_dir / "parent.png" + gone_copy.write_bytes(b"\x89PNG\r\n\x1a\n gone") + kept_copy.write_bytes(b"\x89PNG\r\n\x1a\n parent") + + def add(path: str, *, is_image: bool, copy: Path | None = None) -> None: + db.add( + Document( + source=PRUNE_SOURCE, + path=path, + full_path=f"/srv/brain/{PRUNE_SOURCE}/{path}", + title=f"Guard probe {path}", + content=PRUNE_CONTENT, + content_hash=f"{abs(hash(path)):064x}"[:64], + is_image=is_image, + image_path=str(copy) if copy else None, + ) + ) + + add("notes/kept.md", is_image=False) + add("notes/gone.md", is_image=False) + add("img/gone.png", is_image=True, copy=gone_copy) + add("deck.pdf", is_image=False) + add("deck.pdf#page=1#img=0.png", is_image=True, copy=kept_copy) + db.commit() + + seen = {(PRUNE_SOURCE, "notes/kept.md"), (PRUNE_SOURCE, "deck.pdf")} + session = SessionLocal() + try: + pruned_toggle_off = _prune( + session, {PRUNE_SOURCE}, seen, images=False, prune_batch=0 + ) + finally: + session.close() + # Toggle OFF: BOTH image docs are invisible to the walk, not deleted — + # and neither copy goes with them. + assert pruned_toggle_off == 1 + assert gone_copy.exists() and kept_copy.exists() + + session = SessionLocal() + try: + pruned_toggle_on = _prune( + session, {PRUNE_SOURCE}, seen, images=True, prune_batch=0 + ) + finally: + session.close() + # Toggle ON: the deleted image file prunes its row AND its copy (the + # markdown went in the run above, so this run's single delete is the + # image's); the embedded image survives on its parent PDF's rel. + assert pruned_toggle_on == 1 + assert not gone_copy.exists() + assert kept_copy.exists() + assert sorted( + path + for (path,) in db.execute( + select(Document.path).where(Document.source == PRUNE_SOURCE) + ) + ) == ["deck.pdf", "deck.pdf#page=1#img=0.png", "notes/kept.md"] diff --git a/tests/integration/test_importer.py b/tests/integration/test_importer.py index 894fc93..38bf2d1 100644 --- a/tests/integration/test_importer.py +++ b/tests/integration/test_importer.py @@ -1287,12 +1287,16 @@ def test_backfill_never_touches_standalone_and_copyless_rows(db, tmp_path) -> No assert s.image_context_backfilled == 0 assert s.images_failed == 0 and s.errors == 0 assert llm.text_parts == [] # zero describe calls, both runs - db.expire(standalone) - db.expire(missing_copy) - assert standalone.image_context is None - assert standalone.content == DESCRIPTION - assert missing_copy.image_context is None - assert missing_copy.content == DESCRIPTION + # Phase 184: an import run is now ALLOWED to release the session it + # walks (the identity-map cadence expunges every object at the source + # boundary), so the two seeded rows are re-read by key instead of + # expiring the objects the run detached. Same assertion, one level + # closer to the database — nothing about the run's output changed. + for row in (standalone, missing_copy): + reloaded = db.get(Document, row.id) + assert reloaded is not None + assert reloaded.image_context is None + assert reloaded.content == DESCRIPTION finally: _cleanup_source(db, source_root.name) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 5f6ccc4..dc28d18 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -906,3 +906,58 @@ def test_ui_customization_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None assert s.footer_text == "Powered by my own models" monkeypatch.setenv("BOR_INPUT_PLACEHOLDER", "") assert _settings().input_placeholder == "" # empty stands + + +# --- Sync session bounds (phase 184) --- + + +def test_sync_session_reset_files_default_and_env_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Phase 184 (LOCKED A3): the import walk releases its session every + ``BOR_SYNC_SESSION_RESET_FILES`` files (default 200). ``0`` is accepted + and means NEVER release — the exact pre-phase-184 behaviour, the + operator's no-redeploy rollback.""" + monkeypatch.delenv("BOR_SYNC_SESSION_RESET_FILES", raising=False) + assert _settings().sync_session_reset_files == 200 + monkeypatch.setenv("BOR_SYNC_SESSION_RESET_FILES", "50") + assert _settings().sync_session_reset_files == 50 + monkeypatch.setenv("BOR_SYNC_SESSION_RESET_FILES", "0") + assert _settings().sync_session_reset_files == 0 + + +def test_sync_session_reset_files_rejects_negative( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A negative cadence is a typo — a ``%`` against it never fires, so + startup refuses naming the field (the ``agent_max_rounds`` pattern, + phase 184).""" + monkeypatch.setenv("BOR_SYNC_SESSION_RESET_FILES", "-1") + with pytest.raises(ValidationError, match="sync_session_reset_files"): + _settings() + + +def test_sync_prune_batch_default_and_env_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Phase 184 (LOCKED A3): the prune commits every + ``BOR_SYNC_PRUNE_BATCH`` deleted documents (default 500). ``0`` is + accepted and means ONE commit at the end of the whole prune — the + exact pre-phase-184 behaviour.""" + monkeypatch.delenv("BOR_SYNC_PRUNE_BATCH", raising=False) + assert _settings().sync_prune_batch == 500 + monkeypatch.setenv("BOR_SYNC_PRUNE_BATCH", "25") + assert _settings().sync_prune_batch == 25 + monkeypatch.setenv("BOR_SYNC_PRUNE_BATCH", "0") + assert _settings().sync_prune_batch == 0 + + +def test_sync_prune_batch_rejects_negative( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A negative batch size is a typo — the batch comparison would never + fire, so startup refuses naming the field (the ``agent_max_rounds`` + pattern, phase 184).""" + monkeypatch.setenv("BOR_SYNC_PRUNE_BATCH", "-1") + with pytest.raises(ValidationError, match="sync_prune_batch"): + _settings() diff --git a/tests/unit/test_settings_catalog.py b/tests/unit/test_settings_catalog.py index a3f0df7..6907c0e 100644 --- a/tests/unit/test_settings_catalog.py +++ b/tests/unit/test_settings_catalog.py @@ -188,6 +188,41 @@ def test_verbatim_display_field_types() -> None: assert Settings.model_fields["show_text_max_blocks"].default == 5 +def test_sync_session_bound_rows_are_plain_live_overridable() -> None: + """Phase 184 (task 01, LOCKED A3/D2): the two sync-run knobs sit in + ``Import & sources`` (beside ``upload_max_mb`` — they tune the sync, not + the walk's content) with the ``Sync session reset files`` / ``Sync prune + batch`` labels and are plain — not ``secret`` (two ints), not + ``restart_required`` (the importer reads ``llm.settings`` per run, so a + saved override applies to the NEXT sync), overridable (the overlay is + generic over ``int`` fields). Both accept ``0`` = the exact pre-phase + behaviour (the no-redeploy rollback); a negative value fails startup, + not the panel.""" + for key, label in ( + ("sync_session_reset_files", "Sync session reset files"), + ("sync_prune_batch", "Sync prune batch"), + ): + meta = CATALOG[key] + assert meta.group == "Import & sources" + assert meta.label == label + assert meta.secret is False + assert meta.restart_required is False + assert meta.overridable is True + assert key not in SECRETS + assert key not in RESTART_REQUIRED + assert key not in NON_OVERRIDABLE + + +def test_sync_session_bound_field_types() -> None: + """Phase 184 (task 01, D2): both knobs are ``int``s with the LOCKED A3 + defaults (``200`` files / ``500`` documents) — the type the panel renders + its input for and the overlay coerces from the stored raw string.""" + assert Settings.model_fields["sync_session_reset_files"].annotation is int + assert Settings.model_fields["sync_prune_batch"].annotation is int + assert Settings.model_fields["sync_session_reset_files"].default == 200 + assert Settings.model_fields["sync_prune_batch"].default == 500 + + def test_labels_nonempty_and_unique() -> None: labels = [meta.label for meta in CATALOG.values()] assert all(label.strip() for label in labels), "empty label in the catalogue"