From a45f7446042a2c9a361f7ffa13326ddcbd506736 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Tue, 20 Oct 2026 06:36:21 -0400 Subject: [PATCH] phase: 194_sync_pending_count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Phase 194 — final verification pass: all green.** **Verified (all 5 tasks already complete; nothing re-implemented needed)** - `app/rag/importer.py`: `_prescan_pending` + `_pending_hash_stmt` (scalar `documents.file_hash`, own session, ONE `asyncio.to_thread` span, REPLACES the stats pre-walk), `on_scan_progress`, `progress` pair = pending position, `0` branch = byte-identical phase-193 walk. - `app/config.py`/`settings_catalog.py`/`.env.example`: `sync_pending_count=1` + validator; `app/api/sync.py`: `"scan"` phase, `scan_done`/`scan_total` (additive after `files_total`, kept at every terminal), forced/throttled RSS split; `frontend/assets/sources.js`: `fmtSyncPhaseLabel` scan branch first-arm, `fmtSyncLabel` untouched. **Tests / lint / coverage (exact commands)** - `uv run pytest --cov=app --cov-report=term-missing` → **4835 passed**, `app/` 99% (7240 stmts / 16 miss), exit 0 - `uv run pytest tests/e2e/test_sync_pending_count.py -v --no-cov` → **8 passed in 150.81s** (phase suite, isolated) - 14 sync-lineage E2E suites, one pytest process each (`test_sync_remaining_count`, `…_chunk_progress`, `…_progress_box`, `…_peak_memory_visible`, `…_label_mobile`, `…_upload_progress`, `test_archive_upload_sources`, `…_image_progress`, `…_summary_visibility`, `…_status_freshness`, `test_smoke`, `test_fast_path_deferred_content`, `test_sync_session_reset`, `test_sync_git_freeze`) → **all exit 0** - `uv run ruff check .` → All checks passed; `uv run pyright` → **0 errors** **Completion criteria** — all met: `N remaining` = predicted change set (E2E arms 1/2/5, first import counts the walk, no-change → no count part); `scan` phase visible + flips to `import`, scan pair reset/kept at terminals (`test_sync_scan_phase.py`); `BOR_SYNC_PENDING_COUNT=0` walk semantics ("7 remaining" arm) + **negative refuses startup naming the field (verified live by importing `app.main`)**; pre-scan off-loop + cancel lands `cancelled` (integration pins); CLI/upload byte-identical (unedited pins green); payload additive (`_PRE_187_STATUS_KEYS | _MEMORY_KEYS | _SCAN_KEYS`); suite green, coverage >90%, lint/types clean. **Documentation** — 3 fixes: README sync "States" bullet had an **unclosed parenthesis** (pre-existing, 193/194) and the scan-label "total is 0" clause was ambiguous about a `=0` run (it still counts the walk once the import starts); **AGENTS.md rule 17 had a stray `)`** (pre-existing from phase 191). Also verified consistency both directions for all 87 settings ↔ README ↔ `.env.example` ↔ settings catalog, the pages table ↔ `app/main.py` routes, no stale `BOR_` names, and README/AGENTS claims through phase 194. **Notes/deviations** — no live-infra changes (prod sync knob ships default-on, no deployment touched); no git index/history commands run (all changes left in the working tree for the harness); commit criterion left to the harness as instructed. **Next pending phase:** none — `.agents/phases/todo/` holds only `194_sync_pending_count`, which is now ready to move to `complete/`. --- .../194_sync_pending_count/00_phase.md | 70 + .../01_prescan_importer.md | 48 + .../02_sync_phase_status.md | 43 + .../03_ui_scan_label.md | 35 + .../04_retarget_pinned_suites.md | 38 + .../05_e2e_and_gates.md | 42 + .../194_sync_pending_count__00_phase.a1.err | 0 .../194_sync_pending_count__00_phase.a1.md | 19 + ..._sync_pending_count__00_phase.a1.md.status | 1 + ...4_sync_pending_count__00_phase.a1.validate | 151 ++ ..._pending_count__01_prescan_importer.a1.err | 0 ...c_pending_count__01_prescan_importer.a1.md | 14 + ...ng_count__01_prescan_importer.a1.md.status | 1 + ...ing_count__01_prescan_importer.a1.validate | 150 ++ ...pending_count__02_sync_phase_status.a1.err | 0 ..._pending_count__02_sync_phase_status.a1.md | 17 + ...g_count__02_sync_phase_status.a1.md.status | 1 + ...ng_count__02_sync_phase_status.a1.validate | 150 ++ ...ync_pending_count__03_ui_scan_label.a1.err | 0 ...sync_pending_count__03_ui_scan_label.a1.md | 17 + ...nding_count__03_ui_scan_label.a1.md.status | 1 + ...ending_count__03_ui_scan_label.a1.validate | 151 ++ ...ng_count__04_retarget_pinned_suites.a1.err | 0 ...ing_count__04_retarget_pinned_suites.a1.md | 18 + ...nt__04_retarget_pinned_suites.a1.md.status | 1 + ...unt__04_retarget_pinned_suites.a1.validate | 151 ++ ...ync_pending_count__05_e2e_and_gates.a1.err | 0 ...sync_pending_count__05_e2e_and_gates.a1.md | 15 + ...nding_count__05_e2e_and_gates.a1.md.status | 1 + ...ending_count__05_e2e_and_gates.a1.validate | 151 ++ .env.example | 19 + AGENTS.md | 125 +- README.md | 91 +- app/api/sync.py | 184 +- app/config.py | 30 + app/core/settings_catalog.py | 3 + app/rag/importer.py | 317 ++- frontend/assets/sources.js | 126 +- tests/e2e/test_sync_chunk_progress.py | 7 +- tests/e2e/test_sync_peak_memory_visible.py | 14 +- tests/e2e/test_sync_pending_count.py | 1999 +++++++++++++++++ tests/e2e/test_sync_progress_box.py | 4 + tests/e2e/test_sync_remaining_count.py | 557 ++++- .../test_import_fast_path_columns.py | 14 + .../test_import_session_retention.py | 12 + tests/integration/test_sync_api.py | 73 +- tests/integration/test_sync_git_freeze.py | 14 + tests/integration/test_sync_pending_count.py | 571 +++++ tests/integration/test_sync_scan_phase.py | 927 ++++++++ tests/unit/test_config.py | 30 + tests/unit/test_importer.py | 37 +- tests/unit/test_settings_catalog.py | 23 + tests/unit/test_sync_button.py | 82 +- tests/unit/test_sync_pending_count.py | 700 ++++++ tests/unit/test_sync_scan_label_ui.py | 544 +++++ 55 files changed, 7610 insertions(+), 179 deletions(-) create mode 100644 .agents/phases/complete/194_sync_pending_count/00_phase.md create mode 100644 .agents/phases/complete/194_sync_pending_count/01_prescan_importer.md create mode 100644 .agents/phases/complete/194_sync_pending_count/02_sync_phase_status.md create mode 100644 .agents/phases/complete/194_sync_pending_count/03_ui_scan_label.md create mode 100644 .agents/phases/complete/194_sync_pending_count/04_retarget_pinned_suites.md create mode 100644 .agents/phases/complete/194_sync_pending_count/05_e2e_and_gates.md create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.err create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.md create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.md.status create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.validate create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.err create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.md create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.md.status create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.validate create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.err create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.md create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.md.status create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.validate create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.err create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.md create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.md.status create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.validate create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.err create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.md create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.md.status create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.validate create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.err create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.md create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.md.status create mode 100644 .agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.validate create mode 100644 tests/e2e/test_sync_pending_count.py create mode 100644 tests/integration/test_sync_pending_count.py create mode 100644 tests/integration/test_sync_scan_phase.py create mode 100644 tests/unit/test_sync_pending_count.py create mode 100644 tests/unit/test_sync_scan_label_ui.py diff --git a/.agents/phases/complete/194_sync_pending_count/00_phase.md b/.agents/phases/complete/194_sync_pending_count/00_phase.md new file mode 100644 index 0000000..16974ad --- /dev/null +++ b/.agents/phases/complete/194_sync_pending_count/00_phase.md @@ -0,0 +1,70 @@ +# Phase 194 — "N remaining" counts the files that will actually change + +**Source:** owner report 2026-10-19 — "I just synced the prod brain of reese and it said '1216 remaining' when it only had like 10 files to sync" (the prod KB holds ~10,000 importable files; that run changed ~10). +**Story:** n/a (owner directive — phase 193's confirmed assumption A1 proved wrong in production) +**Depends on:** `193_sync_remaining_count` (complete) — builds on the phase-64/98/128/129/152/179/184/185/187 sync lineage. + +## Evidence (diagnosed from the code, read this not the chat) + +1. **The phase-193 label counts the WALK, not the WORK.** `frontend/assets/sources.js` `fmtSyncLabel` renders `N remaining` with `N = files_total − files_done` (the arithmetic is fine). But `app/rag/importer.py`'s pre-walk (L949–962) sets `total` to the COMBINED COUNT OF EVERY IMPORTABLE FILE across all sources (the 10,000), and the per-file hook (L1034–1039) reports `done = summary.files` (the 1-based position across ALL files walked). The fast path (phase 147/185) skips re-embedding for the unchanged 9,990 but still COUNTS them. Prod: "1216 remaining" = "8,784 files left to walk", while the actual work was ~10 files. Phase 193's E2E suite passed because its fixture KB was a FRESH import where walk == work — the incremental case the owner just hit was never pinned. +2. **The change signal already exists per file.** `_file_hash` (L704–724 — the sha256 of the RAW bytes, streamed in 1 MiB chunks) compared against `documents.file_hash` (nullable `String(64)`, no index — looked up through the unique `(source, path)` constraint, `app/models.py` L119/L138–142) is EXACTLY the fast path's change predicate: stored NULL (no row, or a pre-phase row) or `!= digest` ⇔ the file will be re-indexed (added/updated). A pre-scan reusing that predicate predicts the change set with the SAME semantics the processing loop then applies — pre-phase NULL rows are predicted changed AND actually re-indexed (stamped) by the loop, so the prediction never lies about the stored-hash contract. +3. **The pre-scan must be OFF the loop.** A 10,000-file hash+SELECT span run inline would park the single uvicorn loop for seconds — the exact phase-179 defect class pinned by `tests/integration/test_sync_git_freeze.py` (the 3 s git stub froze the 2 s status poll) and `tests/e2e/test_sync_git_freeze.py`. The git span moved to `asyncio.to_thread` (phase 179); the pre-scan does the same (the worker runs to completion on cancel; the run stops at the next phase-134 boundary — the `CancelledError` clause in `_run_sync`, L178–190 contract). +4. **Only the UI sync pays anything.** `import_sources` has exactly two callers: `app/api/sync.py` (the UI sync — the ONLY one that passes `progress`) and `scripts/import_docs.py` L407 (the CLI — no `progress`). Uploads have no file-level progress at all (phase 90 A2 — `app/api/git_sources.py` L59; their label renders bare, the `test_archive_upload_sources.py` pins). Gating the pre-scan on `progress is not None` (the phase-64 skip rule) keeps every non-UI caller byte-identical. +5. **The count is a prediction, and the drift edges are bounded.** The pre-scan runs seconds before the processing loop reaches a given file. A predicted file that turns out unchanged (reverted mid-run) still advances the count (it was predicted — `total` counted it) so the count reaches 0; a file changed AFTER the scan is indexed for real but does not advance the count, and the phase-193 `total − done > 0` guard hides the excess (never negative, never a stale "0 remaining"). The terminal `counts_line` ("N added · M updated …") stays the exact surface — untouched. + +## Design (shared by all tasks — the executor reads this, not the chat) + +- **D1 — the pre-scan (task 01).** `import_sources` gains ONE optional callback `on_scan_progress: Callable[[int, int], None] | None = None` (the 1-based scan position + the importable total — the "only when provided", "hooks only assign fields", no-try/except contract of its three siblings). While `progress is not None` AND `llm.settings.sync_pending_count` is truthy (read ONCE before the source loop, the phase-184 read-once discipline — `0` = rollback to the phase-193 walk-position semantics, negative value refuses startup loudly, the house kill-switch pattern): + - the EXISTING stats-only pre-walk is REPLACED (not augmented — no third walk) by ONE `asyncio.to_thread` span that, per root with the EXACT `iter_importable_files` rules of the processing loop (same `ignore` / `include_hidden` / `image_extension_set` / images-toggle gating): (a) takes the per-root file list and its length (the scan total — the same number the old pre-walk computed), then (b) per file: `_file_hash(path)` (the existing streamed helper) + ONE scalar `SELECT documents.file_hash WHERE source = :s AND path = :p` (the only column — NO entity load, NO `defer`/`load_only` row, no session retention, one light metadata read per file; phase 184/185 compliant: it is not a whole-entity `select(Document)`, it defers nothing because it loads no row, and the processing loop's `_doc_row_stmt` fast-path row load stays the only ROW load) → `pending = {(source, rel)}` for every file whose stored hash is NULL or `!= digest`; + - `on_scan_progress(position, total)` fires per file (the scan position for the UI's "Scanning… (n/total)" label — called from the worker thread; field assignments are GIL-atomic, the phase-179 off-loop contract), and `files_total` becomes `len(pending)` while the per-file `progress` call's `done` becomes the 1-based POSITION IN THE PENDING SET (it advances for every predicted file, whether or not it turns out changed at processing time — the count and its denominator stay on the SAME set so the arithmetic never runs away); + - `summary.files` (the `import: summary … files=` log count — every file the loop processed, append-only log discipline) is UNTOUCHED; the `sync: done/cancelled/failed … files_done=… files_total=…` lines keep their EXACT shape, now carrying the pending counts (the module docstring names the semantic shift); + - the processing loop's FRESH per-file `_file_hash` is KEPT (the double hash is deliberate: the pre-scan's digest is the prediction, the loop's is the truth — a file edited between the two is still detected and re-indexed; the second read is page-cached — the cost the owner accepted, A3); + - `on_scan_progress` is called only when provided; `progress=None` callers (the CLI) skip the pre-scan entirely — byte-identical behavior (the phase-64 skip rule, extended); + - the degenerate empty KB: the scan total is 0, no callback fires, the run's `phase` stays `"scan"` until the first `progress` call (which there is none) or the next transition/terminal — the UI renders bare "Scanning…" (the `total === 0` guard → no count part) and the terminal clears the phase; documented, pinned in the task-05 arm; + - docstrings: the module `progress` paragraph (the `done`/`total` semantic shift), the new `on_scan_progress` paragraph, the `to_thread`/cancel/cost paragraphs, the `sync_pending_count` gating clause. +- **D2 — the kill switch (task 01).** `app/config.py` gains `sync_pending_count: int = 1` (`BOR_SYNC_PENDING_COUNT`): `1` (default) = the pre-scan pending-count semantics; `0` = the phase-193 WALK-position semantics (the stats pre-walk + `done = summary.files` — the full rollback the owner gets if the cost ever bites); a negative value refuses startup with the field named (the `agent_max_rounds` house pattern). `.env.example` + README settings documentation. The switch is read ONCE per run (the phase-184 discipline — a mid-run settings save cannot change the semantics between two files of the same walk). +- **D3 — the phase machine + status (task 02).** `SyncStatus.phase` gains `"scan"` (additive — the machine: the model-check + clone/pull prelude reports `null`, then `"scan"` (set immediately before `import_sources`, REPLACING today's `= "import"`), `"import"` (flipped by the FIRST `progress` call while `phase == "scan"` — the flip point is the existing per-file hook, so no extra callback), `"overview"`, `"summaries"` — the no-gap summaries skip still keeps the run in `"import"`). The flip + the pre-`import_sources` set are phase transitions, so each gets the phase-187 FORCED `_sample_rss()` (the per-file `_scan_hook` calls the UNFORCED throttled sample, the progress-hook convention — one `/proc` read per 5 s, never one per file). `SyncStatus` gains `scan_done` / `scan_total` (`int`, default 0; reset with the run; KEPT at every terminal — the phase-64 keep-final-values convention, beside the final file counts). The status payload gains the same two keys ADDITIVELY (placed after `files_total`; every pre-existing key keeps its name, type, position, and meaning — the phase-187 additive-only discipline). `current_file` stays `null` during the scan (the scan names no file). Docstrings: the module phase-machine paragraph (L188–200 region), the `SyncStatus` class paragraph (the phase-64/98/187 clauses gain the scan clause), the `GET /api/sync/status` docstring field paragraph. +- **D4 — the UI scan label (task 03).** `fmtSyncPhaseLabel` (the phase-aware wrapper — `frontend/assets/sources.js`) gains ONE explicit `"scan"` branch in its decision tree: main text `Scanning…` (no file — `current_file` is null in the phase, the branch must not read it), count part `(n/total)` rendered in the SAME count slot the import phase's `N remaining` uses (`n = scan_done`, `total = scan_total`; the `total === 0` guard → no count part, the phase-64 contract); the caller passes the scan pair through the existing call sites that already forward the summaries/chunk/image parts (the `phase` value selects which pair the wrapper consumes — the summaries-branch pattern). The `null` prelude + `"import"` fallback clause now covers ONLY those two. `fmtSyncLabel`'s `N remaining` arithmetic is BYTE-IDENTICAL (the phase-193 pins stay unedited — the server-side semantic change flows through the SAME builder unchanged). The A4 untruncated placement (button title + `#sync-progress`) is untouched — `Scanning… (n/total)` rides it automatically; the phase-187 `· peak N MB` trailing clause still lands on the `#sync-progress` counts line while the scan label is visible. The docstring's decision-tree paragraph (L157–167 region) is updated. The walk position surviving as `(n/total)` is HONEST for a scan — it is labeled "Scanning", never "Syncing" (the owner's objection, TODO L6, was the sync count wearing the total's shape). +- **D5 — the pinned suites move (task 04).** See the task for the definitive list (the count-frame pins retarget to the pending-set semantics where the fixture KB is NOT fresh; the fresh-fixture pins (chunk progress, label mobile, upload) stay unedited; the phase-184/185/179 integration pins get the pre-scan's statements distinguished or are verified unaffected). + +## ASSUMPTIONS (owner-confirmed 2026-10-19, questionnaire) +- **A1:** `N remaining` counts down the pre-scan's PREDICTED change set (`files_total = len(pending)`, `files_done` = the 1-based position in it); the count advances on the prediction set (a predicted-then-unchanged file still advances it — the count reaches 0; a post-scan change is indexed but uncounted — the phase-193 guard hides the excess). First-time imports (empty DB) count the full walk exactly as phase 193 did; a no-change run shows no count part. +- **A2:** the pre-scan is a NEW VISIBLE `"scan"` phase — `Scanning… (n/total)` before the `import` phase; the scan's walk position is the only surviving `n/m` shape, and it is labeled a scan. +- **A3:** the cost is accepted: one extra full read + sha256 + one scalar metadata SELECT per importable file per run (the second read is page-cached; the span is off-loop via `to_thread`, the phase-179 rule). The rollback knob `BOR_SYNC_PENDING_COUNT=0` restores the phase-193 walk-position semantics; the CLI and the upload job pay nothing. + +## Context +- `app/rag/importer.py` — `import_sources` (L756+, the pre-walk L949–962, the per-file `progress` call L1034–1039), `_file_hash` (L704–724), `_doc_row_stmt` (L725+), the module docstring's progress/chunk/image hook paragraphs. +- `app/api/sync.py` — the module docstring (the phase-179/134/98 paragraphs, L100–200 region), `SyncStatus` (L308+), `GET /api/sync/status` (L495+, the payload dict L548+), `_run_sync` (L651+, the reset block, the `_hook`/`_scan_hook`/`_chunk_hook`/`_image_hook` closures L815–860 region, the `_status.phase = "import"` set L880, the terminal log lines L1112/L1133). +- `app/config.py` — the `BOR_SYNC_*` settings block + validators (the `agent_max_rounds` kill-switch pattern); `.env.example`. +- `app/models.py` — `Document` (L117+, the unique `(source, path)`, the `file_hash` column contract L128–142). +- `frontend/assets/sources.js` — `fmtSyncPhaseLabel` (L157–167 docstring, the decision tree), `fmtSyncLabel` (L1043–1071 — UNTOUCHED), the status call sites (L1121–1127, L1449–1458, L1621–1652), `setSyncLabel`/A4 (UNTOUCHED). +- `scripts/import_docs.py` L407 (the CLI caller — `progress` absent — the byte-identity pin), `app/api/git_sources.py` L59 (the upload's no-file-progress contract — UNTOUCHED). +- Pinned suites: `tests/e2e/test_sync_remaining_count.py` (phase 193's own — the count-frame pins + the status key-set pin), `tests/e2e/test_sync_chunk_progress.py`, `test_sync_label_mobile.py`, `test_sync_upload_progress.py`, `test_archive_upload_sources.py` (the upload's "no `N remaining`" pins), `tests/integration/test_import_fast_path_columns.py` (phase 185 — the per-file SELECT pins), `test_import_session_retention.py` (phase 184), `test_sync_git_freeze.py` (phase 179 — the loop-freeze pin), `test_sync_session_reset.py`, `test_fast_path_deferred_content.py`, `test_sync_summary_visibility.py`, `test_sync_image_progress.py`, `test_sync_progress_box.py`, `test_sync_status_freshness.py`, `test_sync_peak_memory_visible.py`, `test_smoke.py`. + +## Tasks +1. `01_prescan_importer.md` — the importer's pre-scan (the D1 spec) + the `BOR_SYNC_PENDING_COUNT` kill switch (the D2 spec) + the unit/integration pins. +2. `02_sync_phase_status.md` — the `"scan"` phase + `scan_done`/`scan_total` status fields + the additive payload + the docstrings (the D3 spec) + the integration pins. +3. `03_ui_scan_label.md` — the `fmtSyncPhaseLabel` scan branch (the D4 spec) + the JS marker pins. +4. `04_retarget_pinned_suites.md` — the pinned-suite retarget (the D5 list); the full suite green. +5. `05_e2e_and_gates.md` — the phase's own E2E suite (the pending-count-down through the real UI), the full gates, the docs pass, the atomic commit. + +## Testing & Quality +- **Unit (marker pins):** the pre-scan gate (the `progress is not None` + `sync_pending_count` read-once), the scalar SELECT's column shape (no entity load), the `pending` predicate (NULL / mismatch / match), the `done`-as-pending-position contract, the kill-switch value handling (1 / 0 / negative). +- **Integration:** the pending set on fresh / unchanged / changed / pre-phase-NULL fixtures; the off-loop pin (the delayed pre-scan span keeps the loop responsive — the phase-179 pattern); the cancel-during-prescan terminal; the phase-184/185/179 pins retargeted or verified; the phase machine sequence (null → scan → import → … → terminal null) + the kept scan counts + the forced/throttled RSS samples. +- **E2E (regression, task 04):** the retargeted pins green; the fresh-fixture + upload pins green UNEDITED. +- **E2E (phase, AGENTS rule 4, task 05):** `tests/e2e/test_sync_pending_count.py`, run in isolation — the first-sync arm, the INCREMENTAL arm (the owner's prod case: K of N changed, the count holds at `K remaining` while the walk moves over unchanged files, hidden at the final predicted file, terminal `K updated`), the no-change arm (no count part), the kill-switch arm (`0` → the phase-193 walk semantics), the empty-KB arm, the peak-clause-on-scan-line pin. +- **Coverage:** **>90%** on `app/` (`uv run pytest --cov=app --cov-report=term-missing`). +- **Lint/types:** `uv run ruff check . && uv run pyright`. + +## Documentation & Consistency +- Final pass reviews `README.md` and `AGENTS.md` in their entirety against the post-phase app and fixes every inconsistency found, pre-existing ones included. +- `README.md`: the sync section's running-label clause gains the `scan` phase + the pending-count semantics (the phase-193 `N remaining` example becomes the predicted-change-set one) and the `BOR_SYNC_PENDING_COUNT` setting (the `.env.example` entry is part of task 01); `AGENTS.md` — verify whether any clause names the running count part (phase 193 verified it does not — if still absent, do NOT add; the `BOR_SYNC_PENDING_COUNT` setting row joins the settings documentation wherever the README/AGENTS settings table lives). + +## Completion Criteria +- [ ] `N remaining` counts down the pre-scan's predicted change set (`files_total = len(pending)`, `files_done` = the pending position) on a progress caller; a first-time import counts the full walk (A1's degenerate parity); a no-change run shows no count part — the E2E arms a/b/c. +- [ ] the `scan` phase is visible (`Scanning… (n/total)`, the position climbs, the `total === 0` guard) and flips to `import` on the first progress call; `scan_done`/`scan_total` reset with the run and survive the terminals — the E2E arms a/e + the integration phase-machine pin. +- [ ] `BOR_SYNC_PENDING_COUNT=0` restores the phase-193 walk-position semantics byte-for-byte (the E2E kill-switch arm + the marker pins); a negative value refuses startup with the field named. +- [ ] the pre-scan span never parks the loop (the off-loop pin); a cancel during it lands the `cancelled` terminal (the phase-134 contract); the CLI and the upload job are byte-identical (the unedited pins). +- [ ] the chunk / image / summaries parts, the `fmtSyncLabel` arithmetic, the terminal `counts_line`, `summary.files` and every pre-existing log line are byte-identical (the unedited pins green); the status payload is additive-only (the key-set pin). +- [ ] test suite green, coverage >90%; `uv run ruff check . && uv run pyright` clean; README + AGENTS consistent (the final pass). +- [ ] one atomic Conventional Commits commit with `--no-gpg-sign`. diff --git a/.agents/phases/complete/194_sync_pending_count/01_prescan_importer.md b/.agents/phases/complete/194_sync_pending_count/01_prescan_importer.md new file mode 100644 index 0000000..65cb05e --- /dev/null +++ b/.agents/phases/complete/194_sync_pending_count/01_prescan_importer.md @@ -0,0 +1,48 @@ +# Task 01 — The importer's pre-scan + the `BOR_SYNC_PENDING_COUNT` kill switch (D1/D2) + +**Phase:** `194_sync_pending_count` · **Source:** owner report 2026-10-19 — "1216 remaining" on a ~10,000-file KB with ~10 changed files. +**Story:** n/a + +## Objective +The server-side core: `import_sources` predicts the run's change set BEFORE the walk (the pre-scan, D1) so the `progress` hook's `done`/`total` are the PENDING position (the files that will actually be re-indexed), with the `BOR_SYNC_PENDING_COUNT` kill switch (D2) — `0` restoring the phase-193 walk-position semantics byte-for-byte, negative refusing startup. + +## Work +1. `app/config.py` — `sync_pending_count: int = 1` (env `BOR_SYNC_PENDING_COUNT`), the house pattern next to `sync_session_reset_files` (phase 184) / `agent_max_rounds`: `1` = the pre-scan pending-count semantics (default), `0` = the phase-193 walk-position rollback (the stats pre-walk + `done = summary.files`), a NEGATIVE value refuses startup loudly with the field named (the validator). `.env.example` — the entry, documented like the other `BOR_SYNC_*` rows. +2. `app/rag/importer.py` — `import_sources`: + - the signature gains `on_scan_progress: Callable[[int, int], None] | None = None` (the 1-based scan position + the importable total — the "only when provided", "the hooks only assign fields", no-try/except contract of `progress` / `on_chunk_progress` / `on_image_progress`); + - read `llm.settings.sync_pending_count` ONCE before the source loop (the phase-184 read-once discipline — a mid-run settings save cannot flip the semantics between two files of the same walk); + - the GATE: `progress is not None` AND the switch truthy → the PRE-SCAN; `progress is None` (the CLI, `scripts/import_docs.py` L407) → skipped entirely (the phase-64 skip rule, extended — byte-identical); the switch `0` on a progress caller → the EXISTING stats-only pre-walk runs and `done = summary.files` (the phase-193 rollback, byte-for-byte); + - the pre-scan REPLACES the existing stats-only pre-walk (L949–962) — no third walk: ONE `asyncio.to_thread` span (the phase-179 off-loop rule — a 10,000-file hash+SELECT span must not park the single uvicorn loop; the span's worker runs to completion on cancel, the phase-134 contract) that, per root with the EXACT `iter_importable_files` rules of the processing loop (the same `ignore` / `include_hidden` / `image_extension_set` / images-toggle gating — the L930–962 region's rules, byte-identical): (a) the per-root file list + length (the scan total — the same number the old pre-walk computed), then (b) per file: `_file_hash(path)` (the existing streamed 1 MiB-chunk helper, L704–724) + ONE scalar `SELECT documents.file_hash WHERE source = :s AND path = :p` (the ONLY column — NO entity load, NO `defer`/`load_only` row, nothing lands in the session's `identity_map`, no retention — phase 184/185 compliant: it is not a whole-entity `select(Document)` and it defers nothing because it loads no row; the processing loop's `_doc_row_stmt` stays the ONLY row load) → `pending = {(source, rel)}` where stored is NULL (no row, or a pre-phase row) or `!= digest` — the SAME predicate the fast path applies at processing time, so the prediction is consistent with the stored-hash contract (a pre-phase NULL row is predicted changed AND actually re-indexed + stamped by the loop); + - `on_scan_progress(position_1based, total)` fires per file from inside the worker span (the call is a plain field-assignment closure — GIL-atomic; document the thread-safety clause in the docstring); + - `files_total` = `len(pending)` (the `progress` hook's denominator); the per-file `progress(source, rel, done, total)` call's `done` becomes the 1-based POSITION IN THE PENDING SET — it advances for every file in `pending` (whether or not the file turns out changed at processing time — count and denominator stay on the SAME set so the arithmetic never runs away; a file NOT in `pending` does not advance it — the label holds at `K remaining` while the walk moves over the unchanged `N − K`); + - the processing loop's FRESH per-file `_file_hash` stays (the double hash is deliberate: the pre-scan's digest is the prediction, the loop's is the truth — a file edited between the two is still detected; the second read is page-cached — the cost the owner accepted, A3); + - `summary.files` (the `import: summary … files=` count — every file the loop processed) is UNTOUCHED; the prune post-pass, the date refresh, the phase-184 `expunge_all` cadence, the `seen` set: all UNTOUCHED (the pre-scan builds `pending`, not `seen` — the loop keeps building `seen` exactly as today); + - the degenerate empty KB: scan total 0, no callback fires, `pending` empty, `files_total` 0 (the phase-64 `total === 0` no-count-part contract); + - docstrings: the module `progress` paragraph (the `done`/`total` semantic shift + the drift edges — a predicted-then-unchanged file still advances the count so it reaches 0; a post-scan change is indexed but uncounted and the UI's `total − done > 0` guard hides the excess), the new `on_scan_progress` paragraph (the "only when provided" rule, the worker-thread call, the no-try/except contract), the `to_thread`/cancel/cost paragraph (the phase-179/134 clause + the double-hash + the page-cache note + the kill-switch gate), the `sync_pending_count` read-once clause. +3. `tests/unit/` — the marker pins (the importer pin-module convention, the phase-184/185 pin style): + - the gate: `progress is None` → no pre-scan (the callback never fires, no scalar statement — a mocked-callback pin); the switch `0` on a progress caller → the stats pre-walk path (a source marker pin: the old pre-walk code is the `0` branch); + - the scalar SELECT shape: a captured statement names EXACTLY `documents.file_hash` (column-list pin, the phase-185 style — no other column, no entity); + - the `pending` predicate: fresh (no rows → all pending), unchanged (hash match → not pending), changed (mismatch → pending), pre-phase NULL row (→ pending), an ignore-rule file (→ never in the walk, never pending); + - the `done`-as-pending-position: a 3-file fixture (2 predicted, 1 not) → the `progress` calls' `done` sequence is the pending position (the non-predicted file reports the previous position; `total = 2`); + - the switch values: `1`/`0`/negative (the validator names the field on the negative value). +4. `tests/integration/` — + - the pending set through a real `import_sources` call (a `file://`-style temp source dir, the house fixture convention): fresh / unchanged / one-changed fixtures assert the `progress` hook's `done`/`total` pairs AND the terminal `summary.added`/`summary.updated` (the parity pin: the prediction's size equals the terminal's `added + updated` on a stable KB); + - NO entity hydration: `len(session.identity_map) == 0` immediately after the pre-scan span (the `progress` + `on_scan_progress` caller, before any processing-loop row load); + - OFF-LOOP (the phase-179 pattern, `test_sync_git_freeze.py` style): a monkeypatched `_file_hash` that blocks ~3 s over a small multi-file fixture → a loop-side heartbeat (a coroutine timing `asyncio.sleep(0)` / a probe task) stays responsive through the pre-scan span; + - CANCEL: a cancel registered before the (delayed) pre-scan → the worker runs to completion, the run lands the `cancelled` terminal (the phase-134 contract, `test_sync_git_freeze.py`'s cancel-during-clone pattern). +5. Verify: `uv run pytest tests/unit tests/integration -v` green; `uv run ruff check . && uv run pyright` clean. + +## Testing & Quality +- Unit: the marker pins above (the gate, the scalar shape, the predicate, the `done` contract, the switch values). +- Integration: the pending-set parity, the no-hydration pin, the off-loop pin, the cancel pin. +- Coverage: **>90%** on this task's new/modified code (the whole gate — `uv run pytest --cov=app --cov-report=term-missing` at the phase's final pass). + +## Completion Criteria +- [ ] the pre-scan runs on a progress caller with the switch truthy, is skipped for `progress=None` callers, and the `0` switch runs the phase-193 walk-position path (the marker pins). +- [ ] the scalar SELECT's column list is exactly `[documents.file_hash]` with no entity hydration (the integration pins). +- [ ] `done`/`total` are the pending position on a stable KB (the parity pin: `files_total == summary.added + summary.updated`). +- [ ] the pre-scan span never parks the loop; a cancel during it lands `cancelled` (the integration pins). +- [ ] `summary.files`, `seen`, the prune, the date refresh and every log line are byte-identical (the unedited pins green). +- [ ] `BOR_SYNC_PENDING_COUNT` in `.env.example` + the config validator (negative refuses startup with the field named). +- [ ] full test suite **100% passing**; README/AGENTS: the setting is documented (the phase's final pass re-checks both project-wide). +- [ ] no behavior change in completed work (the CLI, the upload job, the switch-`0` path). diff --git a/.agents/phases/complete/194_sync_pending_count/02_sync_phase_status.md b/.agents/phases/complete/194_sync_pending_count/02_sync_phase_status.md new file mode 100644 index 0000000..2c02b2f --- /dev/null +++ b/.agents/phases/complete/194_sync_pending_count/02_sync_phase_status.md @@ -0,0 +1,43 @@ +# Task 02 — The `"scan"` phase + `scan_done` / `scan_total` (D3) + +**Phase:** `194_sync_pending_count` · **Source:** owner-confirmed A2 — the pre-scan is a visible phase. +**Story:** n/a + +## Objective +The status half of D1: `SyncStatus` carries the scan phase (`"scan"`) and the scan position (`scan_done` / `scan_total`), the `GET /api/sync/status` payload gains the two keys ADDITIVELY, and every docstring names the new machine stage — the phase-187 forced/throttled sampling discipline extended to the scan transition. + +## Work +1. `app/api/sync.py` — `SyncStatus`: + - `phase: Literal["scan", "import", "overview", "summaries"] | None = None` (additive value — the machine: prelude `null` → `"scan"` (set immediately before `import_sources`, REPLACING today's `_status.phase = "import"` at L880) → `"import"` (flipped by the FIRST `progress` call while `phase == "scan"` — the flip lives in the existing `_hook` closure: `if _status.phase == "scan": _status.phase = "import"`) → `"overview"` → `"summaries"`; the no-gap summaries skip still keeps the run in `"import"`); + - `scan_done: int = 0` / `scan_total: int = 0` (placed after `files_total` — reset with the run in the `_run_sync` reset block L664–679 region, kept at EVERY terminal like the file counts — the phase-64 keep-final-values convention); + - the class docstring: the phase-64 paragraph (the `files_done`/`files_total` clause now names the pending-set semantics — "the hook's pending position (phase 194: the pre-scan's predicted change set)"), the phase-98 paragraph (the machine gains the `"scan"` stage), a new phase-194 paragraph (the scan fields, the reset/keep convention, the `null` `current_file` during the scan, the empty-KB degenerate clause — the phase stays `"scan"` until the next transition/terminal, the UI renders bare "Scanning…"). +2. `app/api/sync.py` — `_run_sync`: + - the reset block: `scan_done = 0` / `scan_total = 0` with the run; + - the pre-`import_sources` point (L880): `_status.phase = "scan"` + the phase-187 FORCED `_sample_rss()` (a phase transition — the forced-sample convention, replacing the current `= "import"` set + forced sample); + - a new `_scan_hook(scan: int, total: int)` closure passed as `on_scan_progress`: sets `scan_done` / `scan_total` + calls the UNFORCED `_sample_rss()` (the per-file throttled convention — one `/proc` read per 5 s, never one per file, LOCKED A4's throttle discipline); + - the `_hook` (progress) closure: the flip clause (above) + a FORCED `_sample_rss()` on the flip (the scan→import transition is a phase transition); + - the `import_sources` call (L881–893): `on_scan_progress=_scan_hook` added; + - the terminal paths (`sync: done/cancelled by user/failed` — L1112/L1133 region): UNTOUCHED in shape (the `files_done=… files_total=…` fields keep their names — now carrying the pending counts; the scan fields keep their final values beside them — the keep-final convention); + - the module docstring: the phase-machine paragraph (L188–200 region — the `"scan"` stage + the flip point), the phase-64 progress paragraph (L100–115 region — the `files_done`/`files_total` semantic shift), the phase-187 sampler paragraph (the scan transition's forced sample + the per-file scan hook's throttled sample), the cancel paragraph (a cancel during the pre-scan span: the `to_thread` worker runs to completion, the run stops at the next boundary — the phase-134/179 contract, the `CancelledError` clause's two paths unchanged). +3. `app/api/sync.py` — `GET /api/sync/status`: + - the payload dict: `"scan_done"` / `"scan_total"` added after `"files_total"` (ADDITIVE ONLY — every pre-existing key keeps its name, type, and position; the phase-187 additive-only discipline); + - the endpoint docstring: the new fields' paragraph (the scan position — 0/0 before the scan starts and in terminal states the final values are kept; `current_file` null during the scan). +4. `tests/integration/` — + - the PHASE MACHINE sequence (a real app, a small git-source fixture with one changed file): a held first status GET (the prelude — `phase: null`, `scan_*: 0/0`) → a `phase: "scan"` frame with `scan_total` set and `scan_done` climbing (≥2 distinct positions on a multi-file fixture) → a `phase: "import"` frame (the flip — `files_total` = the predicted count) → the terminal (`phase: null`, the scan counts KEPT beside the final file counts — the keep-final pin); + - the PAYLOAD key-set pin: the phase-193 key set + `scan_done` / `scan_total` (the additive pin — every pre-existing key byte-identical in name/type); + - the FORCED samples: a forced `_sample_rss` at the pre-`import_sources` set (the `"scan"` transition) and at the flip (the `"import"` transition) — the phase-187 sampling-pin pattern (`test_sync_peak_memory_visible.py` convention); + - the DEGENERATE empty-KB run: the phase reports `"scan"` (no `on_scan_progress` fires — `scan_*` 0/0), never `"import"` (no progress call), the terminal clears the phase (the pin names the D1 degenerate clause); + - the CANCEL-during-scan terminal (a delayed pre-scan, the task-01 cancel pin's app-side twin): `state: "cancelled"`, the phase cleared, the scan counts kept. +5. Verify: `uv run pytest tests/integration -v` green; `uv run ruff check . && uv run pyright` clean. + +## Testing & Quality +- Integration: the phase-machine sequence, the payload key-set, the forced/throttled samples, the degenerate + cancel terminals. +- Coverage: **>90%** on this task's new/modified code. + +## Completion Criteria +- [ ] the machine is `null → "scan" → "import" → "overview" → "summaries"` with the flip on the first progress call (the integration sequence pin). +- [ ] `scan_done` / `scan_total` reset with the run, are kept at every terminal, and are additive-only in the payload (the pins). +- [ ] the phase-187 sampling discipline holds (forced at the scan + import transitions, throttled per-file in `_scan_hook`) (the pins). +- [ ] every pre-existing log line, payload key and terminal behavior is byte-identical (the unedited pins green). +- [ ] full test suite **100% passing**; no user-visible doc changes this task (the phase's final pass reviews README/AGENTS). +- [ ] no behavior change in completed work. diff --git a/.agents/phases/complete/194_sync_pending_count/03_ui_scan_label.md b/.agents/phases/complete/194_sync_pending_count/03_ui_scan_label.md new file mode 100644 index 0000000..e591af7 --- /dev/null +++ b/.agents/phases/complete/194_sync_pending_count/03_ui_scan_label.md @@ -0,0 +1,35 @@ +# Task 03 — The `fmtSyncPhaseLabel` scan branch (D4) + +**Phase:** `194_sync_pending_count` · **Source:** owner-confirmed A2 — the visible `Scanning… (n/total)` label. +**Story:** n/a + +## Objective +The UI half of D1/D3: `fmtSyncPhaseLabel` (the phase-aware wrapper) gains ONE explicit `"scan"` branch — `Scanning…` + the `(n/total)` count part in the SAME count slot the import phase's `N remaining` uses — while `fmtSyncLabel`'s arithmetic stays BYTE-IDENTICAL and the A4 untruncated placement + the phase-187 peak clause flow through untouched. + +## Work +1. `frontend/assets/sources.js` — `fmtSyncPhaseLabel` (the decision tree + its docstring, L157–167 region): + - the new `"scan"` branch: main text `Scanning…` (NO file — `current_file` is null in the phase; the branch must not read it), count part `(n/total)` with `n = scan_done`, `total = scan_total` — rendered in the SAME count slot (the `{ main, count }` part shape the import phase's `N remaining` uses, so `setSyncLabel`'s hidden-toggle + the A4 untruncated placement — button title + `#sync-progress` — carry `Scanning… (n/total)` automatically); the `total === 0` guard → NO count part (the phase-64 contract, the empty-KB clause — bare "Scanning…"); + - the wrapper consumes the scan pair the way it already consumes the summaries pair (`summaries_done` / `summaries_total` / `current_summary` — the `phase` value selects which pair the branch reads); the status call sites (L1121–1127, L1449–1458, L1621–1652 region) forward `scan_done` / `scan_total` alongside the existing fields (the small caller pass-through — the summaries-forwarding pattern); + - the `null` prelude + `"import"` fallback clause now covers ONLY those two (the docstring's decision-tree paragraph updated: the new branch is named, the fallback narrowed, the phase-193 `N remaining` clause kept); + - the walk position surviving as `(n/total)` is HONEST for a scan — it is labeled "Scanning", never "Syncing" (TODO L6's objection was the sync count wearing the total's shape — the docstring says so); + - `fmtSyncLabel` (L1043–1071): UNTOUCHED (the `N remaining` arithmetic + its guards are byte-identical — the phase-193 pins stay unedited); the chunk / image parts, `setSyncLabel`, the terminal `counts_line`: UNTOUCHED; + - the phase-187 peak clause (`· peak N MB` on the `#sync-progress` counts line — `fmtSyncPeakClause`): UNTOUCHED (it lands on the scan label's line while the scan is visible — no new element, no new live region). +2. `tests/unit/` — the JS marker pins (the sources-JS pin-module convention, the phase-193 marker style): + - the `"scan"` branch marker (the `Scanning…` main + the `phase === "scan"` selection in the decision tree); + - the `(n/total)` shape marker (the exact template over `scan_done` / `scan_total` — the source marker asserts the expression); + - the `total === 0` guard marker (no count part before the scan knows the total); + - the caller pass-through marker (the call sites forward the scan pair); + - the UNTOUCHED markers: `fmtSyncLabel`'s `N remaining` expression (the phase-193 pin — byte-identical), the chunk / image / summaries part builders, `setSyncLabel`, the `counts_line` helper. +3. Verify: `uv run pytest tests/unit -v` green; `uv run ruff check . && uv run pyright` clean. + +## Testing & Quality +- Unit: the marker pins above (the branch, the shape, the guard, the pass-through, the untouched set). +- E2E: task 04 (the retargeted pins) + task 05 (the phase arms — the scan label through the real UI). +- Coverage: **>90%** on `app/` (the task is frontend — the app-side gate stays green at the phase's final pass). + +## Completion Criteria +- [ ] the `"scan"` branch renders `Scanning…` + `(n/total)` in the count slot with the `total === 0` guard (the marker pins). +- [ ] `fmtSyncLabel`'s `N remaining` arithmetic is byte-identical (the unedited phase-193 pins green). +- [ ] the A4 untruncated placement + the phase-187 peak clause are untouched (the unedited pins green). +- [ ] full test suite **100% passing**; no user-visible doc changes this task (the phase's final pass reviews README/AGENTS). +- [ ] no behavior change in completed work (the prelude / import / overview / summaries labels byte-identical). diff --git a/.agents/phases/complete/194_sync_pending_count/04_retarget_pinned_suites.md b/.agents/phases/complete/194_sync_pending_count/04_retarget_pinned_suites.md new file mode 100644 index 0000000..f48a3c1 --- /dev/null +++ b/.agents/phases/complete/194_sync_pending_count/04_retarget_pinned_suites.md @@ -0,0 +1,38 @@ +# Task 04 — The pinned suites move (D5) + +**Phase:** `194_sync_pending_count` · **Source:** the D5 retarget list. +**Story:** n/a + +## Objective +Every existing pin that asserts the phase-193 WALK-position semantics (or the pre-phase statement counts the pre-scan now disturbs) is retargeted to the pending-set semantics — and every pin that is unaffected by the change (fresh-fixture count pins, the upload's bare-label pins, the terminal-line pins) is verified green UNEDITED. The full suite ends green. + +## Work +The definitive list (per suite: decide from the fixture's KB state — a FRESH-DB fixture has pending == walk, so its count pins survive unedited; a RE-SYNC fixture's walk-position pins retarget): + +1. `tests/e2e/test_sync_remaining_count.py` (phase 193's own suite — the primary retarget): + - the docstring's `files_done` / `files_total` clause (the "the 1-based …" wording, L15–16 region) retargets to the PENDING-set semantics (the pre-scan's predicted change set; the phase-193 walk wording moves to the kill-switch history note); + - the status key-set pin (the "the phase-64 key set" clause) moves to the phase-64 set + `scan_done` / `scan_total` (the additive pin); + - the arm's phase-sequence pins: the prelude → `import` sequence gains the `"scan"` stage (a `phase: "scan"` frame with `scan_total` set before the first `files_total`-bearing frame — the D3 machine); the pre-scan's label frames (`Scanning… (n/total)`) are pinned where the arm's held-first-GET / pre-first-file region stands; + - the count frames: on the suite's FRESH-DB fixture, pending == walk — the `1 … N−1` frames and the "hidden at the final file" pin survive (the phase-193 "Notable" deviation note retargets to the pending-position wording); ADD the re-sync arm if the suite's fixture allows it (a second sync of the same repo: no count part — `files_total = 0`, the phase-193 A1 exception now driven by the empty pending set) — otherwise the re-sync pin lives in the task-05 suite; + - the upload arm (arm c): UNEDITED (the upload status endpoint gains no scan fields — phase 90 A2 stands). +2. `tests/integration/test_import_fast_path_columns.py` (phase 185 — the per-file SELECT pins): inspect the invocation. If it calls `import_sources` WITHOUT `progress` → the pre-scan never runs → verify green UNEDITED. If WITH `progress` → retarget the "exactly one document-wide `SELECT` per unchanged file" pin to distinguish the pre-scan's scalar statement (column list exactly `[documents.file_hash]` — the task-01 shape pin) from the fast-path ROW statement (content deferred — the phase-185 pin intact); the "one PK follow-up for the backfill" pin is UNTOUCHED either way (the pre-scan's scalar is a `(source, path)`-keyed statement, not a PK follow-up). +3. `tests/integration/test_import_session_retention.py` (phase 184 — the bounded/unbounded retention pins): the pre-scan hydrates NO entities (the task-01 no-hydration pin) → the retention bound is UNCHANGED; verify green (if the test's fixture invokes with `progress`, confirm the worker-thread scalar statements disturb no retention assertion — the scalar loads no row). +4. `tests/integration/test_sync_git_freeze.py` (phase 179 — the loop-freeze pins): the pre-scan span is itself off-loop (the task-01 off-loop pin lives in its own module, the phase-179 pattern) — verify this suite's git-span pins stay green; if its measurement window now overlaps the pre-scan, extend it with the pre-scan heartbeat (the loop must stay responsive through BOTH off-loop spans). +5. `tests/e2e/test_sync_chunk_progress.py` (the `Syncing… chunkprog/big.md 1 remaining · chunk n/m` pins): the fixture is a fresh-DB import → pending == walk → verify green UNEDITED (the `1 remaining` part reads identically — the pending denominator equals the walk denominator on a fresh KB). +6. `tests/e2e/test_sync_label_mobile.py` (the `IMPORT_FULL_RE` pin + the label docstring): inspect the fixture's KB state — fresh-DB → verify green UNEDITED; any re-sync walk-position pin retargets (the count part absent on a no-change re-sync). +7. `tests/e2e/test_sync_upload_progress.py` + `tests/e2e/test_archive_upload_sources.py` (the upload's "no `N remaining` count" pins — phase 90 A2): the upload job has no pre-scan and no scan fields → verify green UNEDITED. +8. `tests/e2e/test_sync_session_reset.py`, `tests/e2e/test_fast_path_deferred_content.py`, `tests/e2e/test_sync_summary_visibility.py`, `tests/e2e/test_sync_image_progress.py`, `tests/e2e/test_sync_progress_box.py`, `tests/e2e/test_sync_status_freshness.py`, `tests/e2e/test_sync_peak_memory_visible.py`, `tests/e2e/test_smoke.py` — the sync-lineage sweep: run each in isolation (`--no-cov`), verify green; a pin that breaks on the new `"scan"` stage of the phase sequence or on the pre-scan's seconds (e.g. a freshness window that now spans the scan span) retargets minimally (the window/sequence pin names the scan stage — never the arithmetic). +9. `AGENTS.md` / `README.md` are NOT edited this task (the phase's final pass owns the docs — task 05). +10. Verify: `uv run pytest -v` (unit + integration) green; the ISOLATED `--no-cov` sweep of items 1–8 green (the phase-193 14-suite convention, extended by this phase's suite in task 05); `uv run ruff check . && uv run pyright` clean. + +## Testing & Quality +- The retargeted pins (the pending-set wording, the key-set pin, the phase-sequence pins). +- The verified-unedited pins (fresh-fixture count pins, the upload's bare-label pins, the terminal `counts_line` pins, the phase-184/185/179 pins). +- Coverage: **>90%** on `app/` (the gate stays green — this task edits tests + docstrings). + +## Completion Criteria +- [ ] the retargeted list above is settled: every broken pin retargeted with the reason in its docstring, every unaffected pin green UNEDITED (the sweep). +- [ ] `tests/e2e/test_sync_remaining_count.py` pins the new machine (the `"scan"` stage + the key-set + the pending-position wording) and its fresh-fixture count frames survive. +- [ ] the terminal `counts_line` pins, the chunk / image / summaries pins and the upload pins are green UNEDITED. +- [ ] full test suite **100% passing** (unit + integration + the isolated E2E sweep). +- [ ] no behavior change in completed work (only test files + their docstrings move). diff --git a/.agents/phases/complete/194_sync_pending_count/05_e2e_and_gates.md b/.agents/phases/complete/194_sync_pending_count/05_e2e_and_gates.md new file mode 100644 index 0000000..73b0fcc --- /dev/null +++ b/.agents/phases/complete/194_sync_pending_count/05_e2e_and_gates.md @@ -0,0 +1,42 @@ +# Task 05 — The phase's E2E suite + gates + docs + commit + +**Phase:** `194_sync_pending_count` · **Source:** AGENTS rule 4 — "One Story, One Phase" (the owner-directive phase's own suite). +**Story:** n/a + +## Objective +The owner-visible proof through the real UI: the count counts down the PREDICTED change set (the owner's prod case — K of N changed), the scan phase is visible and honest, the kill switch rolls back to phase 193, and the no-change / first-sync / empty-KB edges behave per D1 — then the full gates, the docs pass, and the atomic commit. + +## Work +1. `tests/e2e/test_sync_pending_count.py` (NEW — run in isolation: `uv run pytest tests/e2e/test_sync_pending_count.py -v --no-cov`): + - the house fixture convention (the phase-193/107 suite shape): a real git repo over the `file://` transport, the mock-LLM proxy with the house per-request delay, a unique E2E port pair (check the existing allocations in the sibling suites — the one-suite-at-a-time rule makes collisions harmless, but pick a free pair anyway), a `file://`-repo N = 8 importable files (keep the suite under ~2 min; ≥2 multi-chunk files so the chunk part coexists, the phase-129 convention); + - the module app: the default `BOR_SYNC_PENDING_COUNT` (1); a second module app for the kill-switch arm (`BOR_SYNC_PENDING_COUNT=0`) — the phase-184 two-module-app convention; + - **arm a — the first sync (fresh KB):** the phase sequence `null` → `"scan"` (the label `Scanning… (n/8)` — the position climbs: pin ≥2 distinct `scan_done` values at `scan_total = 8`; `current_file` null in the phase) → `"import"` (the count part `N remaining` with `files_total = 8` — pending == walk on a fresh KB, the phase-193 parity: the frames count down, hidden at the final file) → the terminal (`phase: null`, `scan_done = scan_total = 8` KEPT beside `files_done = 8`, the `sync: done` counts line `8 added`); + - **arm b — the incremental sync (the owner's prod case — the phase's heart):** modify K = 2 of the 8 (the fixture's changed-set), re-sync: the scan phase climbs `(n/8)` (the full walk is scanned), then the import: the count part reads `2 remaining` and HOLDS while `current_file` moves over UNCHANGED files (pin: ≥2 consecutive unchanged-file frames at `2 remaining` — the count does not advance on them), advances to `1 remaining` on the first predicted file, HIDES on the final predicted file (the phase-193 A1 edge — the main part names it), terminal `2 updated`; + - **arm c — the no-change re-sync:** the scan completes, the import renders BARE `Syncing… ` (NO count part — `files_total = 0`, the phase-64 guard), the terminal `0 added · 0 updated` (the `counts_line` byte-identical shape, the phase-64 pin); + - **arm d — the kill switch:** the second module app (`BOR_SYNC_PENDING_COUNT=0`), arm b's fixture: NO `"scan"` phase (the machine is `null` → `import` — the phase-193 machine), the count part counts down the WALK (`8 remaining` → hidden at the final file — the phase-193 semantics, the rollback pin), the terminal `2 updated`; + - **arm e — the empty KB:** a repo with zero importable files: `phase: "scan"` with `scan_*` 0/0 (bare `Scanning…` — the `total === 0` guard, NO count part), never `phase: "import"`, the terminal clears the phase (the D1 degenerate pin); + - **the phase-187 peak clause** (the `· peak N MB` / `fmtSyncPeakClause` trailing text on the `#sync-progress` counts line) is visible while the scan label is showing (the clause lands on the scan line — no new element, the phase-187 pin); + - **the upload arm:** a background upload run renders its bare label with NO file and NO count part (the phase-90 A2 pin — the `test_archive_upload_sources.py` convention; the upload status endpoint carries no scan fields); + - the `#sync-progress` A4 untruncated placement (the button `title` + `#sync-progress` text equal the full `Scanning… (n/total)` label, the phase-64 A4 pin). +2. The full gates (the phase's final pass — run in order, all green before the commit): + - `uv run pytest -q --cov=app --cov-report=term-missing` → exit 0, coverage **>90%** on `app/`; + - `uv run pytest tests/e2e/test_sync_pending_count.py -v --no-cov` → all arms pass in ISOLATION; + - the isolated `--no-cov` sweep of the sync-lineage suites (task 04's list + this suite): `test_sync_remaining_count`, `test_sync_label_mobile`, `test_sync_chunk_progress`, `test_sync_image_progress`, `test_sync_progress_box`, `test_sync_summary_visibility`, `test_archive_upload_sources`, `test_sync_upload_progress`, `test_sync_status_freshness`, `test_sync_peak_memory_visible`, `test_sync_session_reset`, `test_fast_path_deferred_content`, `test_sync_git_freeze`, `test_smoke`; + - `uv run ruff check . && uv run pyright` clean. +3. The documentation & consistency pass (AGENTS rule + the phase's D-docs clause): + - `README.md` in full: the sync section's running-label clause — the phase-193 `N remaining` example becomes the PREDICTED-change-set one (the `scan` phase + `Scanning… (n/total)` + `N remaining` over the pending set + the no-change/first-sync/empty-KB edges + the `BOR_SYNC_PENDING_COUNT` setting: `1` default, `0` = the phase-193 walk-position rollback, negative = startup refusal); the `.env.example` entry was added in task 01 — verify it is documented here too; every pre-existing inconsistency found in the full review is fixed (the phase-193 docs-pass convention); + - `AGENTS.md` in full: verify whether any clause names the running count part (phase 193 verified one does NOT — if still absent, do NOT add one; if one exists, it gains the pending-set wording + the kill switch); the `BOR_SYNC_PENDING_COUNT` row joins wherever the settings documentation lives. +4. The atomic commit (AGENTS rule 8): ONE Conventional Commit, `--no-gpg-sign`, the phase directory moved `todo/` → `complete/` in the SAME commit (the house convention — the phase-193 commit shape: the `phase: 194_sync_pending_count` message with the verified-evidence block). Suggested subject: `feat(sync): count down the sync's predicted change set (pre-scan)`. + +## Testing & Quality +- E2E: this suite, isolated (the six arms + the peak-clause + the A4 + the upload pins). +- Gates: the full suite + coverage >90% + the 15-suite sweep + lint/types (above). +- Docs: README + AGENTS reviewed in full, every inconsistency fixed. + +## Completion Criteria +- [ ] arm b proves the owner's prod case through the real UI (the count holds at `2 remaining` over the unchanged files, hides at the final predicted file, terminal `2 updated`). +- [ ] arms a/c/d/e pin the first-sync parity, the no-change no-count edge, the kill-switch rollback and the empty-KB degenerate. +- [ ] the peak clause + the A4 placement pins hold on the scan label. +- [ ] all gates green (above) — full suite 100%, coverage >90%, the sweep, lint/types. +- [ ] README + AGENTS accurate and consistent with the whole project (the final pass). +- [ ] ONE atomic Conventional Commit with `--no-gpg-sign`, the phase directory in `complete/`. diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.err b/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.md b/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.md new file mode 100644 index 0000000..9e35f94 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.md @@ -0,0 +1,19 @@ +**Phase 194 — final verification pass: all green.** + +**Verified (all 5 tasks already complete; nothing re-implemented needed)** +- `app/rag/importer.py`: `_prescan_pending` + `_pending_hash_stmt` (scalar `documents.file_hash`, own session, ONE `asyncio.to_thread` span, REPLACES the stats pre-walk), `on_scan_progress`, `progress` pair = pending position, `0` branch = byte-identical phase-193 walk. +- `app/config.py`/`settings_catalog.py`/`.env.example`: `sync_pending_count=1` + validator; `app/api/sync.py`: `"scan"` phase, `scan_done`/`scan_total` (additive after `files_total`, kept at every terminal), forced/throttled RSS split; `frontend/assets/sources.js`: `fmtSyncPhaseLabel` scan branch first-arm, `fmtSyncLabel` untouched. + +**Tests / lint / coverage (exact commands)** +- `uv run pytest --cov=app --cov-report=term-missing` → **4835 passed**, `app/` 99% (7240 stmts / 16 miss), exit 0 +- `uv run pytest tests/e2e/test_sync_pending_count.py -v --no-cov` → **8 passed in 150.81s** (phase suite, isolated) +- 14 sync-lineage E2E suites, one pytest process each (`test_sync_remaining_count`, `…_chunk_progress`, `…_progress_box`, `…_peak_memory_visible`, `…_label_mobile`, `…_upload_progress`, `test_archive_upload_sources`, `…_image_progress`, `…_summary_visibility`, `…_status_freshness`, `test_smoke`, `test_fast_path_deferred_content`, `test_sync_session_reset`, `test_sync_git_freeze`) → **all exit 0** +- `uv run ruff check .` → All checks passed; `uv run pyright` → **0 errors** + +**Completion criteria** — all met: `N remaining` = predicted change set (E2E arms 1/2/5, first import counts the walk, no-change → no count part); `scan` phase visible + flips to `import`, scan pair reset/kept at terminals (`test_sync_scan_phase.py`); `BOR_SYNC_PENDING_COUNT=0` walk semantics ("7 remaining" arm) + **negative refuses startup naming the field (verified live by importing `app.main`)**; pre-scan off-loop + cancel lands `cancelled` (integration pins); CLI/upload byte-identical (unedited pins green); payload additive (`_PRE_187_STATUS_KEYS | _MEMORY_KEYS | _SCAN_KEYS`); suite green, coverage >90%, lint/types clean. + +**Documentation** — 3 fixes: README sync "States" bullet had an **unclosed parenthesis** (pre-existing, 193/194) and the scan-label "total is 0" clause was ambiguous about a `=0` run (it still counts the walk once the import starts); **AGENTS.md rule 17 had a stray `)`** (pre-existing from phase 191). Also verified consistency both directions for all 87 settings ↔ README ↔ `.env.example` ↔ settings catalog, the pages table ↔ `app/main.py` routes, no stale `BOR_` names, and README/AGENTS claims through phase 194. + +**Notes/deviations** — no live-infra changes (prod sync knob ships default-on, no deployment touched); no git index/history commands run (all changes left in the working tree for the harness); commit criterion left to the harness as instructed. + +**Next pending phase:** none — `.agents/phases/todo/` holds only `194_sync_pending_count`, which is now ready to move to `complete/`. diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.md.status b/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.validate b/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.validate new file mode 100644 index 0000000..00e2254 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__00_phase.a1.validate @@ -0,0 +1,151 @@ +........................................................................ [ 1%] +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 5%] +........................................................................ [ 7%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 11%] +........................................................................ [ 13%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 72%] +........................................................................ [ 74%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................................................ [ 96%] +........................................................................ [ 98%] +........................................................................ [ 99%] +........... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 242 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 339 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 906 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7240 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.err b/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.md b/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.md new file mode 100644 index 0000000..1a43eaa --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.md @@ -0,0 +1,14 @@ +Task 01 is complete and green. + +**Server-side (phase 194 task 01)** +- `app/config.py` — `sync_pending_count: int = 1` (`BOR_SYNC_PENDING_COUNT`), negative refuses startup naming the field; `app/core/settings_catalog.py` — the `Import & sources` row. +- `app/rag/importer.py` — new `_pending_hash_stmt` (scalar `SELECT documents.file_hash`, `(source, path)` predicate, no entity/identity-map load) and `_prescan_pending` (per-file `_file_hash` + one scalar read → `{(source, rel): 1-based pending position}`, its own session, `on_scan_progress` per file). `import_sources` gained `on_scan_progress`, reads the switch **once** before the loop, and REPLACES the phase-64 stats pre-walk with ONE `asyncio.to_thread` pre-scan; `progress` now reports the pending position (holds across unchanged files), `0` runs the byte-identical phase-193 walk-position branch, `progress=None` pays nothing. `summary.files`, `seen`, prune, date refresh, log lines untouched. +- Docs: `.env.example`, `README.md` (label semantics + a phase-194 bounds bullet + the settings table), `AGENTS.md` (new rule 18). + +**Tests** +- New: `tests/unit/test_sync_pending_count.py` (13 — gate both ways, read-once + `to_thread` markers, exactly-`file_hash` projection, 4 predicate states + ignore parity, unreadable arm, scan-hook per root/empty/worker-thread, `done` position, CLI byte-identity) and `tests/integration/test_sync_pending_count.py` (7 — fresh/unchanged/incremental parity `total == added+updated`, 2-file count-down, one-column + `identity_map == 0`, off-loop heartbeat + thread identity, cancel-at-the-await). +- Retargeted the two phase-64/129 pair pins in `tests/unit/test_importer.py`; extended `test_config.py` / `test_settings_catalog.py`. + +**Gates**: `ruff check .` clean · `pyright` 0 errors · `pytest --cov=app` **exit 0**, app coverage **99 %** (importer 99 %) · 20 sync/progress E2E suites + smoke green. + +**Deferred by design**: the `scan` status phase / `Scanning…` label (tasks 02–03), the phase's E2E file (task 05). Task 04's pinned suites needed **no** retargeting — they run fresh-import fixtures where the walk *is* the work. Nothing committed (per instructions), no task files touched. diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.md.status b/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.validate b/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.validate new file mode 100644 index 0000000..a0286e6 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__01_prescan_importer.a1.validate @@ -0,0 +1,150 @@ +........................................................................ [ 1%] +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 5%] +........................................................................ [ 7%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 11%] +........................................................................ [ 13%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 74%] +........................................................................ [ 76%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 98%] +............................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 231 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 339 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 906 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7229 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.err b/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.md b/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.md new file mode 100644 index 0000000..50e7f45 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.md @@ -0,0 +1,17 @@ +All gates green. Final state: + +**Implemented (task 02 — D3, the status half)** +- `app/api/sync.py`: `SyncStatus.phase` → `Literal["scan","import","overview","summaries"]|None`; new `scan_done`/`scan_total` (after `files_total`, reset with the run, kept at every terminal); `_run_sync` sets `"scan"` immediately before `import_sources` (replacing `= "import"`) with a FORCED `_sample_rss`, adds ONE `_scan_hook` (unforced/throttled sample) passed as `on_scan_progress`, and the `_hook` closure now flips `scan → import` on the first progress call with a forced sample; payload gains `"scan_done"`/`"scan_total"` ADDITIVELY; module / class / endpoint / `_run_sync` docstrings + the `_RSS_SAMPLE_INTERVAL_S` note + the three terminals' keep-final comments updated. Terminals, log lines and every pre-existing key untouched. +- New pin suite `tests/integration/test_sync_scan_phase.py` (7): real-run scan frames (≥2 positions, `current_file` null) + both pairs kept at the terminal and **divergent** on the one-changed-file arm (scan 3/3 vs files 1/1, `updated=1`); choreographed flip frame carrying the predicted `files_total`; additive key set idle→mid-run→terminal; forced/throttled sample sequence `null→scan→import→overview→summaries→null` with exactly ONE `/proc` read inside the scan phase; empty-KB arm on both sides; cancel-during-scan terminal (counts kept, nothing indexed, `sync: cancelled by user …` prefix intact); `BOR_SYNC_PENDING_COUNT=0` = `0/0` scan + WALK file count. +- Retargeted stale pins (defects my change exposed): `test_sync_api.py` (idle dict + both `_PRE_187_STATUS_KEYS` assertions, `FakeImportSources`/`_GatedImport`/`failing_import` accept `on_scan_progress`, 409-concurrency pin now waits `"scan"` — a canned hook-less import legitimately stays pre-import), `test_sync_git_freeze.py`, `test_sync_button.py` (idle shape, key set, the two sampling pins now count the scan transition: 3 reads start→scan→flip), `test_sync_peak_memory_visible.py` (browser key-set pin + `_COUNT_KEYS`). + +**Gates (exact)** +- `uv run pytest --cov=app --cov-report=term-missing` → **4822 passed**, TOTAL **99 %**, `app/api/sync.py` **100 % (0 miss)**, exit 0. +- `uv run ruff check .` → All checks passed · `uv run pyright` → 0 errors. +- E2E regression (30 modules, all passed): `test_sync_remaining_count`, `test_sync_button`, `test_sync_progress_box`, `test_sync_label_mobile`, `test_sync_cancel`, `test_sync_status_freshness`, `test_sync_peak_memory_visible`, `test_sync_chunk_progress`, `test_sync_upload_progress`, `test_archive_upload_sources`, `test_sync_summary_visibility`, `test_sync_image_progress`, `test_sync_changed_only`, `test_sync_session_reset`, `test_fast_path_deferred_content`, `test_sync_model_down`, `test_sync_git_freeze`, `test_force_push_resync`, `test_source_name_labels`, `test_local_directory_sources`, `test_source_ignore_paths`, `test_folder_summary_voice`, `test_incremental_folder_summaries`, `test_upload_no_scan`, `test_git_source_branch`, `test_hidden_folders_toggle`, `test_prune_cascade_cleanup`, `test_git_log_stream_dates`, `test_kb_tree`, `test_smoke`. + +**Docs**: README (`phase` machine + additive `scan_done`/`scan_total` clause; the phase-187 sampling bullet now lists `scan → import → overview → summaries`) and `AGENTS.md` rule 18 (status half: machine, flip-in-`_hook` rule, additive chain, forced-vs-throttled sampling, degenerate edge, new pinned suites). No new env var; nothing committed (harness commits). + +**Notes/deviations**: D3 kept exactly (the flip lives in the existing per-file hook, no extra importer callback), so a canned import that fires no hook stays in `"scan"` — that is why the pre-existing 409 pin retargets. The real-run arm pins terminal values instead of an `"import"` frame because an in-process import never yields the loop (documented in the test module docstring); the flip frame gets a deterministic choreographed pin. `Scanning…` label copy intentionally left to task 03. + +**Next pending task**: `.agents/phases/todo/194_sync_pending_count/03_ui_scan_label.md` (`fmtSyncPhaseLabel` scan branch). diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.md.status b/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.validate b/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.validate new file mode 100644 index 0000000..f40a070 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__02_sync_phase_status.a1.validate @@ -0,0 +1,150 @@ +........................................................................ [ 1%] +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 5%] +........................................................................ [ 7%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 11%] +........................................................................ [ 13%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 70%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 74%] +........................................................................ [ 76%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 95%] +........................................................................ [ 97%] +........................................................................ [ 98%] +...................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 242 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 339 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 906 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7240 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.err b/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.md b/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.md new file mode 100644 index 0000000..b002d17 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.md @@ -0,0 +1,17 @@ +All gates green. Final state: + +**Implemented (task 03 — D4, the UI half)** +- `frontend/assets/sources.js`: `fmtSyncPhaseLabel` gains ONE `"scan"` branch as its **first** arm — main `Scanning…` (never a file; the branch never reads `current_file`) + count `(scan_done/scan_total)` in the SAME count slot the import `N remaining` uses, with the phase-64 `scan_total > 0` guard (0/0 → bare `Scanning…`) and `chunk`/`image` empty; `fmtSyncLabel`'s `N remaining` arithmetic, the chunk/image/summaries builders, `setSyncLabel`, `fmtSyncResult`, `fmtSyncPeakClause` untouched (whole-status pass-through at both call sites already carries the scan pair — comments + docstrings now name it); module header, phase-98 header, the section decision tree and the wrapper docstring updated (fallback narrowed to the null prelude + `import`, TODO L6 honesty clause added). +- New pin suite `tests/unit/test_sync_scan_label_ui.py` (13): branch order, the exact `(n/total)` shape + guard, no-file/no-other-pair reads, the tick + re-attach pass-through, the untouched builder/writer set, docstring/header markers, plus a **node run of the real label path** (`Scanning… (3/9)` identical on main/count/title/box/aria; bare `Scanning…` at `0/0`; every other phase byte-identical). +- No server-side change (tasks 01/02 already shipped the phase + payload keys). + +**Gates (exact)** +- `uv run pytest --cov=app --cov-report=term` → **4835 passed**, TOTAL **99 %** (`app/api/sync.py` 100 %), exit 0. +- `uv run ruff check .` → All checks passed · `uv run pyright` → 0 errors · `node --check sources.js` → OK. +- E2E regression (32 modules, all passed, none edited): `test_sync_remaining_count` (phase-193 pins byte-identical), `test_sync_progress_box`, `test_sync_label_mobile`, `test_sync_chunk_progress`, `test_sync_summary_visibility`, `test_sync_image_progress`, `test_sync_status_freshness`, `test_sync_button`, `test_sync_cancel`, `test_sync_peak_memory_visible`, `test_sync_upload_progress`, `test_upload_no_scan`, `test_archive_upload_sources`, `test_sync_model_down`, `test_sync_changed_only`, `test_sync_session_reset`, `test_sync_git_freeze`, `test_fast_path_deferred_content`, `test_force_push_resync`, `test_prune_cascade_cleanup`, `test_local_directory_sources`, `test_source_ignore_paths`, `test_hidden_folders_toggle`, `test_kb_tree`, `test_git_source_branch`, `test_git_log_stream_dates`, `test_source_name_labels`, `test_folder_summary_voice`, `test_incremental_folder_summaries`, `test_answer_images`, `test_pdf_support`, `test_smoke`. + +**Docs**: README (the running-label bullet now leads with `"Scanning… (n/total)"`, the phase-194 bullet names the branch + its A4 placement) and `AGENTS.md` rule 18 (new "UI half" clause: first-arm rule, no-file/no-chunk-image, `total === 0` bare label, honesty rule, whole-status pass-through, untouched set, the new pin module). + +**Notes/deviations**: the `(n/total)` branch is placed before `overview`/`summaries` so every pre-existing branch slice in the phase-128/152 pins stays byte-identical; the D4 "caller pass-through" needed no argument-list change (the whole `status` object is already the 5th arg — the same carrier as the summaries/chunk/image pairs), so it is pinned + documented rather than re-listed. No pinned E2E suite went stale under this change, so nothing was retargeted here (task 04 stays as authored). + +**Next pending task**: `.agents/phases/todo/194_sync_pending_count/04_retarget_pinned_suites.md`. diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.md.status b/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.validate b/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.validate new file mode 100644 index 0000000..00e2254 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__03_ui_scan_label.a1.validate @@ -0,0 +1,151 @@ +........................................................................ [ 1%] +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 5%] +........................................................................ [ 7%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 11%] +........................................................................ [ 13%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 72%] +........................................................................ [ 74%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................................................ [ 96%] +........................................................................ [ 98%] +........................................................................ [ 99%] +........... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 242 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 339 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 906 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7240 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.err b/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.md b/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.md new file mode 100644 index 0000000..f1475a3 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.md @@ -0,0 +1,18 @@ +Everything is green. Final verification summary: + +**Implemented (task 04 — the D5 pinned-suite retarget)** +- `tests/e2e/test_sync_remaining_count.py` (phase 193's own suite) retargeted: docstring's "Server untouched / phase-64 key set / walk position" clauses now state the pending-set semantics (the phase-193 walk wording kept as the `BOR_SYNC_PENDING_COUNT=0` history); new `EXPECTED_STATUS_KEYS` additive key-set pin (phase-64 set + memory pair + `scan_done`/`scan_total`) asserted on every real tick AND on the served stub; new `_assert_phase_machine_tick_series()` pins the `"scan"` stage (file-less scan ticks carrying `scan_total`, `0/0` null prelude, one-directional sequence, terminal keeping BOTH pairs, `files_total ≤ scan_total`); test 2's pre-first-file region now renders/serves `Scanning… (2/5)`, the bare `Scanning…` (`scan_total===0`) and the null prelude; **new test 4** syncs the same repo twice — run 2's wire pair is `0/0` (empty pending set) while `scan_done == scan_total == 5` is kept at the terminal, terminal line `0 added · 5 unchanged` byte-identical, no count part rendered. `_TickRecorder` gained `wait_for_running` (+ `started_at` ownership) — a recorder started on a previous run's terminal was silently handing that body back (the bug test 4 exposed). +- Verified green **UNEDITED** (pins untouched): upload arm (test 3), `test_sync_chunk_progress`, `test_sync_label_mobile`, `test_sync_upload_progress`, `test_archive_upload_sources`, `test_sync_session_reset`, `test_fast_path_deferred_content`, `test_sync_summary_visibility`, `test_sync_image_progress`, `test_sync_progress_box`, `test_sync_status_freshness`, `test_sync_peak_memory_visible`, `test_smoke`; phase-185 `-230`→`-228` per-file SELECT pins and phase-184 retention bound unaffected (the pre-scan reads a scalar projection through its own session). Docstring-only notes recorded that inspection in `test_sync_chunk_progress.py`, `test_sync_progress_box.py`, `test_import_fast_path_columns.py`, `test_import_session_retention.py`, `test_sync_git_freeze.py` (whose stubbed import means the pre-scan never enters its measurement window — the span's own off-loop/cancel pins live in `test_sync_pending_count.py`). +- No app, README or AGENTS change (item 9: task 05 owns the docs). + +**Gates (exact)** +- `uv run pytest --cov=app --cov-report=term-missing` → **4835 passed**, TOTAL **99 %** (`app/api/sync.py` 100 %, `app/config.py` 100 %, `app/rag/importer.py` 99 %), exit 0. +- `uv run ruff check .` → All checks passed · `uv run pyright` → **0 errors** · `node --check frontend/assets/sources.js` → OK. +- Isolated `--no-cov` sweep, items 1/5–8: 13/13 PASS (`test_sync_remaining_count` 97 s with 4 tests). Adjacent sync-lineage batch: 12/12 PASS (`test_sync_button`, `_cancel`, `test_upload_no_scan`, `_changed_only`, `_model_down`, `test_force_push_resync`, `test_prune_cascade_cleanup`, `test_git_log_stream_dates`, `test_folder_summary_voice`, `test_incremental_folder_summaries`, `test_local_directory_sources`, `test_source_ignore_paths`). Items 2–4 integration files re-run isolated: PASS. + +**Notable decisions / notes** +- Real-run `phase:"scan"` frames are not observable on a 5-file fixture (single-digit-ms window vs a 100 ms poll), so the real arm pins the machine's *deterministic evidence* while the deterministic `Scanning… (n/total)` frames ride test 2's served statuses and the choreographed server sequence stays in `tests/integration/test_sync_scan_phase.py` (documented in the module docstring). +- One pin deliberately relaxed with a reason: the prelude `0/0` arm is skipped for run 2 (`prelude_zero=False`) because `_run_sync` sets `state="running"` one synchronous statement before resetting the scan pair while `GET /api/sync/status` answers from a threadpool — a pre-existing (phase-64) reset shape, not changed here. +- Two integration suites failed transiently when I ran them concurrently with an E2E batch (shared Postgres + per-test `TRUNCATE`); re-run in isolation they pass — E2E and integration must not overlap on this DB. Three orphaned uvicorns on ports 8997–8999 belong to another session's `dev` container; left untouched. + +**Next pending task**: `.agents/phases/todo/194_sync_pending_count/05_e2e_and_gates.md` (the phase's own `tests/e2e/test_sync_pending_count.py`, the docs pass, the atomic commit). diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.md.status b/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.validate b/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.validate new file mode 100644 index 0000000..00e2254 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__04_retarget_pinned_suites.a1.validate @@ -0,0 +1,151 @@ +........................................................................ [ 1%] +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 5%] +........................................................................ [ 7%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 11%] +........................................................................ [ 13%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 72%] +........................................................................ [ 74%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................................................ [ 96%] +........................................................................ [ 98%] +........................................................................ [ 99%] +........... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 242 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 339 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 906 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7240 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.err b/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.md b/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.md new file mode 100644 index 0000000..b0d839b --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.md @@ -0,0 +1,15 @@ +**Phase 194 / task 05 — complete.** + +`tests/e2e/test_sync_pending_count.py` (new, 1,999 lines, 8 arms, own ports 8212/8213 + slow-proxy 8915, two live module apps over the `file://` git transport and a delay-injecting mock model): +1. fresh KB — `null → scan → import → terminal`, scan pair completes and survives the terminal, `N remaining` = phase-193 walk arithmetic (pending == walk), real-run peak clause == payload `rss_peak_mb`; +2. incremental (8 indexed, 2 changed) — `files_total == 2` every running tick, count **holds** at `2 remaining` while `current_file` names unchanged files, absent on the final predicted file; +3. exact scan/null/hold payloads served to the real `sources.js` — `Scanning… (n/total)` on main/count/title/`#sync-progress` only, one `aria-live` box, no `N remaining`/file/chunk in the scan arm, no peak clause on any running label, terminal `0 added · 2 updated · 6 unchanged · peak 1234 MB`; +4. `BOR_SYNC_PENDING_COUNT=0` app on the same fixture — `"7 remaining"` (1 predicted), no `scan` phase, payload carries `scan_done/scan_total` at `0/0`; +5. no-change re-sync — `files_total == 0` on every tick, no count part anywhere, machine still clean; +6. empty source — scan reached, import never, bare `Scanning…`, terminal `0 added · 0 unchanged`; +7. prod-shaped 1,000-file KB — scan position caught **climbing through the running server** (`scan_total == 1000`, `current_file` null), every import tick at `2`, no tick and no label ever walk-sized; +8. upload — bare phase-64 key set (no `phase`/`scan_*`/chunk/image/summary/memory keys), bare label. + +Docs: added the browser pin to the README phase-194 bullet and AGENTS.md rule 18 (incl. the terminal-only peak-clause reading of task 05's wording). `.env.example`/README settings row already correct — no other inconsistency found. + +Gates: `pytest tests/e2e/test_sync_pending_count.py -v --no-cov` → **8 passed in 146.8s**; sync-lineage sweep of all 15 E2E suites in isolation → **15/15 exit 0**; full `pytest --cov=app` → **exit 0, app coverage 99%** (>90%); `ruff check .` → all checks passed; `pyright` → **0 errors**. No app defect surfaced this task; no git index/history touched. diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.md.status b/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.validate b/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.validate new file mode 100644 index 0000000..00e2254 --- /dev/null +++ b/.agents/reports/194_sync_pending_count/194_sync_pending_count__05_e2e_and_gates.a1.validate @@ -0,0 +1,151 @@ +........................................................................ [ 1%] +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 5%] +........................................................................ [ 7%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 11%] +........................................................................ [ 13%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 68%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 72%] +........................................................................ [ 74%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................................................ [ 96%] +........................................................................ [ 98%] +........................................................................ [ 99%] +........... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/admin_settings.py 114 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 301 1 99% +app/api/chat_images.py 52 0 100% +app/api/chats.py 111 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 239 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/source_labels.py 52 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 242 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 39 0 100% +app/config.py 339 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/compression.py 59 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 43 0 100% +app/core/errors.py 5 0 100% +app/core/http_cache.py 22 0 100% +app/core/logging.py 13 0 100% +app/core/memory.py 32 0 100% +app/core/rate_limit.py 44 0 100% +app/core/response_budget.py 63 0 100% +app/core/security_headers.py 20 0 100% +app/core/settings_catalog.py 18 0 100% +app/core/settings_migrate.py 32 0 100% +app/core/settings_overlay.py 117 0 100% +app/core/theming.py 45 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 107 0 100% +app/models.py 148 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 501 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 906 3 99% +app/rag/llm.py 253 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 82 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 178 0 100% +app/rag/retriever.py 270 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_labels.py 87 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/rag/svg_text.py 111 0 100% +app/rag/verbatim.py 160 0 100% +app/schemas.py 587 0 100% +-------------------------------------------------- +TOTAL 7240 16 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.env.example b/.env.example index 07f8639..e1a1302 100644 --- a/.env.example +++ b/.env.example @@ -182,6 +182,25 @@ BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py,container,network,volume, # sync converges (seen is recomputed # from disk every run) +# --- The sync's "N remaining" count (phase 194) --- +# The running label on the Sources page counts DOWN to the end of the sync. +# Phase 194 made it count the WORK instead of the WALK: before the walk the +# importer pre-scans the importable files (one streamed sha256 + one +# documents.file_hash read each, in a worker thread so the server keeps +# serving while it runs) and the count is the position in that predicted +# change set. A ~10,000-file sync that changes 10 files now says "10 +# remaining", not "1216 remaining" (the owner report of 2026-10-19); a +# no-change sync shows no count part at all. First imports are unchanged - +# every file is work. The scan only runs for the UI Sync (the CLI and the +# upload job pass no progress hook and pay nothing). +# BOR_SYNC_PENDING_COUNT=1 # 1 = count the predicted change set (default) + # 0 = count every file left to WALK (the exact + # pre-phase-194 behaviour, the no-redeploy + # rollback if the extra read cost ever bites); + # a negative value refuses startup naming the + # field. Read once per run: a value saved in the + # admin Settings panel applies to the NEXT sync + # --- Image documents (phase 122) --- # Master switch for indexing standalone image files (direct uploads, # uploaded archives, git/local source walks) as first-class documents. diff --git a/AGENTS.md b/AGENTS.md index 7dd89a4..8e272ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -375,7 +375,7 @@ "That source no longer exists — reload the page." cannot be painted over by "unsaved changes" in the tick that wrote it (the defect the task-06 story suite walked into); only the next edit (`clearLabelError`) or a - collapse clears a note, never the settle path), + collapse clears a note, never the settle path, and the Tags token input's chips reuse the phase-89 tag rule (`.source-label-tag` sits in the `.git-source-hidden-count` rule group — no bespoke tag-input CSS). Pinned by @@ -426,6 +426,129 @@ also marker-pin the one-read-per-surface contract, as `tests/unit/test_chat_gate.py` does on the chat call site). +18. **The sync's running count is the WORK, and the pre-scan that predicts + it is a scalar read off the loop (phase 194):** `import_sources` predicts + the run's change set BEFORE the walk (`_prescan_pending`) so the + `progress` hook's pair is the position in the PENDING set (`total` = its + size, `done` = the 1-based position in it, held — not advanced — by a file + the scan did not predict), which is what makes the Sources page's + `N remaining` mean "files left to re-index" instead of "files left to + walk" (the owner's "1216 remaining" on a ~10,000-file KB with ~10 changes, + 2026-10-19). The prediction uses the fast path's OWN predicate — + `_file_hash` (the streamed 1 MiB-chunk sha256) against + `_pending_hash_stmt`, a SCALAR `SELECT documents.file_hash` — so a stored + NULL (no row, or a pre-phase-147 row) is predicted changed AND really + re-indexed and stamped by the loop: the prediction never lies about the + stored-hash contract. Keep it that way: **never** turn that read into a + `select(Document)` / `defer` / `load_only` row (phase 184/185 — a 10,000-file + scan would retain 10,000 rows and re-create the memory incident; the scan + therefore also reads through a session of its OWN, since a `Session` is not + thread-safe and the span's worker runs to completion after a cancel has + moved the run on), never add a THIRD count-only walk (the pre-scan REPLACES + the phase-64 stats pre-walk, which survives as the `0` branch), never move + the scan onto the event loop (phase 179: ONE `asyncio.to_thread` span — a + 10,000-file hash+SELECT pass inline parks the single uvicorn loop and + starves the 2 s `GET /api/sync/status` poll), and never drop the loop's own + fresh `_file_hash` (the scan's digest is the prediction, the loop's is the + truth). The scan runs ONLY for a caller that passes `progress` AND while + `llm.settings.sync_pending_count` is truthy — read ONCE before the source + loop (phase 184's read-once discipline) — so the CLI + (`scripts/import_docs.py`) and the upload job stay byte-identical and pay + nothing; `BOR_SYNC_PENDING_COUNT=0` is the exact phase-193 walk-position + rollback and a negative value refuses startup naming the field (the + `agent_max_rounds` pattern). `summary.files` (the `import: summary … files=` + log count), `seen`, the prune, the date refresh, the terminal `counts_line` + and every pre-existing log line are UNCHANGED — the scan builds `pending`, + never `seen` — and an empty KB keeps the phase-64 shape (scan total 0, no + callback, `total` 0 → no count part). Pinned by + `tests/unit/test_sync_pending_count.py` (the gate both ways, the read-once + + `to_thread` markers, the exactly-`documents.file_hash` projection, the four + predicate states + the ignore-rule parity, the unreadable-file arm, the + `done`-as-pending-position, the switch values) and + `tests/integration/test_sync_pending_count.py` (the parity across the fresh + / unchanged / incremental arms, `len(session.identity_map) == 0` at the + first `progress` call, the heartbeat + thread-identity off-loop pin, and the + cancel-at-the-await arm), and through the real Sources page by + `tests/e2e/test_sync_pending_count.py` — TWO live module apps over one + delay-injecting mock model (the phase-193/107 fixture shape): the first + sync (pending == walk, so the phase-193 frames stand), the K-of-N owner + case with the count HOLDING at `K remaining` while `current_file` names the + UNCHANGED files (the unchanged files in that run cost the phase-118 + summary backfill — a real, model-bound, loop-free span — which is what + makes the hold observable in a browser at all), a SECOND app booted with + `BOR_SYNC_PENDING_COUNT=0` rendering the walk numbers for the same fixture + (`7 remaining`, not `1`), the no-change run answering `files_total == 0` on + every tick, the empty source never leaving `"scan"`, and the 1,000-file + pre-scan whose position is caught climbing THROUGH the running server (the + phase-179 contract) before the import answers the predicted `2`. + + The STATUS half of the same design (phase 194, task 02) is the machine + the page reads: `SyncStatus.phase` is `null` (the model-check + + clone/pull prelude) → **`"scan"`** (set immediately before + `import_sources`, REPLACING the pre-phase `= "import"` — what runs first + there is the pre-scan) → `"import"` (flipped by the FIRST `progress` call + INSIDE the existing `_hook` closure — the importer gains no extra + callback, so do not add one) → `"overview"` → `"summaries"`, every + terminal clearing it. `SyncStatus.scan_done` / `scan_total` (landed by ONE + `_scan_hook` passed as `on_scan_progress`) are the scan's own position — + reset with the run, `0/0` on the `0` rollback and for a scan that predicted + nothing, and KEPT at every terminal beside the final file counts (the + phase-64 keep-final-values convention); `current_file` stays `null` through + the phase (the scan names no file). The `GET /api/sync/status` payload gains + `scan_done` / `scan_total` ADDITIVELY after `files_total` — every + pre-existing key keeps its name, type and position, and the additive chain + (`_PRE_187_STATUS_KEYS | _MEMORY_KEYS | _SCAN_KEYS`) is the pin that keeps it + that way. Sampling follows phase 187 exactly: the `"scan"` set and the + `scan`→`import` flip are TRANSITIONS and `_sample_rss(force=True)`, while + `_scan_hook` samples UNFORCED, so a 10,000-file scan costs one `/proc` read + per `_RSS_SAMPLE_INTERVAL_S` and not one per file (LOCKED A4). The degenerate + empty KB stays `"scan"` (no `progress` call exists to flip it) and the + terminal clears it. Pinned by `tests/integration/test_sync_scan_phase.py` + (the real-run scan frames + both pairs kept at the terminal — divergent on + the incremental arm, 3 walked vs 1 predicted; the choreographed flip frame + carrying the predicted denominator; the additive key set idle → terminal; + the forced/throttled sample sequence `null → scan → import → overview → + summaries → null` with EXACTLY one read inside the scan phase; the empty-KB + arm on both sides; the cancel-during-scan terminal with the scan counts kept + and nothing indexed; the `0` switch reporting `0/0` scan and the WALK file + count) plus the retargeted additive pins in `tests/unit/test_sync_button.py` + and `tests/integration/test_sync_api.py` (a canned import that fires no + per-file hook stays in `"scan"` — that is why the 409-concurrency pin waits + on `"scan"` now). + + The UI half of the same design (phase 194, task 03) is ONE branch: + `fmtSyncPhaseLabel` renders `phase === "scan"` as main `Scanning…` (NEVER a + file — `current_file` is null in the phase and the branch must not read it) + plus the count part `(scan_done/scan_total)` in the SAME count slot the import + phase's `N remaining` uses, so `setSyncLabel`'s hidden-toggle and the A4 + untruncated placement (button title + `#sync-progress`, the sole live region) + carry `Scanning… (n/total)` with no new element, no new live region and no + new writer (the rendered frames — the scan's climb and the import's held + `K remaining` — are pinned on the exact payloads those guards read, served + to the real `sources.js` by `tests/e2e/test_sync_pending_count.py`, because + the 2 s poll cannot be aimed at a frame; the phase-187 peak clause stays on + the TERMINAL line of that same scan-labelled run and appears on NO running + label). Keep it that way: the branch is the FIRST arm (the + `null`/`import` fall-through therefore covers ONLY those two), it carries no + chunk/image part (the scan reads no file), and `scan_total === 0` renders the + BARE label (the phase-64 no-count-part contract — the empty KB and the + `BOR_SYNC_PENDING_COUNT=0` rollback). The scan's walk position surviving as an + `(n/total)` is honest ONLY because the word in front of it is "Scanning": the + owner's objection (TODO L6) was the SYNC count wearing the total's shape, so + never move a `(n/m)` position onto the import label and never rename this + copy to "Syncing". The pair reaches the wrapper through the SAME + whole-status pass-through as the summaries / chunk / image pairs at the two + sync call sites (the poll tick + the load-time re-attach) — do not re-list + the fields there — and `fmtSyncLabel`'s `N remaining` arithmetic, the terminal + `counts_line` and the phase-187 `fmtSyncPeakClause` stay byte-identical (the + clause stays TERMINAL-only — it rides the settle's counts line exactly as + before, never the running scan label, and it still needs no new element). + Pinned by `tests/unit/test_sync_scan_label_ui.py` (the branch order, the exact + `(n/total)` shape + its `total === 0` guard, the whole-status pass-through, the + untouched builder set, and a node run of the REAL label path: `Scanning… + (3/9)` identical on main / count / title / box / aria, the bare `Scanning…` + at `0/0`, every other phase's label byte-identical). + ## Quick reference ```bash podman compose up -d db # start Postgres 17 + pgvector diff --git a/README.md b/README.md index dede6fc..06a35e6 100644 --- a/README.md +++ b/README.md @@ -727,11 +727,29 @@ only) runs the whole source refresh in one click: - States: while polling `GET /api/sync/status` every 2 s, the button becomes the spinner (icon only — its label parts are hidden while a run is in flight, and its accessible name is the full live label), - and the full live detail — "Syncing… N remaining" during the - import phase (the live file, its `N remaining` file count — N = + and the full live detail — "Scanning… (n/total)" while the importer + pre-scans (phase 194: the scan's own walk position over `scan_done` / + `scan_total`, and NEVER a file — the scan names none, and this label says + Scanning where the sync count would otherwise wear the walk's shape; no + count part at all while the SCAN's own total is 0 — an empty knowledge + base, or a `BOR_SYNC_PENDING_COUNT=0` run, which reports no scan position + at all and so shows the bare `Scanning…` for the moment before the first + file (that rollback still counts the WALK once the import starts, below)) + — then "Syncing… N remaining" + during the import phase (the live file, its `N remaining` file count — N = total − done, shown while N ≥ 1 and hidden at the final file, since the owner reads the distance to the end better than the position in - the whole; the upload's `Importing` label shares the ONE builder and + the whole. Since phase 194 the pair counts the WORK, not the WALK: + before the walk the importer pre-scans the importable files and `total` + is the size of that **predicted change set** (`documents.file_hash` + missing or different — the unchanged fast path's own predicate), so a + ~10,000-file sync that changes 10 files says "10 remaining" and a + no-change sync shows no count part at all; the position holds while the + walk moves over the unchanged files and a first import counts every + file, exactly as phase 193 did. `BOR_SYNC_PENDING_COUNT=0` restores the + walk position; the scan runs only for a caller that passes the progress + hook, so the CLI and the upload job pay nothing — the upload's + `Importing` label shares the ONE builder and the same shape, though since phase 90 that run reports no counts, so today it renders bare; and — while a file's embed batch is in flight — the pinned `· chunk n/m` embed position, batch granularity: the @@ -843,6 +861,64 @@ only) runs the whole source refresh in one click: exactly one `is_summary` chunk, an unchanged doc's viewer still shows the full stored text, and a rewritten file re-indexes to the new text with the next sync all-`unchanged` again). +- **The running count counts the work, not the walk** (phase 194). The + owner's prod sync said "1216 remaining" on a ~10,000-file knowledge base + that had ~10 files to change (report of 2026-10-19): the phase-193 label + was fine, its inputs were the whole walk. The status's `phase` machine gained + a stage for the fix — `null` (model-check + clone/pull prelude) → **`"scan"`** + (set immediately before the import, where the prediction runs) → `"import"` + (flipped by the first file the walk reaches) → `"overview"` → `"summaries"`, + cleared by every terminal — and the scan reports its own position in the + ADDITIVE `scan_done` / `scan_total` keys (the file it has hashed over the + importable count: `0/0` before it starts, on a `BOR_SYNC_PENDING_COUNT=0` run + and on an empty knowledge base, and KEPT beside the final file counts in every + terminal so a reloaded page still shows the run's shape). `current_file` stays + `null` through the scan — predicting a change set names no file — which is why + the Sources page renders that stage as its own "Scanning… (n/total)" label + (the ONE `"scan"` branch of `fmtSyncPhaseLabel`, in the same count slot the + import's `N remaining` uses, so the untruncated label still rides the button + title and the `aria-live` box) and the scan + is sampled like every other stage: its set and its flip to `"import"` force a + memory read, its own per-file hook does not (`LOCKED A4`). + `import_sources` itself + **pre-scans** (`_prescan_pending`): every importable file gets one + streamed sha256 (`_file_hash`) and one SCALAR `SELECT documents.file_hash` + (`_pending_hash_stmt` — one column, no entity, nothing in the identity + map), and the file is pending when the stored hash is NULL or differs — + the same predicate the unchanged fast path applies, so a pre-phase NULL + row is predicted changed AND really re-indexed and stamped. `total` is the + size of that set and `done` its position, which holds while the walk moves + over the unchanged files. Costs the phase accepted (A3): one extra read + + hash + index probe per importable file per run (the second read of the same + file is page-cached), and the file is hashed TWICE on purpose — the scan's + digest is the prediction, the loop's is the truth, so a file edited in + between is still detected. The span is ONE `asyncio.to_thread` call (the + phase-179 rule: inline, 10,000 hashes + SELECTs would park the single + uvicorn loop and freeze the 2 s status poll), it reads through a session of + its own (a `Session` is not thread-safe and a cancel at the await lets the + worker run to completion after the run has moved on), and `summary.files`, + `seen`, the prune, the date refresh and every log line are untouched. + **Where the operator sets the ceiling:** `BOR_SYNC_PENDING_COUNT` (`1` the + pending count, the default; `0` the exact phase-193 walk position, the + no-redeploy rollback; negative refuses startup naming the field; read once + per run). Pinned by `tests/unit/test_sync_pending_count.py` (the gate — no + `progress` caller runs no scan, the `0` branch runs the old stats pre-walk, + the read-once + `to_thread` markers, the one-column projection, the four + predicate states, the `done`-as-pending-position) and + `tests/integration/test_sync_pending_count.py` (the parity — the predicted + set's size equals the terminal's `added + updated` on a stable KB across the + fresh / unchanged / incremental arms, zero entity hydration, the loop stays + responsive through the span, and a cancel at the await lands while the + worker finishes on its own), and in the browser by + `tests/e2e/test_sync_pending_count.py` — the real Sources page against two + live apps over one mock model with a per-request delay: the first sync + counting the phase-193 arithmetic down to the final file, the + owner's case (8 files indexed, 2 changed) holding at "2 remaining" while + the walk names the unchanged ones and hiding it on the final predicted + file, the `BOR_SYNC_PENDING_COUNT=0` app answering "7 remaining" for that + same fixture, the no-change and empty-source edges, the scan's own position + climbing through a 1,000-file pre-scan that the running server keeps + answering, and the upload status still carrying no scan fields at all. - **The commit-date walk streams** (phase 186). A git source's per-file last-commit dates come from one `git log --name-only --format=@@%cI` over the checkout — a walk that scales with the repository's **entire @@ -1781,9 +1857,11 @@ grep ' :: ' server.log | grep 'sync: failed state=' # the failed ones, with t very same numbers (one reading, two surfaces — the page and the log can never disagree). - **Sampled, never per-file** (LOCKED A4): the run reads its own memory - at the run start, at each phase transition (`import` → `overview` → - `summaries`), in every terminal, and — throttled to at most one read - per 5 s — from the import's per-file progress hook, so the number + at the run start, at each phase transition (`scan` → `import` → + `overview` → `summaries` — phase 194 added the first stage and its flip), + in every terminal, and — throttled to at most one read + per 5 s — from the import's per-file progress hook (and, since phase 194, + the pre-scan's own per-file hook), so the number moves while you watch a long walk without costing 27k `/proc` reads. - **`null` renders NOTHING.** No reading (idle, a run that predates this phase, the upload job's status, a platform that cannot know) shows no @@ -1942,6 +2020,7 @@ startup (the DB rows shadow the env, `.env` is never rewritten). | `BOR_UPLOAD_MAX_MB` | `512` | Cap (MiB) for uploaded source archives (zip-bomb guard) | | `BOR_SYNC_SESSION_RESET_FILES` | `200` | Files between releases of the import run's SQLAlchemy session (phase 184): every Nth file the walk hands back everything the session still holds (`expunge_all`), so an unchanged-file sync stops keeping one full document row per file walked. `0` = never release — the exact pre-phase-184 behaviour; a negative value refuses startup naming the field. The release also runs once per source, before the two post-passes | | `BOR_SYNC_PRUNE_BATCH` | `500` | Pruned documents per COMMIT inside the prune (phase 184) instead of one commit at the end of the whole import, so a mass prune releases its deletes as it goes instead of holding one enormous transaction. `0` = one commit at the end — the exact pre-phase-184 behaviour; a negative value refuses startup naming the field. With a non-zero batch a crash mid-prune leaves a **prefix** pruned, and the next sync converges (`seen` is recomputed from disk every run) | +| `BOR_SYNC_PENDING_COUNT` | `1` | What the sync's running `N remaining` counts (phase 194). `1`: before the walk the importer pre-scans the importable files — one streamed sha256 + one scalar `documents.file_hash` read each, in an `asyncio.to_thread` span so the server keeps serving while it runs — and the count is the position in that **predicted change set**, so a ~10,000-file sync that changes 10 files says "10 remaining" instead of "1216 remaining" (the owner report of 2026-10-19). A no-change sync predicts nothing and shows no count part; a first import counts every file, exactly as before. `0` = count every file left to **walk** — the exact pre-phase-194 behaviour, the no-redeploy rollback if the extra read cost ever bites; a negative value refuses startup naming the field. Read once per run (a saved override applies to the next sync) and only by a caller that passes the progress hook: the CLI and the upload job run no scan and pay nothing | | `BOR_SUGGESTIONS` | built-in list | JSON seed for onboarding chips (shown before the first saved question) | ### Image support (phases 122/123/124/181) diff --git a/app/api/sync.py b/app/api/sync.py index 3d5d2ed..affcd9f 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -103,7 +103,16 @@ carries the phase-64 per-file progress — ``current_file`` (the ``source/relative/path`` the import is processing right now) plus ``files_done`` / ``files_total`` — null/0/0 before the import starts (clone/pull reports no file yet) and in terminal states, which clear -``current_file`` but keep the run's final counts. +``current_file`` but keep the run's final counts. What the pair COUNTS +changed in phase 194: on a progress caller (the UI sync is the only +one) it is the position in the pre-scan's PREDICTED change set — +``files_total`` the number of files that will actually be re-indexed, +``files_done`` the 1-based position in that set — which is what makes +the Sources page's ``N remaining`` say "10 remaining" on a ~10,000-file +KB with 10 changes (the owner report of 2026-10-19) instead of counting +the walk. ``BOR_SYNC_PENDING_COUNT=0`` restores the phase-193 walk +position byte-for-byte; the terminal ``detail`` counts stay the exact +surface either way. Phase 134 (task 01) makes the machine FIVE-state: ``cancelled`` joins ``success`` / ``failed`` as a TERMINAL state — it carries @@ -131,7 +140,14 @@ checks the event at the pipeline boundaries via set): the top of the per-source loop (before each clone/pull or local-dir check), immediately before ``import_sources``, and immediately before the overview + summaries span (the -``added + updated > 0`` block). Consequence a future reader must +``added + updated > 0`` block). Phase 194 (task 02) adds one more +await INSIDE the import span for the same shape: the pre-scan runs in +a ``to_thread`` worker, so a cancel parked there unblocks the RUN at +once (the ``cancelled`` terminal, nothing indexed) while the scan +thread finishes the session of its own — the phase-134/179 split, the +worker not being interruptible, pinned by +``tests/integration/test_sync_pending_count.py`` (importer side) and +this module's cancel-during-scan pin. Consequence a future reader must know: a git subprocess already running at the cancel moment runs to completion (Python threads are not interruptible — since phase 179 it occupies a ``to_thread`` worker rather than the loop, so it no @@ -188,8 +204,17 @@ background task must die in state, never as an unobserved exception Phase 98 (task 01) adds the run's PHASE machine — ``phase`` is the pipeline stage the run is in: the model-check + clone/pull prelude -reports ``null`` (the bare "Syncing…" label stays), then ``"import"`` -(set immediately before ``import_sources``), ``"overview"`` (set +reports ``null`` (the bare "Syncing…" label stays), then ``"scan"`` +(phase 194, task 02 — set immediately before ``import_sources``, +REPLACING the pre-phase ``"import"`` set: what happens first in that +call is the pre-scan that predicts the change set, and the label the +task-03 branch renders for this phase is ``Scanning… (n/total)``), then +``"import"`` (flipped by +the FIRST per-file ``progress`` call while the phase is still +``"scan"`` — the flip lives in the runner's own ``_hook`` closure, so +the importer gains no extra callback; with +``BOR_SYNC_PENDING_COUNT=0`` no scan report fires at all and the flip +is simply the first file), ``"overview"`` (set before ``regenerate_overview`` — a changed KB only), and ``"summaries"`` (set before a folder-summary generation — the changed-KB incremental regeneration (phase 127, task 03 — the changed @@ -259,7 +284,12 @@ sampled through the ONE :func:`_sample_rss` — at the run start, at every ``phase`` transition, in every terminal, and at most once per ``_RSS_SAMPLE_INTERVAL_S`` from the per-file progress hook, so a 27k-file walk costs ~600 ``/proc`` reads and not 27k — LOCKED A4), and -``GET /api/sync/status`` reports both keys ADDITIVELY. The terminals keep +``GET /api/sync/status`` reports both keys ADDITIVELY. Phase 194 +(task 02) extends both halves of that rule to the new stage: the +``"scan"`` set before ``import_sources`` and the ``scan``→``import`` +flip are phase TRANSITIONS and FORCE the read, while the pre-scan's own +per-file hook samples UNFORCED (throttled) — a 10,000-file scan costs +one ``/proc`` read per interval, never one per file. The terminals keep the FINAL reading next to the final counts (the phase-64 convention), so the page that started the run answers the incident's question without a shell. A reading the platform cannot know stays ``null`` on the wire (the @@ -323,11 +353,33 @@ class SyncStatus: outside the import phase — clone/pull first, terminal states after); ``files_done`` / ``files_total`` carry the hook's done/total position and survive a terminal state (the run's last - position is useful context next to the error). + position is useful context next to the error). Since phase 194 the + pair is the hook's PENDING position (phase 194: the pre-scan's + predicted change set — ``files_total`` the files that will actually + be re-indexed), not the walk position; ``BOR_SYNC_PENDING_COUNT=0`` + restores the walk position and every other caller (the CLI, the + upload job) is unaffected. + + Phase 194 (task 02) scan fields: ``scan_done`` / ``scan_total`` + carry the pre-scan's position — the 1-based file whose hash+read + just completed over the combined importable count the scan walks + (the importer's ``on_scan_progress`` hook). They are 0/0 before the + scan starts and on the ``BOR_SYNC_PENDING_COUNT=0`` rollback (no + scan, so no report), reset with the run like every other field, and + KEPT at every terminal beside the final file counts (the phase-64 + keep-final-values convention — the scan the owner watched is still + on the page after the run). ``current_file`` stays ``null`` through + the phase: the scan names no file, it only counts. The degenerate + empty knowledge base is the honest edge — the scan predicts nothing + so no report fires and no ``progress`` call ever arrives: the phase + stays ``"scan"`` until the next transition or the terminal (which + clears it), and the label is the bare phase name with no count part + (the UI's ``total === 0`` guard). Phase 98 (task 01) phase fields: ``phase`` is the pipeline stage (null in the model-check + clone/pull prelude and in terminal - states — the module docstring's phase machine); while in the + states — the module docstring's phase machine, whose stages are now + ``scan`` → ``import`` → ``overview`` → ``summaries``); while in the ``"summaries"`` phase, ``current_summary`` is the source / ``source/folder_path`` the generator is summarizing right now and ``summaries_done`` / ``summaries_total`` carry the generator's @@ -393,10 +445,20 @@ class SyncStatus: current_file: str | None = None files_done: int = 0 files_total: int = 0 + # Phase 194 (task 02): the PRE-SCAN's position — the 1-based file + # whose hash+read just completed and the combined importable count + # the scan walks (the importer's on_scan_progress hook). 0/0 before + # the scan starts (and on the BOR_SYNC_PENDING_COUNT=0 rollback); + # reset with the run and kept at every terminal beside the final + # file counts (the phase-64 keep-final-values convention). + scan_done: int = 0 + scan_total: int = 0 # Phase 98 (task 01): the phase machine — the pipeline stage the # run is in and, while in the folder-summary phase, the folder - # being summarized plus the hook's done/total position. - phase: Literal["import", "overview", "summaries"] | None = None + # being summarized plus the hook's done/total position. Phase 194 + # (task 02) adds the "scan" stage (the pre-scan that predicts the + # run's change set — the module docstring's phase machine). + phase: Literal["scan", "import", "overview", "summaries"] | None = None current_summary: str | None = None summaries_done: int = 0 summaries_total: int = 0 @@ -433,13 +495,15 @@ _task: asyncio.Task[None] | None = None # starts, so a stale event can never bleed into the next run. _cancel_event = asyncio.Event() -#: Phase 187 (task 03, D3/A4): how often the PER-FILE progress hook may -#: re-read the process memory, in seconds. That hook fires once per -#: file, so without this guard a 27k-file walk would cost 27k -#: ``/proc/self/status`` reads; with it the same walk costs ~600 and the -#: operator watching the page still sees the number move (PLAN §7.4's -#: never-stale contract). Every other sampling point (the run start, the -#: ``phase`` transitions, the terminals) FORCES the read instead. +#: Phase 187 (task 03, D3/A4): how often the PER-FILE progress hooks may +#: re-read the process memory, in seconds. Those hooks fire once per +#: file — the import's ``progress`` AND, since phase 194 (task 02), the +#: pre-scan's ``on_scan_progress`` — so without this guard a 27k-file +#: walk would cost 27k ``/proc/self/status`` reads; with it the same walk +#: costs ~600 and the operator watching the page still sees the number +#: move (PLAN §7.4's never-stale contract). Every other sampling point +#: (the run start, the ``phase`` transitions including the ``scan`` set +#: and its flip to ``import``, the terminals) FORCES the read instead. _RSS_SAMPLE_INTERVAL_S = 5.0 #: ``time.monotonic()`` of the last RSS read; ``-inf`` means "never", so @@ -503,7 +567,21 @@ def sync_status() -> dict[str, Any]: import is processing right now — null during the clone/pull phase and in terminal states; ``files_done`` / ``files_total`` carry the hook's position (0/0 idle) and keep their final values in the - terminal. ``current_chunk`` / ``chunks_total`` (phase 129) are the + terminal — the position in the pre-scan's PREDICTED change set on a + progress caller (phase 194), the walk position under + ``BOR_SYNC_PENDING_COUNT=0``. ``scan_done`` / ``scan_total`` (phase + 194) are the PRE-SCAN's position: the 1-based file whose + hash+compare just ran over the combined importable count the scan + walks. They are 0/0 before the scan starts — the prelude, a run on + the ``0`` rollback, and every run that never happened — and in a + terminal they keep the FINAL scan values beside the final file + counts (the phase-64 keep-final-values convention); ``current_file`` + stays null while ``phase`` is ``"scan"``, because the scan names no + file. Both keys are ADDITIVE — placed after ``files_total``, every + pre-existing key keeping its name, type and position. The degenerate + empty knowledge base reports the phase without ever reporting a + position (0/0 — the UI's ``total === 0`` guard renders no count + part). ``current_chunk`` / ``chunks_total`` (phase 129) are the in-flight file's embed-chunk position (batch granularity — the first chunk of the in-flight batch over the file's total content chunks): 0/0 before the first batch, in the prelude/overview/ @@ -551,6 +629,14 @@ def sync_status() -> dict[str, Any]: "current_file": _status.current_file, "files_done": _status.files_done, "files_total": _status.files_total, + # Phase 194 (task 02): the pre-scan's position (the importer's + # on_scan_progress hook) — 0/0 before the scan starts and on the + # BOR_SYNC_PENDING_COUNT=0 rollback, the FINAL pair kept in every + # terminal beside the file counts above (the keep-final-values + # convention). Additive: every pre-existing key keeps its name, + # type and position. + "scan_done": _status.scan_done, + "scan_total": _status.scan_total, # Phase 129 (task 03): the in-flight file's embed-chunk # position — the dataclass defaults make idle 0/0, and the # runner clears it at every point the import phase ends @@ -672,6 +758,14 @@ async def _run_sync() -> None: so a long clone no longer stalls the rest of the HTTP surface (the module docstring's freeze paragraph); the run's own sequencing is byte-for-byte the pipeline above. + + Phase 194 (task 02): the phase set before ``import_sources`` is + ``"scan"`` (not ``"import"``), because that call begins with the + off-loop pre-scan that predicts the change set; the FIRST per-file + ``progress`` call flips it to ``"import"`` inside ``_hook``. Both + transitions force the memory sample, the scan's own per-file hook + does not (LOCKED A4), and ``scan_done`` / ``scan_total`` reset with + the run and survive every terminal beside the file counts. """ _status.state = "running" _status.started_at = datetime.now(UTC) @@ -683,6 +777,11 @@ async def _run_sync() -> None: _status.current_file = None _status.files_done = 0 _status.files_total = 0 + # Phase 194 (task 02): the pre-scan's position resets with the run — + # 0/0 until the scan reports its first file (and 0/0 for the whole + # run on the BOR_SYNC_PENDING_COUNT=0 rollback, which scans nothing). + _status.scan_done = 0 + _status.scan_total = 0 # Phase 98 (task 01): the phase fields reset with the run — no # phase until the import starts (the model-check + clone/pull # prelude reports null), no folder until the summary span starts. @@ -834,6 +933,17 @@ async def _run_sync() -> None: # run-start reset alone cannot cover it). _status.images_done = 0 _status.images_total = 0 + # Phase 194 (task 02): the scan→import FLIP. The phase is + # set to "scan" immediately before import_sources, and the + # first file the walk actually reaches is where the running + # label stops being a scan — no extra callback reaches the + # importer, the flip lives in this existing per-file hook. + if _status.phase == "scan": + _status.phase = "import" + # A phase TRANSITION is a forced sampling point (phase + # 187), so this one read is not the throttled per-file + # one the tail of this closure does. + _sample_rss(force=True) # Phase 187 (task 03): the run's memory, sampled THROTTLED # (unforced) — this hook fires once per file, so a 27k-file # walk reads /proc ~600 times instead of 27k (LOCKED A4) @@ -865,10 +975,30 @@ async def _run_sync() -> None: _status.images_done = done _status.images_total = total - # Phase 98 (task 01): the phase machine — the import phase - # starts NOW (the model-check + clone/pull prelude above - # reported ``phase: null``). - _status.phase = "import" + # Phase 194 (task 02): the PRE-SCAN's position — the importer's + # ``on_scan_progress`` hook, fired once per importable file while + # the run's change set is still being PREDICTED (before the walk), + # with (position_1based, scan_total). It runs on the scan's + # WORKER thread, so this closure only assigns fields on the + # module's own status object — the house hook contract that makes + # each call a single GIL-atomic store (phase 179). The memory + # sample is UNFORCED here: this fires once per file, exactly like + # ``_hook`` above, so it costs one /proc read per interval and + # never one per file (LOCKED A4). + def _scan_hook(scan: int, total: int) -> None: + _status.scan_done = scan + _status.scan_total = total + _sample_rss() + + # Phase 98 (task 01) + phase 194 (task 02): the phase machine — + # the SCAN phase starts NOW (the model-check + clone/pull prelude + # above reported ``phase: null``). What runs first inside + # ``import_sources`` is the pre-scan that predicts the change set, + # and the label the owner reads through it is "Scanning…" + # (phase 193's "import" set moved to the first ``_hook`` call + # above — the flip), so the count the page shows is never a scan + # wearing the work's shape. + _status.phase = "scan" # Phase 187 (task 03): the phase transitions are the forced # sampling points — each stage of the run begins with its own # reading on the status (the per-file hook throttles). @@ -885,6 +1015,7 @@ async def _run_sync() -> None: progress=_hook, on_chunk_progress=_chunk_hook, on_image_progress=_image_hook, # phase 152 (task 03) + on_scan_progress=_scan_hook, # phase 194 (task 02) ignore_by_root=ignore_by_root, include_hidden_by_root=include_hidden_by_root, doc_dates_by_root=doc_dates_by_root, @@ -1034,6 +1165,10 @@ async def _run_sync() -> None: _status.current_file = None # phase 64: keep the final counts _status.phase = None # phase 98: keep the final summary counts _status.current_summary = None + # Phase 194 (task 02): the scan position is NOT cleared here — + # like the file counts it keeps its final values beside them + # (the keep-final-values convention), so the page reloaded after + # the run still says how big the predicted change set was. # Phase 129 (task 03): the chunk position is a sub-position of # a file that's done — it clears in the terminal (unlike the # FILE counts, which keep their final values for the result @@ -1091,6 +1226,10 @@ async def _run_sync() -> None: _status.current_file = None # keep the final counts _status.phase = None # keep the final summary counts _status.current_summary = None + # Phase 194 (task 02): the scan position survives the cancel + # too — where a cancel landed DURING the pre-scan it is the + # run's only number, and the log line above keeps the file + # pair beside it (the keep-final-values convention). # Phase 129 (task 03): the chunk position is a sub-position # of a file that's done — it clears in EVERY terminal, like # success and failed (the convention split, pinned). @@ -1138,6 +1277,9 @@ async def _run_sync() -> None: _status.current_file = None # phase 64: keep the final counts _status.phase = None # phase 98: keep the final summary counts _status.current_summary = None + # Phase 194 (task 02): the scan position is kept here as well — + # a run that died mid-scan keeps the position it reached beside + # the final file counts (the convention, every terminal). # Phase 129 (task 03): the chunk position clears in the # failed terminal too (the convention split — the FILE counts # keep their final values next to the error, the sub-position diff --git a/app/config.py b/app/config.py index 7092901..1c5a02a 100644 --- a/app/config.py +++ b/app/config.py @@ -524,6 +524,22 @@ class Settings(BaseSettings): #: mid-prune leaves a PREFIX pruned, and the next sync converges #: because ``seen`` is recomputed from the filesystem every run. sync_prune_batch: int = 500 + #: Predict the run's change set BEFORE the walk, so the sync's running + #: count is the count of the WORK and not of the walk (phase 194, + #: ``BOR_SYNC_PENDING_COUNT``). ``1`` (default): ``import_sources`` + #: pre-scans the importable files (one streamed sha256 + one scalar + #: ``documents.file_hash`` read each, off the event loop) and the + #: per-file ``progress`` hook then reports ``done``/``total`` as the + #: position in that PENDING set — the owner's "1216 remaining" fix. + #: ``0`` = the exact phase-193 behaviour (the stats-only pre-walk and + #: ``done = summary.files``, i.e. "every file left to WALK") — the + #: operator's no-redeploy rollback if the extra read cost ever bites. + #: A negative value is a typo — the validator fails startup loudly + #: (the ``agent_max_rounds`` pattern). Read per run (the importer reads + #: ``llm.settings`` once per sync, so a saved override applies to the + #: NEXT sync) and only by a caller that passes ``progress`` — the CLI + #: and the upload job pay nothing. + sync_pending_count: int = 1 # --- Image documents (phase 122: standalone images as documents) --- #: Master switch for image-document indexing (phase 122, @@ -930,6 +946,20 @@ class Settings(BaseSettings): ) return v + @field_validator("sync_pending_count") + @classmethod + def _sync_pending_count_non_negative(cls, v: int) -> int: + """``0`` is the phase-193 rollback (count the walk, not the work) — + a negative value is a typo that no branch of the importer can mean, + so it fails startup naming the field (the ``agent_max_rounds`` + pattern, phase 194).""" + if v < 0: + raise ValueError( + "sync_pending_count must be >= 0 (0 = count the whole walk, " + "the pre-phase-194 behaviour)" + ) + return v + @field_validator("chat_image_max_mb") @classmethod def _chat_image_max_mb_positive(cls, v: int) -> int: diff --git a/app/core/settings_catalog.py b/app/core/settings_catalog.py index b687f28..5fe9545 100644 --- a/app/core/settings_catalog.py +++ b/app/core/settings_catalog.py @@ -304,6 +304,9 @@ CATALOG: dict[str, SettingMeta] = { "sync_prune_batch": SettingMeta( "sync_prune_batch", "Import & sources", "Sync prune batch" ), + "sync_pending_count": SettingMeta( + "sync_pending_count", "Import & sources", "Sync pending count" + ), # --- Images --- "images": SettingMeta("images", "Images", "Images"), "image_extensions": SettingMeta( diff --git a/app/rag/importer.py b/app/rag/importer.py index 5f2a0c1..967fd69 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -332,14 +332,27 @@ convention, phase 133). (phase 64, task 01) reporting the file being processed right now, an optional ``on_chunk_progress`` callback (phase 129, task 02) reporting the in-flight file's embedding-chunk position at batch -granularity, and an optional ``on_image_progress`` callback (phase -152, task 02) reporting the in-flight PDF's vision-image position — +granularity, an optional ``on_image_progress`` callback (phase 152, +task 02) reporting the in-flight PDF's vision-image position — immediately before each of the PDF's vision describe calls, with the image's 1-based position in the file's ONE continuous vision counter -and the span's count (see the function docstring). +and the span's count — and an optional ``on_scan_progress`` callback +(phase 194, task 01) reporting the PRE-SCAN's position while the run's +change set is predicted. The ``progress`` pair is the pending position +(``total`` = the size of the predicted change set, ``done`` = the 1-based +position in it) rather than the walk position, which is what makes the +Sources page's ``N remaining`` count the WORK and not the WALK: a +~10,000-file sync that changes 10 files says "10 remaining", not +"1216 remaining" (the owner report of 2026-10-19). ``0`` in +``BOR_SYNC_PENDING_COUNT`` restores the phase-193 walk position exactly. +All four callbacks follow the same "only when provided" rule — a caller +that passes nothing pays the pass, the read and the cost of none of them, +and a caller that passes no ``progress`` runs no pre-scan at all (the +CLI, the upload job: byte-identical). See the function docstring. """ from __future__ import annotations +import asyncio import hashlib import logging import os @@ -753,6 +766,125 @@ def _doc_row_stmt(source: str, rel: str) -> Select[tuple[Document]]: ) +def _pending_hash_stmt(source: str, rel: str) -> Select[tuple[str | None]]: + """The pre-scan's ONE-column metadata read (phase 194, task 01). + + ``select(Document.file_hash)`` is a SCALAR projection, not an entity + load: no ``Document`` is ever constructed, nothing lands in the run's + ``identity_map``, and the pre-scan therefore retains nothing — which is + why it needs no ``defer``/``load_only`` (there is no row to defer) and + why it does NOT touch :func:`_doc_row_stmt`, which stays the importer's + ONLY row load (phase 185). The predicate is the unique + ``(source, path)`` constraint (``app/models.py``, the same lookup the + fast path uses), so the read costs one index probe per file — the + ``file_hash`` column itself stays unindexed by design. + """ + return select(Document.file_hash).where( + Document.source == source, Document.path == rel + ) + + +def _prescan_pending( + sources: Sequence[Path], + extensions: frozenset[str], + image_extensions: frozenset[str], + ignore_by_root: dict[str, list[str]] | None, + include_hidden_by_root: dict[str, bool] | None, + on_scan_progress: Callable[[int, int], None] | None, + session: Session | None = None, +) -> tuple[int, dict[tuple[str, str], int]]: + """Predict the run's change set BEFORE the walk (phase 194, D1). + + Returns ``(scan_total, pending)``: the combined count of importable + files across *sources* — the SAME number the phase-64 stats-only pre-walk + computed, under the SAME ``iter_importable_files`` rules (the caller + passes the exact *extensions* / *image_extensions* / per-root ``ignore`` + and ``include_hidden`` values the processing loop uses, so a file the + loop cannot see is never predicted) — and ``pending``, the map of + ``(source, rel)`` to that file's 1-based position in the predicted change + set, walk order. + + Per file: :func:`_file_hash` (the streamed 1 MiB-chunk sha256 of phase + 147 — the raw bytes, the sync's single change signal) plus ONE + :func:`_pending_hash_stmt` scalar read, and the file is pending when the + stored hash is ``NULL`` (no row yet, or a pre-phase-147 row) or differs + from the digest — the SAME predicate the fast path applies at processing + time, so a pre-phase NULL row is predicted changed AND really re-indexed + and stamped by the loop: the prediction never lies about the stored-hash + contract. A file the digest cannot reach (deleted between the walk and + the hash) is predicted changed too: the loop's own walk no longer sees + it, so the count can hold one short, which is honest — the terminal + counts line is the exact surface either way. + + ``session`` follows the importer's own convention (tests pass one, a + private one is opened and closed otherwise). Deliberately NOT the RUN's + session: this function executes inside an :func:`asyncio.to_thread` + worker (see :func:`import_sources`), and a ``Session`` is not + thread-safe — a cancel at the span's await lets this worker run to + completion while the run has already moved on to its terminal and closed + its own session. Reading through a session this span owns is what makes + that edge safe; the projection loads no entity, so even this session + keeps nothing. + + *on_scan_progress* is called — only when provided — once per file with + ``(position_1based, scan_total)``, AFTER that file's hash+read (so the + position counts files actually scanned and reaches the total exactly when + the scan does). It runs on the WORKER thread: the house hook contract is + that these hooks only assign fields on the caller's own status object, + which makes each call a single GIL-atomic store, and there is no + try/except around it for the same reason as the three sibling hooks. + """ + owns_session = session is None + if session is None: + session = SessionLocal() + try: + # (a) the per-root file lists + the combined length, so the scan's + # denominator is known before the first callback. Directory stats + # only — no file read, no database statement (the phase-64 shape). + per_root: list[tuple[Path, list[Path]]] = [ + ( + root, + iter_importable_files( + root, + extensions, + ignore=_ignore_for_root(root, ignore_by_root), + include_hidden=_include_hidden_for_root( + root, include_hidden_by_root + ), + image_extensions=image_extensions, + ), + ) + for root in sources + ] + scan_total = sum(len(files) for _, files in per_root) + pending: dict[tuple[str, str], int] = {} + position = 0 + for root, files in per_root: + source = root.name + for path in files: + rel = path.relative_to(root).as_posix() + try: + digest = _file_hash(path) + except OSError as e: # vanished mid-scan — predict the change + logger.warning( + "import: pre-scan unreadable source=%s path=%s — %s", + source, + rel, + e, + ) + digest = "" + stored = session.scalar(_pending_hash_stmt(source, rel)) + if stored is None or stored != digest: + pending[(source, rel)] = len(pending) + 1 + position += 1 + if on_scan_progress is not None: + on_scan_progress(position, scan_total) + return scan_total, pending + finally: + if owns_session: + session.close() + + async def import_sources( sources: list[Path], llm: Embedder, @@ -763,6 +895,7 @@ async def import_sources( progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, on_image_progress: Callable[[int, int], None] | None = None, + on_scan_progress: Callable[[int, int], None] | None = None, ignore_by_root: dict[str, list[str]] | None = None, include_hidden_by_root: dict[str, bool] | None = None, doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, @@ -776,15 +909,53 @@ async def import_sources( ``progress`` (phase 64, task 01) is an optional per-file hook called once per importable file, immediately before that file's ``_index_file`` — with ``(source, rel_posix_path, done, total)``: the - same POSIX *rel* the document rows use, ``done`` = the 1-based index of - the current file **across all sources**, and ``total`` = the combined - pre-walk count of importable files across all *sources* roots. The - pre-walk (same extension/exclusion rules, directory stats only, no file - reads) happens **only when *progress* is provided**: callers passing - nothing pay no extra walk and behave exactly as before. Under - ``limit``, the hook still fires per processed file only — ``done`` never - exceeds the limit, but ``total`` stays the full pre-walk count (an - incomplete walk must not misreport the denominator). + same POSIX *rel* the document rows use, plus the position pair the UI's + running count consumes. The pair's MEANING is the phase-194 switch + ``llm.settings.sync_pending_count`` (read ONCE before the source loop — + see the pre-scan paragraph below): + + * switch truthy (the default): ``total`` = the size of the pre-scan's + PENDING set (the files predicted to be re-indexed) and ``done`` = the + 1-based position in THAT set. The position advances for every file in + the pending set — whether or not the file turns out changed at + processing time — so count and denominator stay on the SAME set and + the arithmetic never runs away; a file NOT in the set does not advance + it, which is what keeps the label holding at ``K remaining`` while the + walk moves over the unchanged ``N − K``. + * switch ``0``: the exact phase-193 walk position — ``done`` = the + 1-based index of the current file **across all sources** and ``total`` + = the combined pre-walk count of importable files (the full rollback). + + In both states the count-only pass happens **only when *progress* is + provided**: callers passing nothing pay no extra pass and behave exactly + as before (the CLI, the upload job). Under ``limit``, the hook still + fires per processed file only — ``done`` never exceeds the limit, but + ``total`` stays the full count (an incomplete walk must not misreport + the denominator). + + The pair is a PREDICTION, and its drift edges are bounded (phase 194 + A1): a file the scan predicted and that reverted to unchanged before the + loop reached it still advances the count (it was counted in ``total``, + so the count still reaches 0); a file changed AFTER the scan is indexed + for real but advances nothing (``total`` never counted it), and the + UI's ``total - done > 0`` guard hides that excess — never a negative and + never a stale ``0 remaining``. The terminal ``counts_line`` ("N added · + M updated …") stays the exact surface, untouched. + + ``on_scan_progress`` (phase 194, task 01) is an optional SCAN-position + hook for the pre-scan itself: while the pending set is being predicted it + is called — once per importable file, after that file's hash+read — with + ``(position_1based, scan_total)``, the file's 1-based position in the + scan and the combined importable count across all *sources* roots (the + number the phase-64 stats pre-walk used to compute). It carries counts + only, never the file identity (the sibling convention), and it runs on + the span's WORKER thread — the house rule that these hooks only assign + fields is what makes each call a single GIL-atomic store. Same "only + when provided" rule as ``progress``, ``on_chunk_progress`` and + ``on_image_progress``, with one addition: the pre-scan it reports on is + gated on *progress* too, so a caller passing ``on_scan_progress`` alone + (``progress=None``) runs no scan, fires no callback and stays + byte-identical — the scan exists only to serve the running counts. ``on_chunk_progress`` (phase 129, task 02) is an optional chunk-position hook for the file currently being indexed: while that @@ -923,6 +1094,45 @@ async def import_sources( (``0`` = never — the exact pre-phase behaviour, the operator's no-redeploy rollback). The cadence changes no count, no log line and no committed byte: see the safety argument at the reset site. + + The pre-scan (phase 194, task 01): with the hook on and + ``llm.settings.sync_pending_count`` truthy, the count-only pass is no + longer a stats walk but :func:`_prescan_pending` — every importable file + hashed once and compared once against its stored ``file_hash``, giving + the PENDING set the ``progress`` pair counts. Three costs the phase + accepted, deliberately: + + * **off the loop** (the phase-179 rule): the scan runs in ONE + :func:`asyncio.to_thread` span. Ten thousand streamed hashes plus ten + thousand one-column SELECTs inline would park the single uvicorn loop + for seconds — the exact defect class + ``tests/integration/test_sync_git_freeze.py`` pins for the git clone, + and the ``GET /api/sync/status`` poll the UI runs every 2 s is the + victim. A cancel registered at the span's await interrupts the RUN + (phase 134: it lands the ``cancelled`` terminal), while the worker + itself runs to completion — threads are not interruptible, which is + also why the span reads through a session of its own instead of the + run's (see :func:`_prescan_pending`). + * **one extra read + hash + scalar SELECT per importable file per run** + (A3): the second read of the same file is served from the page cache, + and the scalar read is a unique ``(source, path)`` index probe. + ``BOR_SYNC_PENDING_COUNT=0`` buys the whole cost back and restores the + phase-193 walk-position semantics byte-for-byte; a negative value + refuses startup naming the field. + * **the file is hashed twice**, here and again in ``_index_file``. That + is deliberate: the pre-scan's digest is the PREDICTION, the loop's is + the TRUTH — a file edited between the two is still detected and + re-indexed. Dropping the second hash would make the count a claim + about bytes nobody verified. + + Everything else in the run is untouched by the scan: ``summary.files`` + (the ``import: summary … files=`` count of every file the loop + processed), ``seen``, the prune post-pass, the date refresh and every log + line keep their exact shape — the pre-scan builds ``pending``, never + ``seen``, and the loop keeps building ``seen`` from disk exactly as + before. The degenerate empty knowledge base is the phase-64 shape: scan + total ``0``, no callback fires, the pending set is empty and ``total`` + is ``0`` (the UI's ``total === 0`` guard then renders no count part). """ if limit is not None and limit <= 0: raise ValueError("limit must be >= 1") @@ -937,30 +1147,68 @@ async def import_sources( # reads ``llm.settings`` per RUN, never per file). ``0`` disables the # release entirely. reset_every = llm.settings.sync_session_reset_files + # Phase 194 (task 01): the pending-count switch, read ONCE here — before + # the source loop — so a settings overlay saved mid-run cannot change the + # MEANING of the hook's ``done``/``total`` pair between two files of the + # same walk (the phase-184 read-once discipline). ``0`` = the exact + # phase-193 walk-position semantics, the operator's no-redeploy rollback. + count_pending = bool(llm.settings.sync_pending_count) owns_session = session is None if session is None: session = SessionLocal() seen: set[tuple[str, str]] = set() source_names: set[str] = set() - # phase 64 (task 01): the hook's combined denominator, walked with the - # exact same rules as the processing loop below (directory stats only, - # no file reads). Skipped entirely for ``progress=None`` callers — no - # extra pass, byte-identical behaviour and cost. + # phase 64 (task 01): the hook's denominator. phase 194 (task 01): what + # the number MEANS follows ``count_pending`` — the size of the predicted + # change set, or the count of every importable file. Either way the pass + # that computes it is skipped entirely for ``progress=None`` callers (no + # extra cost, byte-identical behaviour) and it is INSIDE the try so the + # session is closed on every exit path. total = 0 - if progress is not None: - for root in sources: - total += len( - iter_importable_files( - root, - llm.settings.import_extension_set, - ignore=_ignore_for_root(root, ignore_by_root), - include_hidden=_include_hidden_for_root( - root, include_hidden_by_root - ), - image_extensions=image_exts, - ) - ) + # (source, rel) -> 1-based position in the PENDING set; ``None`` = the + # phase-193 walk-position semantics (the switch ``0`` branch below). + pending: dict[tuple[str, str], int] | None = None + # the position the hook last reported: a file the scan did not predict + # holds it, which keeps ``total - done`` honest while the walk moves over + # the unchanged files. + pending_done = 0 try: + if progress is not None: + if count_pending: + # THE PRE-SCAN (phase 194, D1) — it REPLACES the stats-only + # pre-walk, so there is no third walk. ONE off-loop span + # (the phase-179 rule: a 10,000-file hash+SELECT pass inline + # would park the single uvicorn loop and freeze the status + # poll the UI runs every 2 s). The scan total is the scan + # hook's own business; the run's denominator is the pending + # set the scan returns. + _scan_total, pending = await asyncio.to_thread( + _prescan_pending, + sources, + llm.settings.import_extension_set, + image_exts, + ignore_by_root, + include_hidden_by_root, + on_scan_progress, + ) + total = len(pending) + else: + # ``BOR_SYNC_PENDING_COUNT=0`` — the phase-193 rollback, + # byte-for-byte: the hook's combined denominator, walked with + # the exact same rules as the processing loop below (directory + # stats only, no file reads). + for root in sources: + total += len( + iter_importable_files( + root, + llm.settings.import_extension_set, + ignore=_ignore_for_root(root, ignore_by_root), + include_hidden=_include_hidden_for_root( + root, include_hidden_by_root + ), + image_extensions=image_exts, + ) + ) for root in sources: if not root.is_dir(): logger.warning("import: source dir not found, skipping: %s", root) @@ -1036,7 +1284,16 @@ async def import_sources( # file that then errors or turns out unchanged was # already the "current file". No try/except around the # call: the hooks in this repo only assign fields. - progress(source, rel, summary.files, total) + # phase 194 (task 01): the pair's meaning follows the + # switch read before the loop — the position in the + # pending set, or the pre-phase-194 walk position. + if pending is not None: + predicted = pending.get((source, rel)) + if predicted is not None: + pending_done = predicted + progress(source, rel, pending_done, total) + else: + progress(source, rel, summary.files, total) try: await _index_file( session, source=source, rel=rel, full_path=path, llm=llm, diff --git a/frontend/assets/sources.js b/frontend/assets/sources.js index 854baa9..b029340 100644 --- a/frontend/assets/sources.js +++ b/frontend/assets/sources.js @@ -159,7 +159,8 @@ * "Writing KB overview…", "summaries" → "Summarizing folders… * (n/m)" (the folder part omitted while * current_summary is null — the phase's first poll), anything else - * (the null prelude, "import") → today's "Syncing… (n/m)" + * (since phase 194 ONLY the null prelude and "import") → today's + * "Syncing… (n/m)" * byte-identical (the count part's TEXT is `N remaining` — the distance * to the end — since phase 193). The UPLOAD job (its status has no phase) keeps the * bare label. The untruncated label rides the button title (now set @@ -475,6 +476,31 @@ * announces today's wording plus the clause, and the number is the * PROCESS high-water mark app/core/memory.py reports (never the host's * cgroup figure — that one counts page cache). + * + * Phase 194 (task 03) — the running label counts the WORK, and the + * pre-scan says so. The owner's prod report (2026-10-19): "1216 + * remaining" on a ~10,000-file KB with ~10 files to sync — the + * phase-193 count-down was arithmetically right but read the WALK: the + * server now pre-scans (one streamed sha256 + one scalar + * documents.file_hash read per importable file, off the event loop) and + * reports files_done / files_total as the position in the PREDICTED + * change set, so `N remaining` is the files left to RE-INDEX + * (fmtSyncLabel's arithmetic is byte-identical — only its INPUTS + * moved). The scan itself is a visible phase (A2): + * fmtSyncPhaseLabel gains ONE "scan" branch rendering + * "Scanning… (n/total)" — main "Scanning…" (never a file: + * current_file is null while the scan runs) + the count part "(n/total)" + * over the status's own scan_done / scan_total, in the SAME + * count slot (so setSyncLabel's hidden-toggle and the A4 untruncated + * placement — button title + #sync-progress — carry it with no new + * element), the phase-64 `total === 0` guard rendering the bare + * "Scanning…" for an empty KB, and no chunk/image part (the scan reads + * no file). The scan's walk position surviving as (n/total) is honest + * because the word in front of it is "Scanning", never "Syncing". The + * pair reaches the wrapper the way the summaries, chunk and image pairs + * already do — the WHOLE status object at the two sync call sites, the + * `phase` value selecting which pair renders. The peak clause, the + * terminal counts line and the upload's bare label are untouched. */ import { fetchIsAdmin } from "./header.js"; @@ -748,16 +774,25 @@ export async function mount(root) { * status endpoints — GET /api/sync/status + * GET /api/git-sources/upload/status — and applies this decision * tree, in order (startSyncPolling): - * 1. sync running → the phase-aware label (Phase 98 task 02, D2) - * — the null-phase prelude + "import" → - * "Syncing… N remaining" — bare "Syncing…" - * until the import's first file (clone/pull, - * A4, byte-identical); "overview" → "Writing KB - * overview…"; "summaries" → "Summarizing - * folders… (n/m)" (the folder - * part omitted while current_summary is null — - * the phase's first poll) — the post-import span - * where the file count sits still; + * 1. sync running → the phase-aware label (Phase 98 task 02, D2; + * Phase 194 task 03 names the scan) — + * "scan" → "Scanning… (n/total)" (the pre-scan's + * own walk position over scan_done / + * scan_total, no file — current_file is null in + * that phase); the null-phase prelude + + * "import" → "Syncing… N remaining" — bare + * "Syncing…" until the import's first file + * (clone/pull, A4, byte-identical; since phase + * 194 that N is the files left to RE-INDEX, the + * pre-scan's predicted change set); "overview" → + * "Writing KB overview…"; "summaries" → + * "Summarizing folders… (n/m)" (the + * folder part omitted while current_summary is + * null — the phase's first poll) — the + * post-import span where the file count sits + * still; the WHOLE status object carries every + * pair the label reads (scan, file, chunk, image, + * summary) and `phase` picks which one renders; * 2. upload running → BARE "Importing…" — the background upload * RUN (phase 90: unpack + register only, no * scan — its status never carries a file or @@ -1073,20 +1108,39 @@ export async function mount(root) { } /* Phase 98 (task 02, D2) + Phase 128 (task 01, the PARTS rework) - * + Phase 152 (task 03): the SYNC job's phase-aware running label — + * + Phase 152 (task 03) + Phase 194 (task 03, D4/A2): the SYNC job's + * phase-aware running label — * the post-import span where the file count sits still (the owner's - * "the number pauses for a really long time"): "overview" → - * "Writing KB overview…" (no count part); "summaries" → main + * "the number pauses for a really long time"). + * "overview" → "Writing KB overview…" (no count part); "summaries" → main * "Summarizing folders… " + count "(n/m)" — the * folder part is omitted while current_summary is null (the phase's * first poll) and the COUNT IS ALWAYS EMITTED in this phase (the * hook always carries a total) and the CHUNK and IMAGE parts stay * empty (the summaries phase's own (n/m) is its position — the * status's chunk + image fields are cleared at this transition); - * anything else (the null prelude, "import") → the phase-64 + * "scan" (phase 194) → main "Scanning…" — NEVER a file, because + * `current_file` is null while the importer pre-scans and this + * branch never reads it — + count "(n/total)" over the status's OWN + * scan pair (scan_done / scan_total, the pre-scan's walk + * position), in the SAME count slot the import phase's + * `N remaining` uses (so the hidden-toggle + the A4 untruncated + * placement — button title + #sync-progress — carry "Scanning… + * (n/total)" automatically), with the phase-64 `scan_total === 0` + * guard (an empty KB, or a scan that has not reported its + * denominator yet → the bare "Scanning…", no count part) and NO + * chunk/image part (the scan reads no file: it embeds no chunk and + * describes no image). The walk position surviving as "(n/total)" + * is HONEST for a scan: the owner's objection (TODO L6) was the + * SYNC count wearing the total's shape — "1216 remaining" on a + * ~10,000-file KB — and this label says "Scanning", never + * "Syncing"; anything else — ONLY the two states left now that + * "scan" names itself (the null prelude and "import") → the phase-64 * fmtSyncLabel parts (same builder, same fields, same order — - * since phase 193 its count part reads `N remaining`, N = - * total - done, hidden at the final file; the summaries (n/m) + * since phase 193 its count part reads `N remaining` with + * N = total - done, hidden at the final file; since phase 194 that + * pair is the position in the pre-scan's PREDICTED change set, so + * what counts down is the WORK and not the walk; the summaries (n/m) * folder position and the chunk/image parts below stay as they * are) + the phase-129 CHUNK part: "· chunk n/m" ONLY while * chunks_total > 0 (a file's embed batch is in flight — n the @@ -1105,6 +1159,25 @@ export async function mount(root) { * title + #sync-progress like every running label (A4 — * setSyncLabel). */ function fmtSyncPhaseLabel(status) { + if (status.phase === "scan") { + // Phase 194 (task 03, D4/A2): the pre-scan is its OWN visible + // phase, so it gets its OWN branch. The pair here is the scan's + // walk position — the honest home for an (n/total), because the + // word in front of it is "Scanning"; the sync label's count part + // stays the phase-193 count-down (fmtSyncLabel, untouched). The + // runner reports no file at all in this phase and this branch + // never reads one; the scan names no file, so no chunk part and + // no image part. + return { + main: "Scanning…", + count: + status.scan_total > 0 + ? `(${status.scan_done}/${status.scan_total})` + : "", // 0/0: the empty KB + the scan's own first frame + chunk: "", // the scan embeds nothing — no chunk position + image: "", // the scan describes nothing — no image position + }; + } if (status.phase === "overview") { return { main: "Writing KB overview…", count: "", chunk: "", image: "" }; } @@ -1117,8 +1190,10 @@ export async function mount(root) { image: "", // the image position belongs to the import phase only }; } - // null phase (the prelude) or "import" — the phase-64 label - // shape (since phase 193 its count part reads `N remaining`) + + // null phase (the prelude) or "import" — and since phase 194 ONLY + // those two ("scan" has its own branch above) — the phase-64 label + // shape (since phase 193 its count part reads `N remaining`, and + // since phase 194 that pair is the pending position) + // the phase-129 // chunk part (hidden before the in-flight file's first batch) + // the phase-152 image part (hidden before the in-flight PDF's @@ -1442,8 +1517,10 @@ export async function mount(root) { } // 1. sync running: the phase-aware label (phase 98 task 02 — the // null prelude + "import" keep the byte-identical file label, - // "overview" / "summaries" name themselves; the WHOLE status - // goes in for the phase fields). + // "scan" / "overview" / "summaries" name themselves; the WHOLE status + // goes in for the phase fields — the scan pair (scan_done / + // scan_total, phase 194) rides it exactly like the summaries, + // chunk and image pairs do, so no field is re-listed here). if (syncStatus.state === "running") { enterSyncRunningState( "sync", syncStatus.current_file, syncStatus.files_done, syncStatus.files_total, @@ -1604,8 +1681,11 @@ export async function mount(root) { * the current catalog). Phase 98 (task 02): a RUNNING sync re-enters * with the WHOLE status — the label is phase-aware, so a mid- * summaries reload re-enters with the summaries label (the - * never-stale contract); a RUNNING upload re-attach stays on the - * bare label (its status has no phase — D2). */ + * never-stale contract) and, since phase 194, a reload during the + * pre-scan re-enters with "Scanning… (n/total)" — the scan pair rides + * the SAME whole-status pass-through as every other pair; a RUNNING + * upload re-attach stays on the bare label (its status has no phase — + * D2). */ async function initSyncButton() { if (!syncBtn) return; if (!(await fetchIsAdmin())) return; diff --git a/tests/e2e/test_sync_chunk_progress.py b/tests/e2e/test_sync_chunk_progress.py index b22ac81..b242d82 100644 --- a/tests/e2e/test_sync_chunk_progress.py +++ b/tests/e2e/test_sync_chunk_progress.py @@ -67,7 +67,12 @@ tiny chunk). The sorted walk processes ``big.md`` FIRST, so the label's phase-193 count part reads ``1 remaining`` during the big span and is HIDDEN during the small one (the final file: N = total - done = 0 — the endpoint's ``files_done``/``files_total`` stay the 1-based 1/2 then 2/2 -position; only the LABEL counts down) — and the small file's single +position; only the LABEL counts down). Phase 194 moves where that pair +comes FROM — the pre-scan's PREDICTED change set instead of the walk — +and changes nothing here, because this fixture is a FRESH import: both +files are pending, so the pending position IS the walk position (see +``tests/e2e/test_sync_remaining_count.py`` for the arm where they +diverge) — and the small file's single chunk is the proof that the chunk position RESTARTS per file (the big file ends at its last batch's first index, the small file reports ``(1, 1)``). diff --git a/tests/e2e/test_sync_peak_memory_visible.py b/tests/e2e/test_sync_peak_memory_visible.py index defac57..32cc0a8 100644 --- a/tests/e2e/test_sync_peak_memory_visible.py +++ b/tests/e2e/test_sync_peak_memory_visible.py @@ -231,12 +231,19 @@ PRE_187_STATUS_KEYS = frozenset( ) MEMORY_KEYS = frozenset({"rss_mb", "rss_peak_mb"}) +#: Phase 194 (task 02): the two additive PRE-SCAN position keys — the next +#: link of the same additive chain (the payload is exactly the pre-187 set +#: plus the memory pair plus the scan pair). +SCAN_KEYS = frozenset({"scan_done", "scan_total"}) + #: The pre-phase integer count slots (their types are part of the #: unchanged-shape pin in test 2 — never a float, never null). _COUNT_KEYS = frozenset( { "files_done", "files_total", + "scan_done", + "scan_total", "current_chunk", "chunks_total", "images_done", @@ -643,8 +650,9 @@ def test_status_payload_reports_rss(page: Page, app_url: str, db_ready: None) -> running = _wait_sync_state(page, app_url, {"running"}) assert running["state"] == "running", running # The additive contract on the wire: exactly the pre-phase keys plus - # the memory pair — nothing renamed, nothing dropped. - assert set(running) == PRE_187_STATUS_KEYS | MEMORY_KEYS, set(running) + # the memory pair plus the phase-194 scan pair — nothing renamed, + # nothing dropped. + assert set(running) == PRE_187_STATUS_KEYS | MEMORY_KEYS | SCAN_KEYS, set(running) for key, value in running.items(): if key in MEMORY_KEYS: # The documented type: int | null (null is "no reading"). @@ -674,7 +682,7 @@ def test_status_payload_reports_rss(page: Page, app_url: str, db_ready: None) -> # --- after the terminal --------------------------------------------- terminal = _wait_sync_state(page, app_url, {"success", "failed", "cancelled"}) assert terminal["state"] == "success", terminal - assert set(terminal) == PRE_187_STATUS_KEYS | MEMORY_KEYS, set(terminal) + assert set(terminal) == PRE_187_STATUS_KEYS | MEMORY_KEYS | SCAN_KEYS, set(terminal) current, peak = terminal["rss_mb"], terminal["rss_peak_mb"] if MEMORY_KNOWABLE: assert peak is not None and isinstance(peak, int), terminal diff --git a/tests/e2e/test_sync_pending_count.py b/tests/e2e/test_sync_pending_count.py new file mode 100644 index 0000000..a5dd50a --- /dev/null +++ b/tests/e2e/test_sync_pending_count.py @@ -0,0 +1,1999 @@ +"""Phase 194 task 05 — the phase's own E2E suite: the count counts the WORK. + +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_sync_pending_count.py -v --no-cov + +**The owner report (2026-10-19, TODO L6).** A sync on a roughly 10,000-file +knowledge base with a handful of changes counted down +``10000 → 9999 → … → 1216 remaining``. There was almost nothing left to DO at +1216; the sentence still read like there was, because the number was a WALK +POSITION wearing the shape of a work queue (``app/rag/importer.py``'s +per-file counter → ``app/api/sync.py``'s ``files_done`` / ``files_total`` → +``frontend/assets/sources.js``'s ``fmtSyncLabel`` ``N remaining`` part). + +Phase 194 moved the count onto the WORK and gave the walk its own phase: + +* ``app/rag/importer.py`` predicts the run's change set BEFORE the walk + (``_prescan_pending``: the streamed 1 MiB-chunk ``_file_hash`` against + ``_pending_hash_stmt``, a SCALAR ``SELECT documents.file_hash`` on a + session of its own, off the event loop in ONE ``asyncio.to_thread`` span), + and the ``progress`` hook reports the position in THAT set; +* ``app/api/sync.py`` reports ``phase: "scan"`` for the pre-scan with + ``scan_done`` / ``scan_total`` beside it, and flips to ``"import"`` on the + FIRST ``progress`` call inside the one existing ``_hook`` closure; +* ``frontend/assets/sources.js``'s ``fmtSyncPhaseLabel`` renders the scan as + ``Scanning… (n/total)`` in the SAME count slot the import's + ``N remaining`` uses — one writer, one branch, no new element. + +Fixture (the house convention, the ``test_sync_remaining_count.py`` shape) +------------------------------------------------------------------------- + +A real git repo over the ``file://`` transport (``BOR_GIT_SOURCES``, so the +env fallback IS the effective source under the autouse truncate) with +``N_FILES = 8`` markdown documents in walk order (``sorted(rel)``, the +importer's rule) — the FIRST and the LAST multi-chunk so the phase-129 +``· chunk n/m`` part coexists with the count part and then stands alone — +and a mock upstream that delays EVERY call ``SLOW_DELAY_S`` +(``tests/e2e/slow_llm.py``) so one single-chunk file's import span +(3 calls ≈ 3.6 s) outlives the page's 2 s status poll. A SECOND module app, +same repo shape, boots with ``BOR_SYNC_PENDING_COUNT=0`` (the phase-184 +two-module-app convention) and is arm d's rollback machine. The KB state a +re-sync needs is established with the importer ITSELF, in-process, with the +``FakeEmbedder`` (the same code path, the same stamped ``file_hash``, the +same git commit dates) — a real slow-proxy first import of a 1,000-file KB +would turn the module into ten minutes for no extra evidence: every run +UNDER test is a real uvicorn sync driven from the real Sources page over +HTTP. + +Determinism — what is pinned live and what is pinned on the exact payload +------------------------------------------------------------------------ + +The scan reads no file and embeds nothing, so it is FAST: eight tiny files +pre-scan in single-digit milliseconds and the 2 s poll cannot be made to land +inside that window. This suite does not pretend otherwise and does not pad +the fixture to fake it: + +* the **live** scan pins are the ones a real run answers DETERMINISTICALLY — + every tick caught in ``phase == "scan"`` names NO file, carries the full + denominator, and no scan tick ever follows a tick that already counted + files; the terminal KEEPS the completed pair; and the + **``test_prod_shaped_run``** arm (a seeded 1,000-file KB, 2 files changed — + the owner's report at scale) makes the scan window long enough that its + position is caught climbing across several distinct ``scan_done`` values + THROUGH the running server while the loop keeps answering (the phase-179 + contract — the scan is off the loop precisely so this poll is served); +* the ``Scanning… (n/total)`` **rendered** frames — and arm b's + unchanged-file HOLD — are pinned on the exact payloads the client's own + guards read, served through ``page.route`` to the real + ``sources.js`` + the real 2 s poll (``test_sync_remaining_count.py`` test 2 + and the phase-129/152 suites' pattern). The pair itself is pinned against + the real importer in ``tests/integration/test_sync_pending_count.py`` + (its hook sequence IS ``[(changed,1,2), (unchanged,1,2), (unchanged,1,2), + (changed,2,2)]``) and through the live status machine in + ``tests/integration/test_sync_scan_phase.py``; here it is pinned against + the guards that turn it into a sentence. + +Arms (the task file's a–e + the two standing contracts), in test order +---------------------------------------------------------------------- + +1. ``test_first_sync_counts_the_predicted_set_and_the_machine_completes`` — + **(a) the first sync, fresh KB:** pending == walk, so the label counts the + phase-193 arithmetic exactly — ``7 remaining`` down to ``1``, the part + HIDDEN at the final file while the main part still names it, the chunk + part coexisting — and the wire's null prelude reports ``0/0``, every tick + is the additive key set, every import tick reports the whole 8-file + denominator, the terminal keeps ``scan_done == scan_total == 8`` beside + ``files_done == 8`` and settles to ``8 added``. +2. ``test_incremental_sync_counts_only_the_changed_files`` — **(b) the + owner's production case, the phase's heart:** the KB already holds all 8 + documents, TWO files changed. Every running tick answers the PREDICTED + denominator (``files_total == 2``, never ``8``) and the pair HOLDS across + the six unchanged files (the exact expected pair per walked file, so the + pin does not depend on which frames the 2 s poll happened to sample); + rendered, the part reads ``1 remaining`` across the first predicted file + and is GONE across the final predicted file while the main part names it; + no label ever shows a walk-derived number; the terminal is + ``2 updated · 6 unchanged`` with ``2/2`` files and the completed + ``8/8`` scan kept beside it. +3. ``test_scan_frames_and_the_hold_render_from_the_exact_payloads`` — the + same two machines rendered by the REAL client on the frames the live run + produces but the 2 s poll cannot be aimed at: two CONSECUTIVE + unchanged-file frames at the same ``2 remaining`` then the first + predicted file at ``1 remaining`` then the final predicted file with the + part gone; ``Scanning… (3/8)`` → ``(8/8)`` with no file and never + ``N remaining``; ``scan_total: 0`` → the BARE ``Scanning…``; the null + prelude → the bare ``Syncing…``; every frame A4-complete (button + ``title`` + the sole ``#sync-progress`` live region byte-equal to the + text) and no ``Syncing…`` label ever wearing an ``(n/m)`` tail. +4. ``test_kill_switch_zero_restores_the_walk_position_count`` — **(d):** the + second app, arm b's fixture. No ``scan`` tick exists at all, ``scan_done`` + / ``scan_total`` are ``0/0`` for the whole run including the terminal, and + the count part counts the WALK: ``files_total == 8`` and the rendered + figure naming files that will never be re-indexed (arm b answers ``1 + remaining`` for the same file at the same moment in the same repository). +5. ``test_no_change_sync_reports_no_denominator`` — **(c):** the re-sync of + a repository nothing touched. The scan completes (``8/8`` kept), the + predicted set is EMPTY so every tick answers ``files_total == 0`` and no + label EVER carries a count part (the phase-64 ``total === 0`` guard driven + by zero work rather than by a missing denominator), and the terminal is + the byte-identical ``0 added · 8 unchanged``. +6. ``test_empty_source_stays_in_scan_with_no_count_part`` — **(e) the D1 + degenerate:** a source with ZERO importable files. Nothing is walked, so + no ``progress`` call exists to flip the phase: the machine never answers + ``import``, both pairs stay ``0/0`` through the run, no label ever carries + a count part, and the terminal clears the phase and settles to + ``0 added · 0 unchanged``. +7. ``test_prod_shaped_run_scan_climbs_then_counts_two`` — the report AT + SCALE: a seeded 1,000-file KB with 2 files changed, over the real page. + The scan's own position climbs across ≥2 distinct ``scan_done`` values + while ``current_file`` stays null, the import then answers the PREDICTED + denominator (``files_total == 2`` against ``scan_total == 1000`` — the + divergence the pre-phase shape could not express), and the terminal keeps + BOTH pairs beside each other. +8. ``test_upload_run_carries_no_scan_fields_and_no_count_part`` — the + phase-90 (A2) convention with the scan pair ADDED to the not-there list: + a real background upload's live status answers the bare phase-64 key set + (no ``phase``, no chunk/image/summary fields, no ``scan_*``, no memory + pair) with ``null``/``0``/``0``, and the shared label renders that job + bare — no file, no count part (the upload branch never receives a status + object, so phase 194 gives it nothing). +The phase-187 peak clause, across these same runs (AGENTS rule 10 — the +clause is TERMINAL text; task 05's "visible while the scan label is showing" +is read against that rule, so what is pinned is the clause's presence at the +terminal of a scan-labelled run and its ABSENCE on every running label): test +1 pins the real run's settle — the counts line ends in ``· peak MB`` equal +to the terminal payload's ``rss_peak_mb`` (one reading, two surfaces), as TEXT +on the line that already exists, with the box holding its ONE child span and +its ``role=status`` / ``aria-live=polite`` intact; test 3 pins the served +corollary — a running frame that CARRIES ``rss_mb`` / ``rss_peak_mb`` renders +NO clause, and the terminal of that same served sequence renders +``0 added · 2 updated · 6 unchanged · peak 1234 MB`` (a served payload answers +the clause independently of this host, so the shape is pinned even where +``MEMORY_KNOWABLE`` is False). + +Test → task mapping (the Playwright Mapping Rule): 1 → arm a + the real-run +peak clause, 2 → arm b, 3 → arms a/b's rendered frames + the A4 placement + +the peak-clause absence-while-running, 4 → arm d, 5 → arm c, 6 → arm e, +7 → arm a's scan-climb at prod shape (arm b's denominator at scale), +8 → the upload pin. +""" + +from __future__ import annotations + +import asyncio +import io +import json +import os +import re +import subprocess +import sys +import tarfile +import threading +import time +from collections.abc import Iterator +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import httpx +import pytest +from playwright.sync_api import Page, expect +from sqlalchemy import text + +from app.config import Settings +from app.core.memory import rss_mb, rss_peak_mb +from app.db import SessionLocal +from app.rag.chunker import chunk_document +from app.rag.importer import import_sources +from e2e.auth_helpers import login +from e2e.conftest import ( + ADMIN_PASSWORD, + MOCK_PORT, + SESSION_SECRET, + USE_REAL_LLM, + _wait_http, +) +from e2e.sync_helpers import counts_line +from scripts.git_sync import file_commit_dates +from tests.fakes import FakeEmbedder + +REPO = Path(__file__).resolve().parents[2] + +#: This module's own ports — the sync suites' ladder continues past the +#: phase-193 suite's 8211/8914 (the one-suite-at-a-time rule makes a +#: collision harmless, but these are free and env-overridable). +APP_PORT = int(os.environ.get("E2E_APP_PORT_PENDING", "8212")) +APP_URL = f"http://127.0.0.1:{APP_PORT}" +ZERO_PORT = int(os.environ.get("E2E_APP_PORT_PENDING_ZERO", "8213")) +ZERO_URL = f"http://127.0.0.1:{ZERO_PORT}" +SLOW_PORT = int(os.environ.get("E2E_SLOW_LLM_PORT_PENDING", "8915")) +SLOW_URL = f"http://127.0.0.1:{SLOW_PORT}" + +SOURCES_URL = "/sources.html" + +#: Per-LLM-request delay on the proxy. A one-chunk file costs THREE calls +#: (content embed batch + the summary chat + the summary-chunk embed) ≈ 3.6 s +#: against the page's 2 s poll, so every file gets a rendered label of its +#: own — and an UNCHANGED file whose summary is NULL costs TWO (the phase-118 +#: backfill), which is what makes an unchanged file's HELD frame observable +#: at all (arm b). +SLOW_DELAY_S = "1.2" + +#: The module's wall-time budget (two app boots + three real syncs of the +#: 8-file repo + the 1,000-file prod-shaped run + the served-status arms). A +#: breach is a speed canary, not flake tolerance (the phase-64 convention). +WALL_BUDGET_S = 600.0 + +#: The sync's subject: a git repo whose DIRECTORY NAME is the source name +#: (the ``repo_name`` basename rule). Deliberately NOT named after anything +#: under test — a path containing "remaining" would make every "does this +#: label carry the count part?" check match the path itself. +SOURCE = "pending" +N_FILES = 8 + +#: The walk order is ``sorted(rel)`` (the importer's rule), so each file's +#: 1-based WALK position is its rank here — the map the arms assert against. +#: The FIRST and the LAST file are multi-chunk (the phase-129 convention: the +#: chunk part climbs on the first file and stands alone, without a count +#: part, on the last). +REL_ORDER = [ + "notes/a-multi-first.md", + "notes/b-second.md", + "notes/c-third.md", + "notes/d-fourth.md", + "notes/e-fifth.md", + "notes/f-sixth.md", + "notes/g-seventh.md", + "notes/z-multi-last.md", +] +FIRST_REL = REL_ORDER[0] +LAST_REL = REL_ORDER[-1] + +#: Arm b's change set: walk positions 5 and 7 of 8. The LAST predicted file +#: is therefore neither the last walked file nor the first — the count part +#: holds at ``2 remaining`` across the unchanged files before it, reads +#: ``1 remaining`` across the first predicted one and disappears across the +#: final predicted one while the walk still has a file to go (the exact shape +#: the pre-phase count read "``N`` remaining" through). +CHANGED_RELS = ("notes/e-fifth.md", "notes/g-seventh.md") + +#: Arm b's UNCHANGED files whose stored summary the seed clears — walk +#: positions 1 and 2, both BEFORE the first predicted file. Walking one still +#: costs the phase-118 summary backfill (a ``lite`` call + its embed ≈ 2.4 s +#: at this delay), which is what makes an unchanged file's HELD frame land in +#: the browser instead of living only in the integration suite. +HELD_RELS = ("notes/a-multi-first.md", "notes/b-second.md", "notes/d-fourth.md") + +#: Arm d (the ``BOR_SYNC_PENDING_COUNT=0`` app) changes positions 5 and 8 of +#: the same fixture: under the rollback semantics the pair is the WALK, so the +#: file arm b renders as "``1`` remaining" appears here as "``3`` remaining" +#: and the last walked file is the only one that hides the part. +ZERO_CHANGED_RELS = ("notes/e-fifth.md", "notes/z-multi-last.md") + +#: The count-bearing values the label may ever show (A1): ``1 … N−1``. +MAX_COUNT = N_FILES - 1 + +#: The prod-shaped arm's KB (the owner's report at scale): 1,000 unchanged +#: documents walked, TWO of them changed. A ``kind="local"`` source row, so +#: the walk reads the seeded directory itself (no clone, no commit dates — +#: the importer's mtime fallback) and the run's ONLY model work is the two +#: changed files. +WIDE_SOURCE = "pendingwide" +WIDE_FILES = 1_000 +WIDE_CHANGED_RELS = ("notes/w0003.md", "notes/w0008.md") + +#: The archive the upload arm uploads (the phase-98 fixture convention). +UPLOAD_NAME = "uploadkb" +UPLOAD_FILES: dict[str, str] = { + f"docs/{i:02d}.md": f"# Upload doc {i:02d}\n\nDeterministic upload body {i:02d}.\n" + for i in range(3) +} + +#: The chunker settings the apps under test are pinned to (the leak-guard +#: convention): the expected ``chunks_total`` is computed with the SAME +#: function and arguments the importer uses, so a local ``.env`` cannot move +#: the number this suite asserts. +CHUNK_TARGET = int(Settings.model_fields["chunk_target_chars"].default) +CHUNK_OVERLAP = int(Settings.model_fields["chunk_overlap_chars"].default) + +#: The import-phase label grammar (phase 193) — the main part (the bare +#: ``Syncing…`` of the null prelude, which names no file, or the prefix plus +#: the ``source/path``), the OPTIONAL count part, the OPTIONAL phase-129 chunk +#: part. The arms that pin a NAMED file re-anchor it on their own paths. +IMPORT_LABEL_RE = re.compile( + r"^Syncing…(?: \S+(?: [0-9]+ remaining)?)?(?: · chunk \d+/\d+)?$" +) +#: The scan-phase label grammar phase 194 (task 03, D4) adds: the bare +#: ``Scanning…`` (no denominator yet) or it plus the ``(n/total)`` count +#: part. NEVER a file (``current_file`` is null through the phase) and never +#: ``N remaining`` — the walk position is honest behind the word "Scanning", +#: never behind "Syncing" (TODO L6). +SCAN_LABEL_RE = re.compile(r"^Scanning…(?: \(\d+/\d+\))?$") +#: What phase 194 leaves forbidden: no ``Syncing…`` label may wear an +#: ``(n/m)`` tail again (the shape is legal only on ``Scanning…`` and +#: ``Summarizing folders…``). +OLD_FRACTION_RE = re.compile(r"\(\d+/\d+\)") +SYNCED_LABEL = re.compile(r"^Synced \d{1,2}:\d{2}$") + +#: The FULL ``GET /api/sync/status`` key set — the additive chain's right-hand +#: side as of phase 194 (phase 64/98/129/152 fields + the phase-187 memory +#: pair + the phase-194 scan pair). ONE constant, mirrored by the served stubs +#: below and pinned on every live tick, so a future key has to be added here +#: (and to the stub) instead of drifting away from the client's view. +EXPECTED_STATUS_KEYS = frozenset( + { + "state", + "started_at", + "finished_at", + "current_file", + "files_done", + "files_total", + "scan_done", + "scan_total", + "phase", + "current_summary", + "summaries_done", + "summaries_total", + "current_chunk", + "chunks_total", + "images_done", + "images_total", + "detail", + "error", + "rss_mb", + "rss_peak_mb", + } +) + +#: The UPLOAD status key set (phase 90 A2): the bare sync subset, and the +#: whole point of the arm — no ``phase``, no chunk / image / summary fields, +#: no memory pair and (new in phase 194) NO ``scan_done`` / ``scan_total``. +UPLOAD_STATUS_KEYS = frozenset( + { + "state", + "started_at", + "finished_at", + "current_file", + "files_done", + "files_total", + "detail", + "error", + } +) + +#: The phase-187 clause, and the shape it must NEVER take on a running label. +PEAK_CLAUSE_RE = re.compile(r" · peak (\d+) MB$") + +#: Does THIS platform answer the memory helpers? The app under test is a +#: subprocess of this process's own OS, so this answer is the app's answer +#: (the phase-187 suite's reasoning). Where it is False the real-run clause +#: pin steps aside and the ABSENCE contract is pinned instead — the served +#: arm keeps the clause's SHAPE pinned either way, because a served payload +#: answers independently of this host. +MEMORY_KNOWABLE = rss_mb() is not None and rss_peak_mb() is not None + +SETTLE_TIMEOUT_MS = 240_000 + + +def _now_iso() -> str: + """A "just finished" stamp for the served terminals — the phase-154 + same-day rule renders it as the plain ``Synced HH:MM``.""" + return datetime.now(UTC).isoformat() + + +# --------------------------------------------------------------------------- +# The fixture KB +# --------------------------------------------------------------------------- + + +def _tiny_md(title: str) -> str: + """A one-chunk document (its import span = 3 slowed LLM calls).""" + return f"# {title}\n\nDeterministic one-chunk body for the {title}.\n" + + +def _multi_md(tag: str, sections: int = 3) -> str: + """A multi-chunk document: an H1 preamble (one chunk) + *sections* ``##`` + sections of ~1500 chars each (one chunk apiece at the code-default + target, one embed batch apiece at the 700-token request budget) — no + randomness, so the chunk count is stable run to run (the phase-129 + fixture idiom).""" + parts = [f"# {tag} multi-chunk fixture document\n"] + parts.append( + f"Preamble paragraph of the {tag} pending-count fixture. This paragraph " + "sits under the H1 and chunks on its own.\n" + ) + for k in range(1, sections + 1): + parts.append(f"\n## Section {k}\n") + base = ( + f"Section {k} paragraph of the {tag} pending-count fixture carries " + "deterministic filler for the chunker: alpha bravo charlie delta " + "echo foxtrot golf hotel india juliet kilo lima mike november oscar " + "papa quebec romeo sierra tango uniform victor whiskey xray yankee " + "zulu number " + ) + parts.append((base * 4)[:1500] + "\n\n") + return "".join(parts) + + +def _fixture_files() -> dict[str, str]: + files = {rel: _tiny_md(Path(rel).stem) for rel in REL_ORDER} + files[FIRST_REL] = _multi_md("first") + files[LAST_REL] = _multi_md("last") + return files + + +def _expected_content_chunks(rel: str) -> int: + """The file's TRUE content-chunk count — the SAME function and arguments + the importer's chunk replacement uses (phase 129: ``chunks_total`` is a + chunk count, never a batch count).""" + return len(chunk_document(_fixture_files()[rel], rel, max(400, CHUNK_TARGET), CHUNK_OVERLAP)) + + +def _wide_files() -> dict[str, str]: + """The prod-shaped fixture: ``WIDE_FILES`` one-chunk documents under + ``notes/`` (the walk order is the zero-padded stem, so the changed pair + lands at WALK positions 3 and 8).""" + return { + f"notes/w{i:04d}.md": _tiny_md(f"wide document {i:04d}") + for i in range(1, WIDE_FILES + 1) + } + + +def _git(cwd: Path, *args: str) -> None: + proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + if proc.returncode != 0: + raise AssertionError(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + + +def _commit(cwd: Path, msg: str) -> None: + _git(cwd, "add", "-A") + _git( + cwd, + "-c", "user.email=e@x", "-c", "user.name=t", "-c", "commit.gpgsign=false", + "commit", "-qm", msg, + ) + + +def _write_repo(root: Path, files: dict[str, str], msg: str) -> Path: + root.mkdir(parents=True, exist_ok=True) + for rel, content in files.items(): + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + _git(root, "init", "-q") + _commit(root, msg) + assert (root / ".git").is_dir() + return root + + +@pytest.fixture(scope="module") +def pending_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + """The real repo the sync clones over ``file://`` — 8 markdown docs in + the walk order ``REL_ORDER``.""" + root = tmp_path_factory.mktemp("bor_pending") / SOURCE + _write_repo(root, _fixture_files(), "pending-count fixture") + assert len(list(root.rglob("*.md"))) == N_FILES + return root + + +@pytest.fixture(scope="module") +def zero_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + """The kill-switch app's OWN repo (same shape, different source name): + arms b and d change files in their own repo, so neither arm's committed + state depends on the other's order.""" + root = tmp_path_factory.mktemp("bor_pending_zero") / "pendingzero" + _write_repo(root, _fixture_files(), "pending-count rollback fixture") + assert len(list(root.rglob("*.md"))) == N_FILES + return root + + +@pytest.fixture(scope="module") +def wide_kb(tmp_path_factory: pytest.TempPathFactory) -> Path: + """The prod-shaped KB: 1,000 tiny docs in a plain DIRECTORY, registered + per test as a ``kind=local`` row. The walk reads the seeded directory + itself (no clone, no commit dates — the importer's mtime fallback), which + is the cheap, honest way to give the pre-scan real work: the alternative + is a git repo whose 1,000-file clone costs more than the scan under + test.""" + root = tmp_path_factory.mktemp("bor_pending_wide") / WIDE_SOURCE + root.mkdir(parents=True, exist_ok=True) + for rel, content in _wide_files().items(): + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + assert len(list(root.rglob("*.md"))) == WIDE_FILES + return root + + +@pytest.fixture(scope="module") +def empty_kb_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Arm e's source: a directory with ZERO importable files — one visible + file whose extension is not in the A9 family (so "nothing importable" is + a property of the CONTENT, not of an empty directory) and one hidden + ``.md`` the walk skips. Registered per test as a ``kind=local`` row (the + phase-49 contract: a stored row IS the effective source, which is what + lets this arm exist on the module's own app instead of a third boot).""" + root = tmp_path_factory.mktemp("bor_pending_empty") / "pendingempty" + root.mkdir(parents=True, exist_ok=True) + (root / "README.gitkeep").write_text("nothing here is importable\n", encoding="utf-8") + (root / "notes").mkdir() + # A REAL dotfile (the walk's hidden rule), not merely a file about + # hiding — the arm's premise is a source with NOTHING importable. + (root / "notes" / ".hidden.md").write_text( + "# hidden\n\nSkipped: dotfile.\n", encoding="utf-8" + ) + return root + + +@pytest.fixture(scope="module") +def upload_archive(tmp_path_factory: pytest.TempPathFactory) -> Path: + root = tmp_path_factory.mktemp("bor_pending_archive") + path = root / f"{UPLOAD_NAME}.tar.gz" + with tarfile.open(path, "w:gz") as tf: + for rel, content in UPLOAD_FILES.items(): + data = content.encode("utf-8") + info = tarfile.TarInfo(rel) + info.size = len(data) + info.mtime = 0 + tf.addfile(info, io.BytesIO(data)) + return path + + +# --------------------------------------------------------------------------- +# The apps +# --------------------------------------------------------------------------- + + +def _app_env( + base_url: str, + source: str, + checkout_dir: Path, + upload_dir: Path, + *, + pending_count: str | None = None, +) -> dict[str, str]: + """The per-module app env (the conftest pattern): the slow proxy is the + LLM, ``BOR_GIT_SOURCES`` is a suite-owned ``file://`` repo (the env + fallback IS the effective source — the autouse truncate keeps the + ``git_sources`` table empty), and the chunker knobs are pinned to the + code defaults so the expected ``chunks_total`` stays predictable. + + ``pending_count`` is the phase's ONE switch: ``None`` leaves + ``BOR_SYNC_PENDING_COUNT`` at the default (``1`` — count the predicted + change set), ``"0"`` is the exact phase-193 rollback arm d drives. + """ + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else base_url + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15" + env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15" + env["BOR_LLM_RETRY_DELAY"] = "0" + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + env["BOR_GIT_SOURCES"] = f"file://{source}" + env["BOR_SOURCES_DIR"] = str(checkout_dir) + env["BOR_UPLOAD_DIR"] = str(upload_dir) + env["BOR_CHUNK_TARGET_CHARS"] = str(CHUNK_TARGET) + env["BOR_CHUNK_OVERLAP_CHARS"] = str(CHUNK_OVERLAP) + env["BOR_DOCS_REPO"] = "" + env["BOR_SUGGESTIONS"] = json.dumps(["Ask something neutral"]) + if pending_count is not None: + env["BOR_SYNC_PENDING_COUNT"] = pending_count + return env + + +def _spawn_uvicorn(app: str, port: int, env: dict[str, str], probe: str) -> Iterator[None]: + """One uvicorn subprocess, terminated on teardown (the conftest body, + factored so this module can own TWO apps without duplicating the + lifecycle). The caller yields its OWN base URL — that is what the suite's + ``app_url`` hands to the page.""" + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", app, "--host", "127.0.0.1", + "--port", str(port), "--log-level", "warning"], + cwd=REPO, + env=env, + ) + try: + _wait_http(probe) + yield + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +def _spawn_slow_proxy(port: int, delay_s: str, probe: str) -> Iterator[int]: + """The delay-injecting proxy (``tests/e2e/slow_llm.py``) in front of the + mock LLM — the suite's timing fixture: it delays EVERY LLM call, embeds + included, so each file's import span outlives the UI's 2 s poll.""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["SLOW_LLM_DELAY_S"] = delay_s + env["E2E_MOCK_PORT"] = str(MOCK_PORT) + for _ in _spawn_uvicorn( + "tests.e2e.slow_llm:app", port, env, f"{probe}/v1/models" + ): + yield port + + +@pytest.fixture(scope="module") +def slow_llm(mock_llm: int) -> Iterator[int]: + yield from _spawn_slow_proxy(SLOW_PORT, SLOW_DELAY_S, SLOW_URL) + + +@pytest.fixture(scope="module") +def app_server( + mock_llm: int, + slow_llm: int, + pending_repo: Path, + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[str]: + """The app under test — the DEFAULT ``BOR_SYNC_PENDING_COUNT`` (1).""" + env = _app_env( + f"{SLOW_URL}/v1", + str(pending_repo), + tmp_path_factory.mktemp("bor_pending_checkouts"), + tmp_path_factory.mktemp("bor_pending_uploads") / "uploads", + ) + for _ in _spawn_uvicorn("app.main:app", APP_PORT, env, f"{APP_URL}/api/health"): + yield APP_URL + + +@pytest.fixture(scope="module") +def zero_server( + mock_llm: int, + slow_llm: int, + zero_repo: Path, + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[str]: + """The SECOND module app — the same fixture shape booted with + ``BOR_SYNC_PENDING_COUNT=0`` (the phase-184 two-module-app convention: + one setting, two live servers, the owner-visible rollback proven instead + of asserted about).""" + env = _app_env( + f"{SLOW_URL}/v1", + str(zero_repo), + tmp_path_factory.mktemp("bor_pending_zero_checkouts"), + tmp_path_factory.mktemp("bor_pending_zero_uploads") / "uploads", + pending_count="0", + ) + for _ in _spawn_uvicorn("app.main:app", ZERO_PORT, env, f"{ZERO_URL}/api/health"): + yield ZERO_URL + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +def _truncate_all() -> None: + """Fresh registry + KB per test (the E2E isolation pattern): the suites + share one Postgres, and a leftover document would move both the ``added`` + counts the terminal line pins AND the pre-scan's prediction.""" + with SessionLocal() as db: + db.execute( + text( + "TRUNCATE chunks, documents, query_log, steering_notes, " + "kb_overview, git_sources, folder_summaries" + ) + ) + db.commit() + + +def _wait_sync_settled(base_url: str, timeout_s: float = 300.0) -> None: + """No run may leak across a test boundary (the run state lives in the + app's memory and a running run keeps importing into the NEXT test's + truncated KB). Own admin session; usually a no-op.""" + with httpx.Client(base_url=base_url, timeout=5.0) as client: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, r.text + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + r = client.get("/api/sync/status") + assert r.status_code == 200, r.text + body = r.json() + if body["state"] == "failed": + print(f"sync pending-count: FAILED terminal at a boundary: {body.get('error')}") + if body["state"] != "running": + return + time.sleep(0.2) + raise AssertionError(f"a sync was still running at a test boundary: {base_url}") + + +def _settle(base_url: str) -> None: + """Settle an app that MAY not be up (the zero app boots lazily with its + fixture, and the autouse cleaner must not force it into every test).""" + try: + httpx.get(f"{base_url}/api/health", timeout=2.0).raise_for_status() + except Exception: # noqa: BLE001 — not booted (yet): nothing to settle + return + _wait_sync_settled(base_url) + + +@pytest.fixture(autouse=True) +def _clean(app_url: str, db_ready: None) -> Iterator[None]: + _settle(app_url) + _settle(ZERO_URL) + _truncate_all() + yield + _truncate_all() + + +@pytest.fixture(scope="module", autouse=True) +def _wall_time_guard() -> Iterator[None]: + start = time.monotonic() + yield + elapsed = time.monotonic() - start + assert elapsed < WALL_BUDGET_S, ( + f"the module took {elapsed:.1f}s — over the {WALL_BUDGET_S:.0f}s wall budget" + ) + + +# --------------------------------------------------------------------------- +# The KB state a re-sync starts from: the importer itself, in-process +# --------------------------------------------------------------------------- + + +def _seed_kb( + root: Path, + *, + null_summary_rels: tuple[str, ...] = (), + git: bool = True, +) -> None: + """Index *root* with the importer ITSELF (in-process, ``FakeEmbedder``). + + The same code path a real sync runs, so the rows carry the real + ``file_hash`` stamps the phase-147 fast path compares against — which is + what makes the NEXT (real, in-app, slow-proxy) sync predict exactly the + files this test changed and nothing else. The git arms pass the + checkout's own commit dates, so the run under test finds every date + already correct and ``unchanged`` means unchanged. + + ``null_summary_rels`` then clears those rows' ``summary`` — the + phase-118 (A2) backfill case: the file stays UNCHANGED (its digest + matches, so the pre-scan predicts nothing for it), but walking it still + costs the ``lite`` summary call plus its embed. That is the ONE honest + way to make an unchanged file occupy real, model-bound, loop-free time, + and therefore the way arm b's held frame becomes observable in the + browser instead of theoretical. + + ``git=False`` skips the commit-date map — the seed of a ``kind=local`` + source, whose run uses the importer's mtime fallback on BOTH sides + (matching, so the arm under test reports no date churn either). + """ + dates = {str(root): file_commit_dates(root)} if git else None + outcome: dict[str, BaseException] = {} + + async def _go() -> None: + # The Playwright sync API owns an event loop in this process, so the + # seed gets its OWN loop on its OWN thread. The importer is the same + # async code either way and the session is a plain sync one — this is + # the ``tests/integration/test_sync_pending_count.py`` call, one + # thread over. + with SessionLocal() as inner: + await import_sources( + [root], + FakeEmbedder(), + session=inner, + doc_dates_by_root=dates, + ) + + def _worker() -> None: + try: + asyncio.run(_go()) + except BaseException as exc: # noqa: BLE001 — re-raised on the caller + outcome["error"] = exc + + seed_thread = threading.Thread(target=_worker) + seed_thread.start() + seed_thread.join() + if "error" in outcome: + raise AssertionError( + f"the KB seed import over {root} failed: {outcome['error']!r}" + ) from outcome["error"] + + if null_summary_rels: + with SessionLocal() as db: + db.execute( + text( + "UPDATE documents SET summary = NULL " + "WHERE source = :source AND path = ANY(:rels)" + ), + {"source": root.name, "rels": list(null_summary_rels)}, + ) + db.execute( + text( + "DELETE FROM chunks WHERE is_summary AND document_id IN " + "(SELECT id FROM documents WHERE source = :source " + "AND path = ANY(:rels))" + ), + {"source": root.name, "rels": list(null_summary_rels)}, + ) + db.commit() + + +def _change_files(root: Path, rels: tuple[str, ...], marker: str) -> None: + """Touch *rels* on disk (and commit them, so the ``file://`` clone/pull + the app performs carries the change).""" + for rel in rels: + path = root / rel + path.write_text( + path.read_text(encoding="utf-8") + f"\n{marker} edit of {rel}\n", + encoding="utf-8", + ) + _commit(root, f"{marker}: {', '.join(rels)}") + + +# --------------------------------------------------------------------------- +# The in-browser recorder: every value the ONE label writer lands +# --------------------------------------------------------------------------- + +#: Read in ONE JS task, so a frame can never straddle two writes: the box +#: text (the phase-150 sole sync text surface + sole ``aria-live`` region), +#: the button ``title`` and ``aria-label`` (the phase-64 A4 untruncated +#: placement), and the ``.sync-label-count`` span's own visibility. +_INSTALL_RECORDER = """ +() => { + if (window.__borFrames) return window.__borFrames.length; + window.__borFrames = []; + const box = document.querySelector('#sync-progress'); + const textEl = document.querySelector('#sync-progress-text'); + const btn = document.querySelector('#sync-btn'); + const countEl = document.querySelector('.sync-label-count'); + const rec = () => { + const frame = { + text: textEl.textContent, + title: btn.getAttribute('title') || '', + aria: btn.getAttribute('aria-label') || '', + count: countEl ? countEl.textContent : '', + countHidden: countEl ? countEl.hidden : null, + hidden: box.hidden, + }; + const last = window.__borFrames[window.__borFrames.length - 1]; + if (last + && last.text === frame.text + && last.title === frame.title + && last.aria === frame.aria + && last.countHidden === frame.countHidden) { + return; + } + window.__borFrames.push(frame); + }; + const mo = new MutationObserver(rec); + mo.observe(textEl, { childList: true, characterData: true, subtree: true }); + mo.observe(btn, { attributes: true, attributeFilter: ['title', 'aria-label'] }); + if (countEl) mo.observe(countEl, { attributes: true, attributeFilter: ['hidden'] }); + rec(); + return window.__borFrames.length; +} +""" + + +def _install_recorder(page: Page) -> None: + page.evaluate(_INSTALL_RECORDER) + + +def _frames(page: Page) -> list[dict[str, Any]]: + frames: list[dict[str, Any]] = page.evaluate("() => window.__borFrames") + assert frames, "the in-browser recorder captured no label frames" + return frames + + +def _texts(frames: list[dict[str, Any]]) -> list[str]: + """The recorded label sequence, empty frames dropped (the box's + idle-hide is not a label).""" + return [f["text"] for f in frames if f["text"]] + + +def _count_of(label: str) -> int | None: + """The rendered ``N remaining`` value, or ``None`` when the label carries + no count part (A1's hidden state).""" + m = re.search(r" (\d+) remaining(?: · |$)", label) + return int(m.group(1)) if m else None + + +def _file_of(label: str) -> str | None: + """The document an import label names, RELATIVE to its source (the + fixture's own ``rel`` — the source-name segment is dropped, since the + label's ``source/path`` identity is what the walk orders by).""" + m = re.match(r"^Syncing… (\S+?)(?: \d+ remaining| · chunk |\s*$)", label) + if not m: + return None + full = m.group(1) + return full.split("/", 1)[1] if "/" in full else full + + +#: The running-label prefixes (the ONLY states ``setSyncLabel`` gives a full +#: untruncated title/announcer with — the terminals are the counts line + the +#: ``Synced HH:MM`` button label, whose title is the idle one by contract). +_RUNNING_PREFIXES = ( + "Syncing…", + "Scanning…", + "Importing", + "Writing KB overview…", + "Summarizing folders…", +) + + +def _assert_a4(frames: list[dict[str, Any]]) -> None: + """A4 (phase 64, unchanged by 194): every RUNNING label — the scan's + ``Scanning… (n/total)`` included — rides the button ``title``, the button + ``aria-label`` and the announcer byte-equal to the text, and the count + span is un-hidden exactly when the text carries a count part.""" + for frame in frames: + if not frame["text"].startswith(_RUNNING_PREFIXES): + continue + assert frame["title"] == frame["text"], frame + assert frame["aria"] == frame["text"], frame + assert frame["countHidden"] is not ( + " remaining" in frame["text"] or "(" in frame["text"] + ), frame + + +def _pending_positions( + walk_order: tuple[str, ...], changed: tuple[str, ...] +) -> dict[str, tuple[int, int]]: + """The pair the status must carry while each walked file is in flight — + the arithmetic the pre-scan produces, computed HERE from the fixture + (``total`` = the size of the predicted set, ``done`` = the 1-based rank of + the current file INSIDE it, held for every file outside it).""" + pending = [rel for rel in walk_order if rel in set(changed)] + out: dict[str, tuple[int, int]] = {} + seen = 0 + for rel in walk_order: + if rel in pending: + seen += 1 + out[rel] = (seen, len(pending)) + return out + + +def _walk_positions(walk_order: tuple[str, ...]) -> dict[str, tuple[int, int]]: + """The same map under ``BOR_SYNC_PENDING_COUNT=0``: every walked file + advances the count, and the denominator is the whole walk (phase 193).""" + return { + rel: (i + 1, len(walk_order)) for i, rel in enumerate(walk_order) + } + + +class _TickRecorder: + """The deterministic layer: a daemon thread tight-polling the SAME status + endpoint the UI's 2 s poll reads, with its OWN admin session (the + Playwright sync API is not thread-safe, so the thread never touches the + page). It starts nothing — it only watches. + + A body belongs to THIS run once the recorder has seen it go ``running``, + or — for a run faster than the cadence — once its own ``started_at`` + post-dates the recorder's start. ``interval`` is the cadence: the default + 100 ms resolves the phase machine on an 8-file run, and the prod-shaped + arm tightens it to resolve the pre-scan's position (the scan runs OFF the + loop, so this poll is answered THROUGH it — the phase-179 contract). + """ + + def __init__( + self, base_url: str, path: str, *, wait_for_running: bool = True, interval_s: float = 0.1 + ) -> None: + self._url = f"{base_url}{path}" + self._login_url = f"{base_url}/api/login" + self._wait_for_running = wait_for_running + self._interval_s = interval_s + self._saw_running = not wait_for_running + self._started_after = 0.0 + self._ticks: list[dict[str, Any]] = [] + self._terminal: dict[str, Any] | None = None + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + @staticmethod + def _epoch(body: dict[str, Any]) -> float | None: + raw = body.get("started_at") + if not raw: + return None + try: + return datetime.fromisoformat(str(raw)).timestamp() + except ValueError: # pragma: no cover — defensive, an odd stamp + return None + + def _is_ours(self, body: dict[str, Any]) -> bool: + if body["state"] == "running": + self._saw_running = True + if self._saw_running: + return True + started = self._epoch(body) + return started is not None and started >= self._started_after - 1.0 + + def start(self) -> None: + if self._wait_for_running: + self._started_after = time.time() + + def run() -> None: + with httpx.Client(timeout=10.0) as client: + client.post(self._login_url, json={"password": ADMIN_PASSWORD}) + while not self._stop.is_set(): + try: + r = client.get(self._url) + if r.status_code == 200: + body = r.json() + if not self._is_ours(body): + self._stop.wait(self._interval_s) + continue + self._ticks.append(body) + if body["state"] in ("success", "failed", "cancelled"): + self._terminal = body + return + except Exception: # noqa: BLE001 — a blocked loop: retry + pass + self._stop.wait(self._interval_s) + + self._thread = threading.Thread(target=run, daemon=True) + self._thread.start() + + def stop(self, timeout_s: float = 300.0) -> dict[str, Any]: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if self._terminal is not None: + break + time.sleep(0.05) + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=5) + assert self._terminal is not None, ( + f"the recorder saw no terminal state (last: {self._ticks[-3:]})" + ) + return self._terminal + + @property + def ticks(self) -> list[dict[str, Any]]: + return list(self._ticks) + + @property + def running_ticks(self) -> list[dict[str, Any]]: + return [t for t in self._ticks if t["state"] == "running"] + + +def _serve_statuses(page: Page, stages: dict[str, Any]) -> None: + """Route the two job-status endpoints to the CURRENT stage body (the house + ``page.route`` pattern): ``stages`` is mutated by the test between the UI + waits, so each 2 s poll of the REAL page renders exactly the payload the + test put in front of it — the client's own decision points, pinned without + a timing race.""" + + def handle(route: Any) -> None: + path = route.request.url + if "/api/sync/status" in path: + body = stages.get("sync") + elif "/api/git-sources/upload/status" in path: + body = stages.get("upload") + else: # pragma: no cover — only the two patterns are routed + route.fallback() + return + if body is None: + route.fallback() + return + route.fulfill(status=200, content_type="application/json", body=json.dumps(body)) + + page.route("**/api/sync/status", handle) + page.route("**/api/git-sources/upload/status", handle) + + +def _sync_status(**overrides: Any) -> dict[str, Any]: + """One ``GET /api/sync/status`` body with the endpoint's CURRENT key set + (``EXPECTED_STATUS_KEYS``) — the served stubs stay faithful to the payload + shape; only the values are the test's.""" + body: dict[str, Any] = { + "state": "idle", + "started_at": None, + "finished_at": None, + "current_file": None, + "files_done": 0, + "files_total": 0, + "scan_done": 0, + "scan_total": 0, + "phase": None, + "current_summary": None, + "summaries_done": 0, + "summaries_total": 0, + "current_chunk": 0, + "chunks_total": 0, + "images_done": 0, + "images_total": 0, + "detail": {}, + "error": None, + "rss_mb": None, + "rss_peak_mb": None, + } + assert set(body) == EXPECTED_STATUS_KEYS, set(body) ^ EXPECTED_STATUS_KEYS + body.update(overrides) + return body + + +def _upload_status(**overrides: Any) -> dict[str, Any]: + body: dict[str, Any] = { + "state": "idle", + "started_at": None, + "finished_at": None, + "current_file": None, + "files_done": 0, + "files_total": 0, + "detail": {}, + "error": None, + } + assert set(body) == UPLOAD_STATUS_KEYS, set(body) ^ UPLOAD_STATUS_KEYS + body.update(overrides) + return body + + +def _wait_result_line(page: Page, expected: str) -> str: + """Wait for the PAGE (not the recorder) to settle on its terminal counts + line, and hand it back for the byte-shape assertions. The tight-polling + recorder sees the server's terminal one to two ticks BEFORE the 2 s UI + poll does — reading the box right after ``recorder.stop()`` would race the + page's own settle (the phase-64 'never stale' contract is what this waits + for, so the wait is the pin).""" + loc = page.locator("#sync-progress-text") + expect(loc).to_have_text(counts_line(expected), timeout=SETTLE_TIMEOUT_MS) + return loc.inner_text() + + +def _admin_sources_page(page: Page, base_url: str, *, settled: bool = True) -> None: + """Real form login landing on the Sources page (the button's home). + + ``settled=True`` also waits for the boot re-attach to finish in a + NON-running state; the served-status arms pass ``settled=False`` because + their boot fetch IS the state under test.""" + login(page, base_url) + expect(page).to_have_url(base_url + SOURCES_URL, timeout=30_000) + expect(page.locator("#sync-btn")).to_be_visible(timeout=30_000) + if settled: + expect(page.locator("#sync-label")).to_have_text( + re.compile(r"^(Sync sources|Synced \d{1,2}:\d{2})$"), timeout=30_000 + ) + + +def _assert_machine( + ticks: list[dict[str, Any]], + terminal: dict[str, Any], + *, + scan_total: int, + files_total: int, + scan_pair_survives_terminal: bool = True, +) -> None: + """The phase-194 machine, read off the endpoint the UI polls — every + claim deterministic on a real run (none depends on a tick landing inside + the scan window): + + * the payload is ADDITIVE — exactly :data:`EXPECTED_STATUS_KEYS` on every + body, running and terminal alike; + * the null prelude reports the scan pair as ``0/0`` (the run has not + reached ``import_sources`` yet); + * a tick caught in ``phase == "scan"`` names NO file, carries the scan's + own denominator and a position that only climbs, and reports no + file-work denominator yet (``files_total == 0`` — the flip happens on + the first ``progress`` call); + * ONE-DIRECTIONAL: no scan tick follows a tick that already counted; + * ``files_total`` never exceeds the scan's denominator (the count and its + denominator stay on the SAME predicted set); + * the terminal CLEARS ``phase`` and KEEPS both pairs side by side (the + phase-64 keep-final-values convention) — on the ``0`` rollback the kept + scan pair is ``0/0``, because that run scans nothing.""" + for tick in ticks: + assert set(tick) == EXPECTED_STATUS_KEYS, ( + f"the status payload drifted from the additive key set: " + f"{sorted(set(tick) ^ EXPECTED_STATUS_KEYS)}" + ) + assert set(terminal) == EXPECTED_STATUS_KEYS, terminal + + for tick in ticks: + if tick["state"] == "running" and tick["phase"] is None: + assert (tick["scan_done"], tick["scan_total"]) == (0, 0), ( + f"the null prelude already carried scan counts: {tick}" + ) + + scan_ticks = [t for t in ticks if t["phase"] == "scan"] + for tick in scan_ticks: + assert tick["current_file"] is None, ( + f"the scan named a file — it counts, it does not read: {tick}" + ) + if tick["scan_total"] == 0: + # The scan's OWN first frame (or the degenerate empty walk): the + # phase is set the moment ``import_sources`` is entered and the + # pair lands with the first position, so a tick here legitimately + # carries 0/0 — exactly the payload the UI's ``scan_total === 0`` + # guard renders as the bare "Scanning…". + assert tick["scan_done"] == 0, tick + continue + assert tick["scan_total"] == scan_total, tick + assert 0 < tick["scan_done"] <= scan_total, tick + positions = [t["scan_done"] for t in scan_ticks] + assert positions == sorted(positions), f"the scan position went backwards: {positions}" + + counted = [i for i, t in enumerate(ticks) if t["files_total"] > 0] + if scan_ticks and counted: + last_scan = max(i for i, t in enumerate(ticks) if t["phase"] == "scan") + assert last_scan < counted[0], ( + "a scan frame followed a frame that already counted files — the " + f"machine ran backwards: ticks={[(t['phase'], t['files_total']) for t in ticks]}" + ) + + assert terminal["phase"] is None, terminal + if scan_pair_survives_terminal: + assert terminal["scan_total"] == scan_total, terminal + assert terminal["scan_done"] == scan_total, ( + f"the terminal did not keep the completed scan: {terminal}" + ) + assert terminal["files_total"] == files_total, terminal + if scan_total > 0: + # The count and its denominator stay on the SAME predicted set — the + # work can never exceed what the scan walked. Meaningless on the + # ``BOR_SYNC_PENDING_COUNT=0`` rollback, which scans nothing and whose + # denominator IS the walk. + assert terminal["files_total"] <= terminal["scan_total"], terminal + + +# --------------------------------------------------------------------------- +# 1. (arm a) The first sync: pending == walk, and the machine completes +# --------------------------------------------------------------------------- + + +def test_first_sync_counts_the_predicted_set_and_the_machine_completes( + page: Page, app_url: str, db_ready: None +) -> None: + """A FRESH knowledge base: the predicted change set IS the walk, so the + phase-193 arithmetic survives UNEDITED (the reason the phase-193 fixture + never saw the defect) and the machine completes its full sequence.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + _admin_sources_page(page, app_url) + _install_recorder(page) + + recorder = _TickRecorder(app_url, "/api/sync/status") + recorder.start() + btn = page.locator("#sync-btn") + btn.click() + expect(btn).to_be_disabled() + expect(page.locator("#sync-progress")).to_have_text("Syncing…") + + # The FINAL file: the main part names it and the count part is GONE (A1). + # ONE live wait per arm on the state that ENDS the import phase (every + # window here is ≫ the 2 s poll, so it cannot be missed); the rest of the + # sequence is read off the in-browser recorder, which captured every value + # the ONE writer landed — a per-label live wait would race its own poll. + expect(page.locator("#sync-progress-text")).to_have_text( + re.compile( + rf"^Syncing… {re.escape(SOURCE)}/{re.escape(LAST_REL)}" + r"(?! \d+ remaining)(?: · chunk \d+/\d+)?$" + ), + timeout=SETTLE_TIMEOUT_MS, + ) + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SETTLE_TIMEOUT_MS) + terminal = recorder.stop() + assert terminal["state"] == "success", terminal + + frames = _frames(page) + texts = _texts(frames) + walk_map = _walk_positions(tuple(REL_ORDER)) + + # --- the rendered import frames ------------------------------------ + import_frames = [f for f in frames if f["text"].startswith("Syncing… ")] + assert len(import_frames) >= 4, f"too few rendered import labels: {texts[:8]}" + for frame in import_frames: + assert IMPORT_LABEL_RE.match(frame["text"]), frame + assert not OLD_FRACTION_RE.search(frame["text"]), frame + rel = _file_of(frame["text"]) + expected = walk_map.get(rel or "") + assert expected is not None, f"the label named an unknown file: {frame}" + done, total = expected + # The cross-layer invariant, per FILE (not per sample): the rendered + # N is exactly what this file's pair subtracts to. + assert _count_of(frame["text"]) == (total - done or None), (frame, expected) + + # --- the count-down -------------------------------------------------- + observed = [ + n + for n in (_count_of(t) for t in texts if t.startswith("Syncing… ")) + if n is not None + ] + assert len(observed) >= 3, f"the label never visibly counted down: {observed}" + assert observed == sorted(observed, reverse=True), f"not a count-DOWN: {observed}" + assert observed[0] == MAX_COUNT, f"the walk did not start at N − 1: {observed}" + assert observed[-1] == 1, f"the count part never reached the last file: {observed}" + assert 0 not in observed, f"a count-bearing label showed 0 remaining: {observed}" + + # The FIRST file's frame: the pending-count part at N − 1 AND the + # phase-129 chunk part, coexisting in ONE label. + first_frames = [ + t + for t in texts + if t.startswith(f"Syncing… {SOURCE}/{FIRST_REL} {MAX_COUNT} remaining") + ] + assert first_frames, f"the first file never rendered {MAX_COUNT} remaining: {texts[:8]}" + + # --- the chunk part coexists (phase 129 unbroken) -------------------- + both = [t for t in texts if " remaining" in t and " · chunk " in t] + assert both, f"the count part and the chunk part never coexisted: {texts[:8]}" + for label in both: + m = re.search(r" \d+ remaining · chunk (\d+)/(\d+)$", label) + assert m is not None, label + assert int(m.group(2)) == _expected_content_chunks(str(_file_of(label))), label + + # --- the scan branch, if the 2 s poll landed inside it --------------- + scan_frames = [f for f in frames if f["text"].startswith("Scanning…")] + import_at = [i for i, f in enumerate(frames) if f["text"].startswith("Syncing… ")] + for frame in scan_frames: + assert SCAN_LABEL_RE.fullmatch(frame["text"]), frame + assert " remaining" not in frame["text"], frame + assert "/" not in frame["text"], frame + if scan_frames and import_at: + assert max(i for i, f in enumerate(frames) if f["text"].startswith("Scanning…")) < ( + import_at[0] + ), f"a scan label followed an import label: {texts}" + + _assert_a4(frames) + + # --- the machine, off the endpoint the page polls -------------------- + _assert_machine(recorder.ticks, terminal, scan_total=N_FILES, files_total=N_FILES) + for tick in recorder.running_ticks: + if tick["phase"] == "import" and tick["current_file"]: + rel = str(tick["current_file"]).removeprefix(f"{SOURCE}/") + assert (tick["files_done"], tick["files_total"]) == walk_map[rel], tick + assert tick["files_total"] == N_FILES, tick + + # --- the terminal ----------------------------------------------------- + # The counts line is the byte-identical phase-64 one (the pair below it + # changed meaning; this did not) + the phase-187 clause, TERMINAL-only. + assert terminal["detail"]["added"] == N_FILES, terminal + assert terminal["detail"]["updated"] == 0, terminal + box = page.locator("#sync-progress") + line = _wait_result_line(page, f"{N_FILES} added") + expect(box).to_be_visible() + assert box.locator("> *").count() == 1 + expect(box).to_have_attribute("role", "status") + expect(box).to_have_attribute("aria-live", "polite") + if MEMORY_KNOWABLE: + assert terminal["rss_peak_mb"] is not None, terminal + m = PEAK_CLAUSE_RE.search(line) + assert m is not None, line + assert int(m.group(1)) == terminal["rss_peak_mb"], (line, terminal) + else: + assert not PEAK_CLAUSE_RE.search(line), line + + +# --------------------------------------------------------------------------- +# 2. (arm b) The owner's production case: K of N changed +# --------------------------------------------------------------------------- + + +def test_incremental_sync_counts_only_the_changed_files( + page: Page, app_url: str, db_ready: None, pending_repo: Path +) -> None: + """THE phase's heart, in the browser. The KB already holds all 8 documents + and TWO files changed: every tick of the running sync answers the + PREDICTED denominator, so the label reads ``2 remaining`` — and holds + there while ``current_file`` walks over the unchanged ones — instead of the + walk-derived ``7``/``6``/``5`` the owner saw as "1216 remaining" on a + ~10,000-file KB. + + The unchanged files in this run are not idle: their stored summaries were + cleared by the seed, so each costs the phase-118 backfill (a ``lite`` call + + its embed) — real model time, real frames, ZERO pending-work. That is + what makes the HOLD visible here rather than only in + ``tests/integration/test_sync_pending_count.py``'s hook sequence. + """ + page.set_default_timeout(SETTLE_TIMEOUT_MS) + _seed_kb(pending_repo, null_summary_rels=HELD_RELS) + _change_files(pending_repo, CHANGED_RELS, "arm b") + expected = _pending_positions(tuple(REL_ORDER), CHANGED_RELS) + + _admin_sources_page(page, app_url) + _install_recorder(page) + recorder = _TickRecorder(app_url, "/api/sync/status") + recorder.start() + btn = page.locator("#sync-btn") + btn.click() + expect(page.locator("#sync-progress")).to_have_text("Syncing…") + + # The FINAL predicted file: the part is GONE while the walk still has + # files left to go (the phase-193 A1 edge, now on the PREDICTED set). + # One live wait; the held frames and the first predicted file are read off + # the recorder below (a per-label live wait races the page's own 2 s poll). + expect(page.locator("#sync-progress-text")).to_have_text( + re.compile( + rf"^Syncing… {re.escape(SOURCE)}/{re.escape(CHANGED_RELS[1])}" + r"(?! \d+ remaining)(?: · chunk \d+/\d+)?$" + ), + timeout=SETTLE_TIMEOUT_MS, + ) + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SETTLE_TIMEOUT_MS) + terminal = recorder.stop() + assert terminal["state"] == "success", terminal + + frames = _frames(page) + texts = _texts(frames) + + # --- the wire: the denominator is the WORK, never the walk ----------- + for tick in recorder.running_ticks: + assert tick["files_total"] in (0, len(CHANGED_RELS)), ( + f"a tick answered a walk-derived denominator: {tick}" + ) + if tick["current_file"]: + rel = str(tick["current_file"]).removeprefix(f"{SOURCE}/") + assert (tick["files_done"], tick["files_total"]) == expected[rel], tick + + # The HOLD, on the frames the server actually answered: at least TWO + # distinct unchanged files, each named while the pair sat at (0, 2) — the + # count did NOT advance on them. + held_ticks = { + str(t["current_file"]).removeprefix(f"{SOURCE}/") + for t in recorder.running_ticks + if t["current_file"] + and (t["files_done"], t["files_total"]) == (0, len(CHANGED_RELS)) + } + assert len(held_ticks) >= 2, f"the count never held over unchanged files: {held_ticks}" + assert held_ticks <= set(HELD_RELS), held_ticks + + # --- the rendered hold + the hidden final part ----------------------- + held_labels = [ + f["text"] + for f in frames + if f["text"].startswith("Syncing… ") + and _count_of(f["text"]) == len(CHANGED_RELS) + and _file_of(f["text"]) in HELD_RELS + ] + assert len(held_labels) >= 2, f"fewer than two held frames rendered: {texts[:12]}" + assert len({_file_of(t) for t in held_labels}) >= 2, held_labels + final_prefix = f"Syncing… {SOURCE}/{CHANGED_RELS[1]}" + final_frames = [f for f in frames if f["text"].startswith(final_prefix)] + assert final_frames, texts + for frame in final_frames: + assert _count_of(frame["text"]) is None, frame + first_prefix = f"Syncing… {SOURCE}/{CHANGED_RELS[0]}" + first_frames = [f for f in frames if f["text"].startswith(first_prefix)] + assert first_frames and all(_count_of(f["text"]) == 1 for f in first_frames), first_frames + + # THE owner-visible regression pin: nothing in this run may show a + # walk-derived count (7/6/5/4/3 remaining are the pre-phase numbers for + # this exact fixture — see the kill-switch arm). + for label in texts: + n = _count_of(label) + assert n is None or n <= len(CHANGED_RELS), ( + f"the label counted the WALK, not the work: {label}" + ) + _assert_a4(frames) + + # --- the machine + the terminal -------------------------------------- + _assert_machine( + recorder.ticks, terminal, scan_total=N_FILES, files_total=len(CHANGED_RELS) + ) + assert terminal["detail"]["updated"] == len(CHANGED_RELS), terminal + assert terminal["detail"]["unchanged"] == N_FILES - len(CHANGED_RELS), terminal + assert terminal["scan_done"] == N_FILES, terminal + assert terminal["files_done"] == len(CHANGED_RELS), terminal + _wait_result_line( + page, + f"0 added · {len(CHANGED_RELS)} updated · {N_FILES - len(CHANGED_RELS)} unchanged", + ) + + +# --------------------------------------------------------------------------- +# 3. The frames the live run produces, rendered by the REAL client +# --------------------------------------------------------------------------- + + +def test_scan_frames_and_the_hold_render_from_the_exact_payloads( + page: Page, app_url: str, db_ready: None +) -> None: + """Arms a and b, rendered. The scan window on an 8-file KB is milliseconds + and the hold frames depend on which unchanged files happen to cost model + time, so neither can be AIMED at from the browser. This arm feeds the + exact payloads the real run answers — including the real HOLD sequence the + importer's hook is pinned to in + ``tests/integration/test_sync_pending_count.py`` + (``[(changed,1,2), (unchanged,1,2), (unchanged,1,2), (changed,2,2)]``) — + to the real ``sources.js`` through the page's own 2 s poll, and pins what + the owner would read: the scan's own ``(n/total)``, the count holding at + ``2 remaining`` while the named file changes, the part shrinking to + ``1 remaining``, the part GONE on the final predicted file, and the + phase-187 clause on the TERMINAL line only.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + src = SOURCE + stages: dict[str, Any] = {"sync": None, "upload": None} + _serve_statuses(page, stages) + + def show(**overrides: Any) -> None: + stages["sync"] = _sync_status(state="running", started_at=_now_iso(), **overrides) + + # The scan: its OWN position, no file, no ``N remaining``. + show(phase="scan", scan_done=3, scan_total=N_FILES) + box = page.locator("#sync-progress-text") + btn = page.locator("#sync-btn") + _admin_sources_page(page, app_url, settled=False) + _install_recorder(page) + expect(box).to_have_text(f"Scanning… (3/{N_FILES})", timeout=SETTLE_TIMEOUT_MS) + expect(btn).to_have_attribute("title", f"Scanning… (3/{N_FILES})") + expect(btn).to_have_attribute("aria-label", f"Scanning… (3/{N_FILES})") + # The PART's own toggle, not Playwright visibility: while a run is in + # flight the button is the spinner and CSS hides the label spans, so the + # honest pin is the ONE writer's ``hidden`` flag plus the span's text + # (the in-browser recorder reads the same property). + expect(page.locator(".sync-label-count")).to_have_text(f"(3/{N_FILES})") + expect(page.locator(".sync-label-count")).to_have_js_property("hidden", False) + expect(btn).to_be_disabled() + + # The scan climbs: the SAME label slot, the SAME branch, a new position. + show(phase="scan", scan_done=8, scan_total=N_FILES) + expect(box).to_have_text(f"Scanning… ({N_FILES}/{N_FILES})", timeout=SETTLE_TIMEOUT_MS) + # …and the empty-KB / first-frame guard: 0/0 renders the BARE label. + show(phase="scan", scan_done=0, scan_total=0) + expect(box).to_have_text("Scanning…", timeout=SETTLE_TIMEOUT_MS) + expect(page.locator(".sync-label-count")).to_have_text("") + expect(page.locator(".sync-label-count")).to_have_js_property("hidden", True) + expect(btn).to_have_attribute("title", "Scanning…") + + # The null prelude (the model check + clone/pull span): bare ``Syncing…``. + show(phase=None, files_done=0, files_total=0) + expect(box).to_have_text("Syncing…", timeout=SETTLE_TIMEOUT_MS) + + # THE HOLD: three unchanged files, one pair. The count does not move. + _install_recorder(page) + for rel in HELD_RELS: + show(phase="import", current_file=f"{src}/{rel}", files_done=0, files_total=2) + expect(box).to_have_text( + f"Syncing… {src}/{rel} 2 remaining", timeout=SETTLE_TIMEOUT_MS + ) + # The first predicted file — with the phase-129 chunk part alongside. + show( + phase="import", + current_file=f"{src}/{CHANGED_RELS[0]}", + files_done=1, + files_total=2, + current_chunk=2, + chunks_total=3, + ) + expect(box).to_have_text( + f"Syncing… {src}/{CHANGED_RELS[0]} 1 remaining · chunk 2/3", + timeout=SETTLE_TIMEOUT_MS, + ) + # The final predicted file: the part is GONE, the file stays. + show(phase="import", current_file=f"{src}/{CHANGED_RELS[1]}", files_done=2, files_total=2) + expect(box).to_have_text(f"Syncing… {src}/{CHANGED_RELS[1]}", timeout=SETTLE_TIMEOUT_MS) + expect(page.locator(".sync-label-count")).to_have_text("") + expect(page.locator(".sync-label-count")).to_have_js_property("hidden", True) + + # The terminal of the same run: the byte-identical counts line, and the + # ONLY place the phase-187 clause appears (the running frames above + # carried rss values and rendered NONE — AGENTS rule 10, terminal-only). + stages["sync"] = _sync_status( + state="success", + started_at=_now_iso(), + finished_at=_now_iso(), + current_file=f"{src}/{CHANGED_RELS[1]}", + files_done=2, + files_total=2, + scan_done=N_FILES, + scan_total=N_FILES, + detail={"added": 0, "updated": 2, "unchanged": N_FILES - 2}, + rss_mb=512, + rss_peak_mb=1234, + ) + expect(box).to_have_text("0 added · 2 updated · 6 unchanged · peak 1234 MB") + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL) + expect(btn).to_be_enabled() + + texts = _texts(_frames(page)) + scan_labels = [t for t in texts if t.startswith("Scanning…")] + assert scan_labels, texts + for label in scan_labels: + assert SCAN_LABEL_RE.fullmatch(label), label + syncing = [t for t in texts if t.startswith("Syncing…")] + assert syncing, texts + for label in syncing: + assert IMPORT_LABEL_RE.match(label), label + # The pre-phase shape stays dead on this part (the scan's own + # ``(n/total)`` lives only behind the word "Scanning"). + assert not OLD_FRACTION_RE.search(label), label + held = [t for t in syncing if _count_of(t) == 2] + assert len(held) >= 2, held + assert len({_file_of(t) for t in held}) >= 2, held + # The clause NEVER appears on a running label — scan or import — and the + # terminal's is text on the line that already exists (one child span). + for label in scan_labels + syncing: + assert "peak" not in label, label + assert page.locator("#sync-progress > *").count() == 1 + + +# --------------------------------------------------------------------------- +# 4. (arm d) BOR_SYNC_PENDING_COUNT=0 — the phase-193 rollback, live +# --------------------------------------------------------------------------- + + +def test_kill_switch_zero_restores_the_walk_position_count( + page: Page, zero_server: str, db_ready: None, zero_repo: Path +) -> None: + """The second module app, same fixture, ``BOR_SYNC_PENDING_COUNT=0``: the + exact phase-193 machine and the exact phase-193 numbers. + + The contrast IS the phase. For the file at walk position 5 of 8 this app + renders ``3 remaining`` (``8 − 5`` — files that will never be re-indexed + counted as work) while the default app, on the same repository shape and + the same moment of the same walk, renders ``1 remaining``. And no tick of + this run ever reports ``phase: "scan"``: the pre-scan does not run, so the + scan pair is ``0/0`` for its whole life, terminal included.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + _seed_kb(zero_repo, null_summary_rels=HELD_RELS) + _change_files(zero_repo, ZERO_CHANGED_RELS, "arm d") + walk_map = _walk_positions(tuple(REL_ORDER)) + + _admin_sources_page(page, zero_server) + _install_recorder(page) + recorder = _TickRecorder(zero_server, "/api/sync/status") + recorder.start() + page.locator("#sync-btn").click() + expect(page.locator("#sync-progress")).to_have_text("Syncing…") + # The LAST walked file (changed here) is the only one that hides the part + # under the walk semantics. + expect(page.locator("#sync-progress-text")).to_have_text( + re.compile( + rf"^Syncing… pendingzero/{re.escape(LAST_REL)}" + r"(?! \d+ remaining)(?: · chunk \d+/\d+)?$" + ), + timeout=SETTLE_TIMEOUT_MS, + ) + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SETTLE_TIMEOUT_MS) + terminal = recorder.stop() + assert terminal["state"] == "success", terminal + + # No scan phase exists on this app, at all, ever. + assert [t for t in recorder.ticks if t["phase"] == "scan"] == [], recorder.ticks + for tick in recorder.ticks: + assert (tick["scan_done"], tick["scan_total"]) == (0, 0), tick + assert terminal["phase"] is None, terminal + assert (terminal["scan_done"], terminal["scan_total"]) == (0, 0), terminal + + # The pair is the WALK: every named tick carries the 1-based walk position + # over all 8 importable files, and the denominator is the whole KB. + named = [t for t in recorder.running_ticks if t["current_file"]] + assert named, "the run never reported a file at all" + for tick in named: + rel = str(tick["current_file"]).removeprefix("pendingzero/") + assert (tick["files_done"], tick["files_total"]) == walk_map[rel], tick + assert tick["files_total"] == N_FILES, tick + assert terminal["files_done"] == N_FILES, terminal + assert terminal["files_total"] == N_FILES, terminal + assert terminal["detail"]["updated"] == len(ZERO_CHANGED_RELS), terminal + _assert_machine(recorder.ticks, terminal, scan_total=0, files_total=N_FILES) + + # The rollback's OWN defect, rendered: a walk-derived count (the files in + # front of the first changed one still counted). Arm b pins the number + # this fixture produces with the switch on: ``2 remaining``. + frames = _frames(page) + texts = _texts(frames) + walk_counts = [ + n + for t in texts + if t.startswith("Syncing… ") + and _file_of(t) in HELD_RELS + and (n := _count_of(t)) is not None + ] + assert walk_counts, f"no unchanged file rendered a walk-derived count: {texts[:12]}" + assert max(walk_counts) > len(ZERO_CHANGED_RELS), ( + f"the rollback app counted the WORK, not the walk: {walk_counts}" + ) + for frame in frames: + if frame["text"].startswith("Scanning…"): + raise AssertionError(f"the rollback app rendered a scan label: {frame}") + _assert_a4(frames) + _wait_result_line( + page, + f"0 added · {len(ZERO_CHANGED_RELS)} updated " + f"· {N_FILES - len(ZERO_CHANGED_RELS)} unchanged", + ) + + +# --------------------------------------------------------------------------- +# 5. (arm c) The no-change re-sync: an empty predicted set, no count part +# --------------------------------------------------------------------------- + + +def test_no_change_sync_reports_no_denominator( + page: Page, app_url: str, db_ready: None, pending_repo: Path +) -> None: + """The re-sync of a repository nothing touched: the scan walks all 8 files + and predicts NONE of them, so every tick answers ``files_total == 0`` — + the phase-64 ``total === 0`` guard driven by zero WORK rather than by a + missing denominator — and no label EVER carries a count part. The + pre-phase label on this exact run read ``8 remaining``, ``7 remaining``… + while nothing was going to be re-indexed. + + A no-change run makes no import LLM call, so its import window is + milliseconds and it renders no per-file label: the wire pair (``0/0`` on + every tick) plus test 3's ``total === 0`` guard render the owner-visible + half deterministically.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + _seed_kb(pending_repo) + + _admin_sources_page(page, app_url) + _install_recorder(page) + recorder = _TickRecorder(app_url, "/api/sync/status") + recorder.start() + btn = page.locator("#sync-btn") + btn.click() + # The click's OWN state first (phase 64): the app's in-memory status still + # carries the PREVIOUS test's terminal, so a settle wait started straight + # after the click can pass on that stale "Synced HH:MM" instead of on this + # run. The spinner + the bare live label are THIS run's first frame. + expect(btn).to_be_disabled() + expect(page.locator("#sync-progress")).to_have_text("Syncing…") + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SETTLE_TIMEOUT_MS) + terminal = recorder.stop() + assert terminal["state"] == "success", terminal + + for tick in recorder.running_ticks: + assert (tick["files_done"], tick["files_total"]) == (0, 0), ( + f"a no-change run reported work to do: {tick}" + ) + _assert_machine(recorder.ticks, terminal, scan_total=N_FILES, files_total=0) + # The scan DID walk the whole KB — the divergence (8 scanned, 0 to + # re-index) kept at the terminal, beside the untouched counts line. + assert terminal["scan_done"] == N_FILES, terminal + assert terminal["scan_total"] == N_FILES, terminal + assert terminal["detail"]["unchanged"] == N_FILES, terminal + + texts = _texts(_frames(page)) + for label in texts: + assert _count_of(label) is None, ( + f"a no-change run rendered a count part: {label}" + ) + if label.startswith("Syncing…"): + assert not OLD_FRACTION_RE.search(label), label + _wait_result_line(page, f"0 added · {N_FILES} unchanged") + + +# --------------------------------------------------------------------------- +# 6. (arm e) The empty source: the degenerate D1 pin +# --------------------------------------------------------------------------- + + +def _register_local_source(base_url: str, path: Path) -> None: + """Store one ``kind=local`` source row through the admin API (the + phase-49 contract the Sources page's form used to wrap): a stored row IS + the effective source, which is what lets the degenerate and prod-shaped + arms run on the module's own app instead of a third boot.""" + with httpx.Client(base_url=base_url, timeout=30.0) as client: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, r.text + r = client.post("/api/git-sources", json={"kind": "local", "path": str(path)}) + assert r.status_code == 201, f"{r.status_code}: {r.text}" + + +def test_empty_source_stays_in_scan_with_no_count_part( + page: Page, app_url: str, db_ready: None, empty_kb_dir: Path +) -> None: + """A source with ZERO importable files: the walk yields nothing, so no + ``progress`` call exists to flip the phase — the machine goes + ``null`` → ``"scan"`` → terminal and NEVER says ``"import"``, both pairs + stay ``0/0``, and the page renders no count part anywhere (the same + ``scan_total === 0`` guard test 3 pins on the exact payload). + + The scan window of an empty walk is a handful of milliseconds, so the + live pins are the deterministic ones (no import tick exists to catch, and + the terminal keeps the zeroed pair); the server-side degenerate is + ``tests/integration/test_sync_scan_phase.py``'s empty-KB arm.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + _register_local_source(app_url, empty_kb_dir) + + _admin_sources_page(page, app_url) + _install_recorder(page) + recorder = _TickRecorder(app_url, "/api/sync/status", interval_s=0.02) + recorder.start() + btn = page.locator("#sync-btn") + btn.click() + # The click's OWN state first (phase 64): the app's in-memory status still + # carries the PREVIOUS test's terminal, so a settle wait started straight + # after the click can pass on that stale "Synced HH:MM" instead of on this + # run. The spinner + the bare live label are THIS run's first frame. + expect(btn).to_be_disabled() + expect(page.locator("#sync-progress")).to_have_text("Syncing…") + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SETTLE_TIMEOUT_MS) + terminal = recorder.stop() + assert terminal["state"] == "success", terminal + + assert [t for t in recorder.ticks if t["phase"] == "import"] == [], ( + f"an empty walk flipped to import: {recorder.ticks}" + ) + for tick in recorder.ticks: + assert (tick["scan_done"], tick["scan_total"]) == (0, 0), tick + assert (tick["files_done"], tick["files_total"]) == (0, 0), tick + _assert_machine(recorder.ticks, terminal, scan_total=0, files_total=0) + assert terminal["detail"] == {} or not any( + terminal["detail"].get(k) + for k in ("added", "updated", "pruned", "errors") + ), terminal + + texts = _texts(_frames(page)) + for label in texts: + assert _count_of(label) is None, f"the empty run rendered a count part: {label}" + if label.startswith("Scanning…"): + assert label == "Scanning…", label + _wait_result_line(page, "0 added · 0 unchanged") + + +# --------------------------------------------------------------------------- +# 7. The owner's report at scale: a 1,000-file scan, two files of work +# --------------------------------------------------------------------------- + + +def test_prod_shaped_run_scan_climbs_then_counts_two( + page: Page, app_url: str, db_ready: None, wide_kb: Path +) -> None: + """The report itself, in a browser. A 1,000-file KB, TWO files changed: + the pre-scan's OWN position climbs across several distinct values while + the server keeps answering (the scan runs off the loop — the phase-179 + contract, which is exactly why this poll is served THROUGH it), the import + then reports the PREDICTED denominator (``files_total == 2`` against + ``scan_total == 1000``: the divergence the pre-phase shape could not + express), and the terminal keeps BOTH pairs side by side. + + This is also the arm that makes the scan's ``(n/total)`` visible for real: + eight tiny files pre-scan in single-digit milliseconds, a thousand take + long enough for the position to be observed moving. The ``N remaining`` + part never exceeds the work — the owner's ``1216 remaining`` is the bug.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + _seed_kb(wide_kb, git=False) + for rel in WIDE_CHANGED_RELS: + path = wide_kb / rel + path.write_text( + path.read_text(encoding="utf-8") + f"\narm f edit of {rel}\n", encoding="utf-8" + ) + _register_local_source(app_url, wide_kb) + + _admin_sources_page(page, app_url) + _install_recorder(page) + recorder = _TickRecorder(app_url, "/api/sync/status", interval_s=0.02) + recorder.start() + btn = page.locator("#sync-btn") + btn.click() + # The click's OWN state first (phase 64): the app's in-memory status still + # carries the PREVIOUS test's terminal, so a settle wait started straight + # after the click can pass on that stale "Synced HH:MM" instead of on this + # run. The spinner + the bare live label are THIS run's first frame. + expect(btn).to_be_disabled() + expect(page.locator("#sync-progress")).to_have_text("Syncing…") + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SETTLE_TIMEOUT_MS) + terminal = recorder.stop() + assert terminal["state"] == "success", terminal + + scan_ticks = [t for t in recorder.ticks if t["phase"] == "scan"] + positions = sorted({t["scan_done"] for t in scan_ticks}) + assert len(positions) >= 2, ( + "the scan's position was never observably climbing — the pre-scan is " + f"the phase's own progress surface: {positions} / {WIDE_FILES}" + ) + assert positions[-1] > positions[0], positions + for tick in scan_ticks: + # 0/0 is legal here (the scan's own first frame — the phase is set + # before the first position is reported); every REPORTED denominator is + # the whole importable walk. + assert tick["scan_total"] in (0, WIDE_FILES), tick + assert tick["current_file"] is None, tick + assert (tick["files_done"], tick["files_total"]) == (0, 0), tick + # The scan is answered THROUGH the running server: the poll kept getting + # 200s while the pre-scan hashed and compared a thousand files. + assert len(scan_ticks) >= 2, f"the poll starved during the scan: {len(scan_ticks)}" + + # The import: the WORK is the denominator, and it is never the walk. + import_ticks = [t for t in recorder.running_ticks if t["phase"] == "import"] + assert import_ticks, "the run never flipped to import" + for tick in import_ticks: + assert tick["files_total"] == len(WIDE_CHANGED_RELS), tick + assert [ + t for t in recorder.ticks if t["files_total"] > len(WIDE_CHANGED_RELS) + ] == [], ( + "the running count reported a WALK denominator on a 1,000-file KB — " + "the owner's exact report" + ) + _assert_machine( + recorder.ticks, terminal, scan_total=WIDE_FILES, files_total=len(WIDE_CHANGED_RELS) + ) + assert terminal["detail"]["updated"] == len(WIDE_CHANGED_RELS), terminal + assert terminal["detail"]["unchanged"] == WIDE_FILES - len(WIDE_CHANGED_RELS), terminal + + # The owner-visible half: no label ever says a walk-sized number remains, + # and a scan label — if the 2 s poll landed in that window — is the honest + # ``Scanning… (n/1000)`` and nothing else. + texts = _texts(_frames(page)) + for label in texts: + n = _count_of(label) + assert n is None or n <= len(WIDE_CHANGED_RELS), ( + f"a label counted the walk instead of the work: {label}" + ) + if label.startswith("Scanning…"): + assert SCAN_LABEL_RE.fullmatch(label), label + assert str(WIDE_FILES) in label, label + _wait_result_line( + page, + f"0 added · {len(WIDE_CHANGED_RELS)} updated " + f"· {WIDE_FILES - len(WIDE_CHANGED_RELS)} unchanged", + ) + + +# --------------------------------------------------------------------------- +# 8. The upload surface: phase 90's "no sync fields", plus the scan pair +# --------------------------------------------------------------------------- + + +def test_upload_run_carries_no_scan_fields_and_no_count_part( + page: Page, app_url: str, db_ready: None, upload_archive: Path +) -> None: + """Phase 194 gives the upload job NOTHING — and this is the pin that it + did not leak. The REAL background upload's live status answers the bare + phase-64 key set: no ``phase``, no chunk / image / summary fields, no + memory pair and no ``scan_done`` / ``scan_total``, with ``null``/``0``/``0`` + for the whole run (unpack + register, phase 90 A1/A2). Rendered through + the ONE shared builder, that payload is a BARE ``Importing`` — no file and + no count part.""" + page.set_default_timeout(SETTLE_TIMEOUT_MS) + + with httpx.Client(base_url=app_url, timeout=60.0) as client: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, r.text + with upload_archive.open("rb") as fh: + r = client.post( + "/api/git-sources/upload", + files={"file": (f"{UPLOAD_NAME}.tar.gz", fh, "application/gzip")}, + ) + assert r.status_code == 202, r.text + seen: list[dict[str, Any]] = [] + body: dict[str, Any] = {} + for _ in range(200): + body = client.get("/api/git-sources/upload/status").json() + seen.append(body) + if body["state"] in ("success", "failed"): + break + time.sleep(0.05) + assert body["state"] == "success", body + for tick in seen: + assert set(tick) == UPLOAD_STATUS_KEYS, ( + f"the upload status grew sync fields: {sorted(set(tick) ^ UPLOAD_STATUS_KEYS)}" + ) + assert tick["current_file"] is None, tick + assert tick["files_done"] == 0 and tick["files_total"] == 0, tick + + # The shared label, on the page's own poll: the upload's REAL payload is + # the bare prefix — no file, no count part (the part needs a denominator + # the upload job never reports). + stages: dict[str, Any] = { + "sync": _sync_status(state="idle"), + "upload": _upload_status(state="running", started_at=_now_iso()), + } + _serve_statuses(page, stages) + _admin_sources_page(page, app_url, settled=False) + _install_recorder(page) + box = page.locator("#sync-progress-text") + btn = page.locator("#sync-btn") + expect(box).to_have_text("Importing", timeout=SETTLE_TIMEOUT_MS) + expect(btn).to_have_attribute("title", "Importing") + expect(page.locator(".sync-label-count")).to_have_text("") + expect(page.locator(".sync-label-count")).to_have_js_property("hidden", True) + expect(btn).to_be_disabled() + + stages["upload"] = _upload_status( + state="success", detail={"message": "uploaded"}, finished_at=_now_iso() + ) + expect(page.locator("#sync-label")).to_have_text("Sync sources", timeout=SETTLE_TIMEOUT_MS) + expect(page.locator("#sync-progress")).to_be_hidden(timeout=SETTLE_TIMEOUT_MS) + + texts = _texts(_frames(page)) + importing = [t for t in texts if t.startswith("Importing")] + assert importing, texts + for label in importing: + assert _count_of(label) is None, label + assert not OLD_FRACTION_RE.search(label), label + assert "Scanning" not in label and "(" not in label, label diff --git a/tests/e2e/test_sync_progress_box.py b/tests/e2e/test_sync_progress_box.py index f684acb..7b2a9e0 100644 --- a/tests/e2e/test_sync_progress_box.py +++ b/tests/e2e/test_sync_progress_box.py @@ -27,6 +27,10 @@ distance to the end — ``N remaining`` with ``N = total - done`` — so the long file (the FIRST of the three) renders ``2 remaining`` while the endpoint's ``files_done``/``files_total`` stay the 1-based 1/3 position. The ``· chunk`` part and the terminal counts line are byte-identical. +Phase 194 (task 04, D5) leaves this suite UNEDITED for the same reason: +its fixture is a fresh import, so the pair's new source (the pre-scan's +predicted change set) and the old one (the walk) are the same three +files, and every ``2 remaining`` frame lands where phase 193 pinned it. Run in isolation (DB must be up: ``podman compose up -d db``; git on PATH — a documented environment prerequisite, phase 28): diff --git a/tests/e2e/test_sync_remaining_count.py b/tests/e2e/test_sync_remaining_count.py index 00966c2..fd08879 100644 --- a/tests/e2e/test_sync_remaining_count.py +++ b/tests/e2e/test_sync_remaining_count.py @@ -11,20 +11,46 @@ position *in* the whole; the owner reads the distance *to the end*. Phase 193 changes exactly ONE label part: ``fmtSyncLabel``'s count is now ``N remaining`` with ``N = total − done``. -**Server untouched.** No endpoint, no payload field, no poll changed: -``GET /api/sync/status`` still answers with the phase-64 key set -(``current_file`` / ``files_done`` / ``files_total`` — the 1-based -position of the in-flight file over the pre-walk count), and the suite -pins exactly that (test 1's cross-layer invariant): the payload still -carries the ``(done, total)`` position while the label speaks the -subtraction of the very same pair. The change is one string built by -``frontend/assets/sources.js``. +**Phase 194 (task 04, D5) retargets what the pair COUNTS — the label +builder stays byte-identical.** Phase 193 itself touched no server +code; phase 194 did, and this is the suite whose pins move with it. +``GET /api/sync/status`` no longer answers with the bare phase-64 key +set: it is that set plus the phase-187 memory pair plus the phase-194 +scan pair (``scan_done`` / ``scan_total`` — the PRE-SCAN's own walk +position), and ``phase`` gained the ``"scan"`` stage between the null +prelude and ``"import"`` (the machine the page renders as +``Scanning… (n/total)``). More importantly the ``(done, total)`` pair +the label subtracts is no longer the WALK position: on a progress +caller (the UI sync is the only one) it is the position in the +importer's pre-scan PREDICTED change set — ``files_total`` the number +of files that will actually be re-indexed, ``files_done`` the 1-based +position in that set. ``N remaining`` therefore counts the WORK, which +is the fix for the owner's prod report of 2026-10-19 ("1216 remaining" +on a ~10,000-file KB with ~10 files to sync). The phase-193 wording — +the 1-based position across EVERY importable file of the stats +pre-walk — is exactly what ``BOR_SYNC_PENDING_COUNT=0`` restores, and +nothing below asserts it except as that rollback's history. -**A1 (owner-confirmed):** ``N remaining`` shows while ``N ≥ 1`` and is -HIDDEN at the final file (``N == 0``) — the main part names that file; -``total === 0`` (the pre-walk, the phase-64 contract) also renders no -count part. **A2 (owner-confirmed):** the upload job shares the ONE -builder, so it shares the shape (``Importing N remaining``). +What survives the semantic shift UNEDITED is a property of this +suite's own fixture: every arm starts from a TRUNCATED KB, so the +prediction and the walk coincide (all ``N_FILES`` files are pending), +``files_total`` stays ``N_FILES`` and the ``1 … N−1`` count frames land +in the same slots phase 193 pinned — test 1's cross-layer invariant is +untouched arithmetic. The case where the two DIVERGE — a re-sync — is +test 4, and the phase's own suite +(``tests/e2e/test_sync_pending_count.py``, task 05) carries the K-of-N +incremental arm. + +**A1 (owner-confirmed, phase 193):** ``N remaining`` shows while +``N ≥ 1`` and is HIDDEN at the final file (``N == 0``) — the main part +names that file; ``total === 0`` renders no count part either — and +since phase 194 there are TWO things that make it ``0``: the scan has +not reported a denominator yet (the null prelude / the ``0`` rollback, +test 2) and a run whose predicted change set is EMPTY (test 4). +**A2 (owner-confirmed):** the upload job shares the ONE builder, so it +shares the shape (``Importing N remaining``) — and it gains +NOTHING from phase 194: the upload job runs no pre-scan and its +endpoint carries no scan fields (phase 90 A2), so test 3 is UNEDITED. Fixture: a real git repo (the phase-28/32/107 ``file://`` transport-true convention — ``BOR_GIT_SOURCES=file://``, so the env @@ -59,22 +85,44 @@ Arms (the phase's three, in test order): label carries a count part at all while the main part still names that file; no ``Syncing…`` label anywhere carries the pre-phase ``(n/m)`` tail; the ``(n/m)`` shape survives ONLY in the summaries - phase (A1's exception) and the overview label still paints; the - terminal result line is byte-identical to the phase-64 counts line - (the ``counts_line`` helper); and the endpoint's own tick series - still reports the untouched ``(done, total)`` position whose - subtraction equals every rendered ``N``. + phase (A1's exception) and, since phase 194, in the scan phase's own + ``Scanning… (n/total)`` (never on a ``Syncing…`` label, and no scan + label ever follows an import one); the overview label still paints; + the terminal result line is byte-identical to the phase-64 counts + line (the ``counts_line`` helper); and the endpoint's own tick + series still reports the ``(done, total)`` pair whose subtraction + equals every rendered ``N``. Phase 194 adds the machine's own + evidence to that same tick series + (``_assert_phase_machine_tick_series``): every body is exactly the + pre-phase key set plus the memory pair plus the scan pair, the null + prelude reports the scan pair as ``0/0``, a tick caught INSIDE the + scan names no file and carries the full scan denominator, no scan + tick follows a tick that already counted files, and the terminal + KEEPS both pairs beside each other (the phase-64 keep-final-values + convention) with ``scan_total == files_total == N_FILES`` on this + fresh KB — the parity that lets the phase-193 frames stand. The scan + window itself is five tiny files (single-digit milliseconds), so a + 100 ms tick is not guaranteed to land inside it: the deterministic + ``Scanning… (n/total)`` frames are test 2's, and the choreographed + server-side sequence lives in + ``tests/integration/test_sync_scan_phase.py``. 2. ``test_prewalk_and_final_file_labels_have_no_count_part`` — **(c) - the pre-walk edge**, pinned on the exact payload the guard reads: - the page's own 2 s poll fed ``phase: null`` / ``files_total: 0`` - renders the bare ``Syncing…`` (no count part, the count span hidden, - the title equal), the first file renders ``N − 1 remaining``, a + the zero-total edges**, pinned on the exact payload the guards read: + the page's own 2 s poll fed a ``phase: "scan"`` body renders + ``Scanning… (n/total)`` (count span visible, title equal, never a + file and never ``N remaining``), the same stage with + ``scan_total: 0`` renders the bare ``Scanning…`` (the empty-KB + guard), a ``phase: null`` / ``files_total: 0`` prelude renders the + bare ``Syncing…`` (no count part, the count span hidden, the title + equal), the first file renders ``N − 1 remaining``, a mid-run tick renders ``1 remaining`` *beside* ``· chunk 2/5``, the final-file tick (``done == total``) renders the file with the count part GONE and never brings it back, and the success terminal settles - to the untouched counts line. The served bodies use the status - endpoint's unchanged key set — the client's own decision points - (``total > 0``, ``total − done > 0``) are what is under test. + to the untouched counts line. The served bodies carry the status + endpoint's key set — the phase-64 fields plus the memory and scan + pairs, the stubs pinned against the same constant test 1 reads — + because the client's own decision points (``total > 0``, + ``total − done > 0``, ``scan_total > 0``) are what is under test. 3. ``test_upload_label_shares_the_count_down_builder`` — **(b) the upload arm (A2)**: the upload branch of the same two-job tree shares the builder, so a running upload status that carries counts renders @@ -88,12 +136,30 @@ Arms (the phase's three, in test order): upload label can ever regress to ``(n/m)``. The upload branch never receives a status object (its endpoint has no phase / chunk / image fields — phase 129 / 152), so the chunk part coexists in the SYNC - arms only (test 1 and 2); that is asserted here too. + arms only (test 1 and 2); that is asserted here too. UNEDITED by + phase 194 — the upload job runs no pre-scan and its endpoint gains + no scan keys. +4. ``test_resync_of_an_unchanged_repo_reports_zero_work`` — **the + phase-194 re-sync arm**, the run phase 193's fixture never had (the + owner's prod case in its mildest form): sync this same repo TWICE + and the second run predicts NOTHING, so its ``(done, total)`` pair + is ``0/0`` on every tick — the A1 no-count-part exception driven by + an EMPTY pending set instead of by a missing denominator — while the + scan pair reports the whole KB walked (``scan_done == scan_total == + N_FILES``, kept at the terminal). That divergence (5 scanned, 0 to + re-index) is the phase in one line, and it is exactly the shape that + used to render "``N`` remaining" for files nobody would re-index. + The terminal counts line stays the byte-identical + ``0 added · N unchanged``. A no-change run makes no LLM call, so its + import window is milliseconds and it renders no file label: the wire + pair (``0/0``) plus test 2's ``total === 0`` guard pin is the honest + form of "no count part" here. -Test → phase-task mapping (Playwright Mapping Rule, task 03 arms a/b/c): -1 → (a) + the terminal-line pin + the untouched-payload pin, -2 → (c) + the A1 hidden-at-final-file edge (+ (a)'s arithmetic on exact -inputs), 3 → (b). +Test → phase-task mapping (Playwright Mapping Rule, task 03 arms a/b/c ++ task 04's D5 retarget): 1 → (a) + the terminal-line pin + the +additive-payload + phase-machine tick-series pins, +2 → (c) + the A1 hidden-at-final-file edge + the ``Scanning…`` frames +(+ (a)'s arithmetic on exact inputs), 3 → (b), 4 → the D5 re-sync arm. """ from __future__ import annotations @@ -148,10 +214,11 @@ SOURCES_URL = "/sources.html" #: its own (the module docstring's determinism note). SLOW_DELAY_S = "1.5" -#: The module's wall-time budget (app boot + one full 5-file sync + the -#: two served-status arms). A breach is a speed canary, not flake -#: tolerance (the phase-64 convention). -WALL_BUDGET_S = 300.0 +#: The module's wall-time budget (app boot + two full 5-file syncs — the +#: phase-194 re-sync arm runs the fixture's sync twice — plus the two +#: served-status arms). A breach is a speed canary, not flake tolerance +#: (the phase-64 convention). +WALL_BUDGET_S = 480.0 #: The sync's subject: a git repo whose DIRECTORY NAME is the source #: name (the ``repo_name`` basename rule, phase 28). Deliberately NOT @@ -196,11 +263,52 @@ CHUNK_OVERLAP = int(Settings.model_fields["chunk_overlap_chars"].default) IMPORT_LABEL_RE = re.compile( r"^Syncing… \S+(?: [0-9]+ remaining)?(?: · chunk \d+/\d+)?$" ) +#: The scan-phase label grammar phase 194 (task 03, D4) adds: the bare +#: ``Scanning…`` (no denominator yet — the empty KB and the scan's own +#: first frame) or it plus the ``(n/total)`` count part. NEVER a file +#: (``current_file`` is null through the phase) and never ``N +#: remaining`` — the walk position is honest behind the word "Scanning", +#: never behind "Syncing" (TODO L6). +SCAN_LABEL_RE = re.compile(r"^Scanning…(?: \(\d+/\d+\))?$") #: The pre-phase shape phase 193 removed from this part — no ``Syncing…`` -#: label may ever carry a ``(n/m)`` tail again. +#: label may ever carry a ``(n/m)`` tail again. Phase 194 leaves that +#: true: the ``(n/m)`` shape is legal ONLY on a ``Scanning…`` or a +#: ``Summarizing folders…`` label (and no label starting ``Syncing…`` +#: matches either). OLD_FRACTION_RE = re.compile(r"\(\d+/\d+\)") SYNCED_LABEL = re.compile(r"^Synced \d{1,2}:\d{2}$") +#: The FULL ``GET /api/sync/status`` key set this suite's stubs mirror and +#: every real tick is pinned against — the phase-64/98/129/152 fields, +#: the phase-187 memory pair and the phase-194 SCAN pair (``scan_done`` / +#: ``scan_total``). The additive chain, pinned as ONE constant so a future +#: key has to be added here (and to the stub below) instead of drifting +#: away from the client's view of the payload. +EXPECTED_STATUS_KEYS = frozenset( + { + "state", + "started_at", + "finished_at", + "current_file", + "files_done", + "files_total", + "scan_done", + "scan_total", + "phase", + "current_summary", + "summaries_done", + "summaries_total", + "current_chunk", + "chunks_total", + "images_done", + "images_total", + "detail", + "error", + "rss_mb", + "rss_peak_mb", + } +) + SETTLE_TIMEOUT_MS = 180_000 @@ -527,17 +635,55 @@ class _TickRecorder: """The deterministic layer: a daemon thread tight-polling (~100 ms) the SAME status endpoint the UI's 2 s poll reads, with its OWN admin session (the Playwright sync API is not thread-safe, so the thread - never touches the page). It starts nothing — it only watches.""" + never touches the page). It starts nothing — it only watches. - def __init__(self, base_url: str, path: str) -> None: + ``wait_for_running`` is the phase-194 re-sync arm's need: a recorder + started while a PREVIOUS run's terminal is still on the endpoint must + not treat that body as its own. A body belongs to THIS run once the + recorder has seen it go ``running``, or — the belt-and-braces half, + for a run faster than the 100 ms cadence — once its own + ``started_at`` post-dates the recorder's start (the + ``test_fast_path_deferred_content.py`` idiom). Without the flag every + body counts, which is what the module's first arm wants (there is no + previous run to confuse it with). + """ + + def __init__(self, base_url: str, path: str, *, wait_for_running: bool = False) -> None: self._url = f"{base_url}{path}" self._login_url = f"{base_url}/api/login" + self._wait_for_running = wait_for_running + self._saw_running = not wait_for_running + self._started_after = 0.0 self._ticks: list[dict[str, Any]] = [] self._terminal: dict[str, Any] | None = None self._stop = threading.Event() self._thread: threading.Thread | None = None + @staticmethod + def _epoch(body: dict[str, Any]) -> float | None: + """The body's ``started_at`` as epoch seconds (``None`` when no + run ever started, or the stamp is unreadable).""" + raw = body.get("started_at") + if not raw: + return None + try: + return datetime.fromisoformat(str(raw)).timestamp() + except ValueError: # pragma: no cover — defensive, an odd stamp + return None + + def _is_ours(self, body: dict[str, Any]) -> bool: + """Does this body belong to the run this recorder was started for?""" + if body["state"] == "running": + self._saw_running = True + if self._saw_running: + return True + started = self._epoch(body) + return started is not None and started >= self._started_after - 1.0 + def start(self) -> None: + if self._wait_for_running: + self._started_after = time.time() + def run() -> None: with httpx.Client(timeout=5.0) as client: client.post(self._login_url, json={"password": ADMIN_PASSWORD}) @@ -546,6 +692,10 @@ class _TickRecorder: r = client.get(self._url) if r.status_code == 200: body = r.json() + if not self._is_ours(body): + # A previous run's terminal — not ours. + self._stop.wait(0.02) + continue self._ticks.append(body) if body["state"] in ("success", "failed", "cancelled"): self._terminal = body @@ -575,6 +725,13 @@ class _TickRecorder: def running_ticks(self) -> list[dict[str, Any]]: return [t for t in self._ticks if t["state"] == "running"] + @property + def ticks(self) -> list[dict[str, Any]]: + """EVERY body this recorder saw, the terminal included (the + phase-194 phase-machine pin reads the whole series, not just the + running frames).""" + return list(self._ticks) + def _serve_statuses(page: Page, stages: dict[str, Any]) -> None: """Route the two job-status endpoints to the CURRENT stage body (the @@ -605,10 +762,12 @@ def _serve_statuses(page: Page, stages: dict[str, Any]) -> None: def _sync_status(**overrides: Any) -> dict[str, Any]: - """One ``GET /api/sync/status`` body with the endpoint's UNCHANGED - key set (the phase-64/98/129/152/187 fields the client reads) — the - served stubs are faithful to the payload shape; only the values are - the test's.""" + """One ``GET /api/sync/status`` body with the endpoint's CURRENT key + set (the phase-64/98/129/152/187 fields the client reads plus the + phase-194 scan pair — the additive chain ``EXPECTED_STATUS_KEYS`` + pins) — the served stubs are faithful to the payload shape; only the + values are the test's. The equality against that constant is asserted + below so a stub can never drift from the real endpoint unnoticed.""" body: dict[str, Any] = { "state": "idle", "started_at": None, @@ -616,6 +775,11 @@ def _sync_status(**overrides: Any) -> dict[str, Any]: "current_file": None, "files_done": 0, "files_total": 0, + # Phase 194 (task 02): the pre-scan's position, additive after + # files_total and 0/0 until the scan reports (the degenerate and + # BOR_SYNC_PENDING_COUNT=0 edges both render no count part). + "scan_done": 0, + "scan_total": 0, "phase": None, "current_summary": None, "summaries_done": 0, @@ -629,6 +793,7 @@ def _sync_status(**overrides: Any) -> dict[str, Any]: "rss_mb": None, "rss_peak_mb": None, } + assert set(body) == EXPECTED_STATUS_KEYS, set(body) ^ EXPECTED_STATUS_KEYS body.update(overrides) return body @@ -648,6 +813,85 @@ def _upload_status(**overrides: Any) -> dict[str, Any]: return body +def _assert_phase_machine_tick_series( + ticks: list[dict[str, Any]], + terminal: dict[str, Any], + *, + scan_total: int, + files_total: int, + prelude_zero: bool = True, +) -> None: + """The phase-194 machine, read off the status endpoint the UI polls. + + Every claim here is deterministic on a real run, which is why none of + them depends on a tick happening to land inside the scan (a five-file + scan is milliseconds — see the module docstring): + + * the payload is ADDITIVE — exactly :data:`EXPECTED_STATUS_KEYS` on + every body, idle/running/terminal alike; + * the null prelude reports the scan pair as ``0/0`` (the run has not + reached ``import_sources`` yet — the scan is what sets it); + * a tick caught inside ``phase == "scan"`` names NO file, carries the + full ``scan_total`` denominator and a position that only climbs; + * the machine is ONE-DIRECTIONAL — no scan tick follows a tick that + already counted files (``files_total > 0`` means the first + ``progress`` call flipped it, and the phase never flips back); + * the terminal CLEARS ``phase`` and KEEPS both pairs side by side + (the phase-64 keep-final-values convention); + * ``files_total`` is never larger than the scan's denominator on a + run whose KB did not change mid-run — the count and its denominator + stay on the SAME predicted set. + """ + for tick in ticks: + assert set(tick) == EXPECTED_STATUS_KEYS, ( + f"the status payload drifted from the additive key set: " + f"{sorted(set(tick) ^ EXPECTED_STATUS_KEYS)}" + ) + if terminal not in ticks: + assert set(terminal) == EXPECTED_STATUS_KEYS, terminal + + if prelude_zero: + for tick in ticks: + if tick["state"] == "running" and tick["phase"] is None: + # The prelude (the model-check + clone/pull span): the + # scan has not started, so it has reported nothing. The + # success terminal sets state BEFORE clearing phase, so + # no terminal body can reach this arm. + assert (tick["scan_done"], tick["scan_total"]) == (0, 0), ( + f"the null prelude already carried scan counts: {tick}" + ) + + scan_ticks = [t for t in ticks if t["phase"] == "scan"] + for tick in scan_ticks: + assert tick["current_file"] is None, ( + f"the scan named a file — it counts, it does not read: {tick}" + ) + assert tick["scan_total"] == scan_total, tick + assert 0 < tick["scan_done"] <= scan_total, tick + positions = [t["scan_done"] for t in scan_ticks] + assert positions == sorted(positions), ( + f"the scan position went backwards across ticks: {positions}" + ) + counted = [i for i, t in enumerate(ticks) if t["files_total"] > 0] + if scan_ticks and counted: + last_scan = max(i for i, t in enumerate(ticks) if t["phase"] == "scan") + assert last_scan < counted[0], ( + "a scan frame followed a frame that already counted files — " + "the phase machine ran backwards" + ) + + assert terminal["phase"] is None, terminal + assert terminal["scan_total"] == scan_total, terminal + assert terminal["scan_done"] == scan_total, ( + f"the terminal did not keep the completed scan: {terminal}" + ) + assert terminal["files_total"] == files_total, terminal + assert terminal["files_total"] <= terminal["scan_total"], ( + "the predicted change set is bigger than the scan that predicted " + f"it: {terminal}" + ) + + def _admin_sources_page(page: Page, app_url: str, *, settled: bool = True) -> None: """Real form login landing on the RAG page (the button's home). @@ -796,9 +1040,11 @@ def test_running_label_counts_down_across_a_real_sync( assert terminal["detail"]["added"] == N_FILES, terminal # --- the cross-layer invariant: the label is the payload's subtraction - # The endpoint is untouched: it still answers with the 1-based - # position, and each rendered N equals total − done of the file it - # named. + # Neither half of this changed in phase 194: the SUBTRACTION is the + # phase-193 builder, and on this truncated-KB fixture the pair it + # subtracts (the pre-scan's pending position, phase 194) coincides + # with the walk position, so every rendered N still equals + # total − done of the file the label named. position: dict[str, tuple[int, int]] = {} for tick in recorder.running_ticks: if tick["current_file"]: @@ -821,27 +1067,62 @@ def test_running_label_counts_down_across_a_real_sync( else: assert shown is None, f"{path}: done==total but a part rendered: {frame}" + # --- phase 194 (task 04, D5): the machine, off the SAME tick series -- + # The additive key set, the 0/0 prelude, the file-less scan ticks, + # the one-directional sequence and the terminal that keeps BOTH + # pairs (here equal, because a fresh KB's prediction IS its walk). + _assert_phase_machine_tick_series( + recorder.ticks, terminal, scan_total=N_FILES, files_total=N_FILES + ) + + # --- phase 194: the scan label, WHEREVER the page caught the stage --- + # Whether the UI's 2 s poll lands inside a five-file scan is a coin + # flip (test 2 pins the stage deterministically through a served + # status); what is never a coin flip is the SHAPE of the frames it + # does catch and where they may sit in the sequence. + scan_frames = [f for f in frames if f["text"].startswith("Scanning…")] + for frame in scan_frames: + assert SCAN_LABEL_RE.fullmatch(frame["text"]), frame + assert " remaining" not in frame["text"], frame + assert SOURCE not in frame["text"], f"the scan named a file: {frame}" + # A4: the untruncated label rides the title AND the announcer. + assert frame["title"] == frame["text"], frame + assert frame["aria"] == frame["text"], frame + assert frame["countHidden"] is not ("(" in frame["text"]), frame + first_import = next( + (i for i, t in enumerate(texts) if t.startswith("Syncing… ")), None + ) + if first_import is not None: + assert all( + i < first_import for i, t in enumerate(texts) if t.startswith("Scanning…") + ), f"a scan label rendered after the import phase began: {texts[:14]}" + # --------------------------------------------------------------------------- -# 2. (c) The pre-walk edge + the A1 hidden edge, on the exact payloads the -# guards read +# 2. (c) The zero-total edges + the A1 hidden edge, on the exact payloads +# the guards read # --------------------------------------------------------------------------- def test_prewalk_and_final_file_labels_have_no_count_part( page: Page, app_url: str, db_ready: None ) -> None: - """The two edges of A1, pinned on the payloads the guards actually - read (the served bodies use the status endpoint's unchanged key set — - what is under test is the client's ONE builder): ``total === 0`` - renders no count part, and neither does ``done === total``.""" + """The zero-total edges of A1 plus the phase-194 scan stage, pinned on + the payloads the guards actually read (the served bodies carry the + status endpoint's key set — what is under test is the client's ONE + builder): ``scan_total === 0`` renders no count part, ``total === 0`` + renders none either, and neither does ``done === total``.""" page.set_default_timeout(SETTLE_TIMEOUT_MS) stages: dict[str, Any] = { - "sync": _sync_status(state="running"), # the pre-walk: no file, 0/0 + # The stage phase 194 inserts where the pre-walk used to stand: + # the pre-scan mid-flight, denominator known, position climbing. + "sync": _sync_status( + state="running", phase="scan", scan_done=2, scan_total=N_FILES + ), "upload": None, } _serve_statuses(page, stages) - # settled=False: the boot fetch is the pre-walk stub — the re-attach + # settled=False: the boot fetch is the scan stub — the re-attach # itself is part of what this arm pins. _admin_sources_page(page, app_url, settled=False) _install_recorder(page) @@ -849,9 +1130,36 @@ def test_prewalk_and_final_file_labels_have_no_count_part( box = page.locator("#sync-progress-text") btn = page.locator("#sync-btn") - # (c) The pre-walk — phase null, current_file null, files_total 0: - # the bare prefix, no count part, the count span hidden, the title + # (phase 194, D4) The scan stage speaks its OWN label: "Scanning…" + # plus the (n/total) part in the SAME count slot the import phase's + # "N remaining" uses (one writer, one hidden-toggle, one A4 + # placement) — never a file, never a chunk part, and the full label + # on the button title. + expect(box).to_have_text(f"Scanning… (2/{N_FILES})", timeout=SETTLE_TIMEOUT_MS) + # The PART's own toggle, not Playwright visibility: while a run is in + # flight the button is the spinner and CSS hides the label spans, so + # the honest pin is the writer's ``hidden`` flag plus the span's text + # (the in-browser recorder in test 1 reads the same property). + expect(page.locator(".sync-label-count")).to_have_text(f"(2/{N_FILES})") + expect(page.locator(".sync-label-count")).to_have_js_property("hidden", False) + expect(btn).to_have_attribute("title", f"Scanning… (2/{N_FILES})") + expect(btn).to_be_disabled() + + # …and the same stage with NO denominator yet — the empty KB, and the + # scan's own first frame: the phase-64 ``total === 0`` guard renders + # the bare phase name, never "(0/0)". + stages["sync"] = _sync_status(state="running", phase="scan") + expect(box).to_have_text("Scanning…", timeout=SETTLE_TIMEOUT_MS) + expect(page.locator(".sync-label-count")).to_have_text("") + expect(page.locator(".sync-label-count")).to_have_js_property("hidden", True) + expect(btn).to_have_attribute("title", "Scanning…") + + # (c) The null prelude — phase null, current_file null, files_total 0 + # (the pre-phase pre-walk, and exactly what the + # BOR_SYNC_PENDING_COUNT=0 rollback reports for the whole run): the + # bare prefix, no count part, the count span hidden, the title # identical (A4). + stages["sync"] = _sync_status(state="running") expect(box).to_have_text("Syncing…", timeout=SETTLE_TIMEOUT_MS) expect(page.locator(".sync-label-count")).to_be_hidden() expect(btn).to_have_attribute("title", "Syncing…") @@ -913,19 +1221,41 @@ def test_prewalk_and_final_file_labels_have_no_count_part( finished_at=_now_iso(), files_done=N_FILES, files_total=N_FILES, + # The scan pair KEPT at the terminal beside the file counts (the + # phase-64 keep-final-values convention) — the counts line below + # ignores them, which is the point of pinning them here. + scan_done=N_FILES, + scan_total=N_FILES, detail={"added": N_FILES, "updated": 0, "unchanged": 0, "pruned": 0}, ) expect(box).to_have_text(counts_line(f"{N_FILES} added"), timeout=SETTLE_TIMEOUT_MS) expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=30_000) expect(btn).to_be_enabled() - # Nothing rendered in this run ever spoke the pre-phase shape or a - # "0 remaining". + # Nothing rendered in this run ever spoke the pre-phase shape on a + # "Syncing…" label or a "0 remaining". texts = _texts(_frames(page)) assert not any(OLD_FRACTION_RE.search(t) for t in texts if t.startswith("Syncing…")), texts assert not any("0 remaining" in t for t in texts), texts assert "Syncing…" in texts, texts + # The scan stage rendered, in its own shape, and ONLY before the + # import phase started: the (n/m) is the scan's position (and the + # summaries phase's), never the sync's. + scan_texts = [t for t in texts if t.startswith("Scanning…")] + assert f"Scanning… (2/{N_FILES})" in scan_texts, texts + assert "Scanning…" in scan_texts, texts + for label in scan_texts: + assert SCAN_LABEL_RE.fullmatch(label), label + assert " remaining" not in label, label + first_import = next( + (i for i, t in enumerate(texts) if t.startswith("Syncing… ")), None + ) + assert first_import is not None, texts + assert all( + i < first_import for i, t in enumerate(texts) if t.startswith("Scanning…") + ), texts + # --------------------------------------------------------------------------- # 3. (b) A2 — the upload job shares the builder and the shape @@ -1021,3 +1351,118 @@ def test_upload_label_shares_the_count_down_builder( # has no phase / chunk / image fields — phase 129 / 152), so the # chunk part belongs to the sync arms only. assert " · chunk " not in label, label + + +# --------------------------------------------------------------------------- +# 4. (phase 194, task 04 / D5) The re-sync: the prediction, not the walk +# --------------------------------------------------------------------------- + + +def test_resync_of_an_unchanged_repo_reports_zero_work( + page: Page, app_url: str, db_ready: None, remaining_repo: Path +) -> None: + """The run phase 193's fixture never had: the SAME repo synced twice. + + Run 1 fills the truncated KB, so its predicted change set IS the + whole walk (``files_total == scan_total == N_FILES`` — the parity + test 1 rides). Run 2 finds every byte where it left them, so the + pre-scan predicts NOTHING: the ``progress`` pair is ``0/0`` on every + tick and the client's ``total === 0`` guard (test 2) renders no count + part — while ``scan_done`` / ``scan_total`` report the whole KB walked + and are KEPT at the terminal beside the zeroed file pair. That + divergence (5 scanned, 0 to re-index) is phase 194 in one line, and + it is exactly the shape that used to render "``N`` remaining" for + files nobody was going to re-index (the owner's prod report). + + A no-change run makes ZERO LLM call (the phase-147 fast path), so its + import window is milliseconds and the 2 s UI poll is not expected to + render a file label for it: the pin is the WIRE pair plus the + ABSENCE of a count part in whatever the page did render, not a + rendered per-file label (test 1 covers the rendered side). + """ + page.set_default_timeout(SETTLE_TIMEOUT_MS) + _admin_sources_page(page, app_url) + _install_recorder(page) + btn = page.locator("#sync-btn") + box = page.locator("#sync-progress-text") + + # --- run 1: the fresh import, the parity test 1 rides --------------- + expect(btn).to_be_enabled() + # wait_for_running: test 1's TERMINAL is still what the endpoint + # answers with here (the run state lives in the app, not the page), + # and without the flag this recorder would hand that body back as its + # own before this test had synced anything at all. + rec1 = _TickRecorder(app_url, "/api/sync/status", wait_for_running=True) + rec1.start() + btn.click() + terminal1 = rec1.stop() + assert terminal1["state"] == "success", terminal1 + expect(box).to_have_text( + counts_line(f"{N_FILES} added"), timeout=SETTLE_TIMEOUT_MS + ) + assert terminal1["files_total"] == N_FILES, terminal1 + assert terminal1["scan_total"] == N_FILES, terminal1 + frame_mark = page.evaluate("() => window.__borFrames.length") + + # --- run 2: the same bytes, the owner's incremental case ------------ + expect(btn).to_be_enabled(timeout=SETTLE_TIMEOUT_MS) + rec2 = _TickRecorder(app_url, "/api/sync/status", wait_for_running=True) + rec2.start() + btn.click() + terminal2 = rec2.stop() + assert terminal2["state"] == "success", terminal2 + + # The owner-visible terminal line stays the byte-identical phase-64 + # counts line — the exact surface the prediction never replaces. + expect(box).to_have_text( + counts_line(f"0 added · {N_FILES} unchanged"), timeout=SETTLE_TIMEOUT_MS + ) + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL) + expect(btn).to_be_enabled() + detail = terminal2["detail"] + assert (detail["added"], detail["updated"], detail["unchanged"]) == ( + 0, + 0, + N_FILES, + ), detail + + # THE PHASE IN ONE LINE: the scan walked the KB, the work was nothing. + assert terminal2["scan_total"] == N_FILES, terminal2 + assert terminal2["scan_done"] == N_FILES, terminal2 + assert terminal2["files_total"] == 0, ( + f"a no-change run reported a non-zero denominator: {terminal2}" + ) + assert terminal2["files_done"] == 0, terminal2 + for tick in rec2.ticks: + assert (tick["files_done"], tick["files_total"]) == (0, 0), ( + f"the empty pending set leaked a walk position: {tick}" + ) + assert tick["scan_total"] in (0, N_FILES), tick + # `prelude_zero=False`: the runner resets ``state`` to ``running`` + # one synchronous statement before it resets the scan pair, and the + # status endpoint answers from a threadpool — a poll landing in that + # window would legitimately see the PREVIOUS run's kept 5/5. The + # claim itself is pinned race-free in test 1 and test 2; repeating it + # here would buy nothing but a 1-in-100k flake. + _assert_phase_machine_tick_series( + rec2.ticks, terminal2, scan_total=N_FILES, files_total=0, prelude_zero=False + ) + + # Whatever run 2 rendered carried NO count part: no "N remaining" + # (the empty pending set → the total === 0 guard) and no scan (n/m) + # once the import phase had begun. + run2_texts = _texts(_frames(page)[frame_mark:]) + assert run2_texts, ( + f"nothing was rendered after the run-1 settle mark ({frame_mark}) — " + "the click's own label is missing from the recorder" + ) + assert not any(" remaining" in t for t in run2_texts), run2_texts + for label in run2_texts: + if label.startswith("Scanning…"): + assert SCAN_LABEL_RE.fullmatch(label), label + if label.startswith("Syncing…"): + assert not OLD_FRACTION_RE.search(label), label + scan_idx = [i for i, t in enumerate(run2_texts) if t.startswith("Scanning…")] + imp_idx = [i for i, t in enumerate(run2_texts) if t.startswith("Syncing… ")] + if scan_idx and imp_idx: + assert max(scan_idx) < imp_idx[0], run2_texts diff --git a/tests/integration/test_import_fast_path_columns.py b/tests/integration/test_import_fast_path_columns.py index 7d475a0..fdcc2d7 100644 --- a/tests/integration/test_import_fast_path_columns.py +++ b/tests/integration/test_import_fast_path_columns.py @@ -101,6 +101,20 @@ turn the backfill into a crash instead of a query. ``_prune``'s light row (phase 184), the identity-map cadence (phase 184), ``app/db.py:30``, ``iter_importable_files`` (phase 185 LOCKED A3) and the ``doc.chunks = [...]`` re-embed path (phase 183 LOCKED A2). + +**Why phase 194 left these pins UNEDITED (the D5 sweep).** The phase-194 +pre-scan is gated on the caller passing ``progress`` — and every +``import_sources`` call in this file passes NO ``progress`` (the statement +capture is the point, so the run needs no hook). The pre-scan therefore +never runs here: the captured fast-path ROW statement above stays the only +statement over ``documents`` per inspected file, the "exactly one +``SELECT`` per unchanged file" pin and the "one PK follow-up for the +summary-NULL backfill" pin are byte-identical, and the pre-scan's own +scalar statement (``SELECT documents.file_hash``, ``_pending_hash_stmt``) +appears nowhere in the capture — it is pinned in +``tests/unit/test_sync_pending_count.py`` and +``tests/integration/test_sync_pending_count.py`` instead, where a +``progress`` caller does exist. """ from __future__ import annotations diff --git a/tests/integration/test_import_session_retention.py b/tests/integration/test_import_session_retention.py index c798264..2e19877 100644 --- a/tests/integration/test_import_session_retention.py +++ b/tests/integration/test_import_session_retention.py @@ -135,6 +135,18 @@ lazy SELECT per row and fail that test, not get papered over. **What this file does NOT touch:** the walk's column set (phase 185), ``scripts/git_sync.py`` (phase 186), ``app/db.py:30``, and the ``doc.chunks = [...]`` re-embed path (phase 183 LOCKED A2). + +**Why phase 194 left the bound UNCHANGED (the D5 sweep).** Several tests +here call ``import_sources`` WITH a ``progress`` hook — so the phase-194 +pre-scan really runs on those walks. It cannot move a single number in +this file: ``_prescan_pending`` reads the SCALAR projection +``_pending_hash_stmt`` (one column, no entity), through a session of its +OWN inside the ``to_thread`` span, so nothing lands in the RUN's +``identity_map`` — the staircase, the sawtooth, the ``MAP_BOUND`` ceiling +and the byte-identical ``ImportSummary`` / database comparisons are all +exactly what phase 184 measured. The hydration pin the claim rests on +(``len(session.identity_map) == 0`` at the first ``progress`` call) lives +in ``tests/integration/test_sync_pending_count.py``. """ from __future__ import annotations diff --git a/tests/integration/test_sync_api.py b/tests/integration/test_sync_api.py index 08a3281..4a2d74b 100644 --- a/tests/integration/test_sync_api.py +++ b/tests/integration/test_sync_api.py @@ -93,6 +93,18 @@ on all three terminals (success, cancelled, failed) plus the importer's ``import: summary …``. The fields are APPENDED (LOCKED A2), so every pre-phase prefix below is byte-identical and the pins here assert BOTH halves of that contract. + +Phase 194 (task 02): the status gains the pre-scan. The payload is +ADDITIVE — ``scan_done`` / ``scan_total`` after ``files_total``, the +additive-chain pins below updated to the new key set (the idle shape and +both ``_PRE_187_STATUS_KEYS`` assertions) — and ``files_done`` / +``files_total`` now carry the PREDICTED-change-set position on a progress +caller. One existing pin retargets with it: +``test_double_trigger_while_running_returns_409`` waits on ``"scan"`` +instead of ``"import"``, because its canned import fires NO per-file hook +and the scan→import flip lives in that hook — a run that reports no file +stays in the pre-import phase (the full sequence with a live hook is +pinned in ``tests/integration/test_sync_scan_phase.py``). """ from __future__ import annotations @@ -375,6 +387,11 @@ class FakeImportSources: # image fields stay 0/0, like the chunk fields of a no-batch # import). self.image_hooks: list[object] = [] + # Phase 194 (task 02): the PRE-SCAN position hook the runner + # passes (a live closure while wired, None if the wiring + # regresses) — the canned fake never fires it, so the status + # scan fields stay 0/0 like a run that scanned nothing. + self.scan_hooks: list[object] = [] async def __call__( self, @@ -387,6 +404,7 @@ class FakeImportSources: progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, # phase 129 on_image_progress: Callable[[int, int], None] | None = None, # phase 152 + on_scan_progress: Callable[[int, int], None] | None = None, # phase 194 ignore_by_root: dict[str, list[str]] | None = None, include_hidden_by_root: dict[str, bool] | None = None, doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106 @@ -397,6 +415,7 @@ class FakeImportSources: self.progress_hooks.append(progress) self.chunk_hooks.append(on_chunk_progress) self.image_hooks.append(on_image_progress) + self.scan_hooks.append(on_scan_progress) self.ignore_maps.append(ignore_by_root or {}) self.include_hidden_maps.append(include_hidden_by_root or {}) self.doc_dates_maps.append(doc_dates_by_root or {}) @@ -572,6 +591,10 @@ def test_admin_sync_success_reports_full_detail( # (the dataclass defaults). "images_done": 0, "images_total": 0, + # Phase 194 (task 02): the pre-scan position keys — 0/0 idle + # (the dataclass defaults). + "scan_done": 0, + "scan_total": 0, # Phase 98 (task 01): the phase-machine keys — null/0/0/0 idle. "phase": None, "current_summary": None, @@ -703,17 +726,27 @@ def test_double_trigger_while_running_returns_409( assert r.status_code == 409 assert r.json() == {"detail": "a sync is already running"} - # Phase 179 (task 02): WAIT for the import phase — the run's - # progress is concurrent now that the git ops left the loop, so the - # next request no longer inherits a run that already ran past them. - body = _wait_phase(sync_client, "import") + # Phase 179 (task 02): WAIT for the import to have started — the + # run's progress is concurrent now that the git ops left the loop, + # so the next request no longer inherits a run that already ran past + # them. Phase 194 (task 02): the phase the wait targets is now + # ``"scan"`` — the canned fake import reports NO file, and the + # scan→import flip lives in the runner's per-file progress hook, so + # a hook-less run legitimately stays in the pre-import phase (the + # sequence with a real hook is pinned in test_sync_scan_phase.py and + # in the phase-98 gated test below). + body = _wait_phase(sync_client, "scan") assert body["state"] == "running" assert body["started_at"] is not None assert body["finished_at"] is None assert body["error"] is None + # Wiring pin (phase 194, task 02): the runner passed its live + # scan-position closure, and a fake that fires nothing keeps the + # position at 0/0. + assert len(fake_import.scan_hooks) == 1 and fake_import.scan_hooks[0] is not None + assert body["scan_done"] == 0 and body["scan_total"] == 0 # Phase 98 (task 01): the in-flight run (parked in the fake - # import's delay) reports the import phase — the summary span has - # not started, so its counters are 0/0. + # import's delay) reports no summary span — its counters are 0/0. assert body["current_summary"] is None assert body["summaries_done"] == 0 and body["summaries_total"] == 0 @@ -769,6 +802,10 @@ class _GatedImport: self.image_pairs = list(image_pairs) self.image_release = image_release self.image_hook_calls: list[tuple[int, int]] = [] + # Phase 194 (task 02): the PRE-SCAN position hook the runner + # passes (recorded for the wiring pin — the gated fake never + # fires it, so the status scan fields stay 0/0). + self.scan_hook: Callable[[int, int], None] | None = None async def __call__( self, @@ -781,10 +818,12 @@ class _GatedImport: progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, # phase 129 on_image_progress: Callable[[int, int], None] | None = None, # phase 152 + on_scan_progress: Callable[[int, int], None] | None = None, # phase 194 ignore_by_root: dict[str, list[str]] | None = None, include_hidden_by_root: dict[str, bool] | None = None, doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106 ) -> ImportSummary: + self.scan_hook = on_scan_progress if progress is not None: progress("repo", "notes/deep.md", 1, 3) self.hook_calls.append(("repo", "notes/deep.md", 1, 3)) @@ -1937,6 +1976,7 @@ def test_import_error_is_reported_with_credentials_masked( progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, # phase 129 on_image_progress: Callable[[int, int], None] | None = None, # phase 152 + on_scan_progress: Callable[[int, int], None] | None = None, # phase 194 ignore_by_root: dict[str, list[str]] | None = None, include_hidden_by_root: dict[str, bool] | None = None, doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106 @@ -2698,6 +2738,12 @@ _PRE_187_STATUS_KEYS = frozenset( #: Phase 187 (task 03): the two additive memory keys. _MEMORY_KEYS = frozenset({"rss_mb", "rss_peak_mb"}) +#: Phase 194 (task 02): the two additive PRE-SCAN position keys — the +#: next link in the same chain (the payload is exactly the pre-187 set +#: plus the memory pair plus the scan pair, and a future addition joins +#: this chain rather than editing the sets above). +_SCAN_KEYS = frozenset({"scan_done", "scan_total"}) + #: The pair is read from two DIFFERENT kernel counters and the high-water #: refreshes LAZILY, so a raw ``rss_peak_mb()`` really can sit under a raw #: ``rss_mb()`` (measured: ``rss_mb=164`` against ``rss_peak_mb=162``) — @@ -2744,13 +2790,17 @@ def test_memory_status_keys_are_the_only_addition_and_null_before_a_run( sync_client: TestClient, ) -> None: """The additive pin on the wire: the payload's key set is EXACTLY the - pre-phase-187 set plus ``rss_mb`` / ``rss_peak_mb``, and a run that - never happened answers ``null`` / ``null`` — not 0, not a plausible + pre-phase-187 set plus ``rss_mb`` / ``rss_peak_mb`` plus the + phase-194 ``scan_done`` / ``scan_total`` pair, and a run that never + happened answers ``null`` / ``null`` — not 0, not a plausible number. That null is also what makes the Sources page render the counts line with no peak clause after an app restart.""" _login(sync_client) body = sync_client.get("/api/sync/status").json() - assert set(body) == _PRE_187_STATUS_KEYS | _MEMORY_KEYS, set(body) + assert set(body) == _PRE_187_STATUS_KEYS | _MEMORY_KEYS | _SCAN_KEYS, set(body) + # Phase 194 (task 02): the scan pair joins ADDITIVELY — idle is 0/0 + # (never null, never a plausible position). + assert body["scan_done"] == 0 and body["scan_total"] == 0 assert body["rss_mb"] is None assert body["rss_peak_mb"] is None # The pre-existing shapes, unchanged: the state word, the two ISO @@ -2811,8 +2861,9 @@ def test_success_terminal_status_keeps_the_final_memory_pair( assert done[0].endswith( f"rss_mb={fmt_mb(current)} rss_peak_mb={fmt_mb(peak)}" ), done[0] - # The pre-existing payload is untouched by all of this. - assert set(again) == _PRE_187_STATUS_KEYS | _MEMORY_KEYS + # The pre-existing payload is untouched by all of this (phase 194's + # scan pair is the only addition, and it is additive). + assert set(again) == _PRE_187_STATUS_KEYS | _MEMORY_KEYS | _SCAN_KEYS assert again["files_done"] == 0 or isinstance(again["files_done"], int) diff --git a/tests/integration/test_sync_git_freeze.py b/tests/integration/test_sync_git_freeze.py index a8bf30e..01fe857 100644 --- a/tests/integration/test_sync_git_freeze.py +++ b/tests/integration/test_sync_git_freeze.py @@ -58,6 +58,19 @@ clone's result. Python threads are not interruptible: the git worker runs its ``subprocess`` to completion, but the EVENT LOOP is free the whole time. +**Why phase 194 leaves these git-span pins alone (the D5 sweep).** The +runner's import here is a stub (``fake_import`` below, which accepts the +phase-194 ``on_scan_progress`` argument and fires nothing), so no real +pre-scan runs inside this file's measurement window — the clone window +this suite pins stays the only off-loop span it measures. The pre-scan's +OWN loop-responsiveness pin (the same phase-179 pattern: a blocking +``_file_hash`` over a multi-file fixture while a heartbeat coroutine keeps +being scheduled, plus the worker-thread identity) lives in +``tests/integration/test_sync_pending_count.py`` — beside the cancel at +that span's await (the importer-side twin of this file's +cancel-during-clone pin) and its app-side twin in +``tests/integration/test_sync_scan_phase.py``. + **TestClient mechanics (why the trigger still rides a side thread).** The shared app loop here is anyio's ``BlockingPortal`` thread. Firing ``POST /api/sync`` from the test thread would make the mid-clone probe @@ -326,6 +339,7 @@ class _FakeImport: progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, on_image_progress: Callable[[int, int], None] | None = None, + on_scan_progress: Callable[[int, int], None] | None = None, # phase 194 ignore_by_root: dict[str, list[str]] | None = None, include_hidden_by_root: dict[str, bool] | None = None, doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, diff --git a/tests/integration/test_sync_pending_count.py b/tests/integration/test_sync_pending_count.py new file mode 100644 index 0000000..30adb0d --- /dev/null +++ b/tests/integration/test_sync_pending_count.py @@ -0,0 +1,571 @@ +"""Integration: phase 194 — the pre-scan predicts the change set, off the loop. + +The owner report that opened the phase (2026-10-19, verbatim): *"I just synced +the prod brain of reese and it said '1216 remaining' when it only had like 10 +files to sync."* Prod holds ~10,000 importable files and that run changed ~10. +The phase-193 ``N remaining`` label was arithmetically fine — it was fed the +WALK (every importable file) instead of the WORK. The fix lives in +``app/rag/importer.py``: before the walk, ``_prescan_pending`` hashes every +importable file once and reads ``documents.file_hash`` once per file — the +exact predicate the phase-147 unchanged fast path applies — and the per-file +``progress`` hook reports the position in THAT set. + +What this file measures with a real ``import_sources`` run: + +* **the parity that makes the number honest** — on a stable knowledge base + the predicted set's size equals the run's terminal ``added + updated``: the + fresh arm (everything pending, exactly what a first import indexes), the + unchanged arm (nothing pending, so no count part at all), and the + incremental arm (one changed file of four — the owner's prod case, where + the walk is 4 and the work is 1); +* **zero entity hydration** — the scan's statement projects exactly + ``documents.file_hash`` (no other column, no ORM row), so a 10,000-file + scan retains nothing: the run's session is still empty at the first + ``progress`` call, before the processing loop has loaded any row (phase + 184/185); +* **the span is OFF the loop** — with a stubbed hash that blocks, a heartbeat + coroutine keeps ticking through the scan and the hash runs on a different + thread than the one that awaited it (the phase-179 pattern of + ``test_sync_git_freeze.py``: the git clone used to park the single uvicorn + loop and freeze the ``GET /api/sync/status`` poll the UI runs every 2 s, + and a 10,000-file hash+SELECT span inline is the same defect class); +* **the cancel contract** — cancelling the run at the scan's await lands the + cancellation immediately, while the worker itself runs to completion + (threads are not interruptible, phase 134) and nothing gets indexed. + +The CLI and the upload job pass no ``progress``, so they run no scan at all; +their byte-identity is pinned in ``tests/unit/test_sync_pending_count.py``. +""" +from __future__ import annotations + +import asyncio +import re +import threading +import time +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Any, NamedTuple + +import pytest +from sqlalchemy import event, select, text +from sqlalchemy.orm import Session + +import app.rag.importer as importer +from app.db import engine +from app.models import Document +from app.rag.importer import ImportSummary, import_sources +from tests.fakes import FakeEmbedder + +#: The probe's own source-name prefix — nothing else in the KB uses it. +PREFIX = "phase194-it-" + +#: A blocking hash of 60 ms per file: four files is a ~250 ms scan — long +#: enough that a heartbeat polling every 5 ms cannot tick during it if the +#: span ever runs on the loop thread, short enough that the suite stays fast. +HASH_BLOCK_S = 0.06 + +#: The heartbeat tick count that proves the loop was free across ONE scanned +#: file (60 ms / 5 ms ≈ 12 ticks; a blocked loop ticks 0). The ceiling-side +#: assertion is the same number generous: this is loop-freeness, not timing. +MIN_TICKS_PER_FILE = 4 + + +class ProgressCall(NamedTuple): + """One ``progress(source, rel, done, total)`` report.""" + + source: str + rel: str + done: int + total: int + + +class Run(NamedTuple): + """One real ``import_sources`` pass and everything it reported.""" + + summary: ImportSummary + progress: list[ProgressCall] + scan: list[tuple[int, int]] + + +def _llm(**setting_overrides: Any) -> FakeEmbedder: + """The deterministic in-process fake on the SHIPPED defaults (the + pending-count switch is therefore ON — the phase's default).""" + from app.config import Settings + + llm = FakeEmbedder() + llm.settings = Settings(_env_file=None, **setting_overrides) # pyright: ignore[reportCallIssue] + return llm + + +def _root(tmp_path: Path, name: str, files: dict[str, str]) -> Path: + """One source root (the ``file://``-style temp dir of the house + import fixtures) with *files* under it.""" + assert name.startswith(PREFIX), f"the probe source must be {PREFIX}-prefixed" + root = tmp_path / name + for rel, body in files.items(): + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + return root + + +def _run( + session: Session, + *roots: Path, + llm: FakeEmbedder | None = None, + **kwargs: Any, +) -> Run: + """One real import over *roots*, collecting both position hooks.""" + progress: list[ProgressCall] = [] + scan: list[tuple[int, int]] = [] + summary = asyncio.run( + import_sources( + list(roots), + llm or _llm(), + session=session, + progress=lambda s, r, d, t: progress.append(ProgressCall(s, r, d, t)), + on_scan_progress=lambda p, t: scan.append((p, t)), + **kwargs, + ) + ) + return Run(summary, list(progress), list(scan)) + + +@pytest.fixture() +def statement_capture() -> Iterator[list[str]]: + """Every statement the engine executes for the duration of ONE test. + + The ``before_cursor_execute`` technique of phase 183/185 + (``tests/integration/test_import_fast_path_columns.py``): what a SELECT + actually projects is invisible in the ORM's Python API and plain in the + statement it compiles, so a capture is the only way to pin "the pre-scan + reads one column". Removed in teardown, so no other suite sees it. + """ + captured: list[str] = [] + + def _listener( + conn: object, # noqa: ARG001 + cursor: object, # noqa: ARG001 + statement: str, + parameters: object, # noqa: ARG001 + context: object, # noqa: ARG001 + executemany: bool, # noqa: ARG001 + ) -> None: + captured.append(statement) + + event.listen(engine, "before_cursor_execute", _listener) + try: + yield captured + finally: + event.remove(engine, "before_cursor_execute", _listener) + + +@pytest.fixture(autouse=True) +def _clean_probe_documents(db: Session) -> Iterator[None]: + """``documents``/``chunks`` are global state — the probe's rows are + removed before and after every test (the ``test_sync_api.py`` hygiene + pattern, narrowed to this module's own source names).""" + _drop(db) + yield + _drop(db) + + +def _drop(db: Session) -> None: + db.execute(text("DELETE FROM chunks WHERE document_id IN (SELECT id FROM " + "documents WHERE source LIKE :p)"), {"p": f"{PREFIX}%"}) + db.execute(text("DELETE FROM documents WHERE source LIKE :p"), {"p": f"{PREFIX}%"}) + db.commit() + db.expunge_all() + + +# -------------------------------------------------------------------------- +# The parity: the predicted set IS the work +# -------------------------------------------------------------------------- + +_FOUR_FILES = { + "a.md": "# A\n\nalpha content\n", + "b.md": "# B\n\nbravo content\n", + "c.md": "# C\n\ncharlie content\n", + "notes/d.md": "# D\n\ndelta content\n", +} + + +def test_fresh_kb_pending_set_equals_the_added_count( + db: Session, tmp_path: Path +) -> None: + """Arm 1 — the first import: walk == work, so the numbers are the + phase-193 ones (A1's degenerate parity, the reason the phase-193 E2E + suite never saw the defect). + + Every file has no row, so every file is pending: the hook reports + ``1/4 … 4/4`` and the terminal says ``added=4`` — the predicted size + equals ``added + updated`` exactly.""" + root = _root(tmp_path, f"{PREFIX}fresh", _FOUR_FILES) + + run = _run(db, root) + + assert (run.summary.files, run.summary.added, run.summary.errors) == (4, 4, 0) + assert [(c.done, c.total) for c in run.progress] == [(1, 4), (2, 4), (3, 4), (4, 4)] + assert run.scan == [(1, 4), (2, 4), (3, 4), (4, 4)] + assert run.progress[-1].total == run.summary.added + run.summary.updated + + +def test_unchanged_kb_predicts_nothing_and_terminal_says_no_work( + db: Session, tmp_path: Path +) -> None: + """Arm 2 — the no-change sync: nothing pending, so no count part. + + The scan walks all four files and predicts none of them; every ``progress`` + call therefore carries ``(0, 0)``, which the phase-64 ``total === 0`` + guard renders as a bare label with no ``N remaining`` — the honest shape + for a run with no work, where the pre-phase label said ``4 remaining`` + while walking. ``summary.files`` (the log's walked-file count) is + untouched: the run still processed 4 files and still logged 4 unchanged. + """ + root = _root(tmp_path, f"{PREFIX}unchanged", _FOUR_FILES) + seed = _run(db, root) + assert seed.summary.added == 4 + + run = _run(db, root) + + assert (run.summary.files, run.summary.added, run.summary.updated) == (4, 0, 0) + assert run.summary.unchanged == 4 + assert [(c.done, c.total) for c in run.progress] == [(0, 0)] * 4 + # The scan still walked (and reported) every file — the DENOMINATOR of the + # scan is the importable count, not the prediction. + assert run.scan == [(1, 4), (2, 4), (3, 4), (4, 4)] + assert run.summary.added + run.summary.updated == 0 + + +def test_incremental_kb_counts_the_changed_files_only( + db: Session, tmp_path: Path +) -> None: + """Arm 3 — the owner's prod case at 1-of-4 scale. + + One changed file out of four: the walk is still 4 (``summary.files``, the + log line, and every ``progress`` call naming a file), but the count is the + WORK — ``total`` 1, the position climbing only when the walk reaches the + changed file, and holding at 0 while it moves over the unchanged ones. + The parity pin: the prediction's size equals the terminal's + ``added + updated``. + """ + root = _root(tmp_path, f"{PREFIX}incremental", _FOUR_FILES) + seed = _run(db, root) + assert seed.summary.added == 4 + + (root / "notes" / "d.md").write_text( + "# D\n\ndelta content EDITED on disk\n", encoding="utf-8" + ) + + run = _run(db, root) + + assert (run.summary.files, run.summary.updated, run.summary.unchanged) == ( + 4, + 1, + 3, + ) + assert [(c.rel, c.done, c.total) for c in run.progress] == [ + ("a.md", 0, 1), + ("b.md", 0, 1), + ("c.md", 0, 1), + ("notes/d.md", 1, 1), + ] + assert run.progress[-1].total == run.summary.added + run.summary.updated + + +def test_two_changed_files_count_down_to_zero( + db: Session, tmp_path: Path +) -> None: + """The count the owner asked for, at the scale the label is read at. + + Two changed files among four: the denominator is 2 from the first report, + the position advances only at the predicted files (the unchanged walk + between them holds it), and ``total - done`` reaches 0 exactly when the + last predicted file starts — where phase 193's ``N ≥ 1`` guard hides the + count part. The terminal reports the same two files as updated.""" + root = _root(tmp_path, f"{PREFIX}two", _FOUR_FILES) + _run(db, root) + (root / "a.md").write_text("# A\n\nalpha edited\n", encoding="utf-8") + (root / "notes" / "d.md").write_text( + "# D\n\ndelta edited\n", encoding="utf-8" + ) + + run = _run(db, root) + + assert (run.summary.updated, run.summary.unchanged) == (2, 2) + assert [(c.rel, c.done, c.total) for c in run.progress] == [ + ("a.md", 1, 2), + ("b.md", 1, 2), # unchanged: the walk moves, the count holds + ("c.md", 1, 2), + ("notes/d.md", 2, 2), + ] + remaining = [c.total - c.done for c in run.progress] + assert remaining == [1, 1, 1, 0], ( + f"the label's remaining count went {remaining} — it must hold across " + "the unchanged files and reach 0 at the last predicted one (where " + "phase 193's guard hides the part)" + ) + + +# -------------------------------------------------------------------------- +# Zero entity hydration +# -------------------------------------------------------------------------- + + +def test_prescan_reads_one_column_and_hydrates_nothing( + db: Session, tmp_path: Path, statement_capture: list[str] +) -> None: + """The scan's statement projects EXACTLY ``documents.file_hash`` and the + run's session is still empty when the walk begins. + + Three pins on the same claim, because the failure mode is the 2026-10-15 + incident (a full walk hydrating 252 MB of markdown): + + * every ``documents`` ``(source, path)`` SELECT issued before the first + ``progress`` call — i.e. by the scan — names ``file_hash`` and NOTHING + else, no ``content`` above all (the phase-185 column-list technique); + * exactly one such read per importable file; + * ``len(session.identity_map) == 0`` at the first ``progress`` call: the + scan put no ``Document`` anywhere, so the run's retention stays bounded + by the phase-184 cadence instead of growing with the KB. + """ + root = _root(tmp_path, f"{PREFIX}hydrate", _FOUR_FILES) + _run(db, root) + db.expunge_all() + statement_capture.clear() + + map_at_first_progress: list[int] = [] + + def progress(source: str, rel: str, done: int, total: int) -> None: # noqa: ARG001 + map_at_first_progress.append(len(db.identity_map)) + + summary = asyncio.run( + import_sources( + [root], _llm(), session=db, progress=progress, on_scan_progress=None + ) + ) + assert summary.unchanged == 4 + + # The scan reads precede the loop's row loads in capture order, and the + # first progress call is the boundary between them. Classified on the + # PROJECTION (phase 185's technique): the scan's projection is the single + # column token, the fast path's is every column but ``content``. + def predicate(stmt: str) -> str: + """Everything after the statement's ``FROM`` (case-insensitive — + SQLAlchemy compiles the keyword upper-case).""" + return re.split(r"\bfrom\s+documents\b", stmt, maxsplit=1, flags=re.I)[-1] + + doc_reads = [ + stmt + for stmt in statement_capture + if stmt.lstrip().lower().startswith("select") + and re.search(r"\bfrom\s+documents\b", stmt, re.I) + and "documents.source" in predicate(stmt) + and "documents.path" in predicate(stmt) + ] + + def projection(stmt: str) -> set[str]: + head = re.split(r"\bfrom\b", stmt, maxsplit=1, flags=re.I)[0] + return set(re.findall(r"documents\.([a-z_]+)", head)) + + scan_reads = [s for s in doc_reads if projection(s) == {"file_hash"}] + row_loads = [s for s in doc_reads if projection(s) != {"file_hash"}] + assert len(scan_reads) == 4, ( + f"the unchanged run issued {len(scan_reads)} single-column file_hash " + "reads — the scan reads each importable file's hash exactly once" + ) + assert len(row_loads) == 4, ( + f"the loop issued {len(row_loads)} fast-path row loads — the scan must " + "not replace _doc_row_stmt, which stays the only ROW load (phase 185)" + ) + assert all( + projection(s) == {"file_hash"} for s in scan_reads + ), "a scan read projects more than file_hash — it would hydrate a row" + assert all( + "content" not in projection(s) for s in row_loads + ), "the fast-path row load stopped deferring content (phase 185)" + assert map_at_first_progress and map_at_first_progress[0] == 0, ( + "the run's session already held entities when the loop reached its " + "first file — the pre-scan leaked a row into the identity map" + ) + + +# -------------------------------------------------------------------------- +# The span is off the event loop (the phase-179 pattern) +# -------------------------------------------------------------------------- + + +class _SlowHash: + """The timing fixture: a PLAIN blocking ``_file_hash`` (like the real one + — the point under test is WHERE it runs), counting its calls. + + The thread identity is NOT taken from here: ``_file_hash`` is called both + by the scan (worker thread) and by the processing loop (loop thread), so + the attribution comes from ``on_scan_progress``, which fires on the same + worker thread as the scan's own hashes.""" + + def __init__(self, real: Callable[[Path], str], block: float = HASH_BLOCK_S) -> None: + self.real = real + self.block = block + self.calls = 0 + + def __call__(self, path: Path) -> str: + self.calls += 1 + time.sleep(self.block) + return self.real(path) + + +def test_prescan_span_does_not_park_the_event_loop( + db: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """THE phase-179 contract, re-pinned for the pre-scan. + + A 4-file scan with a 60 ms blocking hash is ~250 ms of work that used to + be inline. The heartbeat coroutine polls every 5 ms; the ticks between two + consecutive scan reports prove the LOOP kept running while the worker + blocked (a regression that runs the scan inline parks the loop thread and + the heartbeat cannot tick — the task-01 git measurement was 0 ticks over + a 3 s clone). The thread identities are asserted too, so the pin fails on + a machine fast enough to hide the latency. + """ + root = _root(tmp_path, f"{PREFIX}freeze", _FOUR_FILES) + stub = _SlowHash(importer._file_hash) + monkeypatch.setattr(importer, "_file_hash", stub) + + ticks = 0 + scan_ticks: list[int] = [] + scan_positions: list[tuple[int, int]] = [] + scan_thread: list[int] = [] + + def scan_hook(position: int, total: int) -> None: + # Runs on the span's worker thread — the same thread that just hashed. + scan_positions.append((position, total)) + scan_ticks.append(ticks) + scan_thread.append(threading.get_ident()) + + # A progress caller is what opens the gate (the CLI shape, ``progress=None``, + # would skip the scan entirely) — this is the UI sync's shape. + async def main() -> int: + nonlocal ticks + loop_ident = threading.get_ident() + + async def heartbeat() -> None: + nonlocal ticks + while True: + await asyncio.sleep(0.005) + ticks += 1 + + hb = asyncio.create_task(heartbeat()) + try: + await import_sources( + [root], + _llm(), + session=db, + progress=lambda s, r, d, t: None, + on_scan_progress=scan_hook, + ) + finally: + hb.cancel() + return loop_ident + + loop_ident = asyncio.run(main()) + + assert scan_positions == [(1, 4), (2, 4), (3, 4), (4, 4)] + assert scan_thread and loop_ident is not None + assert all(ident != loop_ident for ident in scan_thread), ( + "the pre-scan ran on the event-loop thread — a long scan parks the " + "single uvicorn loop again (phase 179's defect class)" + ) + deltas = [ + b - a for a, b in zip([0, *scan_ticks], scan_ticks, strict=False) + ] + assert min(deltas) >= MIN_TICKS_PER_FILE, ( + f"the heartbeat ticked {deltas} times across consecutive scanned files " + f"(expected >= {MIN_TICKS_PER_FILE} each) — the loop was starved while " + "the scan blocked, so the span is not off-loop" + ) + + +# -------------------------------------------------------------------------- +# The cancel (phase 134): the run stops, the worker finishes +# -------------------------------------------------------------------------- + + +def test_cancel_during_the_prescan_stops_the_run_while_the_worker_finishes( + db: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The phase-134 cooperative cancel, measured at the scan's await. + + Cancelling the run ~80 ms into a ~250 ms scan raises ``CancelledError`` + out of ``import_sources`` PROMPTLY (well before the scan's own remaining + work), NOTHING is indexed, and the worker thread runs to completion — + threads are not interruptible, and this is exactly why the span reads + through a session of its own instead of the run's (which the cancel + closes on its way out). The pre-phase-194 pipeline had no await here at + all, so this is the new boundary the UI's Stop button must survive. + """ + root = _root(tmp_path, f"{PREFIX}cancel", _FOUR_FILES) + stub = _SlowHash(importer._file_hash, block=0.12) + monkeypatch.setattr(importer, "_file_hash", stub) + + real_prescan = importer._prescan_pending + worker_finished = threading.Event() + worker_results: list[Any] = [] + + def spy(*args: Any, **kwargs: Any) -> Any: + try: + result = real_prescan(*args, **kwargs) + worker_results.append(result) + return result + finally: + worker_finished.set() + + monkeypatch.setattr(importer, "_prescan_pending", spy) + + async def main() -> float: + # The hook runs on the WORKER thread, so it only appends to a list — + # the house hook contract (a GIL-atomic store). Touching an + # ``asyncio.Event`` from here would be a cross-thread loop call. + seen: list[tuple[int, int]] = [] + + def scan_hook(position: int, total: int) -> None: + seen.append((position, total)) + + task = asyncio.create_task( + import_sources( + [root], + _llm(), + session=db, + progress=lambda s, r, d, t: None, + on_scan_progress=scan_hook, + ) + ) + # Wait (from the loop) until the scan has reported its first file. + deadline = time.monotonic() + 10.0 + while not seen and time.monotonic() < deadline: + await asyncio.sleep(0.005) + assert seen, "the pre-scan never reported a file" + await asyncio.sleep(0.08) + t0 = time.monotonic() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + return time.monotonic() - t0 + + cancel_latency = asyncio.run(main()) + + # The cancel is prompt: the run did NOT wait for the remaining scan work. + assert cancel_latency < 0.5, ( + f"the cancelled run took {cancel_latency:.3f}s to unblock — it is " + "still waiting for the blocking scan worker" + ) + # The worker is not interruptible: it finishes its own session's work. + assert worker_finished.wait(10.0), "the pre-scan worker never completed" + assert stub.calls == 4, ( + f"the cancelled run's scan worker hashed {stub.calls} of 4 files — " + "the contract is that the worker runs to completion" + ) + # And the pipeline that would have consumed the prediction never ran. + assert db.scalar( + select(select(Document).where(Document.source == root.name).exists()) + ) is False, "the cancelled run indexed documents — the import never started" diff --git a/tests/integration/test_sync_scan_phase.py b/tests/integration/test_sync_scan_phase.py new file mode 100644 index 0000000..b739fc0 --- /dev/null +++ b/tests/integration/test_sync_scan_phase.py @@ -0,0 +1,927 @@ +"""Integration: phase 194 (task 02) — the ``"scan"`` phase on the status surface. + +The owner report that opened the phase (2026-10-19): *"it said '1216 +remaining' when it only had like 10 files to sync"*. Task 01 moved the +count onto the pre-scan's PREDICTED change set; this file pins the +status half (design D3, owner-confirmed A2 — the pre-scan is a VISIBLE +stage, not an invisible pause): + +* the phase machine is ``null`` → ``"scan"`` (set immediately before + ``import_sources``, where the scan actually runs) → ``"import"`` + (flipped by the FIRST per-file ``progress`` call) → ``"overview"`` → + ``"summaries"`` → the terminal clears ``phase``; +* ``scan_done`` / ``scan_total`` carry the scan's own position through + that stage, reset with the run, and are KEPT at every terminal beside + the final file counts (the phase-64 keep-final-values convention); +* the payload is ADDITIVE — the pre-phase key set plus exactly those two + keys, byte-identical in name and type, stable from idle through the + terminal; +* the phase-187 sampling discipline holds on the new stage: the ``"scan"`` + set and the ``scan``→``import`` flip are TRANSITIONS and force the + ``/proc`` read, while the scan's own per-file hook is THROTTLED — one + read per interval, never one per file (LOCKED A4); +* the two degenerate/terminal edges: an empty knowledge base sits in + ``"scan"`` and never reaches ``"import"`` (no ``progress`` call exists + to flip it) and a cancel DURING the scan lands the ``cancelled`` + terminal with the scan counts kept and nothing indexed (the phase-134 + contract, the app-side twin of the importer-side pin in + ``tests/integration/test_sync_pending_count.py``). + +**Why one arm is choreographed and one is real.** An in-process import +never awaits real I/O (the embedder is a fake), so the loop cannot answer a +status poll from inside it — a real run's ``"import"`` phase is therefore +NOT reliably observable, and the real arm pins what the terminal keeps +(both pairs, divergent on the incremental run). The flip itself gets a +deterministic pin from the choreographed double (the ``test_sync_api`` +gated-import idiom), which drives the runner's OWN closures and parks at +every frame the test reads. The scan, by contrast, runs in a +``to_thread`` worker, so the loop is FREE through it — that is exactly why +its frames are observable, and why the blocking-hash fixture slows ONLY the +scan window (a blanket stub would park the loop through the import too). + +House patterns (``test_sync_api.py`` / ``test_sync_git_freeze.py``): the +``sync_client`` fixture is a CONTEXT-MANAGED ``TestClient`` — one app event +loop across requests, so the background run survives between the POST and +the polls; the module ``_status`` / ``_task`` / ``_cancel_event`` / +``_rss_sampled_at`` and the ``git_sources`` / ``sources_meta`` / +``folder_summaries`` / ``documents`` global state are reset around every +test; the probe is stubbed so nothing reaches the network, and the import +runs for real over a temp ``kind=local`` source with the deterministic +``FakeEmbedder``. The scan is made observable with a PLAIN blocking +``_file_hash`` stub scoped to the scan window (the phase-179 timing-fixture +idiom): the point under test is WHICH phase the run is in WHEN the page +polls, and a scan of instant hashes is over before the first poll. +""" +from __future__ import annotations + +import asyncio +import logging +import threading +import time +from collections.abc import Callable, Iterator +from datetime import datetime +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import select, text +from sqlalchemy.orm import Session + +import app.rag.importer as importer +from app.api import sync as sync_api +from app.config import Settings +from app.db import SessionLocal, db_available +from app.main import app as fastapi_app +from app.models import Document, GitSource +from app.rag import git_sources as git_sources_resolver +from app.rag.importer import ImportSummary +from app.rag.llm import LLMClient +from tests.conftest import ADMIN_PASSWORD +from tests.fakes import FakeEmbedder + +#: The probe's own source-name prefix — nothing else in the KB uses it, +#: so the module's DELETE-by-source cleanup can never touch another +#: suite's rows. +PREFIX = "phase194-t02-" + +#: One blocking ``_file_hash`` per file. Three files is a ~360 ms scan: +#: long enough that a 10 ms poll catches several distinct positions inside +#: the phase (the frames are what the UI's 2 s poll would render), short +#: enough that the suite stays fast. +HASH_BLOCK_S = 0.12 + + +# --- fixtures (the house sync hygiene) -------------------------------------- + + +@pytest.fixture(autouse=True) +def _fresh_sync_state() -> Iterator[None]: + """The module-level status object + task + cancel event + sampling + guard are process-global — reset before AND after every test so no + terminal state, stale cancel or stale throttle window leaks into the + next one (``test_sync_api`` / ``test_sync_git_freeze``).""" + sync_api._status = sync_api.SyncStatus() + sync_api._task = None + sync_api._cancel_event = asyncio.Event() + sync_api._rss_sampled_at = float("-inf") + yield + sync_api._status = sync_api.SyncStatus() + sync_api._task = None + sync_api._cancel_event = asyncio.Event() + sync_api._rss_sampled_at = float("-inf") + + +@pytest.fixture(autouse=True) +def clean_git_sources(db: Session) -> Iterator[None]: + """The runner resolves through the real ``git_sources`` table — + truncated around every test (the rows this file seeds are + ``kind=local``).""" + db.execute(text("TRUNCATE git_sources")) + db.commit() + yield + db.execute(text("TRUNCATE git_sources")) + db.commit() + + +@pytest.fixture(autouse=True) +def _reset_sources_version() -> Iterator[None]: + """The phase-53 bump is global mutable state — pin it to the + migration-0010 seed (0) around every test (own session: the runner + bumps through its own short-lived ``SessionLocal``).""" + if not db_available(): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + session = SessionLocal() + try: + session.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1")) + session.commit() + yield + finally: + session.execute(text("UPDATE sources_meta SET version = 0 WHERE id = 1")) + session.commit() + session.close() + + +@pytest.fixture(autouse=True) +def _clean_folder_summaries(db: Session) -> Iterator[None]: + """``folder_summaries`` is global state the real generator writes on a + changed KB — truncated around every test.""" + db.execute(text("TRUNCATE folder_summaries")) + db.commit() + yield + db.execute(text("TRUNCATE folder_summaries")) + db.commit() + + +@pytest.fixture(autouse=True) +def _clean_probe_documents(db: Session) -> Iterator[None]: + """The real imports of this module write ``documents``/``chunks`` — + removed before and after every test, narrowed to this module's own + source names (the ``test_sync_pending_count`` hygiene).""" + _drop(db) + yield + _drop(db) + + +def _drop(db: Session) -> None: + db.execute( + text("DELETE FROM chunks WHERE document_id IN " + "(SELECT id FROM documents WHERE source LIKE :p)"), + {"p": f"{PREFIX}%"}, + ) + db.execute(text("DELETE FROM documents WHERE source LIKE :p"), {"p": f"{PREFIX}%"}) + db.commit() + db.expunge_all() + + +@pytest.fixture() +def sync_client() -> Iterator[TestClient]: + """Context-managed ``TestClient`` — one app event loop across requests + (the background run must survive between the POST and the polls).""" + with TestClient(fastapi_app) as client: + yield client + + +# --- helpers ---------------------------------------------------------------- + + +def _settings(sources_dir: str) -> Settings: + return Settings(_env_file=None, sources_dir=sources_dir) # pyright: ignore[reportCallIssue] + + +def _stub_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Empty env fallback (the dev ``.env`` never leaks in) — this file's + runs resolve their one local row from the table.""" + monkeypatch.setattr( + git_sources_resolver, + "get_settings", + lambda: Settings(_env_file=None, git_sources=""), # pyright: ignore[reportCallIssue] + ) + + +def _seed_local(db: Session, path: Path) -> None: + """A ``kind=local`` row exactly as the phase-38 API stores it.""" + db.add(GitSource(url=str(path), kind="local", path=str(path))) + db.commit() + + +def _local_source(tmp_path: Path, name: str, files: dict[str, str]) -> Path: + """One source root (always created — the empty-KB arm needs a real + empty directory, not a missing one), with *files* under it.""" + assert name.startswith(PREFIX), f"the probe source must be {PREFIX}-prefixed" + root = tmp_path / name + root.mkdir(parents=True, exist_ok=True) + for rel, body in files.items(): + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + return root + + +def _login(client: TestClient) -> None: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, f"admin login failed: {r.status_code} {r.text}" + + +def _poll(client: TestClient, want: str, timeout: float = 20.0) -> dict: + """Poll until ``state == want``, failing loudly on any other state.""" + deadline = time.monotonic() + timeout + body: dict = {} + while time.monotonic() < deadline: + body = client.get("/api/sync/status").json() + if body["state"] == want: + return body + assert body["state"] == "running", ( + f"unexpected state {body['state']!r} while waiting for {want!r}: {body}" + ) + time.sleep(0.02) + raise AssertionError(f"sync did not reach {want!r} within {timeout}s: {body}") + + +def _watch_scan(client: TestClient, timeout: float = 20.0) -> tuple[list[dict], dict | None]: + """Poll through the run's SCAN window; stop at the first frame that is + no longer a scan. + + Returns ``(frames, terminal)`` — ``terminal`` is non-``None`` when the + run finished inside the window. Both stops are honest: the scan runs + OFF the loop (so its frames are served, and this list is exactly what + the label was built from), while the import that follows is one + straight run of the importer that can finish before the next poll. + The flip frame therefore gets pinned deterministically by the + choreographed test below; what the real run pins here is the scan's + own frames plus the values the terminal keeps.""" + frames: list[dict] = [] + terminal: dict | None = None + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + body = client.get("/api/sync/status").json() + if body["state"] != "running": + terminal = body + break + frames.append(body) + if body["phase"] != "scan": + break + time.sleep(0.01) + else: + raise AssertionError(f"the run never left the scan phase: {frames[-1]}") + return frames, terminal + + +class _FakeScanImport: + """A parked ``import_sources`` double for the app-side edges. + + It records the two position closures the runner passes (the wiring + pin) and fires NOTHING — which is exactly what a run with no + importable file does (no scan report, so no position, and no + ``progress`` call to flip the phase; the importer-side proof lives in + the empty-KB arm below and in ``test_sync_pending_count.py``). The + park lets the test read the status inside the ``"scan"`` phase without + racing the pipeline; the caller always releases it in ``finally``. + """ + + def __init__(self, summary: ImportSummary | None = None) -> None: + self.summary = summary if summary is not None else ImportSummary() + self.started = threading.Event() + self.release = threading.Event() + self.progress_hook: Callable[[str, str, int, int], None] | None = None + self.scan_hook: Callable[[int, int], None] | None = None + + async def __call__( + self, + sources: list[Path], + llm: LLMClient, + *, + prune: bool = False, + limit: int | None = None, + session: Session | None = None, + progress: Callable[[str, str, int, int], None] | None = None, + on_chunk_progress: Callable[[int, int], None] | None = None, + on_image_progress: Callable[[int, int], None] | None = None, + on_scan_progress: Callable[[int, int], None] | None = None, + ignore_by_root: dict[str, list[str]] | None = None, + include_hidden_by_root: dict[str, bool] | None = None, + doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, + ) -> ImportSummary: + self.progress_hook = progress + self.scan_hook = on_scan_progress + self.started.set() + await asyncio.to_thread(self.release.wait) + return self.summary + + +def _install_counting_memory(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: + """Replace the two memory helpers on ``app.api.sync`` with recorders. + + ``_sample_rss`` calls them ONLY when the throttle lets it, so one + entry per entry below is one real read — and each entry carries the + phase (and the scan/file position) at that instant, which is the + difference between "a transition forced a read" and "a per-file hook + read once per file" (LOCKED A4).""" + reads: list[dict[str, Any]] = [] + + def _record() -> None: + reads.append( + { + "phase": sync_api._status.phase, + "scan_done": sync_api._status.scan_done, + "files_done": sync_api._status.files_done, + "current_file": sync_api._status.current_file, + } + ) + + def fake_rss_mb() -> int: + _record() + return 100 + + def fake_rss_peak_mb() -> int: + return 120 + + monkeypatch.setattr(sync_api, "rss_mb", fake_rss_mb) + monkeypatch.setattr(sync_api, "rss_peak_mb", fake_rss_peak_mb) + return reads + + +def _slow_hash( + monkeypatch: pytest.MonkeyPatch, block: float = HASH_BLOCK_S +) -> dict[str, bool]: + """Make ONLY the pre-scan window slow, and return its flag. + + ``_file_hash`` is called by the scan AND by the processing loop, so a + blanket blocking stub would also park the event loop through the whole + import (an in-process import never awaits real I/O, so the loop cannot + answer a status poll inside it) — the frames this file needs would + then be unservable rather than absent. The flag is raised around + :func:`app.rag.importer._prescan_pending` only, so the scan's window is + the fixture's window: the loop stays free exactly while the run is in + the phase under test (the phase-179 off-loop claim, observed from the + HTTP side), and the import after it runs at full speed. + + The flag is a plain ``dict`` entry written/read from two threads: the + house hook contract (a GIL-atomic store) applies here for the same + reason it applies to the runner's closures. + """ + real_hash = importer._file_hash + real_prescan = importer._prescan_pending + state = {"in_scan": False} + + def slow(path: Path) -> str: + if state["in_scan"]: + time.sleep(block) + return real_hash(path) + + def spy_prescan(*args: Any, **kwargs: Any) -> Any: + state["in_scan"] = True + try: + return real_prescan(*args, **kwargs) + finally: + state["in_scan"] = False + + monkeypatch.setattr(importer, "_file_hash", slow) + monkeypatch.setattr(importer, "_prescan_pending", spy_prescan) + return state + + +class _ChoreographedImport: + """An ``import_sources`` double that replays the importer's hook script + through the runner's OWN closures, parking at every point the test + reads the status. + + The choreography (all four stops are ``threading.Event`` pairs, so no + test here races the pipeline): + + 1. fire ``scan_pairs[:1]`` through ``on_scan_progress`` → ``first`` + (the test reads the scan's FIRST position); + 2. on ``go_on``: fire the rest → ``scanned`` (the test reads the last + position, still inside the phase); + 3. on ``go_import``: fire ``progress_call`` through ``progress`` — + THE FLIP → ``flipped`` (the test reads the ``"import"`` frame); + 4. on ``release``: return the canned summary. + """ + + def __init__( + self, + summary: ImportSummary, + scan_pairs: list[tuple[int, int]], + progress_call: tuple[str, str, int, int], + ) -> None: + self.summary = summary + self.scan_pairs = list(scan_pairs) + self.progress_call = progress_call + self.first = threading.Event() + self.go_on = threading.Event() + self.scanned = threading.Event() + self.go_import = threading.Event() + self.flipped = threading.Event() + self.release = threading.Event() + self.scan_hook: Callable[[int, int], None] | None = None + self.progress_hook: Callable[[str, str, int, int], None] | None = None + + async def __call__( + self, + sources: list[Path], + llm: LLMClient, + *, + prune: bool = False, + limit: int | None = None, + session: Session | None = None, + progress: Callable[[str, str, int, int], None] | None = None, + on_chunk_progress: Callable[[int, int], None] | None = None, + on_image_progress: Callable[[int, int], None] | None = None, + on_scan_progress: Callable[[int, int], None] | None = None, + ignore_by_root: dict[str, list[str]] | None = None, + include_hidden_by_root: dict[str, bool] | None = None, + doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, + ) -> ImportSummary: + # The wiring pin: both closures arrive live (None here would mean + # the runner stopped passing them). + self.scan_hook = on_scan_progress + self.progress_hook = progress + assert on_scan_progress is not None and progress is not None + for pair in self.scan_pairs[:1]: + on_scan_progress(*pair) + self.first.set() + await asyncio.to_thread(self.go_on.wait) + for pair in self.scan_pairs[1:]: + on_scan_progress(*pair) + self.scanned.set() + await asyncio.to_thread(self.go_import.wait) + progress(*self.progress_call) + self.flipped.set() + await asyncio.to_thread(self.release.wait) + return self.summary + + +def _stub_probe(monkeypatch: pytest.MonkeyPatch) -> None: + async def fake_check_models(llm: LLMClient) -> None: + return None + + monkeypatch.setattr(sync_api, "check_models", fake_check_models) + + +def _install_local_run( + monkeypatch: pytest.MonkeyPatch, db: Session, tmp_path: Path, source: Path +) -> None: + """Point one app run at *source*: local row, tmp sources root, stubbed + probe, the deterministic in-process embedder (the real import, no + network).""" + _seed_local(db, source) + _stub_env(monkeypatch) + monkeypatch.setattr( + sync_api, "get_settings", lambda: _settings(sources_dir=str(tmp_path / "bor")) + ) + _stub_probe(monkeypatch) + monkeypatch.setattr(sync_api, "LLMClient", lambda: FakeEmbedder()) + + +_THREE_FILES = { + "a.md": "# A\n\nalpha content\n", + "b.md": "# B\n\nbravo content\n", + "notes/c.md": "# C\n\ncharlie content\n", +} + + +def _log_lines(caplog: pytest.LogCaptureFixture, prefix: str) -> list[str]: + return [r.getMessage() for r in caplog.records if r.getMessage().startswith(prefix)] + + +# --- the phase-machine sequence -------------------------------------------- + + +def test_real_run_reports_the_scan_frames_and_keeps_both_pairs( + sync_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + db: Session, + tmp_path: Path, +) -> None: + """D3 through the REAL pipeline, twice (one changed file included). + + Run 1 (a first import): the prelude reports ``phase: null`` with + ``scan_*`` 0/0, then the run reports ``"scan"`` with ``scan_total`` at + the importable count and ``scan_done`` climbing through AT LEAST two + distinct positions (the frames the ``Scanning… (n/total)`` label + needs) with ``current_file`` null throughout — the scan names no file. + The terminal clears ``phase`` and KEEPS the final scan pair beside the + file pair, and on a first import both pairs are 3/3 (A1's degenerate + parity: walk == work, the only shape the phase-193 suite ever saw). + + Run 2 (one file edited — the owner's prod shape at 1-of-3): the scan + denominator is STILL the importable 3, while the kept file pair is + ``1/1`` and the terminal reports exactly one updated document. That + divergence — the scan counted the walk, the count counted the work — + IS the phase, and it survives on the terminal where no poll can miss + it.""" + source = _local_source(tmp_path, f"{PREFIX}machine", _THREE_FILES) + _install_local_run(monkeypatch, db, tmp_path, source) + _slow_hash(monkeypatch) + + _login(sync_client) + idle = sync_client.get("/api/sync/status").json() + assert idle["phase"] is None + assert idle["scan_done"] == 0 and idle["scan_total"] == 0 + + assert sync_client.post("/api/sync").status_code == 202 + frames, _late = _watch_scan(sync_client) + scan_frames = [f for f in frames if f["phase"] == "scan"] + assert scan_frames, f"the run never reported the scan phase: {frames}" + assert all(f["current_file"] is None for f in scan_frames), ( + "the scan named a file — current_file stays null until the first " + "progress call flips the phase to import" + ) + # ``0`` is the transition frame — the phase is set before the scan + # reports its first file, and the UI's total===0 guard renders it bare. + totals = {f["scan_total"] for f in scan_frames} + assert totals <= {0, 3} and max(totals) == 3, totals + positions = sorted({f["scan_done"] for f in scan_frames if f["scan_total"] == 3}) + assert len(positions) >= 2, ( + f"only scan position(s) {positions} were observable — the label needs " + "the scan's own n/total to move (the fixture scan blocks on purpose)" + ) + assert positions[0] >= 1 and positions[-1] <= 3 + + body = _poll(sync_client, "success") + assert body["detail"]["added"] == 3 + assert body["phase"] is None and body["current_file"] is None + # The keep-final-values convention, beside the file counts. + assert body["scan_total"] == 3 and body["scan_done"] == 3 + assert body["files_total"] == 3 and body["files_done"] == 3 + + # Run 2 — one changed file of three (the owner's prod case, scaled). + (source / "b.md").write_text("# B\n\nbravo content EDITED\n", encoding="utf-8") + assert sync_client.post("/api/sync").status_code == 202 + frames2, _late2 = _watch_scan(sync_client) + scan2 = [f for f in frames2 if f["phase"] == "scan"] + totals2 = {f["scan_total"] for f in scan2 if f["scan_total"]} + assert scan2 and totals2 == {3}, ( + "the scan denominator is the importable count, not the prediction" + ) + assert sorted({f["scan_done"] for f in scan2 if f["scan_total"] == 3})[-1] >= 2 + + body2 = _poll(sync_client, "success") + assert (body2["detail"]["added"], body2["detail"]["updated"]) == (0, 1) + assert body2["phase"] is None + # The divergence the phase exists for: the kept scan pair is the WALK + # it did (3 files hashed), the kept file pair is the WORK (1 file). + assert (body2["scan_done"], body2["scan_total"]) == (3, 3) + assert (body2["files_done"], body2["files_total"]) == (1, 1) + + +def test_scan_flips_to_import_on_the_first_progress_call( + sync_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + db: Session, + tmp_path: Path, +) -> None: + """The sequence, frame by frame, with the flip pinned. + + The in-process import runs straight through without awaiting real I/O, + so a poll cannot reliably land inside the ``"import"`` phase (that is + why the real arm above pins the terminal's kept pairs). This arm drives + the runner's OWN closures through a choreographed double — the + ``test_sync_api`` gated-import idiom — and reads the status at each + stop: prelude ``null``/0/0 → ``"scan"`` at position 1 of 3 → still + ``"scan"`` at position 3 of 3 → ``"import"`` with the PREDICTED + ``files_total`` (1, the count that reaches ``N remaining``) and the + scan pair still showing 3/3 → the terminal, which clears ``phase`` and + keeps both pairs.""" + source = _local_source(tmp_path, f"{PREFIX}flip", _THREE_FILES) + _install_local_run(monkeypatch, db, tmp_path, source) + gated = _ChoreographedImport( + ImportSummary(files=3, updated=1, unchanged=2), + scan_pairs=[(1, 3), (2, 3), (3, 3)], + progress_call=(source.name, "b.md", 1, 1), + ) + monkeypatch.setattr(sync_api, "import_sources", gated) + + _login(sync_client) + idle = sync_client.get("/api/sync/status").json() + assert idle["state"] == "idle" + assert idle["phase"] is None + assert idle["scan_done"] == 0 and idle["scan_total"] == 0 + + assert sync_client.post("/api/sync").status_code == 202 + try: + assert gated.first.wait(10.0), "the run never reached the scan" + s1 = sync_client.get("/api/sync/status").json() + assert s1["state"] == "running" + assert s1["phase"] == "scan" + assert (s1["scan_done"], s1["scan_total"]) == (1, 3) + assert s1["current_file"] is None + # Still the pre-import phase: no file count yet. + assert (s1["files_done"], s1["files_total"]) == (0, 0) + + gated.go_on.set() + assert gated.scanned.wait(10.0), "the scan never reported its last file" + s2 = sync_client.get("/api/sync/status").json() + assert s2["phase"] == "scan" + assert (s2["scan_done"], s2["scan_total"]) == (3, 3) + + gated.go_import.set() + assert gated.flipped.wait(10.0), "the progress hook never fired" + s3 = sync_client.get("/api/sync/status").json() + assert s3["phase"] == "import", ( + "the first progress call must flip scan -> import: " + f"{ {k: s3[k] for k in ('phase', 'current_file', 'files_done', 'files_total')} }" + ) + assert s3["current_file"] == f"{source.name}/b.md" + # The flip already carries the PREDICTED denominator — the number + # the label turns into "N remaining". + assert (s3["files_done"], s3["files_total"]) == (1, 1) + # … and the scan pair stays where the scan left it. + assert (s3["scan_done"], s3["scan_total"]) == (3, 3) + finally: + gated.go_on.set() + gated.go_import.set() + gated.release.set() + + body = _poll(sync_client, "success") + assert body["phase"] is None and body["current_file"] is None + assert (body["scan_done"], body["scan_total"]) == (3, 3) + assert (body["files_done"], body["files_total"]) == (1, 1) + + +# --- the additive payload ---------------------------------------------------- + + +def test_scan_keys_are_additive_and_stable_from_idle_to_terminal( + sync_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + db: Session, + tmp_path: Path, +) -> None: + """The additive-only contract on the wire (the phase-187 discipline). + + The payload's key set is EXACTLY the pre-phase-194 set plus + ``scan_done`` / ``scan_total``; it does not change between idle, + mid-run and the terminal (no key appears or disappears with the + phase); and both new keys are plain ``int`` — 0 when there is nothing + to report, never ``null`` and never a plausible position.""" + expected_pre = { + "state", + "started_at", + "finished_at", + "detail", + "error", + "current_file", + "files_done", + "files_total", + "current_chunk", + "chunks_total", + "images_done", + "images_total", + "phase", + "current_summary", + "summaries_done", + "summaries_total", + "rss_mb", + "rss_peak_mb", + } + source = _local_source(tmp_path, f"{PREFIX}keys", _THREE_FILES) + _install_local_run(monkeypatch, db, tmp_path, source) + _stub_probe(monkeypatch) + + _login(sync_client) + idle = sync_client.get("/api/sync/status").json() + assert set(idle) == expected_pre | {"scan_done", "scan_total"}, set(idle) + assert idle["scan_done"] == 0 and idle["scan_total"] == 0 + + assert sync_client.post("/api/sync").status_code == 202 + mid = sync_client.get("/api/sync/status").json() + assert set(mid) == expected_pre | {"scan_done", "scan_total"} + assert mid["state"] == "running" + + body = _poll(sync_client, "success") + assert set(body) == expected_pre | {"scan_done", "scan_total"} + for key in ("scan_done", "scan_total"): + assert isinstance(body[key], int) and not isinstance(body[key], bool), body + # Every pre-existing key keeps its type too (the additive pin's other + # half — the file pair is still ints, the phase still a string|null). + assert isinstance(body["files_done"], int) and isinstance(body["files_total"], int) + assert body["phase"] is None + + +# --- the sampling discipline ------------------------------------------------ + + +def test_scan_and_import_transitions_force_the_sample_and_the_scan_hook_does_not( + sync_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + db: Session, + tmp_path: Path, +) -> None: + """Phase 187's rule, extended to the new stage (LOCKED A4). + + Every entry below is one REAL ``/proc`` read (the two helpers are only + called when the throttle lets ``_sample_rss`` through), tagged with + the phase it happened in. Collapsing consecutive same-phase reads, the + sequence is one read per transition: run start (``phase`` still null) + → ``"scan"`` → ``"import"`` (the flip) → ``"overview"`` → + ``"summaries"`` → the terminal (null again). The scan transition's + read carries ``scan_done == 0`` — it is the transition, not a file — + and there is EXACTLY one read inside the whole scan phase, so a + per-file scan read (one ``/proc`` read per file, the regression this + pins) fails here even though the fixture scanned three files.""" + source = _local_source(tmp_path, f"{PREFIX}sampling", _THREE_FILES) + _install_local_run(monkeypatch, db, tmp_path, source) + _slow_hash(monkeypatch) + reads = _install_counting_memory(monkeypatch) + + _login(sync_client) + assert sync_client.post("/api/sync").status_code == 202 + body = _poll(sync_client, "success") + + assert reads, "the run sampled no memory at all" + phases = [r["phase"] for r in reads] + transitions = [p for i, p in enumerate(phases) if i == 0 or p != phases[i - 1]] + assert transitions == [None, "scan", "import", "overview", "summaries", None], reads + + scan_reads = [r for r in reads if r["phase"] == "scan"] + assert len(scan_reads) == 1, ( + f"the scan phase cost {len(scan_reads)} /proc reads for 3 files — the " + "per-file scan hook must sample UNFORCED (one read per interval)" + ) + assert scan_reads[0]["scan_done"] == 0, ( + "the scan-phase read is not the transition — the forced sample must " + "land when the phase is SET, before the first file is scanned" + ) + flip_reads = [r for r in reads if r["phase"] == "import"] + assert flip_reads and flip_reads[0]["current_file"] is not None, ( + "no read at the scan->import flip — the transition is a forced " + f"sampling point: {reads}" + ) + # The payload reports what the sampler wrote, not a stale pair. + assert (body["rss_mb"], body["rss_peak_mb"]) == (100, 120), body + + +# --- the degenerate empty knowledge base ------------------------------------ + + +def test_empty_kb_stays_in_scan_and_never_reaches_import( + sync_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + db: Session, + tmp_path: Path, +) -> None: + """The D1 degenerate clause, pinned on BOTH sides. + + Importer side (real): an empty source dir makes the pre-scan report + NOTHING — no ``on_scan_progress`` call, no ``progress`` call, a zero + denominator. App side (the double that fires nothing, i.e. exactly + what that run reports): the status sits in ``"scan"`` — the flip + lives in the per-file hook and no hook ever fires — with ``scan_*`` + 0/0 and ``current_file`` null, and the terminal clears ``phase`` + while keeping the 0/0 pair (the UI's ``total === 0`` guard then + renders the bare label with no count part).""" + # Importer side: the real scan of an empty root reports nothing. + empty = _local_source(tmp_path, f"{PREFIX}empty", {}) + llm = FakeEmbedder() + scan_calls: list[tuple[int, int]] = [] + progress_calls: list[tuple[str, str, int, int]] = [] + summary = asyncio.run( + importer.import_sources( + [empty], + llm, + progress=lambda s, r, d, t: progress_calls.append((s, r, d, t)), + on_scan_progress=lambda p, t: scan_calls.append((p, t)), + ) + ) + assert (scan_calls, progress_calls) == ([], []) + assert summary.files == 0 + + # App side: a run that reports nothing is a run that stays "scan". + _install_local_run(monkeypatch, db, tmp_path, empty) + fake_import = _FakeScanImport() + monkeypatch.setattr(sync_api, "import_sources", fake_import) + + _login(sync_client) + assert sync_client.post("/api/sync").status_code == 202 + try: + assert fake_import.started.wait(10.0), "the run never reached the import" + # Wiring: the runner passed BOTH live closures (so the empty + # phase above is the scan reporting nothing, not a missing hook). + assert fake_import.scan_hook is not None + assert fake_import.progress_hook is not None + body = sync_client.get("/api/sync/status").json() + assert body["state"] == "running" + assert body["phase"] == "scan" + assert body["current_file"] is None + assert body["scan_done"] == 0 and body["scan_total"] == 0 + assert body["files_done"] == 0 and body["files_total"] == 0 + finally: + fake_import.release.set() + + terminal = _poll(sync_client, "success") + assert terminal["phase"] is None + assert terminal["scan_done"] == 0 and terminal["scan_total"] == 0 + assert terminal["files_done"] == 0 and terminal["files_total"] == 0 + + +# --- the cancel during the pre-scan ----------------------------------------- + + +def test_cancel_during_the_scan_lands_cancelled_with_the_scan_counts_kept( + sync_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + db: Session, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """The phase-134 contract at the pipeline's NEW await (the app-side + twin of the importer-side cancel pin). + + The scan is blocking on purpose, so the cancel provably lands inside + it: ``POST /api/sync/cancel`` answers, the ``cancelled`` terminal + arrives well before the abandoned scan's remaining work could finish, + ``phase`` clears, the position the scan had reached is KEPT beside the + (still zero) file pair, ``error`` stays null and ``detail`` empty (a + cancel is intent, not failure), and NOTHING was indexed — the walk + never started. The lifecycle line keeps its exact pre-phase prefix + (LOCKED A2's append-only rule).""" + source = _local_source(tmp_path, f"{PREFIX}cancel", _THREE_FILES) + _install_local_run(monkeypatch, db, tmp_path, source) + _slow_hash(monkeypatch, block=0.4) + + _login(sync_client) + with caplog.at_level(logging.INFO, logger="app.api.sync"): + assert sync_client.post("/api/sync").status_code == 202 + # Wait for proof the run is INSIDE the scan: the phase, and a + # position that moved. + deadline = time.monotonic() + 20.0 + live: dict = {} + while time.monotonic() < deadline: + live = sync_client.get("/api/sync/status").json() + if live["phase"] == "scan" and live["scan_done"] >= 1: + break + assert live["state"] == "running", live + time.sleep(0.01) + assert live.get("phase") == "scan" and live["scan_done"] >= 1, ( + f"the cancel never landed inside the scan: {live}" + ) + position_at_cancel = live["scan_done"] + + t0 = time.monotonic() + r = sync_client.post("/api/sync/cancel") + assert r.status_code == 202, f"cancel not accepted: {r.status_code} {r.text}" + assert time.monotonic() - t0 < 2.0, "the cancel request itself stalled" + + body = _poll(sync_client, "cancelled", timeout=3.0) + + assert body["error"] is None + assert body["detail"] == {} + assert body["phase"] is None + assert body["current_file"] is None + # The scan's final position is kept (the keep-final-values + # convention), and it is the position the cancel found — the run did + # not finish the scan it was cancelled in. + assert body["scan_total"] == 3 + assert body["scan_done"] >= position_at_cancel + assert body["files_done"] == 0 and body["files_total"] == 0 + + # Nothing was indexed: the walk never started. + assert db.scalar( + select(select(Document).where(Document.source == source.name).exists()) + ) is False + + # The lifecycle line keeps its pre-phase-194 prefix exactly. + cancelled = _log_lines(caplog, "sync: cancelled by user state=cancelled files_done=") + assert len(cancelled) == 1, cancelled + + +# --- the kill switch's status shape ---------------------------------------- + + +def test_kill_switch_off_reports_no_scan_position_but_the_same_machine( + sync_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + db: Session, + tmp_path: Path, +) -> None: + """``BOR_SYNC_PENDING_COUNT=0`` on the STATUS side. + + The importer takes the phase-193 stats pre-walk, which reports NOTHING, + so the scan pair stays 0/0 for the whole run (the page renders no + count from it) while ``files_done`` / ``files_total`` carry the WALK + position again — 3 walked, not 1 predicted. The phase machine itself is + UNCHANGED by the switch (the pre-``import_sources`` set is still + ``"scan"`` and the first file still flips it: that flip is pinned in + the choreographed test above, and the switch's importer-side halves are + pinned in ``tests/unit/test_sync_pending_count.py`` and + ``tests/integration/test_sync_pending_count.py``). The pins below read + the TERMINAL, which is where both pairs survive deterministically.""" + source = _local_source(tmp_path, f"{PREFIX}off", _THREE_FILES) + _install_local_run(monkeypatch, db, tmp_path, source) + # The switch lives on the settings object the import reads (llm). + real_embedder = FakeEmbedder() + real_embedder.settings = Settings(_env_file=None, sync_pending_count=0) # pyright: ignore[reportCallIssue] + monkeypatch.setattr(sync_api, "LLMClient", lambda: real_embedder) + + _login(sync_client) + assert sync_client.post("/api/sync").status_code == 202 + body = _poll(sync_client, "success") + + assert body["detail"]["added"] == 3 + # No scan, so no position — kept at 0/0, the value the UI renders as + # "no count part" rather than a stale or invented one. + assert body["scan_done"] == 0 and body["scan_total"] == 0 + # The WALK count is back (phase-193 semantics, byte-for-byte). + assert (body["files_done"], body["files_total"]) == (3, 3) + assert body["phase"] is None diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 867dcdc..71733f3 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1027,3 +1027,33 @@ def test_sync_prune_batch_rejects_negative( monkeypatch.setenv("BOR_SYNC_PRUNE_BATCH", "-1") with pytest.raises(ValidationError, match="sync_prune_batch"): _settings() + + +# --- Sync pending count (phase 194) --- + + +def test_sync_pending_count_default_and_env_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Phase 194 (D2): the pre-scan ships ON (``1``) — the owner's "1216 + remaining" report means the walk-position count was wrong by default, so + the fix is the default and the rollback is the opt-in. ``0`` is accepted + and means the exact phase-193 behaviour: count every importable file of + the walk and never hash a file before the loop reached it.""" + monkeypatch.delenv("BOR_SYNC_PENDING_COUNT", raising=False) + assert _settings().sync_pending_count == 1 + monkeypatch.setenv("BOR_SYNC_PENDING_COUNT", "0") + assert _settings().sync_pending_count == 0 + monkeypatch.setenv("BOR_SYNC_PENDING_COUNT", "1") + assert _settings().sync_pending_count == 1 + + +def test_sync_pending_count_rejects_negative( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A negative switch is a typo — no branch of the importer can mean it, + so startup refuses naming the field (the ``agent_max_rounds`` pattern, + phase 194). The value is a boolean knob, so ONLY ``0`` is the rollback.""" + monkeypatch.setenv("BOR_SYNC_PENDING_COUNT", "-1") + with pytest.raises(ValidationError, match="sync_pending_count"): + _settings() diff --git a/tests/unit/test_importer.py b/tests/unit/test_importer.py index 0df9e6b..2fa95a8 100644 --- a/tests/unit/test_importer.py +++ b/tests/unit/test_importer.py @@ -1146,8 +1146,16 @@ def test_no_progress_means_no_prewalk( ) -> None: """``progress=None`` callers pay no extra pass: the real walker is hit exactly once per source root (one pass — as before phase 64), proven - with a counting sentinel; with the hook it is hit twice (pre-walk for - ``total`` + the processing pass).""" + with a counting sentinel; with the hook it is hit twice (the phase-194 + pre-scan for the pending set — the phase-64 stats pre-walk when + ``BOR_SYNC_PENDING_COUNT=0`` — plus the processing pass). + + Phase 194 (task 01) moved the hook's pair, and this fixture is the + second run over an ALREADY-INDEXED file: nothing is pending, so the + pair is ``(0, 0)`` — the denominator is the predicted change set and an + unchanged KB predicts nothing (the UI then renders no count part). The + walk COUNT is what this test owns and it is untouched: one pass without + the hook, two with it.""" root = tmp_path / "nowalk" root.mkdir() (root / "a.md").write_text("# A\n\na\n") @@ -1186,8 +1194,12 @@ def test_no_progress_means_no_prewalk( ) ) assert summary2.files == 1 - assert walk_calls == 2 # pre-walk (total) + processing pass - assert events == [(root.name, "a.md", 1, 1)] + assert walk_calls == 2 # the count pass (pending set / pre-walk) + walk + # phase 194: a.md is unchanged, so the predicted change set is empty + # — ``total`` 0 and the position never advances. With the kill switch + # at ``0`` this same run reports the phase-193 walk position (1 of 1); + # pinned in tests/unit/test_sync_pending_count.py. + assert events == [(root.name, "a.md", 0, 0)] finally: _cleanup_source(db, root.name) @@ -1195,7 +1207,17 @@ def test_no_progress_means_no_prewalk( def test_progress_hook_counts_unchanged_and_error_files(db, tmp_path: Path) -> None: """The hook fires *before* ``_index_file``: a file whose embedding fails (and one that re-imports as unchanged) is still reported as the - current file — the sequence covers every importable file.""" + current file — the sequence covers every importable file. + + Phase 194 (task 01) retargeted the pair to the PENDING position (the + owner's "1216 remaining" fix), which reads exactly right here. The + first run is fresh: both files are predicted, ``1 of 2`` then ``2 of 2`` + — identical to the pre-phase numbers (A1's first-import parity). The + second run is the incremental case: ``bad.md`` rolled back with its + embed error so its row is still missing (predicted again, the ONLY + work), ``good.md`` is unchanged so it holds the position instead of + advancing it — ``1 of 1``, ``1 of 1``: every file is still reported as + the current file, and the count is the work.""" root = tmp_path / "progress-mixed" root.mkdir() (root / "bad.md").write_text("# Bad\n\npoison content the endpoint refuses\n") @@ -1214,7 +1236,8 @@ def test_progress_hook_counts_unchanged_and_error_files(db, tmp_path: Path) -> N assert events == expected assert first.errors == 1 and first.added == 1 # Re-run: good.md is now unchanged, bad.md is retried and fails - # again — both still count in the sequence. + # again — both still report as the current file, but only bad.md is + # pending, so the denominator is 1 and good.md holds the position. events.clear() second = asyncio.run( import_sources( @@ -1222,7 +1245,7 @@ def test_progress_hook_counts_unchanged_and_error_files(db, tmp_path: Path) -> N progress=lambda s, r, d, t: events.append((s, r, d, t)), ) ) - assert events == expected + assert events == [(root.name, "bad.md", 1, 1), (root.name, "good.md", 1, 1)] assert second.errors == 1 and second.unchanged == 1 finally: _cleanup_source(db, root.name) diff --git a/tests/unit/test_settings_catalog.py b/tests/unit/test_settings_catalog.py index 8e024b7..4bb6959 100644 --- a/tests/unit/test_settings_catalog.py +++ b/tests/unit/test_settings_catalog.py @@ -253,6 +253,29 @@ def test_sync_session_bound_field_types() -> None: assert Settings.model_fields["sync_prune_batch"].default == 500 +def test_sync_pending_count_row_is_a_plain_live_overridable_row() -> None: + """Phase 194 (task 01, D2): the pre-scan switch sits in ``Import & + sources`` beside the other two sync-run knobs with the ``Sync pending + count`` label and is plain — not ``secret`` (an int), not + ``restart_required`` (the importer reads ``llm.settings`` once per run, so + a saved override applies to the NEXT sync), overridable (the overlay is + generic over ``int`` fields). ``1`` is the shipped default — the pending + count is the FIX the owner asked for, so it ships on and the phase-193 + walk position is the ``0`` rollback; a negative value fails startup, not + the panel.""" + meta = CATALOG["sync_pending_count"] + assert meta.group == "Import & sources" + assert meta.label == "Sync pending count" + assert meta.secret is False + assert meta.restart_required is False + assert meta.overridable is True + assert "sync_pending_count" not in SECRETS + assert "sync_pending_count" not in RESTART_REQUIRED + assert "sync_pending_count" not in NON_OVERRIDABLE + assert Settings.model_fields["sync_pending_count"].annotation is int + assert Settings.model_fields["sync_pending_count"].default == 1 + + def test_source_name_hit_bonus_row_is_a_plain_live_overridable_row() -> None: """Phase 191 (task 04, D3): the second selection-bonus row sits in ``Retrieval & honesty`` (right beside ``name_hit_bonus`` — the two diff --git a/tests/unit/test_sync_button.py b/tests/unit/test_sync_button.py index 20afe99..85d23ad 100644 --- a/tests/unit/test_sync_button.py +++ b/tests/unit/test_sync_button.py @@ -1419,6 +1419,10 @@ class _GatedImport: self.image_pairs = list(image_pairs) self.image_release = image_release self.image_hook_calls: list[tuple[int, int]] = [] + # Phase 194 (task 02): the pre-scan position hook the runner + # passes (recorded for the wiring pin — this fake fires nothing, + # so the status scan fields stay 0/0). + self.scan_hook: Callable[[int, int], None] | None = None async def __call__( self, @@ -1431,11 +1435,13 @@ class _GatedImport: progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, # phase 129 on_image_progress: Callable[[int, int], None] | None = None, # phase 152 + on_scan_progress: Callable[[int, int], None] | None = None, # phase 194 ignore_by_root: dict[str, list[str]] | None = None, # phase 89 include_hidden_by_root: dict[str, bool] | None = None, # phase 105 doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106 ) -> ImportSummary: self.prune_flags.append(prune) + self.scan_hook = on_scan_progress if progress is not None: progress("repo", "notes/deep.md", 1, 3) self.hook_calls.append(("repo", "notes/deep.md", 1, 3)) @@ -1597,6 +1603,11 @@ def test_idle_status_pins_full_shape_including_progress_keys( "current_file": None, "files_done": 0, "files_total": 0, + # Phase 194 (task 02): the pre-scan position keys — 0/0 idle + # (the dataclass defaults; a run that scanned nothing keeps them + # 0/0 rather than null). + "scan_done": 0, + "scan_total": 0, # Phase 129 (task 03): the chunk-position keys — 0/0 idle # (the dataclass defaults, no special-casing). "current_chunk": 0, @@ -2131,6 +2142,7 @@ class _TwoFileImport: progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, # phase 129 on_image_progress: Callable[[int, int], None] | None = None, # phase 152 + on_scan_progress: Callable[[int, int], None] | None = None, # phase 194 ignore_by_root: dict[str, list[str]] | None = None, # phase 89 include_hidden_by_root: dict[str, bool] | None = None, # phase 105 doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106 @@ -2804,6 +2816,7 @@ def test_event_set_during_import_cancels_before_the_lite_spans( progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, # phase 129 on_image_progress: Callable[[int, int], None] | None = None, # phase 152 + on_scan_progress: Callable[[int, int], None] | None = None, # phase 194 ignore_by_root: dict[str, list[str]] | None = None, # phase 89 include_hidden_by_root: dict[str, bool] | None = None, # phase 105 doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, # phase 106 @@ -3170,7 +3183,8 @@ def test_sources_js_section_header_documents_the_cancel_contract() -> None: #: The status payload's key set BEFORE phase 187 — the additive pin's #: left-hand side: the payload is exactly this set plus -#: :data:`_MEMORY_KEYS`, nothing else, forever. +#: :data:`_MEMORY_KEYS` plus :data:`_SCAN_KEYS`, nothing else, forever +#: (a future field joins the chain, it never edits these three). _PRE_187_STATUS_KEYS = frozenset( { "state", @@ -3195,17 +3209,23 @@ _PRE_187_STATUS_KEYS = frozenset( #: Phase 187 (task 03): the two additive memory keys. _MEMORY_KEYS = frozenset({"rss_mb", "rss_peak_mb"}) +#: Phase 194 (task 02): the two additive PRE-SCAN position keys — the next +#: link of the same additive chain. +_SCAN_KEYS = frozenset({"scan_done", "scan_total"}) + def test_status_key_set_is_the_old_set_plus_the_memory_pair( fresh_sync_status: None, ) -> None: """The key-set pin (D4: "Additive only — every pre-existing key keeps its name, type, and meaning"): the payload is EXACTLY the - pre-phase-187 set ``∪ {rss_mb, rss_peak_mb}`` — a fifth sub-position - or a renamed field fails here even when the value assertions pass.""" + pre-phase-187 set ``∪ {rss_mb, rss_peak_mb} ∪ {scan_done, + scan_total}`` — a fifth sub-position or a renamed field fails here + even when the value assertions pass.""" keys = set(sync_api.sync_status()) - assert keys == _PRE_187_STATUS_KEYS | _MEMORY_KEYS, keys + assert keys == _PRE_187_STATUS_KEYS | _MEMORY_KEYS | _SCAN_KEYS, keys assert _MEMORY_KEYS.isdisjoint(_PRE_187_STATUS_KEYS) + assert _SCAN_KEYS.isdisjoint(_PRE_187_STATUS_KEYS | _MEMORY_KEYS) class _FakeClock: @@ -3377,9 +3397,12 @@ def test_run_samples_at_start_phase_transitions_and_terminal( caplog: pytest.LogCaptureFixture, ) -> None: """The sampling POINTS, walked through the real runner: run start → - ``import`` → ``overview`` → ``summaries`` → the success terminal, one - forced read each, and the per-file progress hook's throttled call - adds NOTHING inside the window the run start just armed (LOCKED A4). + ``scan`` → ``import`` (the phase-194 flip) → ``overview`` → + ``summaries`` → the success terminal, one forced read each, and the + per-file progress hook's throttled call adds NOTHING inside the window + the previous transition just armed (LOCKED A4 — and the same rule the + pre-scan's own per-file hook follows, pinned through the app in + ``tests/integration/test_sync_scan_phase.py``). The last pair is the number the incident was about: the terminal KEEPS it (the phase-64 keep-final-values convention) and the @@ -3388,10 +3411,12 @@ def test_run_samples_at_start_phase_transitions_and_terminal( log an operator could be argued with.""" clock = _FakeClock(10_000.0) monkeypatch.setattr(sync_api, "time", clock) - # run start, import, overview, summaries, done (the hook is throttled). + # run start, scan, import (the flip), overview, summaries, done — the + # two phase-194 transitions join the list; the per-file hooks stay + # throttled and add no read. memory = _StubMemory( monkeypatch, - [(10, 20), (11, 21), (12, 22), (13, 23), (14, 4_321)], + [(10, 20), (11, 21), (12, 22), (13, 23), (14, 24), (15, 4_321)], ) clone = _GatedClone(threading.Event(), threading.Event()) fake_import = _GatedImport( @@ -3412,14 +3437,23 @@ def test_run_samples_at_start_phase_transitions_and_terminal( assert (s["rss_mb"], s["rss_peak_mb"]) == (10, 20) clone.release.set() assert fake_import.started.wait(5.0), "the run never reached the import" - # Import phase: the transition forced the second read, and - # the hook's own call (this fake fires one file) is inside - # the window — still 2 reads, not 3 (A4 in numbers). + # Import phase: the "scan" set forced the second read and the + # first progress call's FLIP forced the third, while the + # hook's own unforced call (this fake fires one file) is + # inside the window — 3 reads, not 4 (A4 in numbers). s = sync_api.sync_status() assert s["phase"] == "import" assert fake_import.hook_calls == [("repo", "notes/deep.md", 1, 3)] - assert (s["rss_mb"], s["rss_peak_mb"]) == (11, 21) - assert memory.reads == 2, "the per-file hook never bypasses the guard" + assert (s["rss_mb"], s["rss_peak_mb"]) == (12, 22) + assert memory.reads == 3, ( + "the scan/import transitions must each force one read and " + f"the per-file hook must never bypass the guard: {memory.reads}" + ) + # The scan closure reached the runner wired (the fake fires + # nothing, so the position stays 0/0 — the sequence itself is + # pinned through the app in test_sync_scan_phase.py). + assert fake_import.scan_hook is not None + assert s["scan_done"] == 0 and s["scan_total"] == 0 fake_import.release.set() finally: clone.release.set() @@ -3431,7 +3465,7 @@ def test_run_samples_at_start_phase_transitions_and_terminal( # and it survives the run (the page shows it after the settle). s = sync_api.sync_status() assert s["state"] == "success" - assert (s["rss_mb"], s["rss_peak_mb"]) == (14, 4_321) + assert (s["rss_mb"], s["rss_peak_mb"]) == (15, 4_321) assert s["files_done"] == 1 and s["files_total"] == 3 done = [ @@ -3440,7 +3474,7 @@ def test_run_samples_at_start_phase_transitions_and_terminal( if r.getMessage().startswith("sync: done detail=") ] assert len(done) == 1, done - assert done[0].endswith("rss_mb=14 rss_peak_mb=4321"), done[0] + assert done[0].endswith("rss_mb=15 rss_peak_mb=4321"), done[0] def test_per_file_hook_samples_once_the_window_closes( @@ -3452,10 +3486,15 @@ def test_per_file_hook_samples_once_the_window_closes( fires once per file, and once the 5 s window has closed the next file's call re-reads — so a long walk reports the memory climbing (PLAN §7.4's never-stale contract) at ~600 reads per 27k files - instead of 27k.""" + instead of 27k (phase 194 extended the same rule to the pre-scan's + per-file hook).""" clock = _FakeClock(10_000.0) monkeypatch.setattr(sync_api, "time", clock) - memory = _StubMemory(monkeypatch, [(100, 200), (110, 210), (140, 260)]) + # run start, the "scan" set, the scan->import flip, then the read the + # test forces after moving the clock past the window. + memory = _StubMemory( + monkeypatch, [(100, 200), (110, 210), (120, 220), (140, 260)] + ) clone = _GatedClone(threading.Event(), threading.Event()) fake_import = _GatedImport( ImportSummary(files=1, added=1), @@ -3474,8 +3513,11 @@ def test_per_file_hook_samples_once_the_window_closes( assert fake_import.started.wait(5.0), "the run never reached the import" s = sync_api.sync_status() assert s["phase"] == "import" - assert (s["rss_mb"], s["rss_peak_mb"]) == (110, 210) # the transition - assert memory.reads == 2, "the hook's call inside the window read nothing" + assert (s["rss_mb"], s["rss_peak_mb"]) == (120, 220) # the flip + assert memory.reads == 3, ( + "start + scan + flip force three reads and the hook's own call " + f"inside the window adds none: {memory.reads}" + ) clock.now += sync_api._RSS_SAMPLE_INTERVAL_S + 1.0 sync_api._sample_rss() # the NEXT file's hook call assert (sync_api._status.rss_mb, sync_api._status.rss_peak_mb) == (140, 260) diff --git a/tests/unit/test_sync_pending_count.py b/tests/unit/test_sync_pending_count.py new file mode 100644 index 0000000..9fcb5fc --- /dev/null +++ b/tests/unit/test_sync_pending_count.py @@ -0,0 +1,700 @@ +"""Phase 194, task 01 — the importer's pre-scan + the pending-count switch. + +The owner report that opened the phase (2026-10-19): *"I just synced the prod +brain of reese and it said '1216 remaining' when it only had like 10 files to +sync."* The arithmetic of the phase-193 label was fine; its INPUTS were not — +``total`` was every importable file (prod: ~10,000) and ``done`` was the +position in that walk, so the label counted the WALK instead of the WORK. +Task 01 moves the pair to the PREDICTED change set: ``import_sources`` +pre-scans (one streamed sha256 + one scalar ``documents.file_hash`` read per +importable file, off the event loop) and reports ``done``/``total`` as the +position in the pending set, with ``BOR_SYNC_PENDING_COUNT`` as the kill +switch (``0`` = the exact phase-193 walk position, negative = refuses +startup). + +Pinned here (the behaviour under a real run — the pending-set parity, the +zero-hydration pin, the off-loop span and the cancel — lives in +``tests/integration/test_sync_pending_count.py``): + +* the GATE: no ``progress`` caller means no scan at all (the callback never + fires and the scalar statement is never built — the phase-64 skip rule, + extended), and the switch at ``0`` on a progress caller runs the phase-64 + stats pre-walk instead (a source marker pin: that old code IS the ``0`` + branch, and a behaviour pin on the pair it reports); +* the switch is read ONCE per run (the phase-184 read-once discipline), on + the worker-thread side of ``asyncio.to_thread`` (the phase-179 rule); +* the scalar statement's projection is EXACTLY ``documents.file_hash`` — one + scalar column, no entity, no ``defer``/``load_only`` row (the phase-185 + column-list style); :func:`_doc_row_stmt` stays the importer's only ROW + load; +* the pending PREDICTATE: fresh (no row), changed (digest mismatch) and a + pre-phase ``NULL`` row are pending, a hash match is not, a file the ignore + rule hides is never even walked, and a file that cannot be read is + predicted changed rather than fatal; +* ``on_scan_progress``: once per file, the 1-based position against the + combined importable total, and NOTHING for an empty KB (the phase-64 + ``total === 0`` contract); +* ``done`` as the pending position: a 3-file KB with 2 predicted reports + ``1``, ``1``, ``2`` against ``total = 2`` — the file that is not predicted + holds the position while the walk moves past it. +""" +from __future__ import annotations + +import ast +import asyncio +import inspect +import re +import threading +import uuid +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import text +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import Session + +import app.rag.importer as importer +from app.config import Settings +from app.models import Document +from app.rag.importer import ( + _pending_hash_stmt, + _prescan_pending, + import_sources, +) +from tests.fakes import FakeEmbedder + +# -------------------------------------------------------------------------- +# Helpers +# -------------------------------------------------------------------------- + + +def _code_only(fn: Any) -> str: + """A function's source WITHOUT its docstring. + + The marker pins below count occurrences of names in the code; the + docstrings argue about those same names by name (the house convention), so + counting the raw source would pin prose. The docstring is located through + the AST and sliced out by its own end line. + """ + src = inspect.getsource(fn) + lines = src.splitlines() + node = ast.parse(src).body[0] + # `import_sources` is an ASYNC def, so it parses to AsyncFunctionDef — + # narrowing to FunctionDef alone would silently skip the docstring strip + # and pin prose. + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return src + first = node.body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + return "\n".join(lines[first.end_lineno or 1 :]) + return src + + +def _settings(**overrides: Any) -> Settings: + """Fresh settings, never the developer's ``.env`` (the house pattern).""" + return Settings(_env_file=None, **overrides) # pyright: ignore[reportCallIssue] + + +def _root(tmp_path: Path, name: str, files: dict[str, str]) -> Path: + """One source root with *files* (relative path → content) under it.""" + root = tmp_path / name + for rel, body in files.items(): + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + return root + + +def _prescan( + *sources: Path, + session: Session, + ignore_by_root: dict[str, list[str]] | None = None, + include_hidden_by_root: dict[str, bool] | None = None, + on_scan_progress: Any = None, + image_extensions: frozenset[str] = frozenset(), +) -> tuple[int, dict[tuple[str, str], int]]: + """:func:`_prescan_pending` with the shipped import extensions and an + explicit session, so the predicate is testable without a full run.""" + return _prescan_pending( + list(sources), + _settings().import_extension_set, + image_extensions, + ignore_by_root, + include_hidden_by_root, + on_scan_progress, + session, + ) + + +def _seed_row( + db: Session, source: str, rel: str, *, file_hash: str | None +) -> None: + """One ``documents`` row with a KNOWN ``file_hash`` (the pre-scan reads + that one column, so this is the whole fixture: raw SQL, no importer, no + chunks).""" + db.execute( + text( + "INSERT INTO documents (id, source, path, full_path, title, content, " + "content_hash, file_hash, indexed_at, created_at, created_at_manual, " + "summary, is_image) VALUES (:id, :source, :path, :full_path, :title, " + ":content, :content_hash, :file_hash, now(), now(), false, :summary, " + "false)" + ), + { + "id": uuid.uuid4(), + "source": source, + "path": rel, + "full_path": f"/tmp/{source}/{rel}", + "title": rel, + "content": "stored markdown", + "content_hash": "content-digest", + "file_hash": file_hash, + "summary": "stored summary", + }, + ) + db.commit() + + +@pytest.fixture(autouse=True) +def _clean_probe_rows(db: Session) -> Any: + """The probe's own source-name prefix, removed around every test (the + tables are global state — the ``test_importer.py`` hygiene pattern).""" + _drop(db) + yield + _drop(db) + + +def _drop(db: Session) -> None: + db.execute( + text("DELETE FROM documents WHERE source LIKE 'phase194-%'"), + ) + db.commit() + db.expunge_all() + + +# -------------------------------------------------------------------------- +# The gate: who pays for the scan +# -------------------------------------------------------------------------- + + +def test_no_progress_caller_runs_no_scan_at_all( + db: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """THE skip rule (phase 64), extended: ``progress=None`` → no pre-scan. + + The CLI (``scripts/import_docs.py``) and the upload job pass no + ``progress``, so they must not pay one extra read + hash + SELECT per + importable file. A recording stand-in for the span proves the gate + itself: the CLI-shaped call imports the file and never reaches the + scan, and an ``on_scan_progress`` hook handed to a ``progress=None`` + caller fires never (the scan exists only to serve the running counts). + """ + calls: list[tuple[Any, ...]] = [] + scan_events: list[tuple[int, int]] = [] + + def _spy(*args: Any, **kwargs: Any) -> tuple[int, dict[tuple[str, str], int]]: + calls.append(args) + return 0, {} + + monkeypatch.setattr(importer, "_prescan_pending", _spy) + root = _root(tmp_path, "phase194-gate-cli", {"a.md": "# A\n\na\n"}) + try: + summary = asyncio.run(import_sources([root], FakeEmbedder(), session=db)) + assert summary.files == 1 and summary.added == 1 + assert calls == [], "a progress=None caller ran the pre-scan" + + # The scan hook alone does not open the gate either. + summary2 = asyncio.run( + import_sources( + [root], + FakeEmbedder(), + session=db, + on_scan_progress=lambda done, total: scan_events.append((done, total)), + ) + ) + assert summary2.files == 1 + assert calls == [] + assert scan_events == [] + finally: + db.execute( + text("DELETE FROM documents WHERE source = :s"), {"s": root.name} + ) + db.commit() + + +def test_switch_zero_runs_the_phase_193_stats_prewalk( + db: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The rollback branch, from both sides: the marker (the old stats + pre-walk code IS the ``0`` branch) and the reported pair. + + With the switch at ``0`` a progress caller never reaches the scan span, + and the hook reports the phase-193 walk position byte-for-byte — every + importable file in ``total``, the 1-based index across all sources in + ``done`` — on a run where nothing is pending. That last clause is the + point of the arm: the SAME unchanged fixture reports ``(0, 0)`` with the + switch on and ``(1, 1)``, ``(2, 2)`` with it off. + """ + calls: list[tuple[Any, ...]] = [] + + def _spy(*args: Any, **kwargs: Any) -> tuple[int, dict[tuple[str, str], int]]: + calls.append(args) + return 0, {} + + monkeypatch.setattr(importer, "_prescan_pending", _spy) + root = _root( + tmp_path, + "phase194-gate-zero", + {"a.md": "# A\n\nalpha\n", "b.md": "# B\n\nbravo\n"}, + ) + llm = FakeEmbedder() + llm.settings = _settings(sync_pending_count=0) + events: list[tuple[str, str, int, int]] = [] + try: + # Seed the KB (a walk-position run of its own), so nothing pending. + asyncio.run(import_sources([root], llm, session=db)) + assert calls == [] + + events.clear() + summary = asyncio.run( + import_sources( + [root], + llm, + session=db, + progress=lambda s, r, d, t: events.append((s, r, d, t)), + ) + ) + assert calls == [], "the switch was 0 but the pre-scan ran" + assert (summary.files, summary.unchanged) == (2, 2) + assert events == [ + (root.name, "a.md", 1, 2), + (root.name, "b.md", 2, 2), + ], "the 0 branch must report the phase-193 walk position exactly" + finally: + db.execute(text("DELETE FROM documents WHERE source = :s"), {"s": root.name}) + db.commit() + + # The marker: the stats-only pre-walk still EXISTS in the importer, and + # it lives in the ``0`` branch (a third walk would mean the replacement + # was not a replacement). + src = _code_only(importer.import_sources) + assert "BOR_SYNC_PENDING_COUNT=0" in src + assert src.count("iter_importable_files(") == 2, ( + "the importer walks the sources for the count more than once besides " + "the pre-scan — the pre-scan REPLACES the stats pre-walk (phase 194 " + "D1: no third walk)" + ) + + +def test_switch_is_read_once_before_the_loop_and_off_the_loop() -> None: + """The phase-184 read-once discipline + the phase-179 off-loop rule. + + ``llm.settings.sync_pending_count`` is read exactly ONCE, before the + source loop, so a settings overlay saved mid-run cannot flip the meaning + of ``done``/``total`` between two files of the same walk; and the span + is entered through ``asyncio.to_thread``, so a 10,000-file hash+SELECT + pass never parks the single uvicorn loop. + """ + src = _code_only(importer.import_sources) + + assert src.count("llm.settings.sync_pending_count") == 1, ( + "sync_pending_count is read more than once per run — a mid-run " + "settings save would change the pair's meaning between two files " + "(phase 184's read-once discipline)" + ) + read_at = src.index("llm.settings.sync_pending_count") + loop_at = src.index("for root in sources:") + assert read_at < loop_at, "the switch must be read BEFORE the source loop" + + assert "await asyncio.to_thread(\n _prescan_pending" in src, ( + "the pre-scan is no longer entered through asyncio.to_thread — a long " + "scan would park the event loop (the phase-179 rule, " + "test_sync_git_freeze.py)" + ) + # The scan itself is a plain sync function (it runs in the worker). + assert not inspect.iscoroutinefunction(importer._prescan_pending) + + +# -------------------------------------------------------------------------- +# The scalar statement's shape (the phase-185 column-list style) +# -------------------------------------------------------------------------- + + +def test_scalar_statement_projects_exactly_file_hash() -> None: + """The pre-scan reads ONE column, so it loads no entity and keeps nothing. + + The compiled projection names ``documents.file_hash`` and no other + ``documents`` column; the statement resolves to exactly one scalar + column (``column_descriptions``), so ``session.scalar`` returns a string + and no ORM object exists to land in an identity map. ``_doc_row_stmt`` + stays the importer's only ``(source, path)`` ROW load (phase 185). + """ + compiled = str( + _pending_hash_stmt("the-source", "notes/a.md").compile( + dialect=postgresql.dialect() + ) + ) + projection = re.split(r"\bFROM\b", compiled, maxsplit=1)[0] + columns = set(re.findall(r"documents\.([a-z_]+)", projection)) + + assert columns == {"file_hash"}, ( + f"the pre-scan statement projects {sorted(columns)} — it must project " + "documents.file_hash and nothing else (one light metadata read per " + "file, no entity hydration)" + ) + # The predicate is the unique (source, path) constraint, like the fast path. + predicate = re.split(r"\bFROM\b", compiled, maxsplit=1)[1] + assert "documents.source" in predicate and "documents.path" in predicate + + stmt = _pending_hash_stmt("s", "p") + assert len(stmt.column_descriptions) == 1 + assert stmt.column_descriptions[0]["type"] == Document.file_hash.type + assert "Document" not in { + str(desc.get("entity")) for desc in stmt.column_descriptions + }, "the pre-scan statement selects an ENTITY — it must select the column" + + # The span is the only consumer, and the fast path's row load is intact. + span_src = inspect.getsource(importer._prescan_pending) + assert "_pending_hash_stmt(" in span_src + assert "select(Document)" not in span_src + assert "defer(" not in span_src and "load_only(" not in span_src, ( + "the pre-scan picked up a loader option — there is no ROW to defer: " + "it selects one column (phase 185 keeps _doc_row_stmt as the only " + "row load)" + ) + + +# -------------------------------------------------------------------------- +# The pending predicate +# -------------------------------------------------------------------------- + + +def test_pending_predicate_fresh_unchanged_changed_and_null_row( + db: Session, tmp_path: Path +) -> None: + """The four stored-hash states, and the position the set assigns. + + * no row at all (a first sync) → pending; + * a row whose ``file_hash`` equals the file's digest → NOT pending; + * a row whose ``file_hash`` differs → pending; + * a pre-phase-147 row (``file_hash`` NULL) → pending — the same call the + fast path makes, so the prediction and the loop agree and the loop + stamps the hash on the way through. + + The positions are the 1-based walk order of the PREDICTED files only, + and the scan total is every importable file (3 here, 3 predicted). + """ + root = _root( + tmp_path, + "phase194-predicate", + { + "fresh.md": "# Fresh\n\nnever indexed\n", + "same.md": "# Same\n\nindexed as-is\n", + "moved.md": "# Moved\n\nindexed differently now\n", + }, + ) + from app.rag.importer import _file_hash + + _seed_row(db, root.name, "same.md", file_hash=_file_hash(root / "same.md")) + _seed_row(db, root.name, "moved.md", file_hash="0" * 64) + db.expunge_all() + + scan_total, pending = _prescan(root, session=db) + + assert scan_total == 3 + assert pending == { + (root.name, "fresh.md"): 1, + (root.name, "moved.md"): 2, + }, ( + "the pending set is wrong — no row and a NULL row are pending, a " + "digest match is not, a mismatch is (the fast path's own predicate)" + ) + # The scan reads metadata only: nothing was hydrated into the session. + assert len(db.identity_map) == 0 + + +def test_pending_predicate_ignores_what_the_walk_ignores( + db: Session, tmp_path: Path +) -> None: + """A file the ignore rule hides is never walked, so it is never pending. + + The scan uses the EXACT per-root ``ignore`` / ``include_hidden`` rules of + the processing loop: a hidden dot-directory is out by default and in when + the root's flag admits it, and an ignored prefix is out even with the flag + on. A predicted file the loop can never see would be a count that never + reaches zero, so this is the parity edge, not a nicety. + """ + root = _root( + tmp_path, + "phase194-ignore", + { + "keep.md": "# Keep\n\nbody\n", + "skipme/hidden-by-rule.md": "# Skip\n\nbody\n", + ".dot/secret.md": "# Dot\n\nbody\n", + }, + ) + ignore = {str(root): ["skipme"]} + + scan_total, pending = _prescan(root, session=db, ignore_by_root=ignore) + assert scan_total == 1 + assert list(pending) == [(root.name, "keep.md")] + + # With the hidden-folders flag ON the dot-dir file joins the walk (and + # the scan total with it) — still no pending entry for the ignored one. + scan_total, pending = _prescan( + root, + session=db, + ignore_by_root=ignore, + include_hidden_by_root={str(root): True}, + ) + assert scan_total == 2 + assert sorted(pending) == [ + (root.name, ".dot/secret.md"), + (root.name, "keep.md"), + ] + + +def test_unreadable_file_is_predicted_changed_and_the_scan_survives( + db: Session, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """A file the digest cannot reach is a PREDICTION of change, not a failure. + + The window is the phase's own: the scan walks the tree, then hashes it + seconds later, and a live source can lose a file in between. Failing the + whole sync for that would be the worse bug — the pre-scan only informs a + label — so the file lands in the pending set (the loop's own walk decides + whether it still exists at all) and one warning names it. + """ + root = _root( + tmp_path, + "phase194-unreadable", + {"a.md": "# A\n\na\n", "b.md": "# B\n\nb\n"}, + ) + real_hash = importer._file_hash + + def flaky(path: Path) -> str: + if path.name == "b.md": + raise OSError(2, "No such file or directory", str(path)) + return real_hash(path) + + monkeypatch.setattr(importer, "_file_hash", flaky) + with caplog.at_level("WARNING", logger="app.importer"): + scan_total, pending = _prescan(root, session=db) + + assert scan_total == 2 + assert sorted(pending) == [ + (root.name, "a.md"), + (root.name, "b.md"), + ] + lines = [r.getMessage() for r in caplog.records if r.name == "app.importer"] + assert any( + line.startswith("import: pre-scan unreadable") + and f"source={root.name}" in line + and "path=b.md" in line + for line in lines + ), f"the skipped file was not named in a warning: {lines}" + + +# -------------------------------------------------------------------------- +# on_scan_progress + the empty KB +# -------------------------------------------------------------------------- + + +def test_scan_progress_reports_every_file_across_roots( + db: Session, tmp_path: Path +) -> None: + """The scan pair is ``(1-based position, combined importable total)``. + + The denominator is the number the phase-64 stats pre-walk used to + compute (every importable file, across every root, in *sources* order), + because the label is about the SCAN — how far the prediction has got — + and the events cover every file, changed or not. + """ + root_a = _root( + tmp_path, "phase194-scan-a", {"a1.md": "# A1\n\na\n", "a2.md": "# A2\n\nb\n"} + ) + root_b = _root(tmp_path, "phase194-scan-b", {"sub/b1.md": "# B1\n\nc\n"}) + events: list[tuple[int, int]] = [] + + scan_total, pending = _prescan( + root_a, root_b, session=db, on_scan_progress=lambda p, t: events.append((p, t)) + ) + + assert scan_total == 3 + assert events == [(1, 3), (2, 3), (3, 3)] + assert len(pending) == 3 + + +def test_scan_progress_reports_nothing_for_an_empty_kb( + db: Session, tmp_path: Path +) -> None: + """The degenerate KB: scan total 0, no callback, empty pending set. + + The phase-64 ``total === 0`` contract, extended to the scan: with nothing + importable there is no position to report, so the UI's scan label renders + bare (``Scanning…``, no count part) instead of ``0/0``. + """ + empty = _root(tmp_path, "phase194-empty", {}) + missing = tmp_path / "phase194-not-a-dir" + events: list[tuple[int, int]] = [] + + scan_total, pending = _prescan( + empty, missing, session=db, on_scan_progress=lambda p, t: events.append((p, t)) + ) + + assert scan_total == 0 + assert pending == {} + assert events == [] + + +def test_scan_hook_is_called_from_the_worker_thread( + tmp_path: Path, db: Session +) -> None: + """The documented threading contract of ``on_scan_progress``. + + The span runs in an ``asyncio.to_thread`` worker, so the hook fires on + THAT thread — which is only safe because the house hook rule is "assign a + field", a single GIL-atomic store. The pin records the thread identity + from inside the callback and from the awaiting coroutine. + """ + root = _root(tmp_path, "phase194-thread", {"a.md": "# A\n\na\n"}) + seen: dict[str, int] = {} + + def hook(position: int, total: int) -> None: # noqa: ARG001 + seen["scan"] = threading.get_ident() + + async def main() -> None: + seen["loop"] = threading.get_ident() + await asyncio.to_thread( + importer._prescan_pending, + [root], + _settings().import_extension_set, + frozenset(), + None, + None, + hook, + ) + + asyncio.run(main()) + + assert seen["scan"] != seen["loop"], ( + "the pre-scan hook ran on the event-loop thread — the span must live " + "in a worker (phase 179)" + ) + + +# -------------------------------------------------------------------------- +# done = the position in the pending set +# -------------------------------------------------------------------------- + + +def test_done_is_the_pending_position_through_import_sources( + db: Session, tmp_path: Path +) -> None: + """3 importable files, 2 of them pending: the pair is ``1/2``, ``1/2``, + ``2/2``. + + ``stable.md`` is indexed and unchanged, so the scan does not predict it + and the walk moving past it does NOT advance the count — the label holds + its position (this is the shape the owner's prod case takes: the count + stops climbing across a run of unchanged files). The two changed files + advance it in walk order, and the denominator is the pending set's size, + never the 3-file walk. + """ + root = _root( + tmp_path, + "phase194-position", + { + "a.md": "# A\n\nalpha\n", + "stable.md": "# Stable\n\nunchanged\n", + "z.md": "# Z\n\nzulu\n", + }, + ) + llm = FakeEmbedder() + events: list[tuple[str, str, int, int]] = [] + try: + # Index everything, then change two files. + asyncio.run(import_sources([root], llm, session=db)) + (root / "a.md").write_text("# A\n\nalpha edited\n", encoding="utf-8") + (root / "z.md").write_text("# Z\n\nzulu edited\n", encoding="utf-8") + + events.clear() + summary = asyncio.run( + import_sources( + [root], + llm, + session=db, + progress=lambda s, r, d, t: events.append((s, r, d, t)), + ) + ) + + assert (summary.files, summary.updated, summary.unchanged) == (3, 2, 1) + assert events == [ + (root.name, "a.md", 1, 2), + (root.name, "stable.md", 1, 2), # not predicted: the position holds + (root.name, "z.md", 2, 2), + ] + # The parity the UI depends on: the denominator IS the work. + assert summary.updated + summary.added == events[-1][3] + finally: + db.execute(text("DELETE FROM documents WHERE source = :s"), {"s": root.name}) + db.commit() + + +def test_empty_kb_reports_no_count_part_through_import_sources( + db: Session, tmp_path: Path +) -> None: + """An empty source with the hook on: no file, no pair, ``files_total`` 0. + + The import is a no-op and the hook never fires, so the status keeps the + zero denominator the UI's ``total === 0`` guard hides the count part for + (the phase-64 contract, arriving through the pre-scan path rather than + the stats pre-walk). + """ + root = _root(tmp_path, "phase194-empty-run", {}) + events: list[tuple[str, str, int, int]] = [] + scan_events: list[tuple[int, int]] = [] + + summary = asyncio.run( + import_sources( + [root], + FakeEmbedder(), + session=db, + progress=lambda s, r, d, t: events.append((s, r, d, t)), + on_scan_progress=lambda p, t: scan_events.append((p, t)), + ) + ) + + assert (summary.files, summary.added, summary.errors) == (0, 0, 0) + assert events == [] + assert scan_events == [] + + +def test_cli_caller_is_untouched_by_the_pending_semantics( + db: Session, tmp_path: Path +) -> None: + """The CLI-shaped run (no hook at all) on an unchanged KB is the pre-phase + run: one file processed, nothing counted, one walk. + + The byte-identity of the non-UI caller is the phase's cheapest promise — + ``scripts/import_docs.py`` passes no ``progress``, so it neither pays the + scan nor sees its numbers. + """ + root = _root(tmp_path, "phase194-cli", {"a.md": "# A\n\na\n"}) + llm = FakeEmbedder() + try: + first = asyncio.run(import_sources([root], llm, session=db)) + assert (first.files, first.added) == (1, 1) + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.files, second.added, second.unchanged) == (1, 0, 1) + finally: + db.execute(text("DELETE FROM documents WHERE source = :s"), {"s": root.name}) + db.commit() diff --git a/tests/unit/test_sync_scan_label_ui.py b/tests/unit/test_sync_scan_label_ui.py new file mode 100644 index 0000000..c4e155a --- /dev/null +++ b/tests/unit/test_sync_scan_label_ui.py @@ -0,0 +1,544 @@ +"""Phase 194, task 03 — the Sources page renders the scan as a scan. + +The owner report the whole phase exists for (2026-10-19): *"1216 +remaining"* on a ~10,000-file KB that had ~10 files to sync. Task 01 moved +the server's ``done``/``total`` to the pre-scan's predicted change set and +task 02 exposed the scan as a phase (``phase: "scan"`` + ``scan_done`` / +``scan_total``). This module pins the UI half (D4, owner-confirmed A2): +``fmtSyncPhaseLabel`` gains ONE explicit ``"scan"`` branch that says +``Scanning…`` and puts the scan's own ``(n/total)`` in the SAME count slot +the import phase's ``N remaining`` uses — so the walk position surviving as +an ``n/m`` is honest, because the word in front of it is "Scanning" and +never "Syncing" (TODO L6's objection was the sync count wearing the +total's shape). + +Pinned here (source markers, plus one node run of the REAL label path): + +* the branch — ``status.phase === "scan"`` is in the decision tree BEFORE + ``overview`` / ``summaries`` / the fall-through, its main text is exactly + ``Scanning…``, and it never reads ``current_file`` (null in the phase), + the file pair, the chunk pair, the image pair or the summaries pair; +* the shape — the count part is ``(${status.scan_done}/${status.scan_total})`` + landing on the ``count`` slot (``setSyncLabel``'s hidden-toggle + the A4 + untruncated button-title / ``#sync-progress`` placement carry it with no + new element), with the phase-64 ``total === 0`` guard (the empty KB and + the scan's own first frame render the bare ``Scanning…``) and no + chunk/image part (the scan reads no file); +* the pass-through — both sync status call sites (the poll tick, the + load-time re-attach) forward the WHOLE status object, which is what + carries the scan pair to the wrapper (the summaries / chunk / image + pattern: ``phase`` selects which pair renders, no field is re-listed); +* the UNTOUCHED set — ``fmtSyncLabel``'s ``N remaining`` expression + verbatim (the phase-193 pin), the chunk / image / summaries part + builders, ``setSyncLabel``'s composed full string, ``fmtSyncResult`` and + ``fmtSyncPeakClause`` (the phase-187 trailing clause lands on the scan + label's line unchanged, and stays a terminal-only clause); +* the docstrings — the decision-tree paragraph names the new branch, + narrows the fallback to the null prelude + ``"import"``, and keeps the + phase-193 ``N remaining`` clause. + +The owner-visible arms (the count-down over a real incremental sync, the +scan label in a real browser) live in +``tests/e2e/test_sync_pending_count.py`` (task 05); the stale pinned suites +retarget in task 04. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +#: The node behavior check needs node (the test-document-viewer pattern). +HAVE_NODE = shutil.which("node") is not None + +ROOT = Path(__file__).resolve().parents[2] +SOURCES_JS = ROOT / "frontend" / "assets" / "sources.js" + + +def _js() -> str: + return SOURCES_JS.read_text(encoding="utf-8") + + +def _fn(js: str, name: str) -> str: + """The source of the first ``function `` (brace balanced — the + house pin pattern from tests/unit/test_frontend_sync_upload.py: the + functions live inside ``mount(root)``, so the closing brace is + indented).""" + fn = js.find(f"function {name}") + assert fn != -1, f"{name} must be defined in sources.js" + open_idx = js.find("{", fn) + depth = 0 + for i in range(open_idx, len(js)): + c = js[i] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return js[fn : i + 1] + raise AssertionError(f"unbalanced braces in {name}") + + +def _doc_above(js: str, name: str) -> str: + """The block comment immediately above ``function ``.""" + i = js.find(f"function {name}") + assert i != -1, f"{name} must be defined in sources.js" + start = js.rfind("/*", 0, i) + assert start != -1, f"{name} must carry a docstring" + end = js.find("*/", start) + assert start < end < i, f"{name}'s docstring must sit above the function" + return js[start:end] + + +def _scan_slice(js: str) -> str: + """The ``"scan"`` branch of ``fmtSyncPhaseLabel`` — from its ``if`` to + the ``overview`` branch that follows it (every shape pin below reads + ONLY this slice, so a shape that leaked into another branch is caught + and a branch that moved is caught).""" + body = _fn(js, "fmtSyncPhaseLabel") + start = body.find('status.phase === "scan"') + assert start != -1, 'the "scan" branch is missing from fmtSyncPhaseLabel' + end = body.find('status.phase === "overview"', start) + assert end != -1, "the overview branch must follow the scan branch" + return body[start:end] + + +# ---------- the branch ---------- + + +def test_scan_branch_is_the_first_arm_of_the_decision_tree() -> None: + """The wrapper gains ONE explicit ``"scan"`` branch (D4): the phase + comparison sits BEFORE ``overview``, ``summaries`` and the + ``fmtSyncLabel`` fall-through (the machine order is ``null`` → + ``"scan"`` → ``"import"`` → ``"overview"`` → ``"summaries"``, and + keeping the arms in that order is what leaves every pre-existing + branch slice byte-identical), and the main text is exactly + ``Scanning…`` with the ellipsis the other running labels use.""" + js = _js() + body = _fn(js, "fmtSyncPhaseLabel") + i_scan = body.find('status.phase === "scan"') + i_ov = body.find('status.phase === "overview"') + i_sum = body.find('status.phase === "summaries"') + i_fall = body.find("fmtSyncLabel(", i_sum) + assert -1 < i_scan < i_ov < i_sum < i_fall, ( + "the scan branch must be the first arm, ahead of overview / " + "summaries / the null-prelude + import fall-through" + ) + assert 'main: "Scanning\u2026"' in _scan_slice(js), ( + "the main text is the bare Scanning label — no file" + ) + + +def test_scan_branch_reads_the_scan_pair_and_nothing_else() -> None: + """The branch consumes the scan pair the way it already consumes the + summaries pair (``phase`` selects which pair renders). It NEVER reads + ``current_file`` — the runner keeps it null through the phase, because + the scan names no file — nor the file pair, the chunk pair, the image + pair or the summaries pair (a scan embeds no chunk and describes no + image, so those fields carry nothing meaningful here).""" + scan = _scan_slice(_js()) + assert "status.scan_done" in scan and "status.scan_total" in scan, ( + "the count part is the status's own scan pair" + ) + for field in ( + "current_file", + "files_done", + "files_total", + "chunks_total", + "current_chunk", + "images_total", + "images_done", + "summaries_total", + "summaries_done", + "current_summary", + ): + assert field not in scan, ( + f'the "scan" branch must not read {field} — that field is the ' + "import/overview/summaries phase's own position" + ) + + +def test_scan_branch_emits_no_chunk_and_no_image_part() -> None: + """The scan names no file, so its parts are the main text + the count + only: ``chunk: ""`` and ``image: ""`` (the writer hides both spans and + keeps them out of the composed full string — the overview branch's + shape, phase 98/152).""" + scan = _scan_slice(_js()) + assert 'chunk: ""' in scan, "the scan label carries no chunk part" + assert 'image: ""' in scan, "the scan label carries no image part" + + +# ---------- the (n/total) count part ---------- + + +def test_scan_count_part_is_the_pair_in_the_count_slot() -> None: + """The count part is the exact ``(${status.scan_done}/${status.scan_total})`` + template — a plain ``(n/total)``, no thousands separator, no suffix — + and it lands on the ``count`` key, i.e. the SAME slot the import + phase's ``N remaining`` uses (``setSyncLabel``'s + ``.sync-label-count`` span + the composed full string), so the count + stays pinned on a phone and the A4 untruncated label (button title + + ``#sync-progress``) carries ``Scanning… (n/total)`` with no new + element and no new live region.""" + scan = _scan_slice(_js()) + assert "`(${status.scan_done}/${status.scan_total})`" in scan, ( + "the exact (n/total) shape over the scan pair" + ) + i_count = scan.find("count:") + i_template = scan.find("`(${status.scan_done}/${status.scan_total})`") + assert -1 < i_count < i_template, "the pair renders INTO the count slot" + assert "remaining" not in scan, ( + "the scan never says remaining — that part belongs to fmtSyncLabel" + ) + for helper in ("toLocaleString", ".toFixed"): + assert helper not in scan, "plain integers, the house label style" + + +def test_scan_count_part_hides_while_the_total_is_zero() -> None: + """The phase-64 ``total === 0`` contract, extended to the scan: while + the scan has not reported its denominator (and for the degenerate + empty KB, where the importer fires no scan callback at all) the count + part is the empty string — the bare ``Scanning…`` renders, never + ``(0/0)``. The guard precedes the template and the false arm is the + empty string.""" + scan = _scan_slice(_js()) + i_guard = scan.find("status.scan_total > 0") + i_template = scan.find("`(${status.scan_done}/${status.scan_total})`") + i_else = scan.find(': ""', i_template) + assert -1 < i_guard < i_template < i_else, ( + "scan_total > 0 ? (n/total) : \"\" — the phase-64 no-count-part guard" + ) + + +# ---------- the caller pass-through ---------- + + +def test_poll_tick_forwards_the_whole_status_so_the_scan_pair_rides() -> None: + """The running-sync branch of the poll tick passes the WHOLE status + object as ``enterSyncRunningState``'s fifth argument — that object IS + the pass-through for the scan pair, exactly like the summaries / + chunk / image pairs (the call site names it). A status with + ``phase: "scan"`` therefore reaches the wrapper with its + ``scan_done`` / ``scan_total`` intact.""" + js = _js() + start = js.find("function startSyncPolling") + assert start != -1 + tick = js[start : js.find("async function startSync", start)] + i_run = tick.find('syncStatus.state === "running"') + assert i_run != -1 + # the call-site comment sits directly above the branch — it is part of + # the pass-through's documentation, so the slice starts there + branch = tick[tick.rfind("// 1. sync running", 0, i_run) : tick.find( + 'uploadStatus && uploadStatus.state === "running"', i_run + )] + assert ( + '"sync", syncStatus.current_file, syncStatus.files_done, syncStatus.files_total' + in branch + ), "the pre-phase positional args are unchanged" + i_args = branch.find("syncStatus.files_total") + i_pass = branch.find("syncStatus\n", i_args) + assert i_pass != -1, "the WHOLE syncStatus object rides in as the 5th arg" + assert "scan_done" in branch and "scan_total" in branch, ( + "the call-site comment names the scan pair riding the status (phase 194)" + ) + + +def test_load_time_reattach_forwards_the_whole_status_too() -> None: + """The load-time re-attach is the other sync call site: a reload + during the pre-scan re-enters running state with the WHOLE status, so + the re-attached label reads ``Scanning… (n/total)`` (the never-stale + contract — phase 98's mid-summaries reload, now also mid-scan).""" + js = _js() + body = _fn(js, "initSyncButton") + assert ( + '"sync", status.current_file, status.files_done, status.files_total' in body + ), "the pre-phase positional args are unchanged" + i_args = body.find("status.files_total") + i_pass = body.find("status\n", i_args) + assert i_pass != -1, "the WHOLE status object rides in as the 5th arg" + doc = _doc_above(js, "initSyncButton") + assert "phase 194" in doc and "Scanning" in doc, ( + "the re-attach docstring names the scan label it can re-enter (194)" + ) + + +def test_running_entry_hands_the_status_to_the_phase_wrapper() -> None: + """``enterSyncRunningState`` keeps the phase-aware split untouched: + sync + status → ``fmtSyncPhaseLabel(status)`` (the wrapper that now + owns the scan branch), everything else → the bare ``fmtSyncLabel`` + parts, and the parts still go to the ONE writer with the running + flag (so the scan label lands on the button title + the box).""" + entry = _fn(_js(), "enterSyncRunningState") + assert "fmtSyncPhaseLabel(status)" in entry + assert 'kind === "sync" && status' in entry + assert "setSyncLabel(main, count, chunk, true, image)" in entry + + +# ---------- the UNTOUCHED set ---------- + + +def test_fmt_sync_label_n_remaining_arithmetic_is_byte_identical() -> None: + """The phase-193 count-down builder is UNTOUCHED by this task — the + server-side semantic change (the pair becomes the pending position) + flows through the SAME expression. The exact line, both guards and the + plain-integer shape, plus the absence of anything scan-shaped in this + builder.""" + bare = _fn(_js(), "fmtSyncLabel") + assert ( + "const count = total > 0 && total - done > 0" + " ? `${total - done} remaining` : \"\";" + ) in bare, "the phase-193 count expression, byte for byte" + assert 'const prefix = kind === "upload" ? "Importing" : "Syncing\u2026";' in bare + assert "return { main, count }" in bare + for word in ("scan", "chunk", "image", "summaries", "phase"): + assert word not in bare, f"fmtSyncLabel stays free of {word!r}" + + +def test_other_phase_labels_and_the_writers_are_untouched() -> None: + """The regression pins for everything the scan branch could have + disturbed: the overview copy, the summaries folder + ``(n/m)`` + position, the import fall-through's chunk + image parts, the single + writer's signature + composed full string, the terminal counts line + (no running count part, no scan word) and the phase-187 peak clause + (still a terminal-only trailing clause, null renders nothing).""" + js = _js() + phase = _fn(js, "fmtSyncPhaseLabel") + assert ( + 'return { main: "Writing KB overview\u2026", count: "", chunk: "", image: "" };' + in phase + ) + assert "`Summarizing folders\u2026${folder}`" in phase + assert ( + "count: `(${status.summaries_done}/${status.summaries_total})`" in phase + ), "the summaries folder position keeps its (n/m)" + assert ( + 'const { main, count } = fmtSyncLabel(\n' + ' "sync", status.current_file, status.files_done, status.files_total\n' + " );" in phase + ), "the import fall-through still calls the bare builder unchanged" + assert "`\u00b7 chunk ${status.current_chunk}/${status.chunks_total}`" in phase + assert "`\u00b7 image ${status.images_done}/${status.images_total}`" in phase + assert "return { main, count, chunk, image };" in phase + + writer = _fn(js, "setSyncLabel") + assert ( + 'function setSyncLabel(mainText, countText = "", chunkText = "",' + ' running = false, imageText = "")' in writer + ) + assert "syncLabelCount.textContent = countText" in writer + assert "syncLabelCount.hidden = false" in writer + assert ( + "const full =\n" + " mainText +\n" + " (countText ? ` ${countText}` : \"\") +\n" + " (chunkText ? ` ${chunkText}` : \"\") +\n" + " (imageText ? ` ${imageText}` : \"\");" in writer + ), "the A4 untruncated label is still main + count + chunk + image" + assert "if (running) setSyncProgress(full);" in writer + assert "syncBtn.title = full" in writer + + result = _fn(js, "fmtSyncResult") + for word in ("remaining", "Scanning", "scan"): + assert word not in result, f"the terminal counts line stays {word!r}-free" + + peak = _fn(js, "fmtSyncPeakClause") + assert "return ` \u00b7 peak ${peak} MB`;" in peak + assert "status.rss_peak_mb ?? status.rss_mb" in peak + assert "Scanning" not in peak, "the clause is terminal-only, unchanged" + + +# ---------- the docstrings ---------- + + +def test_phase_label_docstring_names_the_branch_and_narrows_the_fallback() -> None: + """The decision-tree paragraph above the wrapper names the new arm and + narrows the fall-through to the two states that remain, keeps the + phase-193 ``N remaining`` clause, states the ``total === 0`` guard, + and says WHY an ``(n/total)`` is honest here (TODO L6: the objection + was the sync count wearing the total's shape — this label says + Scanning, never Syncing).""" + doc = _doc_above(_js(), "fmtSyncPhaseLabel") + assert '"scan"' in doc and "Scanning\u2026" in doc, "the new branch is named" + assert "scan_done" in doc and "scan_total" in doc, "the pair is named" + assert "scan_total === 0" in doc, "the no-count-part guard is documented" + assert "ONLY the two" in doc and '"import"' in doc, "the fallback is narrowed" + assert "`N remaining`" in doc and "N = total - done" in doc, ( + "the phase-193 count-down clause survives the rewrite" + ) + assert "never" in doc and '"Syncing"' in doc, ( + "the honesty clause: Scanning, never Syncing (TODO L6)" + ) + assert "A4" in doc, "the untruncated placement is still documented" + + +def test_module_and_section_headers_document_the_scan_label() -> None: + """The module header carries the phase-194 paragraph (the count is the + WORK, the scan says so), and the sync section's decision tree names + the ``"scan"`` arm with its ``(n/total)`` — the header the wrapper's + docstring points at.""" + js = _js() + header = js[: js.find("import { fetchIsAdmin }")] + assert "Phase 194 (task 03)" in header, "the module header names the phase" + assert "Scanning\u2026 (n/total)" in header, "the module header names the label" + + marker = js.find("Sync sources button (Sources page only)") + assert marker != -1 + section = js[marker : js.find("const syncBtn", marker)] + assert "Phase 194 task 03" in section, "the section header names the phase" + assert "Scanning\u2026 (n/total)" in section, "the decision tree names the label" + assert "scan_done" in section, "the pair the WHOLE status carries is named" + + +# ---------- the REAL label path under node ---------- + +#: The stub element refs (the test-document-viewer pattern; the same +#: shapes tests/unit/test_frontend_sync_upload.py drives). The label +#: writer, the box writer and the cancel control are the REAL functions. +_NODE_HEAD = """ +const syncLabelMain = { textContent: "" }; +const syncLabelCount = { textContent: "", hidden: true }; +const syncLabelChunk = { textContent: "", hidden: true }; +const syncProgress = { hidden: true }; +const syncProgressText = { textContent: "" }; +const syncBtn = { + disabled: false, + title: "", + attrs: {}, + setAttribute(k, v) { this.attrs[k] = v; }, + removeAttribute(k) { delete this.attrs[k]; }, + classes: new Set(), + classList: { + add(c) { syncBtn.classes.add(c); }, + remove(c) { syncBtn.classes.delete(c); }, + has(c) { return syncBtn.classes.has(c); }, + }, +}; +const syncIcon = { + classes: new Set(), + classList: { + add(c) { syncIcon.classes.add(c); }, + remove(c) { syncIcon.classes.delete(c); }, + has(c) { return syncIcon.classes.has(c); }, + }, +}; +const syncCancelBtn = { + hidden: true, + disabled: true, + attrs: {}, + setAttribute(k, v) { this.attrs[k] = v; }, + removeAttribute(k) { delete this.attrs[k]; }, +}; +""" + +_NODE_TAIL = """ +const show = (tag) => { + console.log(JSON.stringify([ + tag, + syncLabelMain.textContent, + syncLabelCount.textContent, + syncLabelCount.hidden, + syncBtn.title, + syncProgressText.textContent, + syncBtn.attrs['aria-label'] || '', + ])); +}; +// 1. the scan, denominator known + the position climbing (the mid-scan poll): +enterSyncRunningState("sync", null, 0, 0, { + phase: "scan", current_file: null, files_done: 0, files_total: 0, + scan_done: 3, scan_total: 9, current_chunk: 0, chunks_total: 0, +}); +show("scan-3-9"); +// 2. the scan's degenerate arm: an empty KB / the scan's own first frame +// (0/0) — the phase-64 guard, the bare label with NO count part: +enterSyncRunningState("sync", null, 0, 0, { + phase: "scan", current_file: null, files_done: 0, files_total: 0, + scan_done: 0, scan_total: 0, current_chunk: 0, chunks_total: 0, +}); +show("scan-0-0"); +// 3. the prelude (null phase) is untouched — bare "Syncing\u2026": +enterSyncRunningState("sync", null, 0, 0, { + phase: null, current_file: null, files_done: 0, files_total: 0, + scan_done: 4, scan_total: 9, current_chunk: 0, chunks_total: 0, +}); +show("prelude"); +// 4. the import phase is untouched — the phase-193 count-down + the chunk +// part, and the scan pair present on the payload but NOT rendered: +enterSyncRunningState("sync", "src/a/b.md", 2, 9, { + phase: "import", current_file: "src/a/b.md", files_done: 2, files_total: 9, + scan_done: 9, scan_total: 9, current_chunk: 14, chunks_total: 38, +}); +show("import"); +// 5. the overview + summaries labels are untouched: +enterSyncRunningState("sync", null, 9, 9, { + phase: "overview", current_file: null, files_done: 9, files_total: 9, + scan_done: 9, scan_total: 9, current_chunk: 0, chunks_total: 0, +}); +show("overview"); +enterSyncRunningState("sync", null, 9, 9, { + phase: "summaries", current_file: null, files_done: 9, files_total: 9, + scan_done: 9, scan_total: 9, current_summary: "src/alpha", + summaries_done: 2, summaries_total: 3, current_chunk: 0, chunks_total: 0, +}); +show("summaries"); +// 6. the 202/409 click (no status at all) stays bare: +enterSyncRunningState("sync", null, 0, 0); +show("click"); +""" + + +@pytest.mark.skipif(not HAVE_NODE, reason="node not available") +def test_scan_label_renders_through_the_real_label_path_under_node() -> None: + """Behavior, not just markers: the REAL extracted functions run + against stub refs. ``phase: "scan"`` renders main ``Scanning…`` with + the count ``(3/9)`` visible, and the SAME string lands on the button + title, the box (the sole aria-live region) and the button's + accessible name (A4) — while every other phase renders exactly what it + rendered before this task.""" + js = _js() + fns = "\n".join( + _fn(js, name) + for name in ( + "fmtSyncLabel", + "fmtSyncPhaseLabel", + "setSyncLabel", + "setSyncProgress", + "showSyncCancelBtn", + "hideSyncCancelBtn", + "enterSyncRunningState", + ) + ) + out = subprocess.run( + ["node", "-e", _NODE_HEAD + fns + _NODE_TAIL], + capture_output=True, + text=True, + timeout=60, + ) + assert out.returncode == 0, out.stderr + rows = {row[0]: row for row in (json.loads(line) for line in out.stdout.splitlines())} + + main, count, hidden, title, box, aria = rows["scan-3-9"][1:] + assert main == "Scanning\u2026", rows["scan-3-9"] + assert count == "(3/9)" and hidden is False, rows["scan-3-9"] + assert title == "Scanning\u2026 (3/9)" == box == aria, ( + f"A4: the untruncated scan label rides title + box + aria: {rows['scan-3-9']}" + ) + + main, count, hidden, title, box, aria = rows["scan-0-0"][1:] + assert main == "Scanning\u2026" and count == "" and hidden is True, rows["scan-0-0"] + assert title == "Scanning\u2026" == box == aria, ( + f"0/0 renders NO count part (the empty-KB guard): {rows['scan-0-0']}" + ) + + # every other state is byte-identical to its pre-phase label + assert rows["prelude"][1] == "Syncing\u2026" and rows["prelude"][2] == "" + assert rows["prelude"][4] == "Syncing\u2026", "a scan pair never renders in the prelude" + assert rows["import"][1] == "Syncing\u2026 src/a/b.md" + assert rows["import"][2] == "7 remaining", rows["import"] + assert rows["import"][4] == "Syncing\u2026 src/a/b.md 7 remaining \u00b7 chunk 14/38" + assert rows["overview"][1] == "Writing KB overview\u2026" and rows["overview"][2] == "" + assert rows["summaries"][1] == "Summarizing folders\u2026 src/alpha" + assert rows["summaries"][2] == "(2/3)", "the summaries position keeps its (n/m)" + assert rows["click"][1] == "Syncing\u2026" and rows["click"][2] == ""