From 12fa69b7ea0567f9feb15b46680617d0aa52be2f Mon Sep 17 00:00:00 2001 From: ducoterra Date: Sat, 3 Oct 2026 01:52:41 -0400 Subject: [PATCH] phase: 152_sync_progress_image_pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All criteria verified. The CLI path is hook-less (only `app/api/sync.py` passes the hook), and the byte-identity is pinned by `test_on_image_progress_none_run_is_byte_identical`. ## Final Report — Phase 152 final verification pass - **Verified (no new code needed):** `SyncStatus.images_done/images_total` + reset/clear rules in `app/api/sync.py`; `on_image_progress` threaded through the importer's three PDF vision spans with the continuous per-file counter; the `_image_hook` closure; `sources.js` `· image n/m` label part (box + title + aria). - **C1** status climbs 1→N + UI shows `· image n/m`: ✅ unit 3/15→15/15, integration 1/15→7/15, E2E 1/4→4/4 (title/aria ride the full string) - **C2** never frozen: ✅ E2E asserts ≥3 distinct positions, starting at 1, non-decreasing, ≤ total, reaching it - **C3** cleared at overview/summaries + all terminals; per-file reset via `_hook`; no stale position for image-less files: ✅ unit pins + E2E settle (image part gone in box/title/aria, endpoint 0/0) - **C4** `/api/sync/status` additive-only: ✅ full-dict pin passes (all pre-existing keys unchanged + the two new 0/0 keys) - **C5** hook-less caller byte-identical: ✅ CLI passes no hook; `test_on_image_progress_none_run_is_byte_identical` green - **C6** `uv run pytest`: **3526 passed, exit 0** (95s); dedicated E2E in isolation: **2 passed** (95s); `uv run ruff check . && uv run pyright`: **clean**; coverage: **99% on `app/` (>90% gate)** - Regression: all five neighbouring E2E suites green in isolation (progress_box 8, chunk_progress 2, label_mobile 4, pdf_support 6, image_documents 4) - **C7** atomic commit: left to the harness (no git index/history touched per rules) - **Docs:** README.md — sync-label paragraph already updated this phase, accurately; all `.env.example` vars documented; routes/commands verified — **no changes needed**. AGENTS.md — all references (PLAN, user_stories, test_smoke.py, .env.example, gpg config, .gitignore) verified — **no changes needed** - **Defects found in prior work:** none (one prior env-leak defect was already fixed inside this phase's task 04, conftest) - No live-infrastructure touched; no ntfy used. **Next pending phase:** none — `todo/` held only this phase --- .../152_sync_progress_image_pass/00_phase.md | 0 .../01_status_image_fields.md | 0 .../02_importer_image_hook.md | 0 .../03_label_and_wiring.md | 0 .../04_unit_tests.md | 0 .../05_e2e_image_progress.md | 0 ..._sync_progress_image_pass__00_phase.a1.err | 0 ...2_sync_progress_image_pass__00_phase.a1.md | 16 + ..._progress_image_pass__00_phase.a1.validate | 120 +++ ..._image_pass__01_status_image_fields.a1.err | 0 ...s_image_pass__01_status_image_fields.a1.md | 19 + ...e_pass__01_status_image_fields.a1.validate | 120 +++ ..._image_pass__02_importer_image_hook.a1.err | 0 ...s_image_pass__02_importer_image_hook.a1.md | 16 + ...e_pass__02_importer_image_hook.a1.validate | 120 +++ ...ess_image_pass__03_label_and_wiring.a1.err | 0 ...ress_image_pass__03_label_and_wiring.a1.md | 11 + ...mage_pass__03_label_and_wiring.a1.validate | 120 +++ ..._progress_image_pass__04_unit_tests.a1.err | 0 ...c_progress_image_pass__04_unit_tests.a1.md | 15 + ...ress_image_pass__04_unit_tests.a1.validate | 120 +++ ..._progress_image_pass__04_unit_tests.a2.err | 0 ...c_progress_image_pass__04_unit_tests.a2.md | 1 + ...ress_image_pass__04_unit_tests.a2.validate | 185 ++++ ...s_image_pass__05_e2e_image_progress.a1.err | 0 ...ss_image_pass__05_e2e_image_progress.a1.md | 19 + ...ge_pass__05_e2e_image_progress.a1.validate | 120 +++ README.md | 9 +- TODO.md | 2 + app/api/sync.py | 125 ++- app/rag/importer.py | 205 +++- frontend/assets/sources.js | 142 ++- frontend/index.html | 5 +- tests/conftest.py | 6 + tests/e2e/test_sync_image_progress.py | 885 ++++++++++++++++++ tests/integration/test_sync_api.py | 353 ++++++- tests/unit/test_frontend_sync_upload.py | 279 +++++- tests/unit/test_importer.py | 509 +++++++++- tests/unit/test_sync_button.py | 582 +++++++++++- 39 files changed, 3971 insertions(+), 133 deletions(-) rename .agents/phases/{todo => complete}/152_sync_progress_image_pass/00_phase.md (100%) rename .agents/phases/{todo => complete}/152_sync_progress_image_pass/01_status_image_fields.md (100%) rename .agents/phases/{todo => complete}/152_sync_progress_image_pass/02_importer_image_hook.md (100%) rename .agents/phases/{todo => complete}/152_sync_progress_image_pass/03_label_and_wiring.md (100%) rename .agents/phases/{todo => complete}/152_sync_progress_image_pass/04_unit_tests.md (100%) rename .agents/phases/{todo => complete}/152_sync_progress_image_pass/05_e2e_image_progress.md (100%) create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.err create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.md create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.validate create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.err create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.md create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.validate create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.err create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.md create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.validate create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.err create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.md create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.validate create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.err create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.md create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.validate create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.err create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.md create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.validate create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.err create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.md create mode 100644 .agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.validate create mode 100644 tests/e2e/test_sync_image_progress.py diff --git a/.agents/phases/todo/152_sync_progress_image_pass/00_phase.md b/.agents/phases/complete/152_sync_progress_image_pass/00_phase.md similarity index 100% rename from .agents/phases/todo/152_sync_progress_image_pass/00_phase.md rename to .agents/phases/complete/152_sync_progress_image_pass/00_phase.md diff --git a/.agents/phases/todo/152_sync_progress_image_pass/01_status_image_fields.md b/.agents/phases/complete/152_sync_progress_image_pass/01_status_image_fields.md similarity index 100% rename from .agents/phases/todo/152_sync_progress_image_pass/01_status_image_fields.md rename to .agents/phases/complete/152_sync_progress_image_pass/01_status_image_fields.md diff --git a/.agents/phases/todo/152_sync_progress_image_pass/02_importer_image_hook.md b/.agents/phases/complete/152_sync_progress_image_pass/02_importer_image_hook.md similarity index 100% rename from .agents/phases/todo/152_sync_progress_image_pass/02_importer_image_hook.md rename to .agents/phases/complete/152_sync_progress_image_pass/02_importer_image_hook.md diff --git a/.agents/phases/todo/152_sync_progress_image_pass/03_label_and_wiring.md b/.agents/phases/complete/152_sync_progress_image_pass/03_label_and_wiring.md similarity index 100% rename from .agents/phases/todo/152_sync_progress_image_pass/03_label_and_wiring.md rename to .agents/phases/complete/152_sync_progress_image_pass/03_label_and_wiring.md diff --git a/.agents/phases/todo/152_sync_progress_image_pass/04_unit_tests.md b/.agents/phases/complete/152_sync_progress_image_pass/04_unit_tests.md similarity index 100% rename from .agents/phases/todo/152_sync_progress_image_pass/04_unit_tests.md rename to .agents/phases/complete/152_sync_progress_image_pass/04_unit_tests.md diff --git a/.agents/phases/todo/152_sync_progress_image_pass/05_e2e_image_progress.md b/.agents/phases/complete/152_sync_progress_image_pass/05_e2e_image_progress.md similarity index 100% rename from .agents/phases/todo/152_sync_progress_image_pass/05_e2e_image_progress.md rename to .agents/phases/complete/152_sync_progress_image_pass/05_e2e_image_progress.md diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.err b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.md b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.md new file mode 100644 index 0000000..81fd2ee --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.md @@ -0,0 +1,16 @@ +All criteria verified. The CLI path is hook-less (only `app/api/sync.py` passes the hook), and the byte-identity is pinned by `test_on_image_progress_none_run_is_byte_identical`. + +## Final Report — Phase 152 final verification pass + +- **Verified (no new code needed):** `SyncStatus.images_done/images_total` + reset/clear rules in `app/api/sync.py`; `on_image_progress` threaded through the importer's three PDF vision spans with the continuous per-file counter; the `_image_hook` closure; `sources.js` `· image n/m` label part (box + title + aria). +- **C1** status climbs 1→N + UI shows `· image n/m`: ✅ unit 3/15→15/15, integration 1/15→7/15, E2E 1/4→4/4 (title/aria ride the full string) +- **C2** never frozen: ✅ E2E asserts ≥3 distinct positions, starting at 1, non-decreasing, ≤ total, reaching it +- **C3** cleared at overview/summaries + all terminals; per-file reset via `_hook`; no stale position for image-less files: ✅ unit pins + E2E settle (image part gone in box/title/aria, endpoint 0/0) +- **C4** `/api/sync/status` additive-only: ✅ full-dict pin passes (all pre-existing keys unchanged + the two new 0/0 keys) +- **C5** hook-less caller byte-identical: ✅ CLI passes no hook; `test_on_image_progress_none_run_is_byte_identical` green +- **C6** `uv run pytest`: **3526 passed, exit 0** (95s); dedicated E2E in isolation: **2 passed** (95s); `uv run ruff check . && uv run pyright`: **clean**; coverage: **99% on `app/` (>90% gate)** +- Regression: all five neighbouring E2E suites green in isolation (progress_box 8, chunk_progress 2, label_mobile 4, pdf_support 6, image_documents 4) +- **C7** atomic commit: left to the harness (no git index/history touched per rules) +- **Docs:** README.md — sync-label paragraph already updated this phase, accurately; all `.env.example` vars documented; routes/commands verified — **no changes needed**. AGENTS.md — all references (PLAN, user_stories, test_smoke.py, .env.example, gpg config, .gitignore) verified — **no changes needed** +- **Defects found in prior work:** none (one prior env-leak defect was already fixed inside this phase's task 04, conftest) +- No live-infrastructure touched; no ntfy used. **Next pending phase:** none — `todo/` held only this phase diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.validate b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.validate new file mode 100644 index 0000000..399df7a --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__00_phase.a1.validate @@ -0,0 +1,120 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................................................ [ 98%] +...................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 196 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 266 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 135 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 764 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 443 0 100% +-------------------------------------------------- +TOTAL 5719 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.err b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.md b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.md new file mode 100644 index 0000000..62a014f --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.md @@ -0,0 +1,19 @@ +Task 01 is complete and everything is green. Final report: + +**Implemented** +- `SyncStatus` gains `images_done`/`images_total` (0/0 defaults) with the phase-152 comment, a dataclass docstring paragraph, and a module-docstring paragraph after the phase-129 one; endpoint docstring extended (additive-only) +- Reset/clear discipline mirrors the chunk position: run-start reset, and `0` at all 7 `current_chunk` clear sites (overview transition, both summaries branches, success, cancelled, failed) — plus the per-file `_hook` closure now resets the image position to 0/0 (extended pairing rule: a `progress` call names the file and clears the image position) +- `GET /api/sync/status` payload gains `images_done`/`images_total` beside the chunk keys; every pre-existing key unchanged +- Tests: full idle-dict pins extended (unit + integration), new per-file-reset test driving the runner's own `_hook` (two-file fake, live 3/15 position cleared when file B is named), new transition+terminal clear test (overview/summaries/success), failed and cancelled terminal tests extended with live positions + +**Results** +- `uv run pytest` → 3513 passed (full suite 100% green) +- `uv run pytest --cov=app --cov-report=term-missing` → 99% on `app/` (`app/api/sync.py` 100%) +- `uv run pytest tests/unit/test_sync_button.py -q` → 59 passed +- `uv run ruff check . && uv run pyright` → clean + +**Docs:** no README/AGENTS.md impact — the change is an additive server-side API key; README documents the UI label (the `· image n/m` part lands in task 03), not the payload schema. + +**Deviations:** none (ASSUMPTION A5 honored — no new `phase` value; importer/label wiring intentionally left to tasks 02/03). + +**Next pending task:** `.agents/phases/todo/152_sync_progress_image_pass/02_importer_image_hook.md` diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.validate b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.validate new file mode 100644 index 0000000..d556e50 --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__01_status_image_fields.a1.validate @@ -0,0 +1,120 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +......................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 196 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 201 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 266 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 135 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 748 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 443 0 100% +-------------------------------------------------- +TOTAL 5700 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.err b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.md b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.md new file mode 100644 index 0000000..d2e1505 --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.md @@ -0,0 +1,16 @@ +Task complete. Final report: + +**Implemented** (phase 152, task 02 — importer's `on_image_progress` hook): +- `import_sources(..., on_image_progress=…)` threaded exactly like `on_chunk_progress`: → `_index_file(on_image=…)` → `_index_pdf_file(on_image=…)` → the three PDF vision spans; standalone image files get no sub-position (A4), counts only (A2) +- Each span fires the hook at the start of each image's slow step with `(base + i + 1, base + span_total)` — embedded pass total = post-dedupe/post-gate `len(images)` (repeat placement adds no position), pictorial = candidate count, fallback = rendered-page count; every call site `if on_image is not None:` (hook-less runs byte-identical) +- Continuous per-file counter: `_index_pdf_file`'s local `images_reported` — each span returns its reported count, the second span starts where the first ended; skipped spans report nothing +- Docstrings updated: module progress paragraph, `import_sources` (new paragraph + corrected the stale "embedded-image docs do not report either" sentence), `_index_file`, `_index_pdf_file`, and the three span docstrings +- 6 new unit tests in `tests/unit/test_importer.py` (PyMuPDF-built PDFs + `_PageVisionLLM`): embedded pass `(1,N)…(N,N)` + dedupe total + fast-path re-sync fires nothing; pictorial→embedded one counter; fallback→embedded continuity (`(1,2),(2,2),(3,3)`); text-only PDF never fires; toggle-off reports nothing; hook-less vs hooked run byte-identical (rows + all summary counters) + +**Test / lint / coverage**: `uv run pytest --cov=app --cov-report=term-missing` → 3519 passed, 0 failed (exit 0), `app/` coverage **99%** (importer.py 99% — only 4 pre-existing lines missed); `uv run ruff check .` clean; `uv run pyright` 0 errors. + +**Docs**: no doc impact (internal importer plumbing; the user-visible `· image n/m` label is task 03's scope). + +**Notable decisions**: hook fires per image slot in the embedded pass (uniform with the other two spans, matches the task's `base + i + 1` formula and its placement in `_index_pdf_embedded_images`); span return shapes extended to `(content, reported)` tuples / `int` (minimal clean shape; both spans are importer-internal, callers updated in place). + +**Next pending task**: `.agents/phases/todo/152_sync_progress_image_pass/03_label_and_wiring.md` diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.validate b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.validate new file mode 100644 index 0000000..eb18d87 --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__02_importer_image_hook.a1.validate @@ -0,0 +1,120 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +............................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 196 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 201 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 266 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 135 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 764 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 443 0 100% +-------------------------------------------------- +TOTAL 5716 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.err b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.md b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.md new file mode 100644 index 0000000..a528631 --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.md @@ -0,0 +1,11 @@ +All gates green. Final report: + +**Task 03 — sync label wiring + `· image n/m` rendering: complete** + +- `app/api/sync.py`: added the third paired closure `_image_hook(done, total)` beside `_hook`/`_chunk_hook` and passed `on_image_progress=_image_hook` into `import_sources` (established-voice pairing comment). +- `frontend/assets/sources.js`: `setSyncLabel` gains fifth param `imageText` appended last to the composed `full` (box + title + running aria-label, no new span — ASSUMPTION A5); `fmtSyncPhaseLabel` import fall-through renders `· image n/m` while `images_total > 0`, overview/summaries return `image: ""`; `enterSyncRunningState` destructures `image = ""` and passes it; all phase-128/129/149/150 label-part comments + `index.html` box comment updated to name the image part. +- Tests: source-level pins in `test_sync_button.py`/`test_frontend_sync_upload.py` (signature, `full` byte-for-byte, part present/absent, branches never read image fields, bare builder image-free) + new node-harness rows pinning the real path behaviorally: both parts at once (`… · chunk 125/126 · image 3/15`) and image-only (independent disappearance); all runner fakes accept the new kwarg; new integration test `test_image_position_climbs_between_polls_mid_run` (1/15 → 7/15 monotonic across two status polls via the runner's own closure, 0/0 at terminal, wiring pinned). +- Results: `uv run pytest --cov=app` → **3521 passed**, `app/api/sync.py` 100%, TOTAL **99%**; `uv run ruff check .` clean; `uv run pyright` 0 errors; sync-label E2Es green in isolation (`test_sync_chunk_progress.py` 2 passed, `test_sync_progress_box.py` 8 passed — pre-phase labels byte-identical). +- Docs: README.md sync-label section now documents `· image n/m`; no AGENTS.md impact. +- No deviations; no CSS/markup changes (position rides the composed string only, per the design). +- Next pending task: `04_unit_tests.md` (status-machine pins + importer hook pairing). diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.validate b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.validate new file mode 100644 index 0000000..afd24ca --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__03_label_and_wiring.a1.validate @@ -0,0 +1,120 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 44%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +................................................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 196 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 266 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 135 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 764 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 443 0 100% +-------------------------------------------------- +TOTAL 5719 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.err b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.md b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.md new file mode 100644 index 0000000..3961daa --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.md @@ -0,0 +1,15 @@ +All 5 new tests pass, full suite green. Work complete — final report: + +## Phase 152, Task 04 — complete + +**Implemented (unit + integration test pins, no app-code changes):** +- `tests/unit/test_sync_button.py`: mid-run pin via the runner's own `_image_hook` closure (`_GatedImport` now fires `image_pairs` — 3/15 → 15/15 climbing between ticks, three positions independent); per-file reset, overview/summaries transition clears, and failed + cancelled terminals all moved onto closure-driven pins (phase-64/129 idiom); new source-level pin of `fmtSyncPhaseLabel`'s image part (guard `images_total > 0`, exact template, `image: ""` in overview/summaries, composed `full` string, `· chunk 125/126 · image 3/15` example) +- `tests/unit/test_importer.py`: hook fires **before** each describe call (interleaving recorded in a fake LLM — `hook(1,N), vision(1), …`); blank-gate interaction (near-white image excluded from `total` by default, counted with `pdf_image_skip_blank=False`) +- `tests/integration/test_sync_api.py`: full sync over a fixture KB with one image-bearing PDF (real `import_sources` + gated mock LLM) — observed `images_done` sequence monotonic, never above `images_total`, ends at it; terminal clears; KB carries PDF + N image docs +- Defect fix (pre-existing failure): the owner's local `.env` `BOR_APP_NAME=Brain of Patrick` broke `test_config_returns_default_app_metadata` — added the `BOR_APP_NAME` default pin to `tests/conftest.py` (established leak-class idiom) + +**Gates (all green):** `uv run pytest -q` exit 0 · `uv run pytest --cov=app --cov-report=term-missing` → TOTAL 99% (>90%; `app/api/sync.py` 100%, `app/rag/importer.py` 99%) · `uv run ruff check .` clean · `uv run pyright` 0 errors + +**Docs:** no doc impact (README's `· image n/m` paragraph landed in task 03; AGENTS.md is protocol-only) + +**Next pending task:** `05_e2e_image_progress.md` (dedicated Playwright suite) diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.validate b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.validate new file mode 100644 index 0000000..399df7a --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a1.validate @@ -0,0 +1,120 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................................................ [ 98%] +...................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 196 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 266 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 135 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 764 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 443 0 100% +-------------------------------------------------- +TOTAL 5719 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.err b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.md b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.md new file mode 100644 index 0000000..cd51484 --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.md @@ -0,0 +1 @@ +Now let me look at the existing test idioms and what's already covered. diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.validate b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.validate new file mode 100644 index 0000000..a3c4dd6 --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__04_unit_tests.a2.validate @@ -0,0 +1,185 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 44%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +.......................................................F................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +................................................................. [100%] +=================================== FAILURES =================================== +___________________ test_config_returns_default_app_metadata ___________________ + +client = +db = + + def test_config_returns_default_app_metadata(client, db: Session) -> None: + """GET /api/config is public (anonymous) and returns exactly six + keys — the phase-39 app metadata, the phase-59 docs flag (inert + false while BOR_DOCS_REPO is empty — the "Save as doc" gating), + the phase-122 images flag (default false in the test env — + LOCKED A3: off by default), and the phase-62 UI customization + strings (composer placeholder, footer line). Phase 91: with an + empty ui_settings table the effective strings are the env defaults + (B1 — DB-over-env, the row absent here); the retired CSS-file + theming's ``theme`` key is gone (task 03 — the six keys are the + entire contract).""" + _clear_ui_settings(db) + r = client.get("/api/config") + assert r.status_code == 200 + body = r.json() + assert set(body) == { + "app_name", "version", "docs_repo_configured", + "images", "input_placeholder", "footer_text", + } +> assert body["app_name"] == "Brain of Reese" +E AssertionError: assert 'Brain of Patrick' == 'Brain of Reese' +E +E - Brain of Reese +E + Brain of Patrick + +tests/integration/test_api.py:69: AssertionError +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 196 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 266 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 135 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 764 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 443 0 100% +-------------------------------------------------- +TOTAL 5719 17 99% +=========================== short test summary info ============================ +FAILED tests/integration/test_api.py::test_config_returns_default_app_metadata +tests FAILED: unit + integration suite did not pass +All checks passed! +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +Traceback (most recent call last): + File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/bin/pyright", line 10, in + sys.exit(entrypoint()) + ~~~~~~~~~~^^ + File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pyright/cli.py", line 31, in entrypoint + sys.exit(main(sys.argv[1:])) + ~~~~^^^^^^^^^^^^^^ + File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pyright/cli.py", line 18, in main + return run(*args, **kwargs).returncode + ~~~^^^^^^^^^^^^^^^^^ + File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pyright/cli.py", line 27, in run + return node.run('node', str(script), *args, **kwargs) + ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/pyright/node.py", line 144, in run + subprocess.run(node_args, **kwargs), + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ + File "/var/home/ducoterra/.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/subprocess.py", line 556, in run + stdout, stderr = process.communicate(input, timeout=timeout) + ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^ + File "/var/home/ducoterra/.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/subprocess.py", line 1214, in communicate + self.wait() + ~~~~~~~~~^^ + File "/var/home/ducoterra/.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/subprocess.py", line 1280, in wait + return self._wait(timeout=timeout) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^ + File "/var/home/ducoterra/.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/subprocess.py", line 2085, in _wait + (pid, sts) = self._try_wait(0) + ~~~~~~~~~~~~~~^^^ + File "/var/home/ducoterra/.local/share/uv/python/cpython-3.13.13-linux-x86_64-gnu/lib/python3.13/subprocess.py", line 2043, in _try_wait + (pid, sts) = os.waitpid(self.pid, wait_flags) + ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ +KeyboardInterrupt +validation FAILED (see output above) diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.err b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.md b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.md new file mode 100644 index 0000000..087c6cf --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.md @@ -0,0 +1,19 @@ +All green. Final report: + +## Task 05 complete — dedicated E2E for the sync image-progress label + +**Implemented** +- `tests/e2e/test_sync_image_progress.py` (new): module app on its own port (8171) + `slow_llm.py` delay proxy (8909, `SLOW_DELAY_S = 1.5 s`) in front of the mock; local-dir source with one PyMuPDF-built PDF (10 × ~4 KB text pages → 20 chunks / 10 embed batches, 4 distinct non-blank images, all surviving the phase-151 gates); ground truth computed in-test via `extract_pdf_text` + `chunk_document` with the importer's exact args, sizing knobs pinned to code defaults (leak-guard pattern) +- `test_label_advances_through_the_image_span` (anti-freeze proof): dense (~300 ms) page sampling of `#sync-progress-text` first-seen order → asserts ≥3 distinct `· image n/4` positions starting at 1, non-decreasing, never above the total, reaching it; the chunk part (`· chunk n/20`, total = the file's TRUE chunk count) appears earlier and coexists on every image-span label; A4 title/aria == box on every running sample; settle = `5 added` counts line, terminal label, parts cleared, image part gone + endpoint 0/0 +- `test_mid_run_reload_re_attaches_with_image_position`: UI-click start, first observed position `1/4`, mid-run `page.reload()` → load-time re-attach renders the current `· image n/4` with A4 title/aria, then settles to the counts line + +**Results** +- `uv run pytest tests/e2e/test_sync_image_progress.py -v --no-cov` (isolated): **2 passed** +- Neighbours (each in isolation): `test_sync_progress_box` 8 ✓, `test_sync_chunk_progress` 2 ✓, `test_sync_label_mobile` 4 ✓, `test_pdf_support` 6 ✓, `test_image_documents` 4 ✓ +- `uv run pytest --cov=app --cov-report=term-missing`: 3526 passed, **TOTAL 99%** (>90%); `uv run ruff check .` clean; `uv run pyright` 0 errors + +**Docs** — no doc impact (test-only task; no new setup/config/conventions). Phase dir move + atomic commit are the harness's, not made here. + +**Decisions** — test 2 starts the sync via the UI click (the page's 2 s poll starts only on click/re-attach — an API-only start left the page unpolled); fixture timing verified against real logs (image pipeline = 1 describe + 2 embed batches = 4.5 s hold > 2 s poll; run ≈ 40 s). + +**Next pending task** — none in `152_sync_progress_image_pass/` (task 05 was the last); phase 152 is ready for the harness to move to `complete/`. diff --git a/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.validate b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.validate new file mode 100644 index 0000000..399df7a --- /dev/null +++ b/.agents/reports/152_sync_progress_image_pass/152_sync_progress_image_pass__05_e2e_image_progress.a1.validate @@ -0,0 +1,120 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 42%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................................................ [ 98%] +...................................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 196 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 204 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 266 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 135 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 764 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 192 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 443 0 100% +-------------------------------------------------- +TOTAL 5719 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/README.md b/README.md index dcaee4d..057fc7a 100644 --- a/README.md +++ b/README.md @@ -334,8 +334,13 @@ whole git-source refresh in one click: phase (the live file, its `(n/m)` file position, and — while a file's embed batch is in flight — the pinned `· chunk n/m` embed position, batch granularity: the first chunk of the in-flight batch over the - file's total content chunks) — shows in a text box under the button - row, wrapping (never clipped); the same untruncated label rides the + file's total content chunks; and — while a PDF's images are being + described — the `· image n/m` vision position: the 1-based image + about to be described over the file's total images, ONE continuous + counter across the PDF's three vision spans, so a long image pass + visibly climbs instead of sitting frozen on the last chunk + position) — shows in a text box under the button row, + wrapping (never clipped); the same untruncated label rides the button title and the box, which is the SOLE `aria-live` region for the sync area (`role="status"`, polite — phase 150 folded the pre-phase result-line announcer into the box). At settle the box diff --git a/TODO.md b/TODO.md index 4640904..761ea6d 100644 --- a/TODO.md +++ b/TODO.md @@ -1 +1,3 @@ # TODO + +- [ ] image extraction from a PDF is throwing a 404 `https://brain.reeseapps.com/api/documents/679071cb-99f7-4c21-af46-2fa8ff916730/pages/1` \ No newline at end of file diff --git a/app/api/sync.py b/app/api/sync.py index 9727404..3eee248 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -188,6 +188,23 @@ counts convention does NOT extend to it: the file counts survive the terminal for the result line's context, the chunk position clears with the file. +Phase 152 (task 01) adds the in-flight file's VISION-image position — +``images_done`` (the 1-based index of the image about to be described +and embedded) + ``images_total`` (the file's total images for this +run — the three PDF vision spans of ONE file share ONE continuous +counter per file; task 02 wires the importer's +``on_image_progress`` hook, task 03 the runner's closure that feeds +these fields). They are 0/0 before the first image, during the +``"overview"`` / ``"summaries"`` phases, and in terminal states — the +same spots ``current_chunk`` clears. Unlike the chunk position (every +file embeds chunks), a file may have NO images at all, so the per-file +``progress`` hook ALSO resets them to 0/0: the pairing rule, extended +— a ``progress`` call names the file and clears the image position, +and every image position until the next ``progress`` call belongs to +it (a file with no images must not inherit the previous PDF's +position). ASSUMPTION A5: no new ``phase`` value — the image spans +live inside ``phase: "import"``. + The ``failed`` state's ``error`` string is masked by the shared sanitizer — the ``user:pass@`` masker now lives in :mod:`app.core.errors` (imported here under the private name ``_sanitize_error``). @@ -272,6 +289,21 @@ class SyncStatus: convention covers the FILE counts only — the chunk position is a sub-position of a file that's done, so it clears in the terminal while ``files_done`` / ``files_total`` keep their final values. + + Phase 152 (task 01) image-position fields: ``images_done`` is + the 1-based index of the vision image about to be described and + ``images_total`` the in-flight file's total images for this run + (the importer's ``on_image_progress`` hook — the three PDF vision + spans of one file share ONE continuous counter, the module + docstring's contract). Like the chunk position, the image + position is a SUB-position of a file that's done: the + keep-final-counts convention covers the FILE counts only, so it + clears with the file — 0/0 before the first image, during the + ``"overview"`` / ``"summaries"`` phases, and in terminal states + (the same spots the chunk position clears) — and the per-file + ``progress`` hook resets it to 0/0 (a ``progress`` call names the + file and clears the image position; a file with no images must + not inherit the previous PDF's position). """ state: Literal["idle", "running", "success", "failed", "cancelled"] = "idle" @@ -298,6 +330,14 @@ class SyncStatus: # prelude/overview/summaries phases, and in terminal states. current_chunk: int = 0 chunks_total: int = 0 + # Phase 152 (task 01): the in-flight file's VISION-image position — + # the 1-based index of the image about to be described and the + # file's total images for this run (the importer's + # on_image_progress hook; the three PDF vision spans share ONE + # continuous counter per file). 0/0 before the first image, during + # the prelude/overview/summaries phases, and in terminal states. + images_done: int = 0 + images_total: int = 0 _status = SyncStatus() @@ -334,6 +374,15 @@ def sync_status() -> dict[str, Any]: user-cancelled run — a terminal state with ``error`` null and ``detail`` empty; the run's final file + summary counts are kept, the chunk position cleared. + + Phase 152 (task 01): ``images_done`` / ``images_total`` are the + in-flight file's vision-image position (the importer's + ``on_image_progress`` hook — the three PDF vision spans of one + file share ONE continuous counter): 0/0 before the first image, in + the prelude/overview/summaries phases, and in terminal states; the + per-file ``progress`` hook clears it at every file boundary (a + file with no images never shows a stale position). Additive only — + every pre-existing key keeps its name, type, and meaning. """ return { "state": _status.state, @@ -350,6 +399,15 @@ def sync_status() -> dict[str, Any]: # (overview/summaries transitions + every terminal). "current_chunk": _status.current_chunk, "chunks_total": _status.chunks_total, + # Phase 152 (task 01): the in-flight file's VISION-image + # position (the importer's on_image_progress hook — the three + # PDF vision spans share ONE continuous counter per file): + # 0/0 before the first image, during the + # prelude/overview/summaries phases, and in terminal states + # (cleared at every point the import phase ends; the per-file + # progress hook re-clears it at every file boundary). + "images_done": _status.images_done, + "images_total": _status.images_total, # Phase 98 (task 01): the phase machine — null/0/0 idle (the # dataclass defaults) and in terminal states (which clear # phase + current_summary but keep the final summary counts). @@ -461,6 +519,11 @@ async def _run_sync() -> None: # run — 0/0 until the import's first embed batch reports. _status.current_chunk = 0 _status.chunks_total = 0 + # Phase 152 (task 01): the image-position fields reset with the + # run — 0/0 until the import's first vision span reports (the + # per-file progress hook re-clears them at every file boundary). + _status.images_done = 0 + _status.images_total = 0 try: settings = get_settings() # Step 1 (phase 41): fail fast — verify both models the sync @@ -556,6 +619,15 @@ async def _run_sync() -> None: _status.current_file = f"{source}/{rel}" _status.files_done = done _status.files_total = total + # Phase 152 (task 01): the pairing rule, extended — a + # progress call names the file AND clears the image + # position: every image position until the next progress + # call belongs to this file. A file with no images must + # not inherit the previous PDF's position (unlike the + # chunk position, not every file embeds chunks, so the + # run-start reset alone cannot cover it). + _status.images_done = 0 + _status.images_total = 0 # Phase 129 (task 03): the in-flight file's embed-chunk # position — the LLMClient.embed hook (task 01) fires before @@ -567,6 +639,21 @@ async def _run_sync() -> None: _status.current_chunk = chunk _status.chunks_total = total + # Phase 152 (task 03): the in-flight file's VISION-image + # position — the importer's ``on_image_progress`` hook (task + # 02) fires before each of the PDF's vision describe calls + # with (position_1based, total), the file's ONE continuous + # counter across its three vision spans (the importer owns + # the continuity — this closure only lands the pair); the + # file's IDENTITY is the sibling ``progress`` hook's, which + # ALSO resets this position (the three hooks are paired: + # ``progress`` names the file and clears the image position, + # ``on_chunk_progress`` reports the embed-batch position, + # ``on_image_progress`` reports the vision-image position). + def _image_hook(done: int, total: int) -> None: + _status.images_done = done + _status.images_total = total + # Phase 98 (task 01): the phase machine — the import phase # starts NOW (the model-check + clone/pull prelude above # reported ``phase: null``). @@ -577,7 +664,12 @@ async def _run_sync() -> None: # imported. _check_cancelled() summary: ImportSummary = await import_sources( - sources, llm, prune=True, progress=_hook, on_chunk_progress=_chunk_hook, + sources, + llm, + prune=True, + progress=_hook, + on_chunk_progress=_chunk_hook, + on_image_progress=_image_hook, # phase 152 (task 03) ignore_by_root=ignore_by_root, include_hidden_by_root=include_hidden_by_root, doc_dates_by_root=doc_dates_by_root, @@ -596,6 +688,11 @@ async def _run_sync() -> None: # values (the phase-64 convention covers files only). _status.current_chunk = 0 _status.chunks_total = 0 + # Phase 152 (task 01): the image position clears at the + # same point (a sub-position of a done file, like the + # chunk position). + _status.images_done = 0 + _status.images_total = 0 overview = await regenerate_overview(llm) # Phase 94 (task 02), phase 96 (task 03), phase 127 (task 03): # the folder summaries — the drill-down ls's per-level @@ -649,6 +746,10 @@ async def _run_sync() -> None: # chunk-position clear as the overview transition. _status.current_chunk = 0 _status.chunks_total = 0 + # Phase 152 (task 01): the image position clears at + # the same point (a sub-position of a done file). + _status.images_done = 0 + _status.images_total = 0 folder_stats = await generate_folder_summaries( fs_db, llm, changed_docs=summary.changed_paths, on_progress=_summary_hook, @@ -670,6 +771,11 @@ async def _run_sync() -> None: # chunk-position clear. _status.current_chunk = 0 _status.chunks_total = 0 + # Phase 152 (task 01): the image position clears + # at the same point (a sub-position of a done + # file). + _status.images_done = 0 + _status.images_total = 0 folder_stats = await generate_folder_summaries( fs_db, llm, only_missing=True, on_progress=_summary_hook ) @@ -716,6 +822,11 @@ async def _run_sync() -> None: # line's context). _status.current_chunk = 0 _status.chunks_total = 0 + # Phase 152 (task 01): the image position is a sub-position of + # a file that's done — it clears in the terminal too (the FILE + # counts keep their final values, the sub-position does not). + _status.images_done = 0 + _status.images_total = 0 _status.detail = { "files": summary.files, "added": summary.added, @@ -754,6 +865,12 @@ async def _run_sync() -> None: # success and failed (the convention split, pinned). _status.current_chunk = 0 _status.chunks_total = 0 + # Phase 152 (task 01): the image position clears in the + # CANCELLED terminal too — every terminal clears it (a + # sub-position of a file that's done/abandoned, the FILE + # counts above keep their final values). + _status.images_done = 0 + _status.images_total = 0 logger.info( "sync: cancelled by user state=cancelled files_done=%d files_total=%d", _status.files_done, _status.files_total, @@ -773,3 +890,9 @@ async def _run_sync() -> None: # of a file that's done/abandoned does not). _status.current_chunk = 0 _status.chunks_total = 0 + # Phase 152 (task 01): the image position clears in the FAILED + # terminal too — the convention split (a sub-position of a + # file that's done/abandoned; the FILE counts keep their final + # values next to the error). + _status.images_done = 0 + _status.images_total = 0 diff --git a/app/rag/importer.py b/app/rag/importer.py index c05083d..ce39042 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -285,10 +285,14 @@ log-only (the sync ``detail`` shape is untouched — the stats log-only convention, phase 133). ``import_sources`` accepts an optional per-file ``progress`` callback -(phase 64, task 01) reporting the file being processed right now, and -an optional ``on_chunk_progress`` callback (phase 129, task 02) +(phase 64, task 01) reporting the file being processed right now, an +optional ``on_chunk_progress`` callback (phase 129, task 02) reporting the in-flight file's embedding-chunk position at batch -granularity (see the function docstring). +granularity, and an optional ``on_image_progress`` callback (phase +152, task 02) reporting the in-flight PDF's vision-image position — +immediately before each of the PDF's vision describe calls, with the +image's 1-based position in the file's ONE continuous vision counter +and the span's count (see the function docstring). """ from __future__ import annotations @@ -663,6 +667,7 @@ async def import_sources( session: Session | None = None, progress: Callable[[str, str, int, int], None] | None = None, on_chunk_progress: Callable[[int, int], None] | None = None, + on_image_progress: Callable[[int, int], None] | None = None, ignore_by_root: dict[str, list[str]] | None = None, include_hidden_by_root: dict[str, bool] | None = None, doc_dates_by_root: dict[str, dict[str, datetime]] | None = None, @@ -708,9 +713,41 @@ async def import_sources( ``1/1``, which IS its only chunk): the document's summary-chunk embed (:func:`_store_summary`) NEVER receives the hook, so the counter counts content chunks only (no ``1/1`` noise after the - real progress ends), and the post-index enrichment pass / the - PDF's embedded-image docs (separate documents, not the walked - file) do not report either. + real progress ends), and the post-index passes (the phase-125 + enrichment, the phase-137 document-context backfill) do not report + either — the PDF's vision describe calls report through the + separate ``on_image_progress`` hook instead (phase 152, below). + + ``on_image_progress`` (phase 152, task 02) is an optional + vision-image-position hook for the PDF file currently being + indexed: while the PDF's vision spans run — the phase-133 + page-screenshot fallback, the phase-140 pictorial sparse-page + pass, and the phase-126 embedded-image pass — it is called + immediately before each of their vision describe calls with + ``(position_1based, total)``: the 1-based position of the image + about to be described and the count for that span — the extracted + image count (post-dedupe, post-blank-gate, phase 151: the work + actually about to happen) for the embedded pass, the candidate- + page count for the pictorial pass, the rendered-page count for the + screenshot fallback. The spans of ONE file are reported as ONE + continuous counter: each span is called with the count already + reported for this file (*base*) and returns the number of images + it reported, and the caller adds them — the pass that runs second + starts where the first ended, so the position climbs across the + file's whole vision span instead of restarting (the fallback and + the pictorial pass are mutually exclusive — the phase-140 + precedence over the garbage gate — and both precede the embedded + pass, which runs after the document commit). A skipped span (the + ``images`` toggle off, zero candidates, a clean text layer) reports + nothing and advances the counter not at all. Standalone image + files get NO sub-position (phase 152 A4 — one image per file, + ``1/1`` per file would be noise; the file count already covers + them), and the hook carries counts only, never the image identity + (A2 — the per-file ``progress`` hook names the file). Same "only + when provided" rule as ``progress`` and ``on_chunk_progress``: + ``None`` (every existing caller — the CLI, the chat path) is a + zero-cost guarded no-op, and the run is byte-identical to a + hook-less run. ``ignore_by_root`` (phase 89, A1/A2) maps ``str(root)`` — the root path string exactly as passed in *sources* — to that source's RAW ignore-path @@ -861,7 +898,7 @@ async def import_sources( await _index_file( session, source=source, rel=rel, full_path=path, llm=llm, summary=summary, raw_date=dates_map.get(rel), - on_chunk=on_chunk_progress, + on_chunk=on_chunk_progress, on_image=on_image_progress, ) except EmbeddingError as e: # A pathological file (e.g. content the embedding endpoint @@ -1078,6 +1115,7 @@ async def _index_file( summary: ImportSummary, raw_date: datetime | None = None, on_chunk: Callable[[int, int], None] | None = None, + on_image: Callable[[int, int], None] | None = None, ) -> None: """Upsert one file: doc row + chunk rows + embeddings, one transaction. @@ -1112,6 +1150,15 @@ async def _index_file( :func:`_index_pdf_file` (its content embed is the primary target). ``None`` (the default) keeps every branch byte-identical. + ``on_image`` (phase 152, task 02) is the in-flight file's + vision-image-position hook (see :func:`import_sources`) — passed + through to the PDF branch only (:func:`_index_pdf_file` — its + three vision spans share ONE continuous counter per file): the + image branch receives NO sub-position (phase 152 A4 — one image + per file, ``1/1`` would be noise; the file count already covers + it) and the text branch has no vision span. ``None`` (the + default) keeps every branch byte-identical. + Image files (phase 122, toggle on) delegate to :func:`_index_image_file` — the binary path (bytes digest, persistent copy, ``content`` = the vision description) — BEFORE @@ -1165,7 +1212,7 @@ async def _index_file( return await _index_pdf_file( session, source=source, rel=rel, full_path=full_path, llm=llm, summary=summary, raw_date=raw_date, file_digest=file_digest, - on_chunk=on_chunk, + on_chunk=on_chunk, on_image=on_image, ) content = full_path.read_text(encoding="utf-8", errors="replace").replace("\x00", "") digest = hashlib.sha256(content.encode("utf-8")).hexdigest() @@ -1401,6 +1448,7 @@ async def _index_pdf_file( file_digest: str, raw_date: datetime, on_chunk: Callable[[int, int], None] | None = None, + on_image: Callable[[int, int], None] | None = None, ) -> None: """The phase-126 PDF branch of :func:`_index_file` — a PDF is indexed from its EXTRACTED TEXT (LOCKED A1/A2/A4), one transaction @@ -1488,6 +1536,26 @@ async def _index_pdf_file( documents — their ``1/1`` positions would be noise after the file's real progress ends, the A6 exclusion). ``None`` keeps the branch byte-identical. + + ``on_image`` (phase 152, task 02) is the in-flight file's + vision-image-position hook (see :func:`import_sources`): it is + forwarded to the three vision spans — the phase-133 fallback + (:func:`_fallback_page_screenshots`), the phase-140 pictorial pass + (:func:`_pictorial_pdf_pass`), and the embedded-image pass + (:func:`_index_pdf_embedded_images`) — each firing it immediately + before its vision describe calls. The spans of this file share + ONE continuous counter: the local *images_reported* starts at 0, + each span is called with the count reported so far as its *base* + (the pass that runs second starts where the first ended), and the + returned reported count is added — the position climbs across the + file's whole vision span instead of restarting. The fallback and + the pictorial pass are mutually exclusive (the phase-140 + precedence over the garbage gate — an unusable layer takes the + fallback, a usable one the pass), both run BEFORE the document + commit, and the embedded pass runs after it; a skipped span + (toggle off, zero candidates, a clean text layer) reports nothing + and advances the counter not at all. ``None`` keeps the branch + byte-identical. """ # Extract the text (fail-soft: an unopenable PDF yields "" — the # document may still carry embedded images worth indexing). @@ -1504,6 +1572,14 @@ async def _index_pdf_file( # — the pre-phase-148 viewer). fallback_fired = False pictorial_described = False + # Phase 152 (task 02): the file's ONE continuous vision-image + # counter — 0 at the top of the file (the per-file pairing rule: + # each file's spans start fresh; the caller's status fields reset + # at the file boundary). Each span below is called with the count + # reported so far as its *base* and returns the number of images + # it reported, which is added — the pass that runs second starts + # where the first ended. Stays 0 while the hook is None. + images_reported = 0 # Phase 133 (task 03): the garbage gate — BEFORE the digest/chunk/ # summary pipeline, so it runs on the FINAL content exactly once. # A clean layer never reaches the fallback (the gate is the only @@ -1511,9 +1587,11 @@ async def _index_pdf_file( if not is_usable_pdf_text(content): summary.pdfs_screenshotted += 1 fallback_fired = True - content = await _fallback_page_screenshots( - session, source=source, rel=rel, full_path=full_path, llm=llm + content, reported = await _fallback_page_screenshots( + session, source=source, rel=rel, full_path=full_path, llm=llm, + on_image=on_image, base=images_reported, ) + images_reported += reported elif llm.settings.pdf_pictorial: # Phase 140 (task 03, LOCKED A1–A4): the pictorial pass — # USABLE layers only (the precedence is locked: the phase-133 @@ -1523,9 +1601,11 @@ async def _index_pdf_file( # pre-phase-140 path); otherwise the FINAL content carries the # described candidates' illustration blocks (A4: the digest # below runs over it, the phase-126/133 convention). - passed = await _pictorial_pdf_pass( - source=source, rel=rel, full_path=full_path, llm=llm, summary=summary + passed, reported = await _pictorial_pdf_pass( + source=source, rel=rel, full_path=full_path, llm=llm, summary=summary, + on_image=on_image, base=images_reported, ) + images_reported += reported if passed is not None: pictorial_described = True content = passed @@ -1650,6 +1730,7 @@ async def _index_pdf_file( await _index_pdf_embedded_images( session, source=source, rel=rel, full_path=full_path, llm=llm, summary=summary, raw_date=raw_date, + on_image=on_image, base=images_reported, ) @@ -1660,7 +1741,9 @@ async def _fallback_page_screenshots( rel: str, full_path: Path, llm: Embedder, -) -> str: + on_image: Callable[[int, int], None] | None = None, + base: int = 0, +) -> tuple[str, int]: """The phase-133 (task 03) page-screenshot fallback — the owner's specified path ("screenshotting the pages and reading each one as an image") for a PDF whose text layer failed @@ -1700,6 +1783,18 @@ async def _fallback_page_screenshots( unchanged row's content digest (the row stays put until the bytes move; the doc is still correct either way). + * phase 152 (task 02): the ``on_image`` hook (the file's + vision-image position, see :func:`import_sources`) is called + immediately before EACH rendered page's vision call with + ``(base + i + 1, base + len(pages))`` — the page's 1-based slot + in the rendered-page list and the span's total (the rendered- + page count), at the START of the page's slow call, never after. + The return is the ``(content, reported)`` pair — *reported* the + number of images this span reported (0 while the hook is + ``None``), the count the caller adds to the file's continuous + counter. A hook-less run is byte-identical (every call site is + guarded). + *session* is part of the signature for symmetry with the other per-file index helpers (the fallback itself needs no DB access — the caller's upsert pipeline owns the row). @@ -1711,14 +1806,22 @@ async def _fallback_page_screenshots( source, rel, len(pages), ) if not pages: - return "" + return "", 0 blocks: list[str] = [] got_any = False + reported = 0 for i, png in enumerate(pages): page = i + 1 if not png: blocks.append(f"page {page} — description unavailable") continue + # Phase 152 (task 02): the file's vision-image position — + # this page's 1-based slot and the span's total (the rendered- + # page count), at the START of the page's slow call, never + # after. + if on_image is not None: + on_image(base + i + 1, base + len(pages)) + reported += 1 desc = await describe_image(llm, data=png, mime="image/png") if desc is None or not desc.strip(): blocks.append(f"page {page} — description unavailable") @@ -1726,10 +1829,13 @@ async def _fallback_page_screenshots( blocks.append(desc.strip()) got_any = True if not got_any: - return "" - return "\n\n".join( - f"{PAGE_SEPARATOR.format(page=page)}\n{block}" - for page, block in enumerate(blocks, start=1) + return "", reported + return ( + "\n\n".join( + f"{PAGE_SEPARATOR.format(page=page)}\n{block}" + for page, block in enumerate(blocks, start=1) + ), + reported, ) @@ -1740,7 +1846,9 @@ async def _pictorial_pdf_pass( full_path: Path, llm: Embedder, summary: ImportSummary, -) -> str | None: + on_image: Callable[[int, int], None] | None = None, + base: int = 0, +) -> tuple[str | None, int]: """The phase-140 (task 03) pictorial pass (LOCKED A1–A4) — the IKEA assembly-instruction class (TODO L5): a PDF whose text layer is USABLE but SPARSE (mostly pictorial pages — 4–73 chars of @@ -1791,6 +1899,19 @@ async def _pictorial_pdf_pass( (+the described page count), plus the ONE loud log line (PLAN §9): ``pdf: pictorial pass source=… path=… — N candidate pages, M described (min_chars=…)``. + * phase 152 (task 02): the ``on_image`` hook (the file's + vision-image position, see :func:`import_sources`) is called + immediately before EACH candidate's vision call with + ``(base + i + 1, base + len(candidates))`` — the candidate's + 1-based slot in the deterministic gate's list and the span's + total (the candidate-page count), at the START of the page's + slow call, never after (an unreadable page — the zero-byte + render placeholder — burns no vision call and reports nothing). + The return is the ``(content-or-None, reported)`` pair — + *reported* the number of images this span reported (0 while the + hook is ``None`` or there are zero candidates), the count the + caller adds to the file's continuous counter. A hook-less run + is byte-identical (every call site is guarded). """ settings = llm.settings page_texts = extract_pdf_page_texts(full_path) @@ -1798,17 +1919,18 @@ async def _pictorial_pdf_pass( page_texts, min_chars=settings.pdf_pictorial_min_chars ) if not candidates: - return None + return None, 0 # ONE render for the whole candidate set (the phase-140 selector — # a 500-page PDF with 30 sparse pages renders 30 PNGs, not 500); # the result is aligned with *candidates* (ascending, one slot # each — a zero-byte placeholder never shifts the positions). pngs = render_pdf_pages(full_path, pages=candidates) descriptions: dict[int, str] = {} + reported = 0 # strict: the render's result is aligned with the candidate list by # contract (one slot per named page, ascending) — a length drift is # a parser regression, fail loud. - for page, png in zip(candidates, pngs, strict=True): + for i, (page, png) in enumerate(zip(candidates, pngs, strict=True)): if not png: # The phase-133 zero-byte placeholder: an unreadable page. logger.warning( @@ -1817,6 +1939,13 @@ async def _pictorial_pdf_pass( page, source, rel, ) continue + # Phase 152 (task 02): the file's vision-image position — + # this candidate's 1-based slot and the span's total (the + # candidate-page count), at the START of the page's slow + # call, never after. + if on_image is not None: + on_image(base + i + 1, base + len(candidates)) + reported += 1 desc = await describe_pdf_page( llm, data=png, @@ -1854,7 +1983,7 @@ async def _pictorial_pdf_pass( if page in descriptions: block += f"\n\n[Page {page} illustration: {descriptions[page]}]" parts.append(f"{PAGE_SEPARATOR.format(page=page)}\n{block}") - return "\n\n".join(parts) + return "\n\n".join(parts), reported def _persist_pdf_pages( @@ -1951,7 +2080,9 @@ async def _index_pdf_embedded_images( llm: Embedder, summary: ImportSummary, raw_date: datetime, -) -> None: + on_image: Callable[[int, int], None] | None = None, + base: int = 0, +) -> int: """The phase-126 (A3) embedded-image pass — one call per (re)indexed PDF, after its document commit. @@ -2034,6 +2165,19 @@ async def _index_pdf_embedded_images( layer: origin-line-only blocks, fail-soft like the rest of the parser. + * phase 152 (task 02): the ``on_image`` hook (the file's + vision-image position, see :func:`import_sources`) is called at + the START of EACH image's pipeline step — before the slow + :func:`_describe_or_skip` seam inside + :func:`_index_one_embedded_image` — with + ``(base + i + 1, base + len(images))``: the image's 1-based slot + in the pass's list (post-dedupe, post-blank-gate, phase 151: + the work actually about to happen) and the span's total. Returns + the number of images reported (0 while the hook is ``None`` or + the toggle is off) — the count the caller adds to the file's + continuous counter. A hook-less run is byte-identical (every + call site is guarded). + ``raw_date`` is the PDF's resolved source date (the pass runs only for (re)indexed PDFs, so it is always concrete) — the image docs inherit the PDF's date (they are part of it). @@ -2048,7 +2192,7 @@ async def _index_pdf_embedded_images( "source=%s path=%s", source, rel, ) - return + return 0 # Phase 137 (task 02/03): the document context's raw material — # the SHARED probe (task 01's per-page text map extracted ONCE per # PDF — the pass reads it per image — plus the phase-133 garbage @@ -2069,12 +2213,23 @@ async def _index_pdf_embedded_images( ) summary.pdf_images_deduped += stats.duplicates summary.pdf_images_blank_skipped += stats.blank_skipped - for img in images: + reported = 0 + for i, img in enumerate(images): + # Phase 152 (task 02): the file's vision-image position — + # this image's 1-based slot in the pass's list (post-dedupe, + # post-blank-gate: the work actually about to happen) and the + # span's total, at the START of the image's pipeline step + # (the slow :func:`_describe_or_skip` seam inside + # :func:`_index_one_embedded_image`), never after. + if on_image is not None: + on_image(base + i + 1, base + len(images)) + reported += 1 await _index_one_embedded_image( session, img, source=source, rel=rel, full_path=full_path, llm=llm, summary=summary, raw_date=raw_date, page_texts=page_texts, text_usable=text_usable, ) + return reported async def _index_one_embedded_image( diff --git a/frontend/assets/sources.js b/frontend/assets/sources.js index 502d496..ae42f97 100644 --- a/frontend/assets/sources.js +++ b/frontend/assets/sources.js @@ -364,7 +364,8 @@ * * Phase 149 (task 01) — the sync detail box + the icon-only running * button: the owner's report was that the full live label (a long - * source/relative/path + the counts + the chunk part) is clipped in + * source/relative/path + the counts + the chunk part + the phase-152 + * image part) is clipped in * the pill even on desktop — the mobile solution (stuffing the parts * in the button) had become the permanent surface. The fix: while a * sync/upload run is in flight, the button shows ONLY the spinning @@ -399,7 +400,8 @@ * global [hidden] rule does the display). Two content modes, decided * at the callsites (the phase-149 "the callsites are the only * decision sites" rule): - * • running mode — setSyncLabel(main, count, chunk, running) writes + * • running mode — setSyncLabel(main, count, chunk, running, image) + * writes * the SAME untruncated full label through setSyncProgress, ONLY * while running (the idle / settle button labels — "Sync sources", * "Synced HH:MM" — must not fill the panel; they stay on the @@ -749,7 +751,12 @@ export async function mount(root) { * pinned "(n/m)" position (hidden when absent); phase 129 (task * 04): + .sync-label-chunk, the pinned "· chunk n/m" embed-batch * position of the in-flight file (hidden in every state but the - * import phase after the file's first batch)), #sync-cancel-btn + * import phase after the file's first batch); phase 152 (task 03): + * the in-flight PDF's "· image n/m" vision-image position gets NO + * span of its own — the running button is the spinner (its parts + * are hidden, phase 149), so the position rides only the composed + * full string (the box / title / running aria-label)), + * #sync-cancel-btn * (the Cancel control — phase 134 task 02, ship-hidden, revealed * only while a sync run is in flight), #sync-icon (the * spinner icon), #sync-progress (Phase 149 task 01 + Phase 150: @@ -811,29 +818,38 @@ export async function mount(root) { return `${pad(d.getHours())}:${pad(d.getMinutes())}`; } - /* Phase 128 (task 01) + Phase 129 (task 04): the label PARTS - * writer — the SINGLE point that writes #sync-label. mainText lands - * in the MAIN part (.sync-label-main — the shrinks-to-fit prefix + - * file/folder, the part CSS ellipsizes); countText lands in the - * COUNT part (.sync-label-count — the pinned "(n/m)" position, - * NEVER clipped); chunkText (phase 129) lands in the CHUNK part - * (.sync-label-chunk — the pinned "· chunk n/m" embed-batch - * position of the in-flight file). The COUNT and CHUNK parts share - * one contract: present → its textContent + hidden=false, absent → - * emptied + hidden=true (each span is hidden whenever there is - * nothing of its kind to show). Then the UNTRUNCATED full label — - * mainText + (countText ? " " + countText : "") + (chunkText ? - * " " + chunkText : "") — rides the button title (A4 — the screen - * reader hears the whole "Syncing… (2/3) · chunk 14/38" - * there, and in the #sync-progress box — the sole aria-live - * region, phase 150 — while a run is in flight); the visible - * spans carry main + count + chunk, the spaces between them are - * the CSS gap. textContent only — the house rule, no HTML strings. - * Every label state goes through here (the running builders, the - * idle / settle labels, the 202/409 click, the re-attach). + /* Phase 128 (task 01) + Phase 129 (task 04) + Phase 152 (task 03): + * the label PARTS writer — the SINGLE point that writes #sync-label. + * mainText lands in the MAIN part (.sync-label-main — the + * shrinks-to-fit prefix + file/folder, the part CSS ellipsizes); + * countText lands in the COUNT part (.sync-label-count — the pinned + * "(n/m)" position, NEVER clipped); chunkText (phase 129) lands in + * the CHUNK part (.sync-label-chunk — the pinned "· chunk n/m" + * embed-batch position of the in-flight file); imageText (phase + * 152) is the in-flight PDF's "· image n/m" vision-image position — + * it gets NO span of its own (the running button is the spinner — + * its parts are hidden, phase 149), it rides ONLY the composed full + * string. The COUNT and CHUNK parts share one contract: present → + * its textContent + hidden=false, absent → emptied + hidden=true + * (each span is hidden whenever there is nothing of its kind to + * show). Then the UNTRUNCATED full label — mainText + (countText ? + * " " + countText : "") + (chunkText ? " " + chunkText : "") + + * (imageText ? " " + imageText : "") — rides the button title (A4 — + * the screen reader hears the whole "Syncing… (2/3) · chunk + * 14/38 · image 3/15" there, and in the #sync-progress box — the + * sole aria-live region, phase 150 — while a run is in flight); the + * visible spans carry main + count + chunk, the spaces between them + * are the CSS gap. textContent only — the house rule, no HTML + * strings. Every label state goes through here (the running + * builders, the idle / settle labels, the 202/409 click, the + * re-attach). * Phase 149 (task 01): the fourth parameter — `running` (the * in-flight flag; the callsites are the ONLY decision sites: the * running entry passes true, the settle passes the default false). + * Phase 152 (task 03): the fifth parameter — `imageText` (the "· + * image n/m" part; the chunk and image parts are distinct and both + * may show, and each disappears independently — the bare builders + * and the non-running calls pass nothing, so both default ""). * Phase 150 (task 02): while running, the SAME untruncated `full` * string goes into the box through the single box writer * (setSyncProgress — textContent only), so the box can never @@ -846,7 +862,7 @@ export async function mount(root) { * non-running call restores the ship value "Sync sources" (the * static HTML attribute is the idle truth — the JS only * overwrites/restores it, never removes it). */ - function setSyncLabel(mainText, countText = "", chunkText = "", running = false) { + function setSyncLabel(mainText, countText = "", chunkText = "", running = false, imageText = "") { if (syncLabelMain) syncLabelMain.textContent = mainText; if (syncLabelCount) { if (countText) { @@ -866,10 +882,16 @@ export async function mount(root) { syncLabelChunk.hidden = true; } } + // Phase 152 (task 03): the image part is the LAST piece of the + // composed full string (after the chunk part) — the box, the + // button title, and the running aria-label all carry it; the + // button's label spans stay as-is (no new span — see the + // docstring). const full = mainText + (countText ? ` ${countText}` : "") + - (chunkText ? ` ${chunkText}` : ""); + (chunkText ? ` ${chunkText}` : "") + + (imageText ? ` ${imageText}` : ""); if (syncBtn) syncBtn.title = full; // A4: the untruncated label rides the title, always // Phase 150 (task 02): the box — the sole sync text surface and // the sole aria-live region — receives the running label through @@ -924,6 +946,8 @@ export async function mount(root) { * 04): this builder still returns NO chunk part — the pinned * "· chunk n/m" part is added by fmtSyncPhaseLabel's import * fall-through (the only site that sees the status's chunk + * fields), and phase 152 (task 03) likewise adds the "· image + * n/m" part there (the only site that sees the status's image * fields), so the upload label and the 202/409 click stay bare. */ function fmtSyncLabel(kind, currentFile, done, total) { const prefix = kind === "upload" ? "Importing" : "Syncing…"; @@ -932,30 +956,38 @@ export async function mount(root) { return { main, count }; } - /* Phase 98 (task 02, D2) + Phase 128 (task 01, the PARTS rework): - * the SYNC job's phase-aware running label — the post-import span - * where the file count sits still (the owner's "the number pauses - * for a really long time"): "overview" → "Writing KB overview…" - * (no count part); "summaries" → main "Summarizing folders… - * " + count "(n/m)" — the folder part is omitted - * while current_summary is null (the phase's first poll) and the - * COUNT IS ALWAYS EMITTED in this phase (the hook always carries a - * total) and the CHUNK part stays empty (the summaries phase's own - * (n/m) is its position — the status's chunk fields are cleared at - * this transition); anything else (the null prelude, "import") → - * the phase-64 fmtSyncLabel parts, byte-identical (same builder, - * same fields, same order) + the phase-129 CHUNK part: "· chunk - * n/m" ONLY while chunks_total > 0 (a file's embed batch is in - * flight — n the 1-based first chunk of the batch, m the file's - * total content chunks, batch granularity), empty before the first - * batch (0/0) so the part hides. The UPLOAD job never passes a + /* Phase 98 (task 02, D2) + Phase 128 (task 01, the PARTS rework) + * + Phase 152 (task 03): the SYNC job's phase-aware running label — + * the post-import span where the file count sits still (the owner's + * "the number pauses for a really long time"): "overview" → + * "Writing KB overview…" (no count part); "summaries" → main + * "Summarizing folders… " + count "(n/m)" — the + * folder part is omitted while current_summary is null (the phase's + * first poll) and the COUNT IS ALWAYS EMITTED in this phase (the + * hook always carries a total) and the CHUNK and IMAGE parts stay + * empty (the summaries phase's own (n/m) is its position — the + * status's chunk + image fields are cleared at this transition); + * anything else (the null prelude, "import") → the phase-64 + * fmtSyncLabel parts, byte-identical (same builder, same fields, + * same order) + the phase-129 CHUNK part: "· chunk n/m" ONLY while + * chunks_total > 0 (a file's embed batch is in flight — n the + * 1-based first chunk of the batch, m the file's total content + * chunks, batch granularity), empty before the first batch (0/0) so + * the part hides + the phase-152 IMAGE part: "· image n/m" ONLY + * while images_total > 0 (a PDF's vision span is in flight — n the + * 1-based image about to be described, m the file's total images — + * ONE continuous counter across the PDF's three vision spans, the + * importer owns the continuity), empty before the first image + * (0/0) so the part hides; the two positions are distinct and both + * may show ("… · chunk 125/126 · image 3/15"), each disappearing + * independently. The UPLOAD job never passes a * status (its endpoint has no phase — D2) and keeps the bare * fmtSyncLabel parts. The untruncated full label rides the button * title + #sync-progress like every running label (A4 — * setSyncLabel). */ function fmtSyncPhaseLabel(status) { if (status.phase === "overview") { - return { main: "Writing KB overview…", count: "", chunk: "" }; + return { main: "Writing KB overview…", count: "", chunk: "", image: "" }; } if (status.phase === "summaries") { const folder = status.current_summary ? ` ${status.current_summary}` : ""; @@ -963,11 +995,14 @@ export async function mount(root) { main: `Summarizing folders…${folder}`, count: `(${status.summaries_done}/${status.summaries_total})`, chunk: "", // this phase's own (n/m) is its position — no chunk part + image: "", // the image position belongs to the import phase only }; } // null phase (the prelude) or "import" — today's byte-identical // sync label (the phase-64 contract, unchanged) + the phase-129 - // chunk part (hidden before the in-flight file's first batch). + // chunk part (hidden before the in-flight file's first batch) + + // the phase-152 image part (hidden before the in-flight PDF's + // first vision describe call). const { main, count } = fmtSyncLabel( "sync", status.current_file, status.files_done, status.files_total ); @@ -975,7 +1010,11 @@ export async function mount(root) { status.chunks_total > 0 ? `· chunk ${status.current_chunk}/${status.chunks_total}` : ""; - return { main, count, chunk }; + const image = + status.images_total > 0 + ? `· image ${status.images_done}/${status.images_total}` + : ""; + return { main, count, chunk, image }; } function fmtSyncResult(detail) { @@ -1011,7 +1050,10 @@ export async function mount(root) { * Phase 129 (task 04): the third, pinned CHUNK part ("· chunk n/m" * — the in-flight file's embed-batch position) is destructured the * same way (absent from the bare builders — the upload job and the - * 202/409 click — so the "" default keeps the part hidden there). + * 202/409 click — so the "" default keeps the part hidden there); + * phase 152 (task 03) destructures the fourth, IMAGE part ("· + * image n/m" — the in-flight PDF's vision-image position) the same + * way (it too rides only the composed full string — no span). * Same never-stale mechanics as before (disabled, aria-busy, * spinning icon, no is-error). Phase 149 (task 01): the writer call * passes `running = true` — the detail box reveals with the full @@ -1025,13 +1067,15 @@ export async function mount(root) { if (!syncBtn) return; syncBtn.disabled = true; syncBtn.setAttribute("aria-busy", "true"); - const { main, count, chunk = "" } = + const { main, count, chunk = "", image = "" } = kind === "sync" && status ? fmtSyncPhaseLabel(status) : fmtSyncLabel(kind, currentFile, done, total); // the parts + the A4 untruncated title / announcer + the phase-149 - // detail box (revealed, full label) + the running aria-label - setSyncLabel(main, count, chunk, true); + // detail box (revealed, full label — phase 152: with the image + // part when a PDF's vision span is in flight) + the running + // aria-label + setSyncLabel(main, count, chunk, true, image); syncBtn.classList.remove("is-error"); if (syncIcon) syncIcon.classList.add("is-spinning"); // Phase 149 (task 01): the icon-only running button — the class diff --git a/frontend/index.html b/frontend/index.html index 276e989..66e40f2 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -483,8 +483,9 @@