From 863519e9803eac8d345d0b5340c33cf2afc36d30 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Thu, 1 Oct 2026 14:23:33 -0400 Subject: [PATCH] phase: 147_sync_changed_only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All defaults match. Verification complete — everything is green with no defects found. **Phase 147 final verification pass — report** - All 3 task files found in `complete/147_sync_changed_only/` → verification pass only; no code changes needed. - Verified: migration 0025 (`documents.file_hash` nullable String(64), no index) + model field; fast path at top of `_index_file` (streamed 1 MiB sha256 → row lookup → `_unchanged_doc`); stamping on all three branches' create/update + shared helper; pre-phase NULL-hash fallthrough; retriever/`content_hash` untouched. - **Tests:** `uv run pytest --cov=app --cov-report=term-missing` → 3438 passed, `app/` coverage **99%** (>90% ✓); `uv run pytest tests/e2e/test_sync_changed_only.py -v --no-cov` → 3 passed in isolation ✓; `uv run ruff check . && uv run pyright` → all checks passed, 0 errors ✓; prior-phase `tests/e2e/test_sync_button.py` → 3 passed (no regression) ✓. - **Completion criteria:** all 7 pass — (1) column + round-trip ✓ (2) unchanged text/image/garbage-PDF/pictorial-PDF cost zero LLM/extraction/vision, `summary.unchanged` counted, rows byte-still ✓ (3) changed file re-indexes + re-stamps; bytes-changed/content-same stamps only ✓ (4) backfill/D1/D4 + log wordings pinned verbatim ✓ (5) `content_hash`/retriever byte-identical ✓ (6) all gates green ✓ (7) no prior-phase behavior change ✓. - **Documentation:** full review of README.md (1,031 lines) and AGENTS.md against the code: every `.env.example`/code `BOR_*` var documented, all config defaults match `app/config.py`, quick-start/QA commands and referenced test files exist, `file://` URL + troubleshooting claims accurate; the phase's README "Updating the documents" edit is accurate. **Verified consistent — no changes needed.** - **Notable:** no live-infrastructure changes (dev Postgres container used for the standard test gates only). Working tree left uncommitted per harness protocol. - **Next pending phase:** `148_pdf_pages_in_sources`. --- .../147_sync_changed_only/00_phase.md | 0 .../01_file_hash_column.md | 0 .../02_unchanged_fast_path.md | 0 .../03_e2e_unchanged_second_run.md | 0 .../147_sync_changed_only__00_phase.a1.err | 0 .../147_sync_changed_only__00_phase.a1.md | 11 + ...47_sync_changed_only__00_phase.a1.validate | 119 ++++ ...c_changed_only__01_file_hash_column.a1.err | 0 ...nc_changed_only__01_file_hash_column.a1.md | 18 + ...nged_only__01_file_hash_column.a1.validate | 119 ++++ ...hanged_only__02_unchanged_fast_path.a1.err | 0 ...changed_only__02_unchanged_fast_path.a1.md | 10 + ...d_only__02_unchanged_fast_path.a1.validate | 119 ++++ ...d_only__03_e2e_unchanged_second_run.a1.err | 0 ...ed_only__03_e2e_unchanged_second_run.a1.md | 19 + ...y__03_e2e_unchanged_second_run.a1.validate | 119 ++++ README.md | 9 +- alembic/versions/0025_documents_file_hash.py | 55 ++ app/models.py | 15 + app/rag/importer.py | 453 ++++++++----- tests/e2e/test_sync_changed_only.py | 408 ++++++++++++ tests/integration/test_importer.py | 26 +- tests/integration/test_migration_0025.py | 366 ++++++++++ tests/unit/test_importer.py | 34 +- .../unit/test_importer_unchanged_fast_path.py | 626 ++++++++++++++++++ tests/unit/test_models.py | 75 +++ tests/unit/test_pictorial_import.py | 46 +- 27 files changed, 2442 insertions(+), 205 deletions(-) rename .agents/phases/{todo => complete}/147_sync_changed_only/00_phase.md (100%) rename .agents/phases/{todo => complete}/147_sync_changed_only/01_file_hash_column.md (100%) rename .agents/phases/{todo => complete}/147_sync_changed_only/02_unchanged_fast_path.md (100%) rename .agents/phases/{todo => complete}/147_sync_changed_only/03_e2e_unchanged_second_run.md (100%) create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.err create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.md create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.validate create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.err create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.md create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.validate create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.err create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.md create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.validate create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.err create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.md create mode 100644 .agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.validate create mode 100644 alembic/versions/0025_documents_file_hash.py create mode 100644 tests/e2e/test_sync_changed_only.py create mode 100644 tests/integration/test_migration_0025.py create mode 100644 tests/unit/test_importer_unchanged_fast_path.py diff --git a/.agents/phases/todo/147_sync_changed_only/00_phase.md b/.agents/phases/complete/147_sync_changed_only/00_phase.md similarity index 100% rename from .agents/phases/todo/147_sync_changed_only/00_phase.md rename to .agents/phases/complete/147_sync_changed_only/00_phase.md diff --git a/.agents/phases/todo/147_sync_changed_only/01_file_hash_column.md b/.agents/phases/complete/147_sync_changed_only/01_file_hash_column.md similarity index 100% rename from .agents/phases/todo/147_sync_changed_only/01_file_hash_column.md rename to .agents/phases/complete/147_sync_changed_only/01_file_hash_column.md diff --git a/.agents/phases/todo/147_sync_changed_only/02_unchanged_fast_path.md b/.agents/phases/complete/147_sync_changed_only/02_unchanged_fast_path.md similarity index 100% rename from .agents/phases/todo/147_sync_changed_only/02_unchanged_fast_path.md rename to .agents/phases/complete/147_sync_changed_only/02_unchanged_fast_path.md diff --git a/.agents/phases/todo/147_sync_changed_only/03_e2e_unchanged_second_run.md b/.agents/phases/complete/147_sync_changed_only/03_e2e_unchanged_second_run.md similarity index 100% rename from .agents/phases/todo/147_sync_changed_only/03_e2e_unchanged_second_run.md rename to .agents/phases/complete/147_sync_changed_only/03_e2e_unchanged_second_run.md diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.err b/.agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.md b/.agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.md new file mode 100644 index 0000000..aa604d9 --- /dev/null +++ b/.agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.md @@ -0,0 +1,11 @@ +All defaults match. Verification complete — everything is green with no defects found. + +**Phase 147 final verification pass — report** + +- All 3 task files found in `complete/147_sync_changed_only/` → verification pass only; no code changes needed. +- Verified: migration 0025 (`documents.file_hash` nullable String(64), no index) + model field; fast path at top of `_index_file` (streamed 1 MiB sha256 → row lookup → `_unchanged_doc`); stamping on all three branches' create/update + shared helper; pre-phase NULL-hash fallthrough; retriever/`content_hash` untouched. +- **Tests:** `uv run pytest --cov=app --cov-report=term-missing` → 3438 passed, `app/` coverage **99%** (>90% ✓); `uv run pytest tests/e2e/test_sync_changed_only.py -v --no-cov` → 3 passed in isolation ✓; `uv run ruff check . && uv run pyright` → all checks passed, 0 errors ✓; prior-phase `tests/e2e/test_sync_button.py` → 3 passed (no regression) ✓. +- **Completion criteria:** all 7 pass — (1) column + round-trip ✓ (2) unchanged text/image/garbage-PDF/pictorial-PDF cost zero LLM/extraction/vision, `summary.unchanged` counted, rows byte-still ✓ (3) changed file re-indexes + re-stamps; bytes-changed/content-same stamps only ✓ (4) backfill/D1/D4 + log wordings pinned verbatim ✓ (5) `content_hash`/retriever byte-identical ✓ (6) all gates green ✓ (7) no prior-phase behavior change ✓. +- **Documentation:** full review of README.md (1,031 lines) and AGENTS.md against the code: every `.env.example`/code `BOR_*` var documented, all config defaults match `app/config.py`, quick-start/QA commands and referenced test files exist, `file://` URL + troubleshooting claims accurate; the phase's README "Updating the documents" edit is accurate. **Verified consistent — no changes needed.** +- **Notable:** no live-infrastructure changes (dev Postgres container used for the standard test gates only). Working tree left uncommitted per harness protocol. +- **Next pending phase:** `148_pdf_pages_in_sources`. diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.validate b/.agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.validate new file mode 100644 index 0000000..246661c --- /dev/null +++ b/.agents/reports/147_sync_changed_only/147_sync_changed_only__00_phase.a1.validate @@ -0,0 +1,119 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +...................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 181 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 134 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 700 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 440 0 100% +-------------------------------------------------- +TOTAL 5545 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.err b/.agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.md b/.agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.md new file mode 100644 index 0000000..a587b98 --- /dev/null +++ b/.agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.md @@ -0,0 +1,18 @@ +Task 01 complete — all criteria met and the project is 100% green. + +**Implemented** +- `alembic/versions/0025_documents_file_hash.py` — `documents.file_hash` VARCHAR(64), nullable, no server default, **no index** (compared in Python after the unique `(source, path)` lookup); downgrade drops it (chains 0024 → 0025) +- `app/models.py` — `Document.file_hash: Mapped[str | None] = mapped_column(String(64), default=None)` directly below `content_hash`, house-style docstring distinguishing it from `content_hash` (final-content digest, indexed, retriever identity — untouched) +- `tests/unit/test_models.py` — column contract (nullable String(64), no server default, unindexed on every axis; `content_hash` stays NOT NULL + indexed) + ORM default-None / explicit round-trip +- `tests/integration/test_migration_0025.py` — house migration idiom against the live dev DB: 0024→0025 contract (incl. UNINDEXED + pre-row NULL backfill + full 0024 schema survival), ORM NULL/explicit round-trip via fresh session, downgrade 0025→0024, round-trip re-upgrade + +**Tests / lint / coverage** +- `uv run pytest --cov=app --cov-report=term-missing` — 3425 passed, `app/` coverage 99% (>90%) +- `uv run pytest tests/integration/test_migration_0025.py tests/unit/test_models.py -v --no-cov` — 19 passed +- `uv run ruff check .` — clean; `uv run pyright` — 0 errors, 0 warnings + +**Docs:** no doc impact (README/AGENTS.md don't enumerate columns or migration numbers; no API/UI/env change) + +**Notable decision:** the dev DB's `documents` indexes carry a one-off `documents_heal_new_…` name prefix from a manual dev-DB repair (not a migration), so the unique-index survival check matches on index *definition* (`USING btree (source, path)`), not the model's nominal index name. + +**Next pending task:** `147_sync_changed_only/02_unchanged_fast_path.md` diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.validate b/.agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.validate new file mode 100644 index 0000000..895a0dc --- /dev/null +++ b/.agents/reports/147_sync_changed_only/147_sync_changed_only__01_file_hash_column.a1.validate @@ -0,0 +1,119 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 42%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 181 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 134 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 440 0 100% +-------------------------------------------------- +TOTAL 5550 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.err b/.agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.md b/.agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.md new file mode 100644 index 0000000..99b009e --- /dev/null +++ b/.agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.md @@ -0,0 +1,10 @@ +All green. Final report: + +**Phase 147, task 02 — unchanged fast path — complete** + +- **`app/rag/importer.py`**: added `_file_hash` (sha256 of raw bytes, streamed 1 MiB chunks); added `_unchanged_doc` — the ONE shared unchanged path (unchanged count + verbatim log wordings, stamp, phase-118 backfill from stored `doc.content`, D1 manual-date lock, D4 date refresh, stamp commit); rewired `_index_file` top: hash → row lookup → D2 mtime fallback → `file_hash` match returns before any dispatch (zero read/extract/vision/embed); all three branches stamp `file_hash` on create/update and their unchanged blocks now call the shared helper; stale re-burn docstrings (module, PDF branch, `_fallback_page_screenshots`, `_pictorial_pdf_pass`) updated to the bytes-decide model. +- **Tests**: new `tests/unit/test_importer_unchanged_fast_path.py` (13 tests: the zero-llm-call matrix for text/image/garbage-PDF/pictorial-PDF, changed-file re-stamp, bytes-changed/content-same PDF stamp, pre-phase NULL-hash stamping incl. image, D1/D4 pins on the fast path, backfill-from-`doc.content`, >1 MiB streamed-hash, caplog wording pin); updated 3 stale pre-phase pins that asserted vision re-issue on re-sync (`test_importer.py` fallback rerun, `test_pictorial_import.py` IKEA rerun, `tests/integration/test_importer.py` real-tar rerun). +- **Results**: `uv run pytest --cov=app --cov-report=term-missing` → 3438 passed, 0 failed, `app/` 99% (importer 99%; remaining misses pre-existing) · `uv run pytest tests/e2e/test_sync_button.py -v --no-cov` and `test_garbage_pdf.py` → 3 passed each in isolation · `uv run ruff check .` clean · `uv run pyright` 0 errors. +- **Docs**: README "Updating the documents" now documents the raw-bytes fast path; AGENTS.md unchanged (no command/convention impact). +- **Decisions**: the D1 path commits the stamp when dirty (a manual-date pictorial PDF must also go fast next sync — "the NEXT sync is always fast"); branch `raw_date` params tightened to non-optional since `_index_file` now owns the D2 fallback (single call site). +- **Next pending task**: `147_sync_changed_only/03_e2e_unchanged_second_run.md` (dedicated Playwright E2E: idempotent second sync). diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.validate b/.agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.validate new file mode 100644 index 0000000..246661c --- /dev/null +++ b/.agents/reports/147_sync_changed_only/147_sync_changed_only__02_unchanged_fast_path.a1.validate @@ -0,0 +1,119 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +...................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 181 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 134 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 700 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 440 0 100% +-------------------------------------------------- +TOTAL 5545 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.err b/.agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.md b/.agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.md new file mode 100644 index 0000000..91cd8e6 --- /dev/null +++ b/.agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.md @@ -0,0 +1,19 @@ +Done. Report: + +**Implemented** — `tests/e2e/test_sync_changed_only.py` (the phase-147 dedicated E2E, the `test_sync_button.py` module-app idiom: own port 8157 via `E2E_APP_PORT_SYNCCHANGED`, real `file://` fixture repo `changed-notes` with 3 committed markdown docs, own `BOR_SOURCES_DIR`, mock LLM): +- `test_second_sync_unchanged`: run 1 → `3 added` + every row's `file_hash` non-NULL; run 2 → settle line `0 added · 3 unchanged` and every row's `indexed_at`/`content_hash`/`file_hash` byte-still (the load-bearing no-re-index assertion) +- `test_change_syncs_only_the_change`: commit a new doc → run 3 → `1 added · 3 unchanged`; new row stamped, pre-existing rows untouched +- `test_changed_file_reindexes`: modify the sentinel + commit → run 4 → `0 added · 1 updated · 3 unchanged`; exactly that row's `indexed_at` bumped, `content_hash`/`file_hash` re-stamped +- One module-scoped truncate (not per-test) — tests 2–3 build on test 1's settled state, as the task requires + +**Tests / lint / coverage** +- `uv run pytest tests/e2e/test_sync_changed_only.py -v --no-cov` → 3 passed (isolation) +- `uv run pytest --cov=app --cov-report=term-missing` → 3438 passed, `app/` 99% (>90%) +- `uv run ruff check . && uv run pyright` → clean, 0 errors +- Pre-existing sync E2Es (`test_sync_button` + `test_force_push_resync` + `test_git_source_dates` + `test_garbage_pdf`) → 12 passed + +**Docs:** no doc impact from this task (per-suite run pattern is generic in README/AGENTS.md; the README fast-path edit is tasks 01–02's, already in the tree) + +**Decisions:** settle-line pins use the exact `fmtSyncResult` house shape — the zero "updated" part is omitted by the renderer, so `0 added · N unchanged` (matches the existing `test_sync_button.py` "0 added · 1 unchanged" pin); the task's `0 added · 0 updated · N` reading is honored via that house shape per the task's own instruction. + +**Next pending task:** none — task 03 is phase 147's final task. diff --git a/.agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.validate b/.agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.validate new file mode 100644 index 0000000..246661c --- /dev/null +++ b/.agents/reports/147_sync_changed_only/147_sync_changed_only__03_e2e_unchanged_second_run.a1.validate @@ -0,0 +1,119 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 18%] +........................................................................ [ 20%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 27%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 35%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 85%] +........................................................................ [ 87%] +........................................................................ [ 90%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 96%] +........................................................................ [ 98%] +...................................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 270 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 181 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 134 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 431 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 77 0 100% +app/rag/importer.py 700 4 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 155 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 440 0 100% +-------------------------------------------------- +TOTAL 5545 17 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/README.md b/README.md index febee91..f424cb0 100644 --- a/README.md +++ b/README.md @@ -187,8 +187,13 @@ catalog, git sources, tuning, and history stay admin-only. ### Updating the documents The knowledge base is refreshed by **re-running the import**. It is -idempotent and delta-based (sha256 per file), so a refresh after a normal -editing session takes seconds: +idempotent and delta-based: each file's RAW BYTES are hashed (sha256, +streamed) and checked against the last index's stamp *before* any +extraction or vision work — an unchanged file costs one hash + one row +lookup (no re-extract, no vision, no re-embed), while a changed file +re-indexes fully. A refresh after a normal editing session takes +seconds — pictorial PDFs included (their expensive first-index vision +burns only once per file version): ```bash # After editing/adding/removing notes: diff --git a/alembic/versions/0025_documents_file_hash.py b/alembic/versions/0025_documents_file_hash.py new file mode 100644 index 0000000..9cb0c46 --- /dev/null +++ b/alembic/versions/0025_documents_file_hash.py @@ -0,0 +1,55 @@ +"""documents.file_hash: the raw source-file-bytes digest (phase 147, task 01) + +Revision ID: 0025 +Revises: 0024 +Create Date: 2026-09-30 + +Phase 147 (sync only what changed — task 01, storage only): + +* ``documents.file_hash`` — VARCHAR(64) NULLABLE, NO server default: + the sha256 of the RAW SOURCE FILE's bytes at index time — the sync's + change signal, compared at the top of the importer's ``_index_file`` + BEFORE the image/text/PDF dispatch (an unchanged file then costs one + streamed hash + one lookup — no extraction, no vision, no embed; + task 02). NULL = "unknown" for every pre-phase row: a backfill is + impossible in a migration (it cannot see the source files on disk), + so the first post-upgrade sync runs the full pipeline once and + STAMPS the hash — fast forever after. + + DISTINCT from ``content_hash`` (0001): that column is the digest of + the FINAL indexed content (a pictorial PDF's includes vision text), + is indexed (a query/exposure surface — the retriever's + dedupe/caching identity, ``app/rag/retriever.py``) and stays + untouched. ``file_hash`` is internal bookkeeping — it is compared in + Python AFTER the unique ``(source, path)`` lookup, so it carries no + index of its own. + +One additive, fully reversible migration (A13); no other schema +change. The fast path itself (task 02) is importer code — this +revision only carries the column. +""" +from __future__ import annotations + +import sqlalchemy as sa + +from alembic import op + +revision = "0025" +down_revision = "0024" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # No index (the fast path compares the value in Python after the + # unique (source, path) lookup — it is never a query surface) and + # no server default (NULL is the meaningful pre-phase state). + op.add_column("documents", sa.Column("file_hash", sa.String(length=64), nullable=True)) + + +def downgrade() -> None: + # The file_hash column is the only 0025 artefact — dropping it + # leaves 0024's schema byte-identical (A13, fully reversible). The + # hash is re-computable from the source files at any time, so no + # data is lost by the drop. + op.drop_column("documents", "file_hash") diff --git a/app/models.py b/app/models.py index e3cab64..8813275 100644 --- a/app/models.py +++ b/app/models.py @@ -110,6 +110,21 @@ class Document(Base): title: Mapped[str] = mapped_column(String(500)) content: Mapped[str] = mapped_column(Text) # full markdown — the RAG context content_hash: Mapped[str] = mapped_column(String(64), index=True) # sha256 for change detection + #: The sha256 of the RAW SOURCE FILE's bytes at index time (phase + #: 147) — the sync's change signal, compared at the top of the + #: importer's ``_index_file`` BEFORE the image/text/PDF dispatch + #: (an unchanged file costs one streamed hash + one lookup — no + #: extraction, no vision, no embed). NULL for pre-phase rows and + #: until the first post-upgrade sync stamps it (no migration + #: backfill is possible — a migration cannot see the source files + #: on disk). DISTINCT from ``content_hash`` — the FINAL indexed + #: content's digest (a pictorial PDF's includes vision text; + #: ``content_hash`` is exposed via ``app/rag/retriever.py`` and is + #: the dedupe/caching identity — untouched). No index: the fast + #: path compares it in Python after the unique ``(source, path)`` + #: lookup — internal bookkeeping, never a query surface (unlike + #: ``content_hash``). + file_hash: Mapped[str | None] = mapped_column(String(64), default=None) indexed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) #: The document's CREATION date (phase 106, D1/D2/D3) — sourced at #: sync time (git last-commit date for git sources, file mtime for diff --git a/app/rag/importer.py b/app/rag/importer.py index c8f39b7..355577a 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -59,6 +59,32 @@ backfill runs BEFORE the ``created_at_manual`` early-return (the manual flag protects the DATE only, D1) and the strict ``is None`` check leaves owner-set summaries (even empty strings, phase 57) alone. +Unchanged fast path (phase 147, task 02): the sync's single change +signal is the sha256 of the RAW source file's bytes +(:func:`_file_hash` — streamed in 1 MiB chunks, a large PDF is never +loaded fully), computed ONCE at the top of :func:`_index_file` BEFORE +the image/text/PDF dispatch and compared against +``documents.file_hash`` (migration 0025 — nullable, no index, internal +bookkeeping; DISTINCT from ``content_hash``, the FINAL indexed +content's digest, which stays the retriever's dedupe/caching identity). +A match returns through :func:`_unchanged_doc` — the ONE shared +unchanged path (the three branches' near-identical unchanged blocks, +one code site): the ``unchanged`` count + log line, the phase-118 (A2) +summary-NULL backfill from the STORED ``doc.content`` (never a +re-read/re-extract/re-describe — the digest-equality invariant), the +phase-106 (D1) manual-date early-return, and the (D4) date refresh +(may go older, counts ``dates_updated``) — with ZERO file read beyond +the streamed hash, ZERO PDF extraction, ZERO vision call, and ZERO +embed. Every create/update STAMPS ``file_hash`` (all three branches), +and the shared helper stamps too (a pre-phase NULL row, or a +bytes-changed/content-same file whose content digest still matches), +so the NEXT sync is always fast. Pre-phase rows (``file_hash`` NULL) +fall through to the branches' content-digest logic exactly as +pre-phase on the first post-upgrade sync (one full pass), then stamp +and go fast. The previous re-burn costs (a garbage PDF's fallback +vision re-issued each sync, a non-deterministic vision model flipping +a pictorial PDF's content digest) are gone: the file bytes decide. + Standalone images (phase 122, LOCKED A3): with the ``images`` toggle (``BOR_IMAGES``) ON, the walk also admits the image extension set (``BOR_IMAGE_EXTENSIONS`` — a SEPARATE set from ``import_extension_set``; @@ -123,11 +149,11 @@ call, the SAME ``---PAGE-N---`` separators, the loud class of PDF whose text layer is garbage. A CLEAN text layer NEVER burns a fallback vision call (the gate is the only gate — the phase-126 happy path is byte-identical), and the digest runs over the -FINAL content exactly once — for a stable file + stable vision model -the digest is byte-stable, so an unchanged re-sync re-imports the row -as unchanged (no re-chunk/re-embed/re-summary, no KB change; the -vision calls are re-issued each sync — the no-cache cost, documented -in :func:`_fallback_page_screenshots`). +FINAL content exactly once — and an UNCHANGED re-sync is caught by the +phase-147 fast path BEFORE this pipeline runs at all (the raw file +bytes are the change signal: no re-extract, no re-render, no +re-issued vision call, no re-chunk/re-embed/re-summary, no KB change — +see :func:`_index_file`). Pictorial sparse pages (phase 140, task 03, LOCKED A1–A4): AFTER the phase-133 gate, a PDF whose text layer is USABLE gets the sparse-page @@ -557,6 +583,33 @@ def iter_importable_files( return files +#: The streamed-hash read size (phase 147, task 02) — 1 MiB: a +#: multi-hundred-MB PDF is hashed in a bounded stream, never loaded +#: fully into memory. +_HASH_CHUNK_SIZE = 1024 * 1024 + + +def _file_hash(path: Path) -> str: + """Phase 147 (task 02): the sha256 of *path*'s RAW bytes, streamed + in 1 MiB chunks (the :data:`_HASH_CHUNK_SIZE` loop — a large file + is never loaded fully into memory; the ``hashlib`` incremental + pattern). + + The sync's SINGLE change signal — computed ONCE at the top of + :func:`_index_file`, before the image/text/PDF dispatch, and + compared against the stamped ``documents.file_hash`` (the fast + path). Raw bytes on purpose (the owner's "hash the document + contents"): every class hashes the same way — images, text, PDFs + alike — and a bytes-identical file is an unchanged file, whatever + its extraction/vision pipeline would produce. + """ + h = hashlib.sha256() + with open(path, "rb") as f: + while chunk := f.read(_HASH_CHUNK_SIZE): + h.update(chunk) + return h.hexdigest() + + async def import_sources( sources: list[Path], llm: Embedder, @@ -876,6 +929,100 @@ async def _replace_content_chunks( return len(chunks_text) +async def _unchanged_doc( + session: Session, + doc: Document, + *, + source: str, + rel: str, + llm: Embedder, + summary: ImportSummary, + raw_date: datetime, + file_digest: str, +) -> None: + """Phase 147 (task 02): the ONE shared unchanged path — the three + branches' near-identical unchanged blocks, one code site, reached + from two places: + + * the FAST PATH at the top of :func:`_index_file` (a + ``doc.file_hash == file_digest`` match — the raw bytes are what + was indexed last time): zero file read beyond the streamed + hash, zero extraction, zero vision, zero embed; + * each branch's CONTENT-digest hit (a pre-phase NULL-hash row, or + a bytes-changed/content-same file whose final content digest + still matches): the branch's own expensive work already ran, + but the row is skipped exactly as pre-phase. + + The preserved semantics, in order (the per-branch blocks' counts, + log wordings, and commits — verbatim): + + 1. ``summary.unchanged`` + the ``import: unchanged`` log line; + 2. the STAMP: ``doc.file_hash = file_digest`` (a pre-phase NULL + row — or a bytes-changed/content-same row — learns its hash + here, so the NEXT sync takes the fast path; a no-op on the + fast path, where the DB value already equals the digest); + 3. the phase-118 (A2) backfill when ``doc.summary is None`` — + from the STORED ``doc.content``: EXACTLY what the file's bytes + produced is already stored (text: the file text; image: the + vision description; PDF: the final constructed content — the + digest-equality invariant), so the backfill never re-reads the + file, never re-extracts, never re-describes (strict + ``is None`` — an owner-set empty string, phase 57, is never + touched); + 4. the phase-106 (D1) early-return — ``created_at_manual`` + protects the DATE: no date write, the manual correction + survives (the phase-97 ``manually_edited`` precedent); the + stamp, if dirty, is the only write, and it commits so the next + sync is fast even for a manual-date row; + 5. the phase-106 (D4) date refresh — ``normalize_doc_date`` of + the raw date, committed + counted in ``dates_updated`` + the + ``import: date-refreshed`` line when it changes (it may go + OLDER — no monotonic guard); + 6. the stamp's final commit — when the row is still dirty after + the date block (the stamp set ``file_hash`` and the date did + not change): quiet bookkeeping — no count, no log (the + ``unchanged`` line already fired; a date-change run already + committed in step 5). + """ + summary.unchanged += 1 + logger.info("import: unchanged source=%s path=%s", source, rel) + # 2 — the stamp (a no-op on the fast path; a real write on the + # branch paths — the next sync is then fast either way). + doc.file_hash = file_digest + # 3 — the phase-118 (A2) summary-NULL backfill, from the stored + # content (see the docstring — never a re-read/re-extract). + if doc.summary is None: + await _store_summary( + session, doc=doc, source=source, rel=rel, content=doc.content, + llm=llm, summary=summary, backfill=True, + ) + # 4 — the phase-106 (D1) manual-date lock: no date write; the + # stamp, if dirty, is the only write (committed — see step 2's + # "next sync takes the fast path"). + if doc.created_at_manual: + if session.is_modified(doc): + session.commit() + return + # 5 — the phase-106 (D4) date refresh: the stored date follows the + # source on every sync and may go OLDER (no monotonic guard); a + # date-only refresh counts ``unchanged`` — never added/updated/ + # pruned, so no ``sources_meta`` bump and no regeneration. + target = normalize_doc_date(raw_date) + if target != doc.created_at: + doc.created_at = target + session.commit() + summary.dates_updated += 1 + logger.info( + "import: date-refreshed source=%s path=%s date=%s", + source, rel, + doc.created_at.isoformat(), + ) + elif session.is_modified(doc): + # 6 — the stamp's commit (the date did not move): quiet + # bookkeeping — the ``unchanged`` line already fired. + session.commit() + + async def _index_file( session: Session, *, @@ -889,6 +1036,23 @@ async def _index_file( ) -> None: """Upsert one file: doc row + chunk rows + embeddings, one transaction. + Phase 147 (task 02): the UNCHANGED FAST PATH. BEFORE the + image/text/PDF dispatch, the file's RAW source bytes are hashed + (:func:`_file_hash` — streamed), the ``(source, path)`` row is + looked up, and the D2 mtime fallback for *raw_date* runs (once, + here — the branches receive the already-computed date) — a + ``doc.file_hash == file_digest`` match returns through the shared + unchanged path (:func:`_unchanged_doc`) with ZERO file read beyond + the streamed hash, ZERO PDF extraction, ZERO vision call, and + ZERO embed (the preserved bookkeeping: the phase-118 summary-NULL + backfill from the STORED ``doc.content``, the phase-106 D1 + manual-date lock, the D4 date refresh). A pre-phase row + (``file_hash`` NULL) or a bytes-changed file falls through to the + branches below, which keep their own lookups + content-digest + checks (the content digest then decides changed vs. the same + shared path) and STAMP ``file_hash`` on every create/update — so + the next sync is always fast. + ``raw_date`` (phase 106, D2) is the file's RAW source date — the git last-commit datetime from the caller's ``doc_dates_by_root`` map, or ``None`` (every non-git case): the file's mtime is read @@ -909,68 +1073,66 @@ async def _index_file( any text read: ``read_text`` is never called for an image. """ settings = llm.settings + # Phase 147 (task 02): the unchanged fast path — the RAW source- + # file-bytes digest is the single change signal, computed ONCE + # here, before the image/text/PDF dispatch, and compared against + # the stamped ``documents.file_hash``. A match returns through the + # shared unchanged path with zero further work — no + # ``read_text``/``read_bytes``, no ``extract_pdf_text``, no + # fallback/pictorial vision, no re-chunk/re-embed/re-summary. + file_digest = _file_hash(full_path) + doc = session.scalar(select(Document).where(Document.source == source, Document.path == rel)) + if raw_date is None: + # D2 fallback: no source date in the map → the file's mtime + # (one stat). Read before the unchanged early-return — the + # fast path (and the branches' unchanged path) refresh the + # stored date from the same source. + raw_date = file_mtime_datetime(full_path) + if doc is not None and doc.file_hash == file_digest: + # The fast path: the raw bytes are what was indexed last time + # — the bookkeeping (backfill / D1 / D4) runs in the shared + # helper, nothing else does. + await _unchanged_doc( + session, doc, source=source, rel=rel, llm=llm, summary=summary, + raw_date=raw_date, file_digest=file_digest, + ) + return # Phase 122 (task 02): the image branch FIRST. Only reachable while # the ``images`` toggle is on — the walk never admits image files # while it is off, and with it off this check is a no-op (the text - # path below stays byte-identical to pre-phase). + # path below stays byte-identical to pre-phase). Phase 147: this + # point is reached only for a pre-phase NULL-hash row or a + # bytes-changed file — the branch's own digest then decides. if settings.images: image_set = settings.image_extension_set if match_extension(full_path, image_set) is not None: return await _index_image_file( session, source=source, rel=rel, full_path=full_path, llm=llm, - summary=summary, raw_date=raw_date, on_chunk=on_chunk, + summary=summary, raw_date=raw_date, file_digest=file_digest, + on_chunk=on_chunk, ) # Phase 126: PDF files take the PDF index path — AFTER the image # branch, BEFORE any text read (a PDF is binary; ``extract_pdf_text`` - # owns it and ``read_text`` is never called for one). + # owns it and ``read_text`` is never called for one). Phase 147: + # reached only for a pre-phase NULL-hash row or a bytes-changed + # file — the branch's own digest then decides. if full_path.suffix.lower() == ".pdf": return await _index_pdf_file( session, source=source, rel=rel, full_path=full_path, llm=llm, - summary=summary, raw_date=raw_date, on_chunk=on_chunk, + summary=summary, raw_date=raw_date, file_digest=file_digest, + on_chunk=on_chunk, ) content = full_path.read_text(encoding="utf-8", errors="replace").replace("\x00", "") digest = hashlib.sha256(content.encode("utf-8")).hexdigest() - doc = session.scalar(select(Document).where(Document.source == source, Document.path == rel)) - if raw_date is None: - # D2 fallback: no source date in the map → the file's mtime - # (one stat). Read before the unchanged early-return — the - # unchanged path refreshes the stored date from the same source. - raw_date = file_mtime_datetime(full_path) if doc is not None and doc.content_hash == digest: - summary.unchanged += 1 - logger.info("import: unchanged source=%s path=%s", source, rel) - # Phase 118 (A2): an unchanged doc whose summary is still NULL - # (a pre-phase-30 row, or an earlier fail-soft miss) gets a - # summary-only backfill — one ``is_summary`` chunk, no content - # re-embed, and NEVER an added/updated/pruned count (so no - # ``sources_meta`` bump, no overview/folder-summary - # regeneration). Strict ``is None``: an empty-string summary is - # owner-set (phase 57) and is never overwritten. BEFORE the - # manual-date early-return: ``created_at_manual`` protects the - # DATE only (phase 106, D1), not the summary. - if doc.summary is None: - await _store_summary( - session, doc=doc, source=source, rel=rel, content=content, - llm=llm, summary=summary, backfill=True, - ) - if doc.created_at_manual: - # D1/D4: the owner's correction survives the sync — no - # write at all (the phase-97 ``manually_edited`` precedent). - return - # D4: the date refreshes on every sync, including unchanged - # files, and may go OLDER (no monotonic guard). A date-only - # refresh is still counted ``unchanged`` — never added/updated/ - # pruned, so no ``sources_meta`` bump and no regeneration. - target = normalize_doc_date(raw_date) - if target != doc.created_at: - doc.created_at = target - session.commit() - summary.dates_updated += 1 - logger.info( - "import: date-refreshed source=%s path=%s date=%s", - source, rel, - doc.created_at.isoformat(), - ) + # Phase 147: the shared unchanged path — the pre-phase + # content-digest hit (a pre-phase NULL-hash row, or a + # bytes-changed/content-same file). The helper stamps the new + # file hash, so the NEXT sync takes the fast path above. + await _unchanged_doc( + session, doc, source=source, rel=rel, llm=llm, summary=summary, + raw_date=raw_date, file_digest=file_digest, + ) return verb = "updated" if doc is not None else "added" @@ -988,6 +1150,9 @@ async def _index_file( title=title, content=content, content_hash=digest, + # Phase 147: the raw-bytes stamp (the next sync's fast + # path). + file_hash=file_digest, indexed_at=datetime.now(UTC), # Phase 106 (D2/D3): the sourced creation date, normalized # (undetermined or future → today). ``created_at_manual`` @@ -1001,6 +1166,8 @@ async def _index_file( doc.title = title doc.content = content doc.content_hash = digest + # Phase 147: re-stamp (the file's bytes changed). + doc.file_hash = file_digest doc.indexed_at = datetime.now(UTC) # Phase 106 (D4): a content change is a new document version — # the date is re-sourced and a previous manual correction is @@ -1186,7 +1353,8 @@ async def _index_pdf_file( full_path: Path, llm: Embedder, summary: ImportSummary, - raw_date: datetime | None = None, + file_digest: str, + raw_date: datetime, on_chunk: Callable[[int, int], None] | None = None, ) -> None: """The phase-126 PDF branch of :func:`_index_file` — a PDF is @@ -1221,12 +1389,13 @@ async def _index_pdf_file( vision call). * the sha256 digest is over the FINAL content (the digest rule is content identity — the same text is the same document version; - the fallback's constructed page descriptions digest exactly the - same way, so an unchanged fallback doc re-syncs as unchanged — - no re-chunk/re-embed/re-summary, no KB change — while the vision - model's replies stay stable; a non-deterministic vision model - re-describes the row each sync, a cost, not a bug; the empty - case digests the empty byte string). + the empty case digests the empty byte string). Phase 147: the + UNCHANGED detection itself is the caller's RAW-BYTES file hash + (the fast path — :func:`_index_file`): an unchanged PDF never + re-extracts, never re-renders, and never re-issues a vision + call (the previous "non-deterministic vision model re-describes + the row each sync" cost is gone — a re-describe now happens + only when the file's BYTES change). * the title is the metadata title or the file stem (:func:`_extract_pdf_title`); the date follows the text path exactly (the phase-106 D2 mtime fallback — PDFs carry no @@ -1237,15 +1406,23 @@ async def _index_pdf_file( retry lives there), and the phase-30/118 (A2) ``lite`` summary path runs on the text — PDFs are no more excluded than any other non-markdown format. - * the unchanged path (same text digest) skips everything — no - re-extract, no re-embed, no image pass — and runs the same - backfill/manual-date/date-refresh dance as the text path. + * the UNCHANGED path (phase 147): the fast path above catches an + unchanged file before any of this runs; a pre-phase NULL-hash + row, or a bytes-changed/content-same PDF, that reaches the + content-digest hit below runs the SAME shared path + (:func:`_unchanged_doc`) — no re-extract, no re-embed, no image + pass, the backfill/manual-date/date-refresh semantics in one + code site — and the helper stamps the new file hash (the next + sync takes the fast path). * THEN the embedded images: :func:`_index_pdf_embedded_images` (A3 — the phase-122 pipeline, gated by the ``images`` toggle). - ``raw_date`` follows the text path: the caller's git last-commit - date or ``None`` (the mtime fallback, read before the unchanged - early-return — the unchanged path refreshes from the same source). + ``raw_date`` is the caller's git last-commit date, or the file's + mtime — :func:`_index_file` computes the D2 fallback (and reads + it before the fast-path check) — and the unchanged path + refreshes from the same source. ``file_digest`` is the caller's + raw-bytes sha256 (phase 147 — the fast-path signal and this + branch's stamp value). ``on_chunk`` (phase 129, task 02) is the in-flight file's chunk-position hook: it is forwarded to the PDF's CONTENT embed @@ -1294,40 +1471,17 @@ async def _index_pdf_file( else hashlib.sha256(b"").hexdigest() ) title = _extract_pdf_title(full_path, fallback=full_path.stem) - if raw_date is None: - # D2 fallback (same as the text path): no source date in the - # map → the file's mtime (one stat). Read before the unchanged - # early-return — the unchanged path refreshes from it. - raw_date = file_mtime_datetime(full_path) - doc = session.scalar(select(Document).where(Document.source == source, Document.path == rel)) if doc is not None and doc.content_hash == digest: - # Unchanged PDF (text digest) — the text path's unchanged branch, - # unmodified in shape: no re-extract, no re-embed, no image pass. - summary.unchanged += 1 - logger.info("import: unchanged source=%s path=%s", source, rel) - # Phase 118 (A2): an unchanged doc whose summary is still NULL - # gets the same best-effort summary pass (see the text path). - if doc.summary is None: - await _store_summary( - session, doc=doc, source=source, rel=rel, content=content, - llm=llm, summary=summary, backfill=True, - ) - if doc.created_at_manual: - # D1/D4: the owner's correction survives the sync — no write - # at all (the text path's manual-date early-return). - return - # D4: the date refreshes on every sync and may go OLDER (no - # monotonic guard); a date-only refresh counts unchanged. - target = normalize_doc_date(raw_date) - if target != doc.created_at: - doc.created_at = target - session.commit() - summary.dates_updated += 1 - logger.info( - "import: date-refreshed source=%s path=%s date=%s", - source, rel, doc.created_at.isoformat(), - ) + # Phase 147: the shared unchanged path — a pre-phase NULL-hash + # row, or a bytes-changed/content-same PDF (the content digest + # still matches). No re-extract, no re-embed, no image pass; + # the helper stamps the new file hash (the next sync takes the + # fast path). + await _unchanged_doc( + session, doc, source=source, rel=rel, llm=llm, summary=summary, + raw_date=raw_date, file_digest=file_digest, + ) return verb = "updated" if doc is not None else "added" @@ -1339,6 +1493,9 @@ async def _index_pdf_file( title=title, content=content, content_hash=digest, + # Phase 147: the raw-bytes stamp (the next sync's fast + # path). + file_hash=file_digest, indexed_at=datetime.now(UTC), created_at=normalize_doc_date(raw_date), ) @@ -1348,6 +1505,8 @@ async def _index_pdf_file( doc.title = title doc.content = content doc.content_hash = digest + # Phase 147: re-stamp (the file's bytes changed). + doc.file_hash = file_digest doc.indexed_at = datetime.now(UTC) # Phase 106 (D4): a content change is a new document version — # the date is re-sourced and a manual correction reset. @@ -1428,18 +1587,18 @@ async def _fallback_page_screenshots( or every page a placeholder) returns ``""``: the existing empty-content path (the doc lands as a zero-text doc, the phase-126 no-extractable-text behavior). - * the digest the caller computes is over the constructed content — - for a stable file + stable vision model the content (and thus - the digest) is byte-stable across re-syncs, so the row - re-imports as UNCHANGED: no re-chunk, no re-embed, no - re-summary, no KB change. The fallback itself RE-RUNS on every - sync of a garbage PDF (the gate is a cheap pure predicate and - the render needs no LLM; the vision calls are re-issued before - the unchanged check can skip them — a documented cost of the - no-cache design, bounded by the page count of the garbage - PDFs). If the production vision model's reply is - non-deterministic, the row is re-described each sync (updated — - a cost, not a bug; the doc is still correct). + * the digest the caller computes is over the constructed content + — and phase 147's fast path makes a fallback re-run moot for an + unchanged file: the raw BYTES are the change signal, so an + unchanged garbage PDF is caught at the top of :func:`_index_file` + BEFORE the gate runs at all (no re-extract, no re-render, no + re-issued vision call — the owner's "really long sync times for + PDFs that are mostly pictoral" complaint, fixed). The fallback's + vision cost now applies to a garbage PDF's FIRST index — and to + a re-index only when the file's bytes change; a + non-deterministic production vision model no longer flips an + unchanged row's content digest (the row stays put until the + bytes move; the doc is still correct either way). *session* is part of the signature for symmetry with the other per-file index helpers (the fallback itself needs no DB access — @@ -1520,10 +1679,12 @@ async def _pictorial_pdf_pass( The assembly with NO descriptions is byte-identical to :func:`extract_pdf_text`'s output (the unit pin). * the caller's digest runs over this FINAL content (the - phase-126/133 convention — A4): a stable file + a stable vision - reply re-syncs as ``unchanged``; a non-deterministic re-describe - re-updates the row — a cost, not a bug (the phase-133 docstring - documents it, A21 endorses it). + phase-126/133 convention — A4) — and phase 147's fast path + catches an unchanged file BEFORE the pass runs at all (the raw + bytes are the change signal): no re-render, no re-issued vision + call, and a non-deterministic vision model no longer re-updates + an unchanged row — the pass burns vision only on a file's first + index and on byte changes. * stats (log-only — the sync ``detail`` API shape is untouched, the phase-133 convention): ``summary.pdfs_pictorial`` (+1 when ≥1 page was described) and ``summary.pdf_pictorial_pages`` @@ -2647,7 +2808,8 @@ async def _index_image_file( full_path: Path, llm: Embedder, summary: ImportSummary, - raw_date: datetime | None = None, + file_digest: str, + raw_date: datetime, on_chunk: Callable[[int, int], None] | None = None, ) -> None: """The phase-122 image branch of :func:`_index_file` — a standalone @@ -2666,11 +2828,14 @@ async def _index_image_file( path running on it (the ``is_summary`` position −1 chunk mirrors ``Document.summary``). - ``raw_date`` follows the text path exactly (the phase-106 D2 - fallback: no source date in the map → the file's mtime, read before - the unchanged early-return because the unchanged path refreshes the - stored date from the same source; the D1 manual-date lock and the - D4 refresh apply unmodified). + ``raw_date`` follows the text path exactly — the CALLER + (:func:`_index_file`) computes the phase-106 D2 fallback (no + source date in the map → the file's mtime) before the fast-path + check, because the unchanged path refreshes the stored date from + the same source; the D1 manual-date lock and the D4 refresh apply + unmodified through the shared :func:`_unchanged_doc`. ``file_digest`` + is the caller's raw-bytes sha256 (phase 147 — the fast-path + signal and this branch's stamp value). Fail-soft (LOCKED A3): a failed/empty description SKIPS the doc entirely (no row, no copy) — ``summary.images_failed`` + a warning, @@ -2698,48 +2863,25 @@ async def _index_image_file( byte-identical. """ settings = llm.settings + # Phase 147: the fast path (:func:`_index_file`) already returned + # for an unchanged file — everything below (``read_bytes``, the + # vision describe, the copy) runs only on the first index or a + # byte change. data = full_path.read_bytes() digest = hashlib.sha256(data).hexdigest() doc = session.scalar(select(Document).where(Document.source == source, Document.path == rel)) - if raw_date is None: - # D2 fallback (same as the text path): no source date in the map - # → the file's mtime (one stat). - raw_date = file_mtime_datetime(full_path) if doc is not None and doc.content_hash == digest: - # Unchanged image (byte digest) — the text path's unchanged - # branch, unmodified in shape. - summary.unchanged += 1 - logger.info("import: unchanged source=%s path=%s", source, rel) - # Phase 118 (A2) backfill, image flavour: an unchanged image doc - # whose summary is still NULL (an earlier fail-soft summary miss - # — for an image, the one remaining failure class: the summary - # chunk's embed) gets the same best-effort summary pass. For an - # image the summary IS the stored description (``doc.content``), - # so the image-aware ``_store_summary`` (task 03) re-stores it - # verbatim with one ``is_summary`` chunk; a failure (the - # embed) keeps the doc as-is (no row mutation) — fail-soft, - # same as the text path. - if doc.summary is None: - await _store_summary( - session, doc=doc, source=source, rel=rel, content=doc.content, - llm=llm, summary=summary, backfill=True, - ) - if doc.created_at_manual: - # D1/D4: the owner's correction survives the sync — no write - # at all (the text path's manual-date early-return). - return - # D4: the date refreshes on every sync, including unchanged - # files, and may go OLDER (no monotonic guard). - target = normalize_doc_date(raw_date) - if target != doc.created_at: - doc.created_at = target - session.commit() - summary.dates_updated += 1 - logger.info( - "import: date-refreshed source=%s path=%s date=%s", - source, rel, doc.created_at.isoformat(), - ) + # Phase 147: the shared unchanged path — a pre-phase NULL-hash + # row, or a bytes-changed/content-same image (the byte digest + # still matches). The helper stamps the new file hash (the + # next sync takes the fast path); for an image the summary is + # the stored description (``doc.content``), so the image-aware + # ``_store_summary`` backfill re-stores it verbatim. + await _unchanged_doc( + session, doc, source=source, rel=rel, llm=llm, summary=summary, + raw_date=raw_date, file_digest=file_digest, + ) return verb = "updated" if doc is not None else "added" @@ -2803,6 +2945,9 @@ async def _index_image_file( title=title, content=content, content_hash=digest, + # Phase 147: the raw-bytes stamp (the next sync's fast + # path). + file_hash=file_digest, indexed_at=datetime.now(UTC), created_at=normalize_doc_date(raw_date), is_image=True, @@ -2818,6 +2963,8 @@ async def _index_image_file( doc.title = title doc.content = content doc.content_hash = digest + # Phase 147: re-stamp (the file's bytes changed). + doc.file_hash = file_digest doc.indexed_at = datetime.now(UTC) # Phase 106 (D4): a content change is a new document version — # the date is re-sourced and a previous manual correction is diff --git a/tests/e2e/test_sync_changed_only.py b/tests/e2e/test_sync_changed_only.py new file mode 100644 index 0000000..0cc1a51 --- /dev/null +++ b/tests/e2e/test_sync_changed_only.py @@ -0,0 +1,408 @@ +"""Phase 147 E2E (Playwright): only what changed is synced. + +Source: ``TODO.md:4`` — "Local sources, like PDFs, are synced every +time even if they haven't changed. This results in really long sync +times for PDFs that are mostly pictoral. Only sync documents if they've +changed since the last sync (hash the document contents to check)." + +Run in isolation (DB must be up: ``podman compose up -d db``; git on +PATH — a documented environment prerequisite, phase 28): + + uv run pytest tests/e2e/test_sync_changed_only.py -v --no-cov + +End-to-end proof through the real UI + the real sync path (a real +``git clone`` of a local ``file://`` fixture repo, a real import +against the mock LLM): the SECOND sync of an unchanged source reports +everything unchanged and touches NO document row (``indexed_at`` / +``content_hash`` / ``file_hash`` all stay still — a re-index would +bump ``indexed_at``); a source change syncs ONLY the changed part (new +file → ``1 added``; modified file → ``1 updated`` with only its own +row moving). NO PDF here — the unit matrix (task 02) carries the +vision-skip proof; this suite pins the WIRE (the owner's visible +``#sync-result`` line) + the UI settle + the row-level proof, fast and +deterministic. + +Per-module app env (the E2E conftest pattern, module-scoped): this +suite's app boots with ``BOR_GIT_SOURCES=file://`` and +its own ``BOR_SOURCES_DIR`` on its own port (env-overridable, the +house port-constant pattern). The session app (no git sources) is +never started in this isolated run, so no port clash. + +The three tests SHARE the settled state on purpose (one module-scoped +truncate, not per-test — the suite's point is the run 2 → run 3 → +run 4 sequence): + +1. ``test_second_sync_unchanged`` — run 1 (clone + import) indexes + ``N_DOCS`` documents and stamps every row's ``file_hash`` non-NULL; + run 2 (unchanged bytes) settles ``0 added · N unchanged`` and moves + NO row (the load-bearing assertion). +2. ``test_change_syncs_only_the_change`` — commit a NEW doc to the + fixture repo; run 3 settles ``1 added · N unchanged``; the new row + appears stamped, the pre-existing rows do not move. +3. ``test_changed_file_reindexes`` — modify the sentinel doc; run 4 + settles ``0 added · 1 updated · N unchanged``; exactly the + sentinel row moves (``indexed_at`` bumped, ``content_hash`` and + ``file_hash`` re-stamped). +""" +from __future__ import annotations + +import os +import re +import subprocess +import sys +from collections.abc import Iterator +from datetime import datetime +from pathlib import Path + +import httpx +import pytest +from playwright.sync_api import Page, expect +from sqlalchemy import select, text + +from app.db import SessionLocal +from app.models import Document +from e2e.auth_helpers import login +from e2e.conftest import ( + ADMIN_PASSWORD, + SESSION_SECRET, + USE_REAL_LLM, + _wait_http, +) + +REPO = Path(__file__).resolve().parents[2] + +# The house port-constant pattern (module app, env-overridable) — the +# session app (8123) is never started in this isolated run. +APP_PORT = int(os.environ.get("E2E_APP_PORT_SYNCCHANGED", "8157")) +APP_URL = f"http://127.0.0.1:{APP_PORT}" + +# The fixture repo's directory name — the source name the import +# records for it (the repo_name basename rule, phase 28). +GIT_SOURCE_NAME = "changed-notes" + +# The three importable fixture docs (task 03: a few importable docs, +# NO PDF — the unit matrix covers the vision-skip proof). +SENTINEL_DOC = "notes/sync-fixture.md" +SECOND_DOC = "notes/second-note.md" +THIRD_DOC = "notes/third-note.md" +#: The doc test 2 commits AFTER run 2 (``1 added``). +NEW_DOC = "notes/new-note.md" +N_DOCS = 3 + +#: "Synced HH:MM" — the local-time last-result label (sources.js's +#: fmtSyncTime), any hour/minute. +SYNCED_LABEL = re.compile(r"Synced \d{1,2}:\d{2}") + +#: Real git clone/pull + import against the mock LLM (+ the KB +#: overview regeneration after a KB-changing run) — generous budget, +#: no client timeout (the test_sync_button.py pattern). +SYNC_TIMEOUT_MS = 90_000 + +#: One document row's row-level proof snapshot (phase 147: the +#: ``file_hash`` stamp joins the pre-phase pair). +DocRow = tuple[datetime | None, str | None, str | None] + + +def _git(cwd: Path, *args: str) -> None: + """Run git in *cwd*; a non-zero exit fails the fixture/test loudly.""" + proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + if proc.returncode != 0: + raise AssertionError(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + + +def _commit(cwd: Path, message: str) -> None: + """The house fixture commit — identity inline, never signed.""" + _git( + cwd, + "-c", "user.email=e@x", "-c", "user.name=t", + "-c", "commit.gpgsign=false", # the fixture commits never sign + "commit", "-qm", message, + ) + + +@pytest.fixture(scope="module") +def sync_git_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + """A real one-commit git repo with ``N_DOCS`` importable docs that + the sync must clone (the test_sync_button.py recipe, module-safe + ``tmp_path_factory`` + real ``git`` subprocess calls).""" + root = tmp_path_factory.mktemp("sync_changed_git") + repo = root / GIT_SOURCE_NAME + (repo / "notes").mkdir(parents=True) + (repo / "notes" / "sync-fixture.md").write_text( + "# Sync fixture note\n" + "\n" + "One small note that exists only to prove the changed-only sync\n" + "end to end: a real git clone, a real import, an idempotent\n" + "second run — and the sentinel test 3 modifies.\n" + "\n" + "Marker: RESE-SYNC-CHANGED-SENTINEL-7f41\n", + encoding="utf-8", + ) + (repo / "notes" / "second-note.md").write_text( + "# Second note\n" + "\n" + "A second unchanged doc — the sync's unchanged count is over\n" + "the whole source, so it must stay still through runs 2–4.\n" + "\n" + "Marker: RESE-SECOND-NOTE-5c0b\n", + encoding="utf-8", + ) + (repo / "notes" / "third-note.md").write_text( + "# Third note\n" + "\n" + "A third unchanged doc (rounds out the source — the settle\n" + "line's unchanged count is the KB size, not a singleton).\n" + "\n" + "Marker: RESE-THIRD-NOTE-91de\n", + encoding="utf-8", + ) + _git(repo, "init", "-q") + _git(repo, "add", "-A") + _commit(repo, "one") + assert (repo / ".git").is_dir() + return repo + + +@pytest.fixture(scope="module") +def app_server(mock_llm: int, sync_git_repo: Path) -> Iterator[str]: + """The real app under test — per-module env: the sync's subject is + a real ``file://`` git source with its own checkout dir (the + conftest session app boots without ``BOR_GIT_SOURCES`` and is never + started in this isolated run).""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = ( + "https://aipi.reeseapps.com/v1" + if USE_REAL_LLM + else f"http://127.0.0.1:{mock_llm}/v1" + ) + # Mock-calibrated threshold (conftest pattern) — keeps every story + # suite's deterministic gate behavior; production default stays 0.62. + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + # Phase 16: admin auth must be set or create_app() refuses to boot. + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + # Phase 147: the sync's subject — one real local repo. + env["BOR_GIT_SOURCES"] = f"file://{sync_git_repo}" + env["BOR_SOURCES_DIR"] = str(sync_git_repo.parent / "checkouts") + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], + cwd=REPO, + env=env, + ) + try: + _wait_http(f"{APP_URL}/api/health") + yield APP_URL + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +@pytest.fixture(scope="module", autouse=True) +def _clean_kb_once(app_url: str) -> Iterator[None]: + """Fresh KB ONCE per module — deliberately NOT per test: the suite's + point is the run 2 → run 3 → run 4 sequence, and tests 2–3 build on + the settled state test 1 leaves behind. git_sources too: the shared + Postgres may carry leftover registry rows from other suites, and + effective_sources prefers the DB list over this app's + BOR_GIT_SOURCES env fallback.""" + body = httpx.get(f"{app_url}/api/health", timeout=5).json() + if body["db"] != "up": + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + with SessionLocal() as db: + db.execute( + text("TRUNCATE chunks, documents, query_log, kb_overview, git_sources") + ) + db.commit() + yield + + +def _doc_rows() -> dict[str, DocRow]: + """Every document row of the fixture source, keyed by path — the + row-level proof (the test_sync_button.py DB-assertion idiom, + extended with the phase-147 ``file_hash`` stamp).""" + with SessionLocal() as db: + docs = db.execute( + select(Document).where(Document.source == GIT_SOURCE_NAME) + ).scalars().all() + return {d.path: (d.indexed_at, d.content_hash, d.file_hash) for d in docs} + + +def _click_sync_and_wait_settle(page: Page) -> None: + """Click the admin sync button and wait for the settle — the + "Syncing…" → "Synced HH:MM" transition with the button re-enabled + at the terminal state (§7.4 never-stale, the test_sync_button.py + idiom).""" + btn = page.locator("#sync-btn") + btn.click() + expect(btn).to_be_disabled() + expect(page.locator("#sync-label")).to_have_text("Syncing…") + expect(page.locator("#sync-label")).to_have_text( + SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS + ) + expect(btn).to_be_enabled() + + +def _expect_settle_line(page: Page, expected: str) -> None: + """The owner's surface: the VISIBLE ``#sync-result`` text in its + house count shape (sources.js's fmtSyncResult — the zero + "updated" part is omitted: ``0 added · N unchanged``, + ``1 added · N unchanged``, ``0 added · 1 updated · N unchanged``).""" + expect(page.locator("#sync-result")).to_have_text( + expected, timeout=SYNC_TIMEOUT_MS + ) + + +def _expect_rows_moved( + before: dict[str, DocRow], after: dict[str, DocRow], *, moved: set[str] +) -> None: + """Exactly the rows in *moved* may differ — every other row's + ``indexed_at``/``content_hash``/``file_hash`` is byte-still (a + re-index would bump ``indexed_at``; the changed-only sync must not + leak into neighbors). New rows may appear only within *moved*; the + suite never expects a prune.""" + gained = set(after) - set(before) + assert gained <= moved, f"unexpected rows added: {sorted(gained)}" + lost = set(before) - set(after) + assert not lost, f"unexpected rows pruned: {sorted(lost)}" + for path, snapshot in before.items(): + if path in moved: + continue + assert after[path] == snapshot, ( + f"{path} moved though only {sorted(moved)} may move: " + f"{snapshot} -> {after[path]}" + ) + + +# --- 1. Run 1 stamps, run 2 re-indexes nothing ----------------------------- + + +def test_second_sync_unchanged(page: Page, app_url: str, db_ready: None) -> None: + """Run 1 (the real clone + import) indexes ``N_DOCS`` docs and + stamps EVERY row's ``file_hash`` non-NULL (the first sync's job); + run 2 (unchanged bytes) settles ``0 added · N unchanged`` and + touches NO document row — ``indexed_at``/``content_hash``/ + ``file_hash`` all still (the load-bearing assertion: a re-index + would bump ``indexed_at`` to now and the owner's long syncs would + be back).""" + login(page, app_url) # lands on /sources.html (the button's home) + + # --- run 1: clone + import (everything is new) ----------------------- + _click_sync_and_wait_settle(page) + _expect_settle_line(page, f"{N_DOCS} added") + after_run1 = _doc_rows() + assert set(after_run1) == {SENTINEL_DOC, SECOND_DOC, THIRD_DOC}, after_run1 + for path, (indexed_at, content_hash, file_hash) in after_run1.items(): + assert indexed_at is not None and content_hash, path + # The first sync stamped the hash on every row (the fast + # path's precondition — NULL would mean run 2 re-indexed). + assert file_hash is not None and len(file_hash) == 64, ( + f"{path} must carry a non-NULL file_hash after run 1" + ) + + # --- run 2: unchanged bytes → zero re-index -------------------------- + _click_sync_and_wait_settle(page) + _expect_settle_line(page, f"0 added · {N_DOCS} unchanged") + after_run2 = _doc_rows() + _expect_rows_moved(after_run1, after_run2, moved=set()) + + +# --- 2. A new file syncs as `1 added`, nothing else moves ------------------ + + +def test_change_syncs_only_the_change( + page: Page, app_url: str, db_ready: None, sync_git_repo: Path +) -> None: + """From the settled state (test 1), commit a NEW doc to the fixture + repo's working tree: run 3 syncs ONLY the change — the settle line + reads ``1 added · N unchanged`` (the unchanged count is still run + 1's doc count), the new row exists stamped, and the pre-existing + rows' ``indexed_at``/``content_hash`` are untouched.""" + login(page, app_url) + before = _doc_rows() + assert len(before) == N_DOCS, "the settled state from test 1 is required" + + (sync_git_repo / "notes" / "new-note.md").write_text( + "# New note\n" + "\n" + "Committed after run 2 — only this file may move in run 3.\n" + "\n" + "Marker: RESE-NEW-DOC-3d8a\n", + encoding="utf-8", + ) + _git(sync_git_repo, "add", "-A") + _commit(sync_git_repo, "add new-note") + + _click_sync_and_wait_settle(page) + _expect_settle_line(page, f"1 added · {N_DOCS} unchanged") + + after = _doc_rows() + assert set(after) - set(before) == {NEW_DOC}, after.keys() + _, new_content_hash, new_file_hash = after[NEW_DOC] + assert new_content_hash, "the added row must carry its content hash" + assert new_file_hash is not None and len(new_file_hash) == 64, ( + "the added row must carry its file_hash stamp" + ) + # The pre-existing rows are byte-still (the added file must not + # drag them into a re-index). + _expect_rows_moved(before, after, moved={NEW_DOC}) + + +# --- 3. A modified file syncs as `1 updated`, only its own row ------------- + + +def test_changed_file_reindexes( + page: Page, app_url: str, db_ready: None, sync_git_repo: Path +) -> None: + """Modify the fixture sentinel doc (append a line) + commit: run 4 + syncs ONLY the change — the settle line reads + ``0 added · 1 updated · N unchanged`` (N = the run-1 count — the + test-2 doc is unchanged too), and EXACTLY the sentinel row moves: + its ``indexed_at`` is bumped, its ``content_hash`` and + ``file_hash`` re-stamped; every other row is byte-still.""" + login(page, app_url) + before = _doc_rows() + assert len(before) == N_DOCS + 1, "the settled state from test 2 is required" + + sentinel = sync_git_repo / "notes" / "sync-fixture.md" + sentinel.write_text( + sentinel.read_text(encoding="utf-8") + + "\nAmended: the sentinel was touched — exactly this row may move.\n", + encoding="utf-8", + ) + _git(sync_git_repo, "add", "-A") + _commit(sync_git_repo, "modify sentinel") + + _click_sync_and_wait_settle(page) + _expect_settle_line(page, f"0 added · 1 updated · {N_DOCS} unchanged") + + after = _doc_rows() + _expect_rows_moved(before, after, moved={SENTINEL_DOC}) + old_indexed_at, old_content_hash, old_file_hash = before[SENTINEL_DOC] + new_indexed_at, new_content_hash, new_file_hash = after[SENTINEL_DOC] + assert new_content_hash != old_content_hash, ( + "the content changed — content_hash must be re-stamped" + ) + assert new_file_hash != old_file_hash, ( + "the bytes changed — file_hash must be re-stamped" + ) + assert ( + old_indexed_at is not None + and new_indexed_at is not None + and new_indexed_at > old_indexed_at + ), "the re-index must bump indexed_at (run 1 stamped it; run 4 re-indexed)" diff --git a/tests/integration/test_importer.py b/tests/integration/test_importer.py index 1e04c88..894fc93 100644 --- a/tests/integration/test_importer.py +++ b/tests/integration/test_importer.py @@ -1636,10 +1636,11 @@ def test_ikea_tar_sparse_pages_are_described_and_rerun_unchanged( (the owner's rationale — more information is better for the embeddings), the title is the metadata title, the digest is over the FINAL content, and a re-sync with the same fake lands - ``unchanged`` (digest stable, byte-stable row, NO re-embed; the - vision calls are re-issued before the unchanged check can skip - them — the documented no-cache cost, the phase-133 convention, - A21 endorses it).""" + ``unchanged`` through the phase-147 FAST PATH (the raw bytes are + the change signal: byte-stable row, NO re-embed, and the vision + calls are NOT re-issued — the pass never runs on an unchanged + file; the pre-phase "vision re-issued before the unchanged check" + no-cache cost is gone).""" pdf_path = ikea_pdf llm = _image_llm( tmp_path, @@ -1724,20 +1725,19 @@ def test_ikea_tar_sparse_pages_are_described_and_rerun_unchanged( f"(min_chars={min_chars})" ] - # The re-sync with the SAME fake: ``unchanged`` — the final - # content is a pure function of the fixture + the canned - # replies, so the digest matches and the row is skipped (no - # re-chunk, no re-embed, no re-summary, no KB change). The - # vision calls ARE re-issued (the documented no-cache cost — - # the phase-133 convention; a non-deterministic vision model - # re-describes the row each sync: a cost, not a bug). + # The re-sync with the SAME fake: ``unchanged`` through the + # phase-147 fast path — the raw bytes are the change signal, + # so the pass never re-runs: no re-render, the vision calls + # are NOT re-issued (the pre-phase "vision re-issued before + # the unchanged check can skip them" no-cache cost is gone), + # no re-chunk, no re-embed, no re-summary, no KB change. embeds1 = len(llm.calls) pictorial1 = len(llm.pictorial_texts) content1, digest1 = doc.content, doc.content_hash s2 = asyncio.run(import_sources([pdf_path.parent], llm, session=db)) assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 1) - assert s2.pdfs_pictorial == 1 # the pass re-fired - assert len(llm.pictorial_texts) == 2 * pictorial1 # re-issued (cost) + assert s2.pdfs_pictorial == 0 # the pass did not re-fire + assert len(llm.pictorial_texts) == pictorial1 # NO re-issued vision assert len(llm.calls) == embeds1, "unchanged: no re-embed" assert s2.changed_paths == [], "unchanged: no KB change recorded" doc2 = db.scalar( diff --git a/tests/integration/test_migration_0025.py b/tests/integration/test_migration_0025.py new file mode 100644 index 0000000..8900d51 --- /dev/null +++ b/tests/integration/test_migration_0025.py @@ -0,0 +1,366 @@ +"""Integration: migration 0025 (documents.file_hash) schema contract +(phase 147, task 01). + +Drives the **real Alembic engine** against the live dev database +(``podman compose up -d db``), mirroring the house pattern of +``test_migration_0024.py`` (information_schema assertions on the state +the migration must leave). The tests target the 0024 → 0025 step +explicitly so later migrations cannot break the pins: + +* upgrade 0024 → 0025 → ``file_hash`` exists with the full contract — + VARCHAR(64), NULLABLE, NO server default (NULL = "unknown" for + pre-phase rows; a migration cannot backfill — it cannot see the + source files on disk) — and is UNINDEXED (the fast path compares it + in Python after the unique ``(source, path)`` lookup — internal + bookkeeping, never a query surface) while the 0024 ``documents`` + schema survives — in particular ``content_hash`` stays NOT NULL and + INDEXED (the retriever's dedupe/caching identity, untouched); +* rows inserted while the DB is at 0024 backfill ``file_hash`` to NULL + — the pre-phase shape, no backfill, no default; +* the ORM contract agrees: a freshly inserted ``Document`` with no + ``file_hash`` reads ``file_hash is None`` through a fresh session + (the pre-phase shape), and an explicit digest round-trips; +* downgrade to 0024 → the column is GONE (A13) while the row + its + content survive; upgrade back to 0025 → the column is back + (round-trip). + +The ``alembic`` fixture guarantees the DB ends at head even if a test +fails or the process is interrupted. +""" +from __future__ import annotations + +import hashlib +import uuid +from collections.abc import Iterator +from typing import Any + +import pytest +from alembic.config import Config +from sqlalchemy import text +from sqlalchemy.orm import Session + +from alembic import command +from app.db import SessionLocal, db_available +from app.models import Document + +SOURCE = "mig0025" +CONTENT = "content here" +CONTENT_HASH = hashlib.sha256(CONTENT.encode()).hexdigest() +FILE_HASH = hashlib.sha256(b"raw source bytes").hexdigest() + + +@pytest.fixture() +def alembic(db: Session) -> Iterator[Config]: + """Real Alembic config bound to the dev DB (URL from app settings). + + Starts at head (repairs an interrupted earlier run); teardown + upgrades to head no matter what happened, so the dev DB is never + left below head. + """ + if not db_available(): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + cfg = Config() # no alembic.ini file — env.py gets the URL from app config + cfg.set_main_option("script_location", "alembic") + command.upgrade(cfg, "head") + try: + yield cfg + finally: + # Release the test session's open transaction BEFORE the repair + # DDL: an idle-in-transaction SELECT holds an ACCESS SHARE lock + # on ``documents``, which would deadlock the repair's + # ``ALTER TABLE`` (0025) forever. + db.rollback() + command.upgrade(cfg, "head") + + +def _version(db: Session) -> str | None: + return db.execute(text("SELECT version_num FROM alembic_version")).scalar() + + +def _column(db: Session, column: str) -> tuple[Any, ...] | None: + """(data_type, is_nullable, column_default, char_max_length) for one + documents column.""" + row = db.execute( + text( + "SELECT data_type, is_nullable, column_default, character_maximum_length" + " FROM information_schema.columns" + " WHERE table_name = 'documents' AND column_name = :c" + ), + {"c": column}, + ).fetchone() + return tuple(row) if row is not None else None + + +def _index_covering(db: Session, column: str) -> str | None: + """The name of any documents index whose definition mentions the + column (None = UNINDEXED).""" + return db.execute( + text( + "SELECT indexname FROM pg_indexes" + " WHERE tablename = 'documents' AND indexdef ILIKE :like" + ), + {"like": f"%{column}%"}, + ).scalar() + + +def _insert_sql(db: Session, path: str) -> uuid.UUID: + """Insert one documents row with the PRE-0025 column set (the 0024 + shape — the file_hash column, when present, is omitted so a NULL + backfill is what the row reads).""" + doc_id = uuid.uuid4() + db.execute( + text( + "INSERT INTO documents (id, source, path, full_path, title, content," + " content_hash) VALUES (:id, :s, :p, :fp, :t, :c, :h)" + ), + { + "id": doc_id, + "s": SOURCE, + "p": path, + "fp": f"/tmp/{path}", + "t": f"Doc {path}", + "c": CONTENT, + "h": CONTENT_HASH, + }, + ) + db.commit() + return doc_id + + +def _delete(db: Session, doc_id: uuid.UUID) -> None: + db.execute(text("DELETE FROM documents WHERE id = :id"), {"id": doc_id}) + db.commit() + + +def _assert_0024_schema_survives(db: Session) -> None: + """The 0024 documents contract must survive the additive upgrade — + in particular ``content_hash`` stays NOT NULL and indexed (the + retriever's identity — untouched by phase 147).""" + content = _column(db, "content_hash") + assert content is not None and content[0] == "character varying" and content[1] == "NO", ( + "documents.content_hash (0001) must keep its 0024 contract" + ) + assert content[3] == 64, "documents.content_hash must keep String(64)" + assert ( + _index_covering(db, "content_hash") is not None + ), "the ix_documents_content_hash index must survive the upgrade" + + indexed = _column(db, "indexed_at") + assert indexed is not None and indexed[0] == "timestamp with time zone", ( + "documents.indexed_at (0001) must survive the upgrade" + ) + created = _column(db, "created_at") + assert created is not None and created[1] == "NO", ( + "documents.created_at (0020) must survive the upgrade" + ) + manual = _column(db, "created_at_manual") + assert manual is not None and manual[0] == "boolean" and manual[1] == "NO", ( + "documents.created_at_manual (0020) must survive the upgrade" + ) + is_image = _column(db, "is_image") + assert is_image is not None and is_image[0] == "boolean" and is_image[1] == "NO", ( + "documents.is_image (0022) must survive the upgrade" + ) + image_path = _column(db, "image_path") + assert image_path is not None and image_path[1] == "YES", ( + "documents.image_path (0022) must survive the upgrade" + ) + image_ctx = _column(db, "image_context") + assert image_ctx is not None and image_ctx[1] == "YES", ( + "documents.image_context (0023) must survive the upgrade" + ) + # Match by definition, not name: the dev DB's ``documents`` indexes + # carry a one-off repair prefix (``documents_heal_new_…_*`` — a + # manual dev-DB heal, never a migration), so the name itself is not + # a stable pin. + uq = db.execute( + text( + "SELECT indexname FROM pg_indexes" + " WHERE tablename = 'documents'" + " AND indexdef LIKE 'CREATE UNIQUE INDEX %USING btree (source, path)'", + ) + ).scalar() + assert uq is not None, ( + "the unique (source, path) index must survive the upgrade" + ) + + +def test_upgrade_to_0025_adds_file_hash(db: Session, alembic: Config) -> None: + """Upgrade 0024 → 0025: ``file_hash`` exists with the full contract + (VARCHAR(64), NULLABLE, NO server default — NULL = "unknown" for + pre-phase rows — and UNINDEXED: the fast path compares it in Python + after the unique ``(source, path)`` lookup), is ABSENT at 0024, and + every pre-0025 row backfills ``file_hash`` to NULL (no backfill + possible — a migration cannot see the source files on disk) — while + the 0024 table contract (incl. the INDEXED ``content_hash``) + survives.""" + command.downgrade(alembic, "0024") # start from the pre-0025 state + assert _version(db) == "0024" + assert _column(db, "file_hash") is None, "file_hash must be absent at 0024" + + pre_id = _insert_sql(db, "pre.md") # the 0024 column set + try: + command.upgrade(alembic, "0025") + assert _version(db) == "0025", "alembic_version must be at 0025" + + col = _column(db, "file_hash") + assert col is not None, "documents.file_hash is missing" + assert col[0] == "character varying", "file_hash must be VARCHAR" + assert col[1] == "YES", "file_hash must be NULLABLE (NULL = unknown)" + assert col[2] is None, ( + "file_hash must carry NO server default — NULL is the meaningful" + " pre-phase state" + ) + assert col[3] == 64, "file_hash must be VARCHAR(64) — a sha256 hex digest" + assert _index_covering(db, "file_hash") is None, ( + "file_hash must be UNINDEXED — it is compared in Python after the" + " unique (source, path) lookup, never a query surface" + ) + + # The pre-0025 row backfilled to NULL (the pre-phase shape — one + # full pipeline pass stamps it on the next sync). + assert db.execute( + text("SELECT content_hash, file_hash FROM documents WHERE id = :id"), + {"id": pre_id}, + ).fetchone() == (CONTENT_HASH, None), ( + "every pre-0025 row must backfill file_hash to NULL — no" + " backfill, no default — while content_hash survives" + ) + + _assert_0024_schema_survives(db) + finally: + _delete(db, pre_id) + + +def test_orm_file_hash_round_trips(db: Session, alembic: Config) -> None: + """The ORM contract agrees with the column contract: a freshly + inserted ``Document`` with NO ``file_hash`` reads ``file_hash is + None`` through a FRESH session (the pre-phase shape — NULL until the + first post-upgrade sync stamps it), and an explicit digest + round-trips the value.""" + command.upgrade(alembic, "head") + row_stamped = Document( + source=SOURCE, + path="stamped.md", + full_path="/tmp/stamped.md", + title="stamped", + content=CONTENT, + content_hash=CONTENT_HASH, + file_hash=FILE_HASH, + ) + row_unstamped = Document( + source=SOURCE, + path="unstamped.md", + full_path="/tmp/unstamped.md", + title="unstamped", + content=CONTENT, + content_hash=CONTENT_HASH, + ) + db.add(row_stamped) + db.add(row_unstamped) + db.commit() + try: + with SessionLocal() as fresh: + reloaded_stamped = fresh.get(Document, row_stamped.id) + assert reloaded_stamped is not None, "the stamped row must be readable" + assert reloaded_stamped.file_hash == FILE_HASH, ( + "the explicit file_hash must round-trip through the DB" + ) + reloaded_unstamped = fresh.get(Document, row_unstamped.id) + assert reloaded_unstamped is not None, "the unstamped row must be readable" + assert reloaded_unstamped.file_hash is None, ( + "a row created without file_hash must read NULL (the pre-phase" + " shape)" + ) + assert reloaded_unstamped.content_hash == CONTENT_HASH, ( + "content_hash must be untouched by the file_hash addition" + ) + finally: + _delete(db, row_stamped.id) + _delete(db, row_unstamped.id) + + +def test_downgrade_to_0024_drops_the_column(db: Session, alembic: Config) -> None: + """Downgrade 0025 → 0024: the file_hash column is gone (A13 — fully + reversible, the hash is re-computable from the source files) while + the row + its content (incl. the stamped file_hash value's sibling + ``content_hash``) survive, and the rest of the 0024 table contract + is intact.""" + command.upgrade(alembic, "head") + row = Document( + source=SOURCE, + path="down.md", + full_path="/tmp/down.md", + title="down", + content=CONTENT, + content_hash=CONTENT_HASH, + file_hash=FILE_HASH, + ) + db.add(row) + db.commit() + try: + command.downgrade(alembic, "0024") + assert _version(db) == "0024" + assert _column(db, "file_hash") is None, "file_hash must be dropped" + + surviving = db.execute( + text( + "SELECT source, path, content, content_hash, is_image, image_path," + " image_context FROM documents WHERE id = :id" + ), + {"id": row.id}, + ).fetchone() + assert surviving is not None and surviving[0] == SOURCE, ( + "the row must survive the column drop" + ) + assert surviving[1] == "down.md" and surviving[2] == CONTENT, ( + "path + content must survive the drop" + ) + assert surviving[3] == CONTENT_HASH, ( + "content_hash (the retriever's identity) must survive the drop" + ) + assert surviving[4] is False and surviving[5] is None and surviving[6] is None, ( + "is_image + image_path + image_context must survive the drop" + ) + assert _index_covering(db, "content_hash") is not None, ( + "the content_hash index must survive the downgrade" + ) + finally: + _delete(db, row.id) + # Repair: the fixture teardown re-upgrades to head. + + +def test_upgrade_round_trip_restores_the_column(db: Session, alembic: Config) -> None: + """Downgrade to 0024, then upgrade back to 0025: ``file_hash`` is + back with the full contract (VARCHAR(64), NULLABLE, NO server + default, UNINDEXED), a row seeded at 0024 reads ``file_hash is + None`` (the pre-phase shape on the round-trip), and the 0024 schema + (incl. the indexed ``content_hash``) is intact.""" + command.downgrade(alembic, "0024") + pre_id = _insert_sql(db, "round.md") + try: + command.upgrade(alembic, "0025") + assert _version(db) == "0025", "round-trip upgrade must land at 0025" + + col = _column(db, "file_hash") + assert col is not None, "documents.file_hash must be back" + assert col[0] == "character varying", "file_hash must be VARCHAR after the round-trip" + assert col[1] == "YES", "file_hash must be NULLABLE after the round-trip" + assert col[2] is None, ( + "file_hash must still carry NO server default after the round-trip" + ) + assert col[3] == 64, "file_hash must be VARCHAR(64) after the round-trip" + assert _index_covering(db, "file_hash") is None, ( + "file_hash must stay UNINDEXED after the round-trip" + ) + + assert db.execute( + text("SELECT file_hash FROM documents WHERE id = :id"), + {"id": pre_id}, + ).scalar() is None, ( + "a row seeded at 0024 must read file_hash NULL after the round-trip" + ) + + _assert_0024_schema_survives(db) + finally: + _delete(db, pre_id) diff --git a/tests/unit/test_importer.py b/tests/unit/test_importer.py index 3c8a01d..e2c2960 100644 --- a/tests/unit/test_importer.py +++ b/tests/unit/test_importer.py @@ -1990,18 +1990,19 @@ def test_fallback_unrenderable_page_gets_the_placeholder( _cleanup_source(db, "halfblind") -def test_fallback_rerun_lands_unchanged_no_reembed( +def test_fallback_rerun_unchanged_fast_path_no_reburn( db, tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: - """Digest stability (the task's re-burn guard): the digest runs - over the constructed fallback content — a re-sync of an unchanged - garbage PDF (same pages + the deterministic fake's same replies) - re-imports as UNCHANGED: the fallback re-runs (the gate + the - render are cheap; the vision calls are re-issued — the documented - no-cache cost), but the digest MATCHES, so the row is skipped — - no re-chunk, no re-embed, no re-summary, no KB change (the row is - not ``updated`` and records no changed path). The doc's content - and digest are byte-stable across the re-sync.""" + """Phase 147 (task 02) — the owner's core complaint, fixed: a + re-sync of an unchanged garbage PDF (same raw bytes) takes the + FAST PATH at the top of ``_index_file`` (the file hash matches) + — the fallback does NOT re-run: the gate never fires + (``pdfs_screenshotted`` stays 0 on run 2), no re-extract, no + re-render, and the vision calls are NOT re-issued (the previous + "vision calls re-issued before the unchanged check can skip them" + no-cache cost is gone). The row re-imports as UNCHANGED — no + re-chunk, no re-embed, no KB change (not ``updated``, no changed + path) — and the doc's content/digest are byte-stable.""" root = tmp_path / "stable" root.mkdir() _build_garbage_pdf(root / "stable.pdf") @@ -2010,19 +2011,18 @@ def test_fallback_rerun_lands_unchanged_no_reembed( try: s1 = asyncio.run(import_sources([root], llm, session=db)) assert (s1.added, s1.pdfs_screenshotted) == (1, 1) - assert llm.vision_calls == 2 + assert llm.vision_calls == 2 # one per page — the first index embeds1 = len(llm.calls) doc1 = _pdf_doc(db, "stable", "stable.pdf") content1, digest1 = doc1.content, doc1.content_hash s2 = asyncio.run(import_sources([root], llm, session=db)) - # The fallback re-ran (gate re-fired, vision re-issued) — but - # the row itself re-imported as unchanged (no re-embed, no - # re-chunk, no KB change: the stable replies give the same - # digest, so the upsert is skipped entirely). + # The fast path fired (file hash match) — the fallback never + # ran on run 2: no gate, no render, no vision re-issued, no + # re-embed, no KB change. assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 1) - assert s2.pdfs_screenshotted == 1 # the gate re-fired - assert llm.vision_calls == 4 # the vision calls were re-issued + assert s2.pdfs_screenshotted == 0 # the gate never re-fired + assert llm.vision_calls == 2 # NO re-issued vision call assert len(llm.calls) == embeds1, "unchanged: no re-embed" assert s2.changed_paths == [], "unchanged: no KB change recorded" diff --git a/tests/unit/test_importer_unchanged_fast_path.py b/tests/unit/test_importer_unchanged_fast_path.py new file mode 100644 index 0000000..532c5be --- /dev/null +++ b/tests/unit/test_importer_unchanged_fast_path.py @@ -0,0 +1,626 @@ +"""Phase 147, task 02 — the unchanged fast path (unit matrix). + +The owner's TODO L4: "Local sources, like PDFs, are synced every time +even if they haven't changed. This results in really long sync times +for PDFs that are mostly pictoral." The fix: the sha256 of the RAW +source file's bytes (:func:`app.rag.importer._file_hash` — streamed in +1 MiB chunks) is computed ONCE at the top of ``_index_file``, BEFORE +the image/text/PDF dispatch, and compared against +``documents.file_hash`` (migration 0025). A match returns through the +shared unchanged path (``_unchanged_doc``) with ZERO llm calls (embed ++ chat, both counted through the fakes), ZERO PDF extraction, and +ZERO vision work — while the preserved unchanged semantics (the +phase-118 summary-NULL backfill, the phase-106 D1 manual-date lock, +the D4 date refresh, the counts, the log wordings) survive verbatim. + +Pinned here (the load-bearing pins are the ZERO-llm-call assertions on +the second sync): + +* the fast path fires for all three document classes (text, image, + garbage-layer PDF, pictorial PDF) and returns before any branch code; +* a changed file (any byte) re-indexes fully and re-stamps the hash; +* a bytes-changed/content-same file (a PDF metadata edit) re-runs the + branch's pipeline but lands unchanged (no content change) and + stamps the new hash — the next sync is fast; +* a pre-phase row (``file_hash`` NULL) runs the branch's content-digest + path (today's behavior) and gets STAMPED — the next sync is fast; +* the D1 manual-date lock and the D4 date refresh (may go older, + counts ``dates_updated``) hold on the fast path; +* the phase-118 backfill on the fast path runs from the STORED + ``doc.content`` (one ``is_summary`` chunk, no content re-embed); +* the streamed hash is transparent (a >1 MiB file hashes identically + to ``hashlib.sha256(bytes).hexdigest()``). + +The E2E surface (the idempotent second sync through the real UI + +mock LLM) is pinned in ``tests/e2e/test_sync_changed_only.py`` +(task 03). +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pymupdf as fitz +import pytest +from sqlalchemy import select + +from app.config import Settings +from app.models import Document +from app.rag.importer import _file_hash, import_sources +from tests.fakes import FakeEmbedder +from tests.unit.test_importer import ( + _build_garbage_pdf, + _PageVisionLLM, + _pdf_settings, + _png_bytes, + _VisionLLM, +) +from tests.unit.test_pictorial_import import ( + _build_ikea_like_pdf, + _PictorialLLM, + _settings, +) + +#: mtime granularity tolerance (the ``test_importer_dates.py`` +#: convention). +_TOL = timedelta(milliseconds=50) + +OLD_2021 = datetime(2021, 6, 1, 12, 0, 0, tzinfo=UTC) + + +def _utime(path: Path, when: datetime) -> None: + ts = when.timestamp() + os.utime(path, (ts, ts)) + + +def _doc(db, source: str, rel: str) -> Document: + doc = db.scalar(select(Document).where(Document.source == source, Document.path == rel)) + assert doc is not None, f"no documents row for ({source!r}, {rel!r})" + return doc + + +def _cleanup_source(db, source: str) -> None: + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + db.delete(doc) + db.commit() + + +# ---------- the streamed hash ------------------------------------------------- + + +def test_file_hash_streamed_matches_direct_hash(tmp_path: Path) -> None: + """The chunking is transparent: a >1 MiB file (1.5 MiB here — + spans multiple 1 MiB reads) hashes identically to + ``hashlib.sha256(bytes).hexdigest()``, an empty file hashes the + empty byte string, and the result is a 64-char lowercase hex + digest (the ``documents.file_hash`` column width).""" + block = hashlib.sha256(b"seed-147").digest() # 32 B + data = block * 49152 # 1.5 MiB — well past one 1 MiB chunk + big = tmp_path / "big.bin" + big.write_bytes(data) + expected = hashlib.sha256(data).hexdigest() + assert len(expected) == 64 and expected == expected.lower() + assert _file_hash(big) == expected + + empty = tmp_path / "empty.bin" + empty.write_bytes(b"") + assert _file_hash(empty) == hashlib.sha256(b"").hexdigest() + + +# ---------- the fast path: zero llm on the second sync ------------------------ + + +def test_text_unchanged_second_sync_costs_zero_llm(db, tmp_path: Path) -> None: + """Text class: index a file, capture the llm call count (embed + + chat both counted); re-run → ``unchanged == 1``, ZERO new llm + calls, ``files == 1``, the doc's ``indexed_at``/``content_hash`` + untouched, and ``file_hash`` stamped after run 1 (equal to + ``_file_hash`` of the raw bytes).""" + root = tmp_path / "fasttext" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\nmarkdown body\n") + llm = FakeEmbedder() + try: + first = asyncio.run(import_sources([root], llm, session=db)) + assert (first.added, first.unchanged, first.files) == (1, 0, 1) + doc = _doc(db, root.name, "note.md") + # Stamped after run 1 — the raw-bytes digest. + assert doc.file_hash == _file_hash(path) + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + indexed1, digest1 = doc.indexed_at, doc.content_hash + + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged, second.files) == (0, 0, 1, 1) + # THE pin: zero new llm calls of any kind. + assert len(llm.calls) == embeds1, "fast path: no re-embed" + assert len(llm.chat_calls) == chats1, "fast path: no lite/vision call" + assert second.changed_paths == [] # no KB change recorded + + db.expire_all() + doc2 = _doc(db, root.name, "note.md") + assert doc2.indexed_at == indexed1 # the re-index did not happen + assert doc2.content_hash == digest1 + assert doc2.file_hash == _file_hash(path) + finally: + _cleanup_source(db, root.name) + + +def test_garbage_pdf_unchanged_second_sync_costs_zero_vision(db, tmp_path: Path) -> None: + """Garbage-layer PDF class (the owner's core complaint): a no-text + layer's PDF indexed through the page-screenshot fallback (two + vision calls); re-run → ``unchanged == 1``, ZERO new llm calls — + the fallback vision did NOT re-fire (the pre-phase "vision + re-issued before the unchanged check can skip them" cost is + gone), the gate never re-fires, and the content row is + byte-identical.""" + root = tmp_path / "fastgarbage" + root.mkdir() + path = root / "garbage.pdf" + _build_garbage_pdf(path) + llm = _PageVisionLLM() + llm.settings = _pdf_settings(tmp_path) + try: + first = asyncio.run(import_sources([root], llm, session=db)) + assert (first.added, first.pdfs_screenshotted) == (1, 1) + assert llm.vision_calls == 2 # one per page — the first index + doc = _doc(db, root.name, "garbage.pdf") + assert doc.file_hash == _file_hash(path) + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + content1, digest1, indexed1 = doc.content, doc.content_hash, doc.indexed_at + + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + assert second.pdfs_screenshotted == 0 # the gate never re-fired + # THE pin: the fallback vision did NOT re-fire. + assert llm.vision_calls == 2, "fast path: no re-issued vision call" + assert len(llm.chat_calls) == chats1 + assert len(llm.calls) == embeds1, "fast path: no re-embed" + + db.expire_all() + doc2 = _doc(db, root.name, "garbage.pdf") + assert doc2.content == content1 # content row byte-identical + assert doc2.content_hash == digest1 + assert doc2.indexed_at == indexed1 + finally: + _cleanup_source(db, root.name) + + +def test_pictorial_pdf_unchanged_second_sync_costs_zero_vision( + db, tmp_path: Path +) -> None: + """Pictorial PDF class (the IKEA assembly-instruction class, + TODO L5): a sparse-page PDF indexed with the pictorial pass ON + (six vision calls); re-run → ``unchanged == 1``, ZERO new llm + calls — the pictorial pass did NOT re-fire (no re-render, no + vision), and the content is byte-stable.""" + root = tmp_path / "fastikea" + root.mkdir() + path = root / "ikea.pdf" + _build_ikea_like_pdf(path) + llm = _PictorialLLM() + llm.settings = _settings(tmp_path) # pdf_pictorial ON (code default) + try: + first = asyncio.run(import_sources([root], llm, session=db)) + assert (first.added, first.pdfs_pictorial, first.pdf_pictorial_pages) == (1, 1, 6) + assert llm.pictorial_calls == 6 # one per candidate — the first index + doc = _doc(db, root.name, "ikea.pdf") + assert doc.file_hash == _file_hash(path) + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + content1, digest1 = doc.content, doc.content_hash + + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + assert second.pdfs_pictorial == 0 # the pass did not re-fire + # THE pin: the pictorial vision did NOT re-fire. + assert llm.pictorial_calls == 6, "fast path: no re-issued vision call" + assert len(llm.chat_calls) == chats1 + assert len(llm.calls) == embeds1, "fast path: no re-embed" + + db.expire_all() + doc2 = _doc(db, root.name, "ikea.pdf") + assert doc2.content == content1 # byte-stable content + assert doc2.content_hash == digest1 + finally: + _cleanup_source(db, root.name) + + +def test_image_unchanged_second_sync_costs_zero_llm(db, tmp_path: Path) -> None: + """Image class: a standalone image indexed through the vision + describe (one chat call); re-run → ``unchanged == 1``, ZERO new + llm calls — the describe did NOT re-fire, no re-embed, and the + row (content + ``indexed_at``) is untouched.""" + root = tmp_path / "fastimg" + root.mkdir() + path = root / "pic.png" + path.write_bytes(_png_bytes(4, 4, (0.2, 0.4, 0.6))) + llm = _VisionLLM() + llm.settings = Settings( + _env_file=None, # pyright: ignore[reportCallIssue] (kwarg exists at runtime) + images=True, + image_dir=str(tmp_path / "images"), + ) + try: + first = asyncio.run(import_sources([root], llm, session=db)) + assert (first.added, first.images_failed) == (1, 0) + doc = _doc(db, root.name, "pic.png") + assert doc.file_hash == _file_hash(path) + assert doc.content == _VisionLLM.DESCRIPTION + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + indexed1 = doc.indexed_at + + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + # THE pin: no re-describe, no re-embed. + assert len(llm.chat_calls) == chats1, "fast path: no re-describe" + assert len(llm.calls) == embeds1, "fast path: no re-embed" + + db.expire_all() + doc2 = _doc(db, root.name, "pic.png") + assert doc2.content == _VisionLLM.DESCRIPTION + assert doc2.indexed_at == indexed1 + finally: + _cleanup_source(db, root.name) + + +# ---------- changed files: full re-index + re-stamp --------------------------- + + +def test_changed_file_reindexes_and_re_stamps(db, tmp_path: Path) -> None: + """A changed file (one byte flipped in a text file) re-indexes + FULLY — the llm calls fire again (embed + summary) — and + ``file_hash`` is RE-STAMPED with the new raw-bytes digest; + ``content_hash`` moves with the content.""" + root = tmp_path / "fastchange" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\nv1\n") + llm = FakeEmbedder() + try: + first = asyncio.run(import_sources([root], llm, session=db)) + assert first.added == 1 + doc = _doc(db, root.name, "note.md") + hash1, content_digest1 = doc.file_hash, doc.content_hash + assert hash1 == _file_hash(path) + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + + path.write_text("# Note\n\nv2\n") # the bytes changed + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 1, 0) + # The full pipeline re-fired. + assert len(llm.calls) > embeds1, "a changed file re-embeds" + assert len(llm.chat_calls) > chats1, "a changed file re-summarizes" + + db.expire_all() + doc2 = _doc(db, root.name, "note.md") + assert doc2.file_hash == _file_hash(path) # re-stamped + assert doc2.file_hash != hash1 + assert doc2.content_hash != content_digest1 + finally: + _cleanup_source(db, root.name) + + +def test_pdf_bytes_changed_content_same_stamps_without_content_change( + db, tmp_path: Path +) -> None: + """The bytes-changed/content-same edge (a PDF metadata edit with + identical extracted text): the fast path does NOT fire (the raw + bytes moved), the branch re-runs its pipeline (the fallback's + vision re-issues — the honest cost of a byte change), but the + CONTENT digest matches — the row is UNCHANGED (no content change, + no re-embed), and the shared helper STAMPS the new hash, so the + NEXT sync takes the fast path (zero further vision).""" + root = tmp_path / "fastmeta" + root.mkdir() + path = root / "meta.pdf" + _build_garbage_pdf(path) + llm = _PageVisionLLM() + llm.settings = _pdf_settings(tmp_path) + try: + first = asyncio.run(import_sources([root], llm, session=db)) + assert (first.added, first.pdfs_screenshotted) == (1, 1) + doc = _doc(db, root.name, "meta.pdf") + content1, digest1 = doc.content, doc.content_hash + hash1 = doc.file_hash + assert llm.vision_calls == 2 + embeds1 = len(llm.calls) + + # A metadata-only edit: the raw bytes change, the extracted + # text (and the rendered pages) do not. + with fitz.open(str(path)) as d: + d.set_metadata({"title": "metadata-only edit"}) + d.saveIncr() + new_hash = _file_hash(path) + assert new_hash != hash1 # the bytes really moved + + second = asyncio.run(import_sources([root], llm, session=db)) + # The branch re-ran (gate re-fired, vision re-issued — the byte + # change's cost) but the content digest matched: the row is + # unchanged, no content change, no re-embed. + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + assert second.pdfs_screenshotted == 1 # the branch re-ran its gate + assert llm.vision_calls == 4 # the fallback re-issued (bytes changed) + assert len(llm.calls) == embeds1, "content-same: no re-embed" + + db.expire_all() + doc2 = _doc(db, root.name, "meta.pdf") + assert doc2.content == content1 # content byte-identical + assert doc2.content_hash == digest1 + assert doc2.file_hash == new_hash # STAMPED by the shared helper + + # The NEXT sync is fast (the stamp landed): zero further vision. + third = asyncio.run(import_sources([root], llm, session=db)) + assert (third.added, third.updated, third.unchanged) == (0, 0, 1) + assert third.pdfs_screenshotted == 0 + assert llm.vision_calls == 4, "the third sync is the fast path" + assert len(llm.calls) == embeds1 + finally: + _cleanup_source(db, root.name) + + +# ---------- pre-phase rows (NULL file_hash) ----------------------------------- + + +def test_pre_phase_null_file_hash_runs_branch_path_and_stamps( + db, tmp_path: Path +) -> None: + """A row that predates the column (``file_hash`` NULL) + an + unchanged file: the fast path cannot fire, so the branch's + content-digest path runs (today's behavior — the digest matches, + no re-embed), and the shared helper STAMPS the hash — the THIRD + sync then takes the fast path (proof the stamp landed).""" + root = tmp_path / "fastlegacy" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\nmarkdown body\n") + llm = FakeEmbedder() + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = _doc(db, root.name, "note.md") + doc.file_hash = None # simulate the pre-phase row + db.commit() + + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + # The content digest matched — no re-embed, no re-summary. + assert len(llm.calls) == embeds1 + assert len(llm.chat_calls) == chats1 + + db.expire_all() + doc2 = _doc(db, root.name, "note.md") + assert doc2.file_hash == _file_hash(path) # stamped by the helper + + # The third sync takes the FAST PATH (the stamp is committed): + third = asyncio.run(import_sources([root], llm, session=db)) + assert third.unchanged == 1 + assert len(llm.calls) == embeds1 + assert len(llm.chat_calls) == chats1 + finally: + _cleanup_source(db, root.name) + + +def test_pre_phase_null_file_hash_image_runs_branch_and_stamps( + db, tmp_path: Path +) -> None: + """The image flavour of the branch path: a pre-phase row + (``file_hash`` NULL) + an unchanged image — the fast path cannot + fire, so the image branch's content-digest check runs (the raw + BYTES digest — the image's content identity — matches BEFORE the + expensive describe, so no re-describe, no re-embed), the row is + unchanged, and the shared helper STAMPS the hash — the next sync + is fast. + """ + root = tmp_path / "fastimglegacy" + root.mkdir() + path = root / "pic.png" + path.write_bytes(_png_bytes(4, 4, (0.2, 0.4, 0.6))) + llm = _VisionLLM() + llm.settings = Settings( + _env_file=None, # pyright: ignore[reportCallIssue] (kwarg exists at runtime) + images=True, + image_dir=str(tmp_path / "images"), + ) + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = _doc(db, root.name, "pic.png") + doc.file_hash = None # simulate the pre-phase row + db.commit() + + chats1 = len(llm.chat_calls) + embeds1 = len(llm.calls) + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + # The branch's byte-digest check matched before the describe — + # zero llm work of any kind. + assert len(llm.chat_calls) == chats1 + assert len(llm.calls) == embeds1, "content-same: no re-embed" + + db.expire_all() + doc2 = _doc(db, root.name, "pic.png") + assert doc2.file_hash == _file_hash(path) # stamped by the helper + assert doc2.content == _VisionLLM.DESCRIPTION + + # The next sync takes the fast path (zero further calls): + third = asyncio.run(import_sources([root], llm, session=db)) + assert third.unchanged == 1 + assert len(llm.chat_calls) == chats1 + assert len(llm.calls) == embeds1 + finally: + _cleanup_source(db, root.name) + + +# ---------- the preserved semantics on the fast path -------------------------- + + +def test_manual_date_row_fast_path_no_date_write(db, tmp_path: Path) -> None: + """D1 on the fast path: an unchanged file carrying the owner's + correction (``created_at_manual``) gets NO date write — the moved + mtime does not refresh it, ``dates_updated`` stays 0, the + correction + the flag survive, and the row's stamp is intact + (zero llm calls).""" + root = tmp_path / "fastmanual" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\ncontent for the KB\n") + llm = FakeEmbedder() + correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC) + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = _doc(db, root.name, "note.md") + doc.created_at = correction + doc.created_at_manual = True + db.commit() + _utime(path, OLD_2021) # the source moved to a DIFFERENT date + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + assert second.dates_updated == 0 # the manual lock withheld it + assert len(llm.calls) == embeds1 and len(llm.chat_calls) == chats1 + + db.expire_all() + doc2 = _doc(db, root.name, "note.md") + assert doc2.created_at == correction # the correction survived + assert doc2.created_at_manual is True # the flag intact + assert doc2.file_hash == _file_hash(path) # the stamp intact + finally: + _cleanup_source(db, root.name) + + +def test_manual_date_null_hash_row_stamps_without_date_write( + db, tmp_path: Path +) -> None: + """D1 on the BRANCH path (a pre-phase NULL-hash row that also + carries the manual correction): the shared helper's stamp is the + only write — it COMMITS (so the next sync is fast even for a + manual-date row) — while the DATE stays the manual value + (``dates_updated`` 0, the flag intact).""" + root = tmp_path / "fastmanualnull" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\ncontent for the KB\n") + llm = FakeEmbedder() + correction = datetime(2023, 5, 5, 9, 30, 0, tzinfo=UTC) + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = _doc(db, root.name, "note.md") + doc.created_at = correction + doc.created_at_manual = True + doc.file_hash = None # pre-phase row + db.commit() + _utime(path, OLD_2021) # a refresh is DUE — the lock must withhold it + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + assert second.dates_updated == 0 + # No summary work of any kind on this run (run 1's summary + # count is the baseline). + assert len(llm.chat_calls) == chats1 + assert len(llm.calls) == embeds1 + + db.expire_all() + doc2 = _doc(db, root.name, "note.md") + assert doc2.created_at == correction # the correction survived + assert doc2.created_at_manual is True + assert doc2.file_hash == _file_hash(path) # the stamp committed + + # The next sync is fast (the stamp landed despite the D1 lock): + third = asyncio.run(import_sources([root], llm, session=db)) + assert third.unchanged == 1 and third.dates_updated == 0 + db.expire_all() + assert _doc(db, root.name, "note.md").created_at == correction + finally: + _cleanup_source(db, root.name) + + +def test_unchanged_fast_path_date_refresh_may_go_older( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """D4 on the fast path: an unchanged file whose mtime-derived date + moved (OLDER — no monotonic guard) refreshes the stored date on + the fast path: ``dates_updated == 1``, the date lands, and zero + llm calls (the refresh is bookkeeping, not a re-index). The + PRE-CHANGE log wordings are pinned verbatim (``import: unchanged`` + + ``import: date-refreshed``).""" + root = tmp_path / "fastdate" + root.mkdir() + path = root / "note.md" + path.write_text("# Note\n\ncontent for the KB\n") + llm = FakeEmbedder() + try: + asyncio.run(import_sources([root], llm, session=db)) + _utime(path, OLD_2021) # the source date moved; content identical + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + + with caplog.at_level(logging.INFO, logger="app.importer"): + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.unchanged) == (0, 0, 1) + assert second.dates_updated == 1 # the refresh ran (fast path) + assert len(llm.calls) == embeds1 and len(llm.chat_calls) == chats1 + # The preserved wordings (the shared helper's log lines, verbatim + # from the per-branch blocks). + messages = [r.getMessage() for r in caplog.records] + assert f"import: unchanged source={root.name} path=note.md" in messages + assert any( + m.startswith(f"import: date-refreshed source={root.name} path=note.md date=") + for m in messages + ) + + db.expire_all() + doc2 = _doc(db, root.name, "note.md") + assert abs(doc2.created_at - OLD_2021) <= _TOL + finally: + _cleanup_source(db, root.name) + + +def test_null_summary_backfill_on_fast_path(db, tmp_path: Path) -> None: + """Phase 118 (A2) via the fast path: an unchanged doc whose + summary is still NULL (the stored content is intact) gets the + summary-only backfill from the STORED ``doc.content`` — one + ``is_summary`` chunk, no content re-embed, counted + ``summary_backfilled`` (never ``summaries``), never + added/updated/pruned. The ``lite`` call fires (the backfill's own + work — the fast path's "zero llm calls" is "unless the backfill + fires"); nothing else does.""" + root = tmp_path / "fastbfill" + root.mkdir() + (root / "note.md").write_text("# Note\n\nmarkdown body\n") + llm = FakeEmbedder() + try: + asyncio.run(import_sources([root], llm, session=db)) + doc = _doc(db, root.name, "note.md") + # Simulate the NULL-summary target row: the content stays, only + # the summary + its chunk go away (the phase-118 convention). + doc.summary = None + for c in [c for c in doc.chunks if c.is_summary]: + doc.chunks.remove(c) + db.commit() + assert doc.file_hash is not None # the fast path can fire + + embeds1, chats1 = len(llm.calls), len(llm.chat_calls) + second = asyncio.run(import_sources([root], llm, session=db)) + assert (second.added, second.updated, second.pruned) == (0, 0, 0) + assert (second.unchanged, second.summary_backfilled, second.summaries) == (1, 1, 0) + assert second.summary_errors == 0 + # Exactly the backfill's own work: one lite call, one embed + # batch (the summary chunk only — no content re-embed). + assert len(llm.chat_calls) == chats1 + 1 + assert len(llm.calls) == embeds1 + 1 + assert second.changed_paths == [] # no KB change recorded + + db.expire_all() + doc2 = _doc(db, root.name, "note.md") + assert doc2.summary is not None + schunks = [c for c in doc2.chunks if c.is_summary] + assert len(schunks) == 1 and schunks[0].position == -1 + assert schunks[0].content == doc2.summary + content = [c for c in doc2.chunks if not c.is_summary] + assert len(content) == 1 and content[0].embedding is not None + finally: + _cleanup_source(db, root.name) diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index b513f4f..9ca8591 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -128,6 +128,81 @@ def test_documents_unique_source_path() -> None: assert uq, "documents must be unique on (source, path) — the upsert key" +def test_documents_file_hash_column_contract() -> None: + """Phase 147 (task 01): ``documents.file_hash`` — the sha256 of the + RAW SOURCE FILE's bytes (the sync's change signal): VARCHAR(64), + NULLABLE (NULL = "unknown" — pre-phase rows until the first + post-upgrade sync stamps it), NO server default, and NO index + (the fast path compares it in Python after the unique + ``(source, path)`` lookup — internal bookkeeping, never a query + surface). ``content_hash`` (0001) stays untouched: NOT NULL and + still indexed — the retriever's dedupe/caching identity.""" + from sqlalchemy import String as SAString + + documents = Base.metadata.tables["documents"] + col = documents.c["file_hash"] + assert col.nullable is True, ("file_hash must be NULLABLE — NULL is the pre-phase state") + assert isinstance(col.type, SAString), "file_hash must be String(64)" + assert col.type.length == 64, "file_hash must be String(64) — a sha256 hex digest" + assert col.server_default is None, ( + "file_hash must carry NO server default — NULL is the meaningful" + " pre-phase state" + ) + # No index: neither the inline flag (None = unspecified) nor any + # table-level index may cover file_hash (unlike content_hash — a + # query/exposure surface). + assert not col.index, ("file_hash must be UNINDEXED — compared in Python") + covering = [ + idx + for idx in documents.indexes + if any(c.name == "file_hash" for c in idx.columns) + ] + assert not covering, "no table-level index may cover file_hash" + uq_covering = [ + c + for c in documents.constraints + if isinstance(c, UniqueConstraint) and any(col.name == "file_hash" for col in c.columns) + ] + assert not uq_covering, "no unique constraint may cover file_hash" + + # content_hash is untouched: still NOT NULL and indexed. + content = documents.c["content_hash"] + assert content.nullable is False, ("content_hash must stay NOT NULL (0001)") + assert content.index is True, "content_hash must stay INDEXED (0001)" + content_covering = [ + idx for idx in documents.indexes if any(c.name == "content_hash" for c in idx.columns) + ] + assert content_covering, "the content_hash index must survive phase 147" + + +def test_document_file_hash_orm_defaults_to_none() -> None: + """A freshly constructed ``Document`` (no DB) exposes ``file_hash``: + the omitted kwarg and explicit ``None`` read NULL (the pre-phase + shape — NULL is backfilled with a real digest only by the first + post-upgrade sync), and a set digest round-trips as-is (the real + DB round-trip is pinned in ``test_migration_0025.py``).""" + from app.models import Document + + base = dict( + source="s", + path="p.md", + full_path="/tmp/s/p.md", + title="p", + content="c", + content_hash="1" * 64, + ) + stamped = Document(**base, file_hash="a" * 64) + assert stamped.file_hash == "a" * 64 + + explicit_null = Document(**base, file_hash=None) + assert explicit_null.file_hash is None + + omitted = Document(**base) + assert omitted.file_hash is None, ( + "an omitted file_hash must read NULL (the pre-phase shape)" + ) + + def test_doc_drafts_token_is_unique_not_null() -> None: """Phase 59: the draft's URL credential — an unguessable uuid4, UNIQUE + NOT NULL (no "un-drafted" state, unlike the NULLable diff --git a/tests/unit/test_pictorial_import.py b/tests/unit/test_pictorial_import.py index 4ee007d..65b8eb4 100644 --- a/tests/unit/test_pictorial_import.py +++ b/tests/unit/test_pictorial_import.py @@ -28,11 +28,12 @@ fixtures; the real tar is pinned in others are described), the unreadable-page placeholder skip (the zero-byte ``render_pdf_pages`` slot burns no vision call, the page keeps its text only). -* the digest-stability rerun — a stable file + a stable vision reply - re-syncs as ``unchanged`` (no re-embed, byte-stable content + - digest); the vision calls are re-issued before the unchanged check - can skip them — the documented no-cache cost (the phase-133 - convention, A21 endorses it). +* the phase-147 rerun — an unchanged file re-syncs through the FAST + PATH (the raw bytes are the change signal): ``unchanged``, no + re-embed, byte-stable content + digest — and the vision calls are + NOT re-issued (the pass never runs on an unchanged file; the + pre-phase "vision re-issued before the unchanged check" no-cache + cost is gone). The E2E surface (the real UI + the mock LLM's ``PICTORIAL_PAGE_MODE`` branch) is pinned in ``tests/e2e/test_pictorial_pdf.py`` (task 04). @@ -565,18 +566,22 @@ def test_unreadable_page_placeholder_skips_the_vision_call( _cleanup_source(db, "halfblind") -def test_rerun_lands_unchanged_with_byte_stable_content( +def test_rerun_lands_unchanged_fast_path_no_revision( db, tmp_path: Path ) -> None: - """Digest stability (A4 — the phase-126/133 convention): the - digest runs over the FINAL content, so a re-sync of an unchanged - IKEA-class PDF (same pages + the stable fake's same replies) - re-imports as UNCHANGED: no re-chunk, no re-embed, no re-summary, - no KB change (not ``updated``, no changed path), and the content + - digest are byte-stable. The vision calls ARE re-issued before the - unchanged check can skip them — the documented no-cache cost (the - phase-133 convention; a non-deterministic vision model re-describes - the row each sync — a cost, not a bug; A21 endorses it).""" + """Phase 147 (task 02) — the owner's "really long sync times for + PDFs that are mostly pictoral" complaint, fixed: a re-sync of an + unchanged IKEA-class PDF (same raw bytes) takes the FAST PATH at + the top of ``_index_file`` (the file hash matches) — the + pictorial pass does NOT re-run: no re-render, the vision calls are + NOT re-issued (``pictorial_calls`` stays at run 1's 6 — the + pre-phase "vision re-issued before the unchanged check can skip + them" no-cache cost is gone), ``pdfs_pictorial`` stays 0 on run 2. + The row re-imports as UNCHANGED: no re-chunk, no re-embed, no + re-summary, no KB change (not ``updated``, no changed path), and + the content + digest are byte-stable. (A non-deterministic vision + model no longer flips the row's digest either — the raw bytes + decide.)""" root = tmp_path / "stable" root.mkdir() pdf = root / "ikea.pdf" @@ -586,17 +591,18 @@ def test_rerun_lands_unchanged_with_byte_stable_content( try: s1 = asyncio.run(import_sources([root], llm, session=db)) assert (s1.added, s1.pdfs_pictorial, s1.pdf_pictorial_pages) == (1, 1, 6) - assert llm.pictorial_calls == 6 + assert llm.pictorial_calls == 6 # one per candidate — first index embeds1 = len(llm.calls) doc1 = _pdf_doc(db, "stable", "ikea.pdf") content1, digest1 = doc1.content, doc1.content_hash s2 = asyncio.run(import_sources([root], llm, session=db)) - # The pass re-ran (gate re-fired, vision re-issued — the - # documented cost) — but the row itself re-imported unchanged. + # The fast path fired (file hash match) — the pass never ran + # on run 2: no render, no vision re-issued, no re-embed, no KB + # change. assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 1) - assert s2.pdfs_pictorial == 1 # the pass re-fired - assert llm.pictorial_calls == 12 # the vision calls were re-issued + assert s2.pdfs_pictorial == 0 # the pass did not re-fire + assert llm.pictorial_calls == 6 # NO re-issued vision call assert len(llm.calls) == embeds1, "unchanged: no re-embed" assert s2.changed_paths == [], "unchanged: no KB change recorded"