From 9633eb7653e8a8d123bd376236355ea2a92a3ac5 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Wed, 30 Sep 2026 19:16:35 -0400 Subject: [PATCH] phase: 140_pictorial_pdf_pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything is verified green. Final report: ## Phase 140 final verification pass — report - Verified all 4 tasks already complete (code in working tree, task files in `complete/`): `pictorial_pages` gate + 2 settings (`BOR_PDF_PICTORIAL`, `BOR_PDF_PICTORIAL_MIN_CHARS` + validator), `PICTORIAL_PAGE_MODE`/`describe_pdf_page` seam + `render_pdf_pages` page selector + mock branch, `_pictorial_pdf_pass` in `_index_pdf_file` with log-only counters, dedicated E2E. No code changes needed. - Test results (exact commands): - `uv run pytest --cov=app --cov-report=term-missing` → **3320 passed, 11 skipped, 0 failed; TOTAL 99%** (>90% gate) - `uv run pytest tests/e2e/test_pictorial_pdf.py -v --no-cov` → **3 passed in isolation** (Playwright, mock-only) - Pre-existing suites unedited: `test_pdf_support.py` 6 passed; `test_garbage_pdf.py` 2 passed + 1 skipped (owner fixture); `test_pdf_image_context.py` 4 passed - `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors - Completion criteria: 1. IKEA tar import repro — pinned by 2 integration tests; **skipped loudly here** (`test_sources/ikea_instructions.tar` absent on this machine — verified repo-wide; same house convention as completed phase-133 HNB tests; runs real on the owner's machine). Equivalent logic fully covered by unit pins on synthetic IKEA-class (vector-drawn, zero-raster) PDFs. 2. Chat grounded on described page — **green** (E2E test b: phrase-only-in-illustration question, PDF in `done.sources`, answer echoes the block) 3. Clean-text no-op (zero vision calls) + garbage precedence pin — **green** (`test_zero_candidates_is_a_noop` unit; `test_garbage_pdf_with_pictorial_on_takes_the_fallback_not_the_pass` integration, verified alone) 4. `BOR_PDF_PICTORIAL=0` byte-identical — **green** (unit pin on synthetic PDF; IKEA variant runs on owner's machine) 5. Full gate + E2E isolation + pre-existing suites — **green** (above) 6. ruff + pyright — **clean** 7. Commit/phase-move — delegated to harness per run rules (left in working tree: 16 modified + 8 new files, uncommitted) - Deviations: none beyond the expected owner-fixture skips (11, all pre-existing fixture-missing skips, including 2 IKEA). - Next pending phase: **none** — `todo/` holds only 140; 139 is already complete. --- .../140_pictorial_pdf_pages/00_phase.md | 0 .../01_settings_and_gate.md | 0 .../02_pictorial_vision_seam.md | 0 .../03_importer_pictorial_pass.md | 0 .../04_e2e_pictorial_pdf.md | 0 .../140_pictorial_pdf_pages__00_phase.a1.err | 0 .../140_pictorial_pdf_pages__00_phase.a1.md | 20 + ..._pictorial_pdf_pages__00_phase.a1.validate | 118 +++ ...ial_pdf_pages__01_settings_and_gate.a1.err | 0 ...rial_pdf_pages__01_settings_and_gate.a1.md | 12 + ...df_pages__01_settings_and_gate.a1.validate | 117 +++ ...pdf_pages__02_pictorial_vision_seam.a1.err | 0 ..._pdf_pages__02_pictorial_vision_seam.a1.md | 1 + ...ages__02_pictorial_vision_seam.a1.validate | 118 +++ ...pdf_pages__02_pictorial_vision_seam.a2.err | 0 ..._pdf_pages__02_pictorial_vision_seam.a2.md | 1 + ...ages__02_pictorial_vision_seam.a2.validate | 118 +++ ...f_pages__03_importer_pictorial_pass.a1.err | 0 ...df_pages__03_importer_pictorial_pass.a1.md | 19 + ...es__03_importer_pictorial_pass.a1.validate | 118 +++ ...f_pages__03_importer_pictorial_pass.a2.err | 0 ...df_pages__03_importer_pictorial_pass.a2.md | 1 + ...es__03_importer_pictorial_pass.a2.validate | 118 +++ ...f_pages__03_importer_pictorial_pass.a3.err | 0 ...df_pages__03_importer_pictorial_pass.a3.md | 1 + ...es__03_importer_pictorial_pass.a3.validate | 118 +++ ...ial_pdf_pages__04_e2e_pictorial_pdf.a1.err | 0 ...rial_pdf_pages__04_e2e_pictorial_pdf.a1.md | 17 + ...df_pages__04_e2e_pictorial_pdf.a1.validate | 118 +++ .env.example | 17 + app/config.py | 40 + app/rag/importer.py | 192 ++++- app/rag/pdf.py | 99 ++- app/rag/summarizer.py | 152 +++- tests/conftest.py | 11 + tests/e2e/conftest.py | 11 + tests/e2e/mock_llm.py | 35 + tests/e2e/test_pictorial_pdf.py | 729 ++++++++++++++++++ tests/integration/test_importer.py | 332 +++++++- .../test_pdf_hnb_guide_regression.py | 13 +- tests/unit/test_importer.py | 24 +- tests/unit/test_pdf.py | 8 +- tests/unit/test_pictorial_gate.py | 196 +++++ tests/unit/test_pictorial_import.py | 607 +++++++++++++++ tests/unit/test_pictorial_seam.py | 450 +++++++++++ 45 files changed, 3895 insertions(+), 36 deletions(-) rename .agents/phases/{todo => complete}/140_pictorial_pdf_pages/00_phase.md (100%) rename .agents/phases/{todo => complete}/140_pictorial_pdf_pages/01_settings_and_gate.md (100%) rename .agents/phases/{todo => complete}/140_pictorial_pdf_pages/02_pictorial_vision_seam.md (100%) rename .agents/phases/{todo => complete}/140_pictorial_pdf_pages/03_importer_pictorial_pass.md (100%) rename .agents/phases/{todo => complete}/140_pictorial_pdf_pages/04_e2e_pictorial_pdf.md (100%) create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.err create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.md create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.validate create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.err create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.md create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.validate create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.err create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.md create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.validate create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.err create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.md create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.validate create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.err create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.md create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.validate create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.err create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.md create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.validate create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.err create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.md create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.validate create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.err create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.md create mode 100644 .agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.validate create mode 100644 tests/e2e/test_pictorial_pdf.py create mode 100644 tests/unit/test_pictorial_gate.py create mode 100644 tests/unit/test_pictorial_import.py create mode 100644 tests/unit/test_pictorial_seam.py diff --git a/.agents/phases/todo/140_pictorial_pdf_pages/00_phase.md b/.agents/phases/complete/140_pictorial_pdf_pages/00_phase.md similarity index 100% rename from .agents/phases/todo/140_pictorial_pdf_pages/00_phase.md rename to .agents/phases/complete/140_pictorial_pdf_pages/00_phase.md diff --git a/.agents/phases/todo/140_pictorial_pdf_pages/01_settings_and_gate.md b/.agents/phases/complete/140_pictorial_pdf_pages/01_settings_and_gate.md similarity index 100% rename from .agents/phases/todo/140_pictorial_pdf_pages/01_settings_and_gate.md rename to .agents/phases/complete/140_pictorial_pdf_pages/01_settings_and_gate.md diff --git a/.agents/phases/todo/140_pictorial_pdf_pages/02_pictorial_vision_seam.md b/.agents/phases/complete/140_pictorial_pdf_pages/02_pictorial_vision_seam.md similarity index 100% rename from .agents/phases/todo/140_pictorial_pdf_pages/02_pictorial_vision_seam.md rename to .agents/phases/complete/140_pictorial_pdf_pages/02_pictorial_vision_seam.md diff --git a/.agents/phases/todo/140_pictorial_pdf_pages/03_importer_pictorial_pass.md b/.agents/phases/complete/140_pictorial_pdf_pages/03_importer_pictorial_pass.md similarity index 100% rename from .agents/phases/todo/140_pictorial_pdf_pages/03_importer_pictorial_pass.md rename to .agents/phases/complete/140_pictorial_pdf_pages/03_importer_pictorial_pass.md diff --git a/.agents/phases/todo/140_pictorial_pdf_pages/04_e2e_pictorial_pdf.md b/.agents/phases/complete/140_pictorial_pdf_pages/04_e2e_pictorial_pdf.md similarity index 100% rename from .agents/phases/todo/140_pictorial_pdf_pages/04_e2e_pictorial_pdf.md rename to .agents/phases/complete/140_pictorial_pdf_pages/04_e2e_pictorial_pdf.md diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.err b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.md b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.md new file mode 100644 index 0000000..fcd6acf --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.md @@ -0,0 +1,20 @@ +Everything is verified green. Final report: + +## Phase 140 final verification pass — report + +- Verified all 4 tasks already complete (code in working tree, task files in `complete/`): `pictorial_pages` gate + 2 settings (`BOR_PDF_PICTORIAL`, `BOR_PDF_PICTORIAL_MIN_CHARS` + validator), `PICTORIAL_PAGE_MODE`/`describe_pdf_page` seam + `render_pdf_pages` page selector + mock branch, `_pictorial_pdf_pass` in `_index_pdf_file` with log-only counters, dedicated E2E. No code changes needed. +- Test results (exact commands): + - `uv run pytest --cov=app --cov-report=term-missing` → **3320 passed, 11 skipped, 0 failed; TOTAL 99%** (>90% gate) + - `uv run pytest tests/e2e/test_pictorial_pdf.py -v --no-cov` → **3 passed in isolation** (Playwright, mock-only) + - Pre-existing suites unedited: `test_pdf_support.py` 6 passed; `test_garbage_pdf.py` 2 passed + 1 skipped (owner fixture); `test_pdf_image_context.py` 4 passed + - `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors +- Completion criteria: + 1. IKEA tar import repro — pinned by 2 integration tests; **skipped loudly here** (`test_sources/ikea_instructions.tar` absent on this machine — verified repo-wide; same house convention as completed phase-133 HNB tests; runs real on the owner's machine). Equivalent logic fully covered by unit pins on synthetic IKEA-class (vector-drawn, zero-raster) PDFs. + 2. Chat grounded on described page — **green** (E2E test b: phrase-only-in-illustration question, PDF in `done.sources`, answer echoes the block) + 3. Clean-text no-op (zero vision calls) + garbage precedence pin — **green** (`test_zero_candidates_is_a_noop` unit; `test_garbage_pdf_with_pictorial_on_takes_the_fallback_not_the_pass` integration, verified alone) + 4. `BOR_PDF_PICTORIAL=0` byte-identical — **green** (unit pin on synthetic PDF; IKEA variant runs on owner's machine) + 5. Full gate + E2E isolation + pre-existing suites — **green** (above) + 6. ruff + pyright — **clean** + 7. Commit/phase-move — delegated to harness per run rules (left in working tree: 16 modified + 8 new files, uncommitted) +- Deviations: none beyond the expected owner-fixture skips (11, all pre-existing fixture-missing skips, including 2 IKEA). +- Next pending phase: **none** — `todo/` holds only 140; 139 is already complete. diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.validate b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.validate new file mode 100644 index 0000000..4a0b7de --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__00_phase.a1.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +...........................................................s.....sss...s [ 54%] +s....................................................................... [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 92%] +...........ss........................................................... [ 95%] +..................................................sss................... [ 97%] +........................................................................ [ 99%] +................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 412 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5488 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.err b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.md b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.md new file mode 100644 index 0000000..d3452be --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.md @@ -0,0 +1,12 @@ +All 9 skips are pre-existing owner-fixture skips (untracked `test_sources/` files), unrelated to this change. Task 01 is complete. + +**Report — Phase 140, Task 01 (settings + sparse-page gate):** +- `app/config.py`: `pdf_pictorial: bool = True` (`BOR_PDF_PICTORIAL`, house kill-switch, default ON per LOCKED A3) + `pdf_pictorial_min_chars: int = 100` (`BOR_PDF_PICTORIAL_MIN_CHARS`, ≥0 validator, negative fails startup loudly naming the field) +- `app/rag/pdf.py`: `pictorial_pages(page_texts, *, min_chars)` — pure, deterministic, no-LLM gate (LOCKED A1); docstring carries the contract + IKEA example; module docstring bullet added +- `.env.example`: both settings documented (neighboring comment-block pattern) +- `tests/unit/test_pictorial_gate.py` (new, 14 tests): defaults + env overrides, validator (0/100 ok, −1 raises named), `<`/`=`/`>` boundary pinned, whitespace-only page, empty dict → `[]`, ascending order, 500-page dict, IKEA example, purity/no-mutation +- **Notable decision:** the spec documents "0 = every page is a candidate (A21)" three places, but strict `<` can't express it (`len < 0` is never true — 0 would silently mean *zero* candidates, the opposite of A21). Implemented 0 (and defensively ≤0) as the documented "describe everything" special case, pinned in a test; strict boundary holds for all positive thresholds per ASSUMPTION A1. +- `uv run pytest tests/unit/test_pictorial_gate.py -v` → 14 passed +- `uv run pytest --cov=app --cov-report=term-missing` → 3297 passed, 9 skipped (pre-existing owner-fixture skips), TOTAL **99%** (>90%); `app/rag/pdf.py` 100% +- `uv run ruff check .` → clean; `uv run pyright` → 0 errors +- Next pending task: `.agents/phases/todo/140_pictorial_pdf_pages/02_pictorial_vision_seam.md` diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.validate b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.validate new file mode 100644 index 0000000..ec3c9f5 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__01_settings_and_gate.a1.validate @@ -0,0 +1,117 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 37%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 50%] +........................................................................ [ 52%] +...........................................................s.....sss...s [ 54%] +s....................................................................... [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 74%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +.........................sss............................................ [ 98%] +.................................................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 412 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 671 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 123 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5444 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.err b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.md b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.md new file mode 100644 index 0000000..9e9c741 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.md @@ -0,0 +1 @@ +Coverage is 99% total with both modified modules at 100%. Now lint + types: diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.validate b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.validate new file mode 100644 index 0000000..15403c9 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a1.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 52%] +...........................................................s.....sss...s [ 54%] +s....................................................................... [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................sss............................. [ 97%] +........................................................................ [ 99%] +......... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 412 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 671 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5454 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.err b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.md b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.md new file mode 100644 index 0000000..9e9c741 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.md @@ -0,0 +1 @@ +Coverage is 99% total with both modified modules at 100%. Now lint + types: diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.validate b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.validate new file mode 100644 index 0000000..15403c9 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__02_pictorial_vision_seam.a2.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 52%] +...........................................................s.....sss...s [ 54%] +s....................................................................... [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................sss............................. [ 97%] +........................................................................ [ 99%] +......... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 412 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 671 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5454 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.err b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.md b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.md new file mode 100644 index 0000000..4dd9d04 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.md @@ -0,0 +1,19 @@ +All green. Final report: + +**Implemented (task 03 — `_index_pdf_file` pictorial pass)** +- `app/rag/importer.py`: `_pictorial_pdf_pass` after the phase-133 gate (usable layers only — locked precedence, no double vision): `pictorial_pages` gate (no LLM) → `render_pdf_pages(pages=candidates)` → `describe_pdf_page` per page (fail-soft) → `[Page N illustration: …]` appended after the page text; digest over final content; zero-candidates no-op (byte-identical); `pdfs_pictorial`/`pdf_pictorial_pages` log-only counters + the one PLAN §9 log line. +- New `tests/unit/test_pictorial_import.py` (7 tests: happy path, assembly byte-identity, kill switch, zero-candidates no-op, fail-soft page, unreadable-page skip, stable re-run). +- `tests/integration/test_importer.py`: IKEA repro (candidates computed in-test, blocks on every sparse page, none on 2–5, chunks carry description, title, re-sync `unchanged`), kill-switch byte-identity, garbage→fallback precedence pin (zero `PICTORIAL_PAGE_MODE` calls). +- Pre-140 suite compatibility: `pdf_pictorial=False` in the PDF test helpers (they pin pre-140 byte-exact shapes), HNB pin's vision counts now seam-filtered, `tests/conftest.py` leak guards, `tests/e2e/conftest.py` pins `BOR_PDF_PICTORIAL=0` for all E2E apps (task 04 opts in explicitly). + +**Gates** +- `uv run pytest tests/unit/test_pictorial_import.py -v` → 7 passed +- `uv run pytest --cov=app --cov-report=term-missing` → 3320 passed, 11 skipped, TOTAL 99% (>90%) +- `uv run ruff check . && uv run pyright` → clean, 0 errors +- E2E unedited, isolated: `test_garbage_pdf.py` 2 passed/1 skipped, `test_pdf_support.py` 6 passed, `test_pdf_image_context.py` 4 passed; real app + mock smoke with `BOR_PDF_PICTORIAL=1` OK. + +**Decisions/deviations** +- IKEA tar absent on this machine → the two repro tests use the phase-133 skipif-with-named-reason convention (verified green against a faithful 24-page synthetic stand-in, removed after). +- "Zero added vision calls" on re-sync is unachievable under LOCKED A4 (digest over final content, pass runs before the unchanged check) — tests assert the phase-133 convention instead: `unchanged`, no re-embed, byte-stable content/digest, vision re-issued (documented cost). + +**Next pending task:** `04_e2e_pictorial_pdf.md` diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.validate b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.validate new file mode 100644 index 0000000..4a0b7de --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a1.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +...........................................................s.....sss...s [ 54%] +s....................................................................... [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 92%] +...........ss........................................................... [ 95%] +..................................................sss................... [ 97%] +........................................................................ [ 99%] +................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 412 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5488 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.err b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.md b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.md new file mode 100644 index 0000000..428828c --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.md @@ -0,0 +1 @@ +(no final assistant message — see the .err log) diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.validate b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.validate new file mode 100644 index 0000000..15403c9 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a2.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 52%] +...........................................................s.....sss...s [ 54%] +s....................................................................... [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................sss............................. [ 97%] +........................................................................ [ 99%] +......... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 412 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 671 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5454 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.err b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.md b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.md new file mode 100644 index 0000000..428828c --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.md @@ -0,0 +1 @@ +(no final assistant message — see the .err log) diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.validate b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.validate new file mode 100644 index 0000000..15403c9 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__03_importer_pictorial_pass.a3.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 13%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 52%] +...........................................................s.....sss...s [ 54%] +s....................................................................... [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 78%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................sss............................. [ 97%] +........................................................................ [ 99%] +......... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 412 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 671 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5454 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.err b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.md b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.md new file mode 100644 index 0000000..ca261a1 --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.md @@ -0,0 +1,17 @@ +Phase 140 task 04 complete — all gates green. + +**Implemented** (only new file: `tests/e2e/test_pictorial_pdf.py`, no other changes): +- Phase-140 dedicated Playwright suite, mock-only, in isolation: module app with explicit `BOR_PDF_PICTORIAL=1` opt-in (conftest pins it off for pre-phase suites) + `pdf` import token +- Synthetic 2-page PDF (PyMuPDF): 700+ char prose page 1 + sparse page 2 ("Illustration 2" + vector rect/arrow, zero raster images), tarred `.tar` (owner's format) +- Real admin flow: UI tar upload (202) → UI Sync button ("1 added"); seed asserts stored content byte-exact (candidates computed in-test from `extract_pdf_page_texts` + `pictorial_pages`) +- (a) viewer: page 1 prose w/o illustration block; page 2 with `[Page 2 illustration: ]`; (b) raw-SSE chat: grounded, one `read` tool frame, `done.sources` = exactly the PDF, answer echoes the canned phrase; (c) sync `detail` = pre-phase 12-key set exactly, pictorial counters absent (log-only) + +**Results:** +- `uv run pytest tests/e2e/test_pictorial_pdf.py -v --no-cov` → 3 passed (×3 stable runs) +- `test_pdf_support.py` 6 passed / `test_pdf_image_context.py` 4 passed / `test_garbage_pdf.py` 2 passed + 1 skipped (documented missing-owner-fixture skip) — all unedited +- `uv run pytest --cov=app --cov-report=term-missing` → 3320 passed, TOTAL 99% (>90%) +- `uv run ruff check . && uv run pyright` → clean + +**Decisions:** question scripted through the house seeded-read convention (phase 119 A1: `done.sources` is read-docs-only; mock echoes the read verbatim) — the only way to satisfy "sources includes the PDF" + "answer echoes the phrase" with the unedited mock; natural part is the task's exact question. Module-granularity KB truncate (tests share one seed). + +**Next pending task:** none — task 04 is the last in phase 140; `todo/` is otherwise empty. diff --git a/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.validate b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.validate new file mode 100644 index 0000000..4a0b7de --- /dev/null +++ b/.agents/reports/140_pictorial_pdf_pages/140_pictorial_pdf_pages__04_e2e_pictorial_pdf.a1.validate @@ -0,0 +1,118 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 6%] +........................................................................ [ 8%] +........................................................................ [ 10%] +........................................................................ [ 12%] +........................................................................ [ 15%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 23%] +........................................................................ [ 25%] +........................................................................ [ 28%] +........................................................................ [ 30%] +........................................................................ [ 32%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 47%] +........................................................................ [ 49%] +........................................................................ [ 51%] +...........................................................s.....sss...s [ 54%] +s....................................................................... [ 56%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 79%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 90%] +........................................................................ [ 92%] +...........ss........................................................... [ 95%] +..................................................sss................... [ 97%] +........................................................................ [ 99%] +................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain-of-reese/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.7-final-0 ________________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 268 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 259 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 183 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 264 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 133 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 412 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 134 0 100% +app/rag/git_sources.py 56 0 100% +app/rag/importer.py 705 9 99% +app/rag/llm.py 246 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 124 0 100% +app/rag/prompts.py 154 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 58 0 100% +app/schemas.py 424 0 100% +-------------------------------------------------- +TOTAL 5488 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.env.example b/.env.example index 021f70c..1c655e3 100644 --- a/.env.example +++ b/.env.example @@ -138,6 +138,23 @@ BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py,container,network,volume, # them in answers). # BOR_ANSWER_IMAGES=0 +# --- Pictorial PDF pages (phase 140: sparse pages are vision-described) --- +# Master switch for the pictorial PDF pass (ON by default — LOCKED A3): +# when an imported PDF's text layer is usable, every page whose STRIPPED +# text is under BOR_PDF_PICTORIAL_MIN_CHARS chars is rendered to a PNG +# and vision-described; the description appends to that page's indexed +# content ([Page N illustration: …]) so the embeddings carry what the +# text layer couldn't — the IKEA assembly-instruction class (20 of 24 +# pages: 4-73 chars of markers around vector pictograms). NO page cap — +# PLAN A21: the LLM calls are free, a slower sync is acceptable. +# 0/false = off -> a PDF's indexed content is byte-identical to +# pre-phase-140 (the kill switch). +# BOR_PDF_PICTORIAL=1 # 0 = disable the pass entirely +# BOR_PDF_PICTORIAL_MIN_CHARS=100 # sparseness bar (stripped chars): + # STRICTLY under this = candidate; + # 0 = every page (A21); negative + # fails startup loudly + # --- Chat image questions (phase 123: attach an image to a question) --- # Gated by the SAME BOR_IMAGES toggle above (enable only when the chat # model supports vision). The question image is stored on the server — diff --git a/app/config.py b/app/config.py index 78cbc08..51b4503 100644 --- a/app/config.py +++ b/app/config.py @@ -396,6 +396,34 @@ class Settings(BaseSettings): #: upload): the served copy must outlive the source file. image_dir: str = "~/bor-sources/images" + # --- Pictorial PDF pages (phase 140: sparse pages rendered + vision-described) --- + #: Master switch for the pictorial PDF pass (phase 140, LOCKED A3, + #: ``BOR_PDF_PICTORIAL``; ``0``/``false`` = off). While on, a PDF + #: whose text layer passes the phase-133 usability gate ALSO gets the + #: sparse-page pass: every page whose stripped text is under + #: :attr:`pdf_pictorial_min_chars` chars (``app.rag.pdf.pictorial_pages`` + #: — the deterministic gate, LOCKED A1) is rendered to a PNG and + #: vision-described, and the description appends to that page's + #: indexed content block (``[Page N illustration: …]``, LOCKED A4) — + #: the embeddings then carry what the thin text layer couldn't (the + #: IKEA assembly-instruction class: 20 of 24 pages are 4–73 chars of + #: markers around vector pictograms). NO page cap: every candidate + #: page is described (PLAN A21 — the LLM calls are free; a slower + #: sync is acceptable, a lower-quality KB is not). While off, a + #: PDF's indexed content is byte-identical to pre-phase-140 (the + #: ``BOR_IMAGES`` naming pattern — the kill switch). + pdf_pictorial: bool = True + #: Sparseness bar for the pictorial pass (phase 140, LOCKED A1, + #: ``BOR_PDF_PICTORIAL_MIN_CHARS``): a page whose STRIPPED text is + #: STRICTLY shorter than this many characters is a pictorial + #: candidate — a page with exactly this many stripped chars is NOT + #: sparse (the boundary is pinned in the unit tests). Default 100 — + #: the IKEA fixture's sparse pages run 4–73 chars and its text-heavy + #: pages 3,091+, so 100 sits between the classes. ``0`` = every page + #: is a candidate (PLAN A21); a negative value is a typo — the + #: validator fails startup loudly (the ``agent_max_rounds`` pattern). + pdf_pictorial_min_chars: int = 100 + # --- Chat image questions (phase 123: attach an image to a question) --- #: Where a user's question image bytes are stored (phase 123, #: ``BOR_CHAT_IMAGE_DIR`` — the ``image_dir`` convention: a sibling of @@ -572,6 +600,18 @@ class Settings(BaseSettings): raise ValueError("read_max_chars must be >= 0 (chars)") return v + @field_validator("pdf_pictorial_min_chars") + @classmethod + def _pdf_pictorial_min_chars_non_negative(cls, v: int) -> int: + """``0`` is the every-page-is-a-candidate mode (PLAN A21 — the + LLM calls are free) — a negative threshold is a typo (the + ``agent_max_rounds`` pattern, phase 140).""" + if v < 0: + raise ValueError( + "pdf_pictorial_min_chars must be >= 0 (0 = every page is a candidate)" + ) + return v + @field_validator("embed_question_max_chars") @classmethod def _embed_question_max_chars_positive(cls, v: int) -> int: diff --git a/app/rag/importer.py b/app/rag/importer.py index b56a314..c8f39b7 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -129,6 +129,28 @@ as unchanged (no re-chunk/re-embed/re-summary, no KB change; the vision calls are re-issued each sync — the no-cache cost, documented in :func:`_fallback_page_screenshots`). +Pictorial sparse pages (phase 140, task 03, LOCKED A1–A4): AFTER the +phase-133 gate, a PDF whose text layer is USABLE gets the sparse-page +pass (:func:`_pictorial_pdf_pass`, gated by the ``pdf_pictorial`` +setting — ``BOR_PDF_PICTORIAL``, the house kill-switch): the +deterministic gate (:func:`app.rag.pdf.pictorial_pages` — NO LLM +call, LOCKED A1) selects the pages whose stripped text is under +``pdf_pictorial_min_chars``; EVERY candidate is rendered (PLAN A21 — +no page cap, no call budget) and vision-described through the phase- +140 seam (:func:`app.rag.summarizer.describe_pdf_page` — the +``PICTORIAL_PAGE_MODE`` one-shot, fail-soft per page), and the +description appends to the page's content block as +``[Page N illustration: …]`` (LOCKED A2 — no LLM judge: every +described page joins the content; the owner's L5 rationale: more +information is better for the embeddings). The precedence is locked: +the pass runs on a USABLE layer ONLY — an unusable layer takes the +phase-133 full-page screenshot fallback and never burns a second +vision pass. Zero candidates → a no-op (no render, no vision call — +a clean-text PDF's path is byte-identical to pre-phase); the +counters ``pdfs_pictorial`` / ``pdf_pictorial_pages`` are log-only +(the sync ``detail`` API shape is untouched, the stats log-only +convention). + Image-anchor context (phase 125, task 03, LOCKED A1/A2/A3): BEFORE the description, the same source's importable markdown files are scanned for ``![alt](path)`` references that name the image (the task-02 @@ -250,6 +272,7 @@ from app.rag.pdf import ( extract_pdf_page_texts, extract_pdf_text, is_usable_pdf_text, + pictorial_pages, render_pdf_pages, ) from app.rag.retriever import TRUNCATION_MARKER @@ -257,6 +280,7 @@ from app.rag.summarizer import ( IMAGE_FALLBACK_MIME, IMAGE_MIMES, describe_image, + describe_pdf_page, generate_summary, ) @@ -341,6 +365,20 @@ class ImportSummary: #: ``---PAGE-N---``-anchored descriptions, or ``""`` when the #: fallback produced nothing — the phase-126 empty-content path). pdfs_screenshotted: int = 0 + #: PDFs (phase 140, task 03, LOCKED A2/A4) whose USABLE text layer + #: carried ≥1 pictorial candidate page (``app.rag.pdf. + #: pictorial_pages`` — the deterministic sparse-page gate, LOCKED + #: A1) whose vision description was appended to the page block + #: (``[Page N illustration: …]``) — the sparse-page pass + #: (:func:`_pictorial_pdf_pass`). Log-only (the sync ``detail`` + #: API shape is untouched — the stats log-only convention, + #: phase 133); ``pdf_pictorial_pages`` counts the described pages. + pdfs_pictorial: int = 0 + #: Pictorial candidate pages described across the run (phase 140, + #: task 03) — one PDF with three described sparse pages counts + #: ``pdfs_pictorial=1`` + ``pdf_pictorial_pages=3`` (A21: no page + #: cap — the count may be large, that is the point). + pdf_pictorial_pages: int = 0 #: PDF-embedded image docs whose stored ``image_context`` was #: backfilled with the document-context block (phase 137, task 03) #: — the one-time best-effort pass; a success never repeats, a @@ -367,7 +405,8 @@ class ImportSummary: "import: summary files=%d added=%d updated=%d unchanged=%d pruned=%d " "errors=%d chunks=%d embed_batches=%d summaries=%d summary_errors=%d " "summary_backfilled=%d dates_updated=%d images_failed=%d " - "image_context_backfilled=%d pdfs_screenshotted=%d formats=%s", + "image_context_backfilled=%d pdfs_screenshotted=%d " + "pdfs_pictorial=%d pdf_pictorial_pages=%d formats=%s", self.files, self.added, self.updated, @@ -383,6 +422,8 @@ class ImportSummary: self.images_failed, self.image_context_backfilled, self.pdfs_screenshotted, + self.pdfs_pictorial, + self.pdf_pictorial_pages, self.format_counts(), ) @@ -1165,6 +1206,19 @@ async def _index_pdf_file( An unopenable PDF — or one whose fallback also produced nothing — yields ``""`` (still a document — it may carry embedded images — with no content chunks; a warning is the signal). + * Phase 140 (task 03, LOCKED A1–A4): when the layer is USABLE and + ``settings.pdf_pictorial`` is on, the sparse-page pass + (:func:`_pictorial_pdf_pass`) renders + vision-describes every + page whose stripped text is under + ``settings.pdf_pictorial_min_chars`` (the deterministic gate — + NO LLM call, A1; every candidate, no page cap — A21/A3) and + appends the description to the page block as + ``[Page N illustration: …]`` (A2/A4). The precedence is locked: + the pass sits AFTER the phase-133 gate, so an unusable layer + (the fallback's full-page screenshots) never burns a second + vision pass; zero candidates → a no-op (the extracted text + stays byte-identical to the pre-phase-140 path — no render, no + vision call). * the sha256 digest is over the FINAL content (the digest rule is content identity — the same text is the same document version; the fallback's constructed page descriptions digest exactly the @@ -1213,6 +1267,20 @@ async def _index_pdf_file( content = await _fallback_page_screenshots( session, source=source, rel=rel, full_path=full_path, llm=llm ) + elif llm.settings.pdf_pictorial: + # Phase 140 (task 03, LOCKED A1–A4): the pictorial pass — + # USABLE layers only (the precedence is locked: the phase-133 + # fallback above already screenshotted an unusable layer, no + # double vision). Zero candidates → a no-op (the pass returns + # None — the extracted text stays byte-identical to the + # pre-phase-140 path); otherwise the FINAL content carries the + # described candidates' illustration blocks (A4: the digest + # below runs over it, the phase-126/133 convention). + passed = await _pictorial_pdf_pass( + source=source, rel=rel, full_path=full_path, llm=llm, summary=summary + ) + if passed is not None: + content = passed if not content: logger.warning( "import: PDF has no extractable text source=%s path=%s", source, rel @@ -1406,6 +1474,128 @@ async def _fallback_page_screenshots( ) +async def _pictorial_pdf_pass( + *, + source: str, + rel: str, + full_path: Path, + llm: Embedder, + summary: ImportSummary, +) -> str | None: + """The phase-140 (task 03) pictorial pass (LOCKED A1–A4) — the + IKEA assembly-instruction class (TODO L5): a PDF whose text layer + is USABLE but SPARSE (mostly pictorial pages — 4–73 chars of + markers around vector-drawn pictograms) gets every sparse page + rendered to a PNG and vision-described, and the description + APPENDS to that page's indexed content block + (``[Page N illustration: …]``) — the page's text stays, so the + embeddings carry what the thin text layer couldn't (the owner's + L5 rationale: "More information is better when generating the + text embeddings"). + + * the gate is :func:`app.rag.pdf.pictorial_pages` over + :func:`app.rag.pdf.extract_pdf_page_texts` — deterministic, NO + LLM call (LOCKED A1): the 1-based page numbers whose stripped + text is under ``settings.pdf_pictorial_min_chars``, ascending. + (The kill-switch itself — ``settings.pdf_pictorial`` — gates the + whole pass in the caller: while off, this function is never + called and the content is byte-identical to pre-phase.) + * ZERO candidates → a NO-OP (returns ``None``): no render, no + vision call — the caller keeps its :func:`extract_pdf_text` + output untouched, so a clean-text PDF's path is byte-identical + to the pre-phase-140 path (the pin). + * a page that could not be rendered (the zero-byte + :func:`render_pdf_pages` placeholder — the index never shifts, + so it stays aligned with the candidates) is skipped with a + warning (the phase-133 handling — the page keeps its text only, + no vision call is burned for it); a page whose vision call + failed or came back blank (:func:`describe_pdf_page` → + ``None``) is skipped with a warning and keeps its text only — + one dead page never kills the pass (LOCKED A2 fail-soft). + * the assembly (LOCKED A4): every page of the + :func:`extract_pdf_page_texts` map (the SAME pages, order and + stripped text as :func:`extract_pdf_text`) as its + :data:`PAGE_SEPARATOR` + the page text + — for the described + candidates — ``"\\n\\n[Page {n} illustration: {description}]"``. + The assembly with NO descriptions is byte-identical to + :func:`extract_pdf_text`'s output (the unit pin). + * the caller's digest runs over this FINAL content (the + phase-126/133 convention — A4): a stable file + a stable vision + reply re-syncs as ``unchanged``; a non-deterministic re-describe + re-updates the row — a cost, not a bug (the phase-133 docstring + documents it, A21 endorses it). + * stats (log-only — the sync ``detail`` API shape is untouched, + the phase-133 convention): ``summary.pdfs_pictorial`` (+1 when + ≥1 page was described) and ``summary.pdf_pictorial_pages`` + (+the described page count), plus the ONE loud log line (PLAN + §9): ``pdf: pictorial pass source=… path=… — N candidate pages, + M described (min_chars=…)``. + """ + settings = llm.settings + page_texts = extract_pdf_page_texts(full_path) + candidates = pictorial_pages( + page_texts, min_chars=settings.pdf_pictorial_min_chars + ) + if not candidates: + return None + # ONE render for the whole candidate set (the phase-140 selector — + # a 500-page PDF with 30 sparse pages renders 30 PNGs, not 500); + # the result is aligned with *candidates* (ascending, one slot + # each — a zero-byte placeholder never shifts the positions). + pngs = render_pdf_pages(full_path, pages=candidates) + descriptions: dict[int, str] = {} + # strict: the render's result is aligned with the candidate list by + # contract (one slot per named page, ascending) — a length drift is + # a parser regression, fail loud. + for page, png in zip(candidates, pngs, strict=True): + if not png: + # The phase-133 zero-byte placeholder: an unreadable page. + logger.warning( + "pdf: skipping pictorial page %d of %s/%s — the render is " + "empty (unreadable page)", + page, source, rel, + ) + continue + desc = await describe_pdf_page( + llm, + data=png, + mime="image/png", + page=page, + page_text=page_texts[page], + ) + if desc is None: + # Fail-soft (LOCKED A2): one dead page never kills the + # pass — the page keeps its text only, the sync continues. + logger.warning( + "pdf: pictorial page %d of %s/%s has no description — " + "the page keeps its text only", + page, source, rel, + ) + continue + descriptions[page] = desc + # The ONE loud log line (PLAN §9): N candidate pages, M described. + logger.info( + "pdf: pictorial pass source=%s path=%s — %d candidate pages, %d " + "described (min_chars=%d)", + source, rel, len(candidates), len(descriptions), + settings.pdf_pictorial_min_chars, + ) + # The stats (log-only — the sync ``detail`` API shape is untouched). + if descriptions: + summary.pdfs_pictorial += 1 + summary.pdf_pictorial_pages += len(descriptions) + # The assembly: the extracted text's page blocks, the described + # candidates carrying their illustration block (A4 — the page's + # text STAYS; the description appends after it). + parts: list[str] = [] + for page, text in page_texts.items(): + block = text + if page in descriptions: + block += f"\n\n[Page {page} illustration: {descriptions[page]}]" + parts.append(f"{PAGE_SEPARATOR.format(page=page)}\n{block}") + return "\n\n".join(parts) + + async def _index_pdf_embedded_images( session: Session, *, diff --git a/app/rag/pdf.py b/app/rag/pdf.py index 4d367ff..a64da1d 100644 --- a/app/rag/pdf.py +++ b/app/rag/pdf.py @@ -29,11 +29,20 @@ package, imported as ``fitz`` — its stable API name). I/O) — REAL content vs. garbage (the pre-126 XML/binary soup, the empty text layer). It is the single trigger for the page-screenshot vision fallback (phase 133 task 03). +* :func:`pictorial_pages` — the phase-140 sparse-page gate: the 1-based + page numbers of a :func:`extract_pdf_page_texts` map whose stripped + text is STRICTLY under *min_chars* chars (ascending) — pure, + deterministic, NO LLM call (LOCKED A1): the "way to determine when + that's necessary" (TODO L5) for the pictorial pass. * :func:`render_pdf_pages` — the fallback's visual input (phase 133 - task 03): one PNG per page (150 DPI, page order); an unreadable page - yields ``b""`` (the caller's placeholder — the page index never - shifts), an unopenable file yields ``[]`` (the caller's existing - empty-content path). + task 03): one PNG per page (150 DPI, page order); the phase-140 + optional 1-based *pages* selector renders ONLY the named pages + (ascending — a 500-page PDF with 30 sparse pages renders 30 PNGs, + not 500), ``None`` = every page (the phase-133 caller's result is + byte-identical); an unreadable (or out-of-range named) page yields + ``b""`` at its position (the caller's placeholder — the result + stays aligned with the requested pages), an unopenable file yields + ``[]`` (the caller's existing empty-content path). Every failure mode is fail-soft: an unreadable file returns an empty result, an unreadable page or image is logged and skipped — this module @@ -43,7 +52,7 @@ from __future__ import annotations import logging import re -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from dataclasses import dataclass from pathlib import Path from typing import Any @@ -264,15 +273,60 @@ def is_usable_pdf_text(text: str) -> bool: return non_printable / len(text) < NONPRINT_CHAR_FRACTION -def render_pdf_pages(pdf_path: Path, dpi: int = 150) -> list[bytes]: - """Render every page of *pdf_path* to a PNG (page order — list index - *i* is page *i*+1). +def pictorial_pages(page_texts: dict[int, str], *, min_chars: int) -> list[int]: + """The 1-based page numbers of *page_texts* that are PICTORIAL + candidates: ``len(text.strip()) < min_chars`` — STRICTLY less than + (a page with exactly *min_chars* stripped chars is NOT sparse; the + boundary is pinned in the unit tests), ascending. + + The deterministic sparse-page gate (phase 140, task 01, LOCKED A1) + — the "way to determine when that's necessary" for the pictorial PDF + pass (TODO L5): NO LLM call, pure, deterministic (no PDF, no I/O — + the input is a :func:`extract_pdf_page_texts` map, so the gate never + re-opens the file). A page that carries no text at all has NO key in + the map and is therefore not a candidate here (the importer's pass + owns the page-total handling); an empty map yields ``[]`` — a + clean-text PDF's path never burns a vision call. + + The IKEA example (the phase-140 fixture, ``min_chars=100``): + ``{1: "BESTÅ", 2: <7 KB of warnings>, 6: "Illustration 6"}`` → + ``[1, 6, …]`` — page 1 (the title page: the logo + product name) IS + a candidate: describing it is useful (the logo + product name) and + never harmful; page 2's 7 KB of multilingual warnings is not. + + ``min_chars=0`` is the documented PLAN A21 special case — **every + page in the map is a candidate** (the "describe everything" mode). + The strict-``<`` test cannot express this (no length is below 0), so + 0 is not "a stricter bar": it is the A21 no-cap endpoint, and a + negative value — unreachable through ``Settings`` (the validator + fails startup loudly) — takes the same branch defensively. + """ + if min_chars <= 0: + return sorted(page_texts) + return sorted( + page for page, text in page_texts.items() if len(text.strip()) < min_chars + ) + + +def render_pdf_pages( + pdf_path: Path, dpi: int = 150, pages: Sequence[int] | None = None +) -> list[bytes]: + """Render the selected pages of *pdf_path* to PNGs. + + *pages* (phase 140, task 02): the optional 1-based selector — + render ONLY the named pages, in ascending order, one PNG each + (a 500-page PDF with 30 pictorial candidates renders 30 PNGs, not + 500). ``None`` (the default) = every page in page order — the + phase-133 caller's result is byte-identical, so list index *i* is + page *i*+1 (with a selector, list index *i* is the *i*-th smallest + NAMED page). The fallback's visual input (phase 133 task 03, ASSUMPTION A8: fixed 150 DPI — the balance of vision legibility vs data-URL - payload). A page that cannot be rendered yields ``b""`` (the - caller's placeholder — the page index NEVER shifts, so - ``---PAGE-N---`` stays truthful; the failure is logged, the + payload). A page that cannot be rendered — including a named page + outside the document's page range — yields ``b""`` at ITS position + (the caller's placeholder — the result's order NEVER shifts, so it + stays aligned with the requested pages; the failure is logged, the :func:`extract_pdf_text` fail-soft style). A file that cannot be OPENED yields ``[]`` (the caller's existing empty-content path — the doc lands with the no-extractable-text behavior, unchanged). @@ -283,18 +337,29 @@ def render_pdf_pages(pdf_path: Path, dpi: int = 150) -> list[bytes]: logger.warning("pdf: could not open %s for page rendering — %s", pdf_path, exc) return [] - pages: list[bytes] = [] + # 1-based page numbers: the default walks every page in page order + # (the phase-133 contract — list index *i* is page *i*+1); a + # selector renders the named pages, ascending (de-duplicated — a + # repeated page number would render it twice for no reason). + numbers = ( + range(1, len(doc) + 1) + if pages is None + else sorted(set(pages)) + ) + rendered: list[bytes] = [] with doc: - for page_num in range(len(doc)): + for page_number in numbers: try: - pages.append(doc[page_num].get_pixmap(dpi=dpi).tobytes("png")) + rendered.append( + doc[page_number - 1].get_pixmap(dpi=dpi).tobytes("png") + ) except Exception as exc: logger.warning( "pdf: could not render page %d of %s — %s", - page_num + 1, pdf_path, exc, + page_number, pdf_path, exc, ) - pages.append(b"") - return pages + rendered.append(b"") + return rendered def extract_pdf_images(pdf_path: Path) -> list[PdfImage]: diff --git a/app/rag/summarizer.py b/app/rag/summarizer.py index fe5d795..9b70306 100644 --- a/app/rag/summarizer.py +++ b/app/rag/summarizer.py @@ -33,10 +33,24 @@ it is not assumed vision-capable). Fail-soft by contract: any client error, empty reply, or non-2xx yields ``None`` — the importer skips the doc, counts ``images_failed``, and the sync continues. -The ``SUMMARY_MODE`` / ``IMAGE_DESCRIPTION_MODE`` markers follow the -``DEFLECT_MODE`` convention: the deterministic E2E mock LLM keys on -them (``tests/e2e/mock_llm.py`` — the image branch is wired by task -06's story suite). +Pictorial PDF pages (phase 140, LOCKED A2): :func:`describe_pdf_page` +is this module's third one-shot generation path — a SINGLE CHAT-model +(vision) call describing ONE rendered sparse PDF page, with the +page's OWN extracted text as context so the model describes what the +text does NOT carry (the IKEA assembly-instruction class: vector +pictograms around a 4–73-char text layer). The description is +APPENDED to the page's indexed content block by the importer (the +``[Page N illustration: …]`` line, LOCKED A4) — the page's text stays; +more information is better for the embeddings (the owner's L5 +rationale). Fail-soft by contract: any client error or empty reply +yields ``None`` — the importer skips that page with a warning and +continues (one dead page never stalls a sync). + +The ``SUMMARY_MODE`` / ``IMAGE_DESCRIPTION_MODE`` / +``PICTORIAL_PAGE_MODE`` markers follow the ``DEFLECT_MODE`` +convention: the deterministic E2E mock LLM keys on them +(``tests/e2e/mock_llm.py`` — the image branch is wired by task +06's story suite, the pictorial branch by phase 140 task 02). """ from __future__ import annotations @@ -92,8 +106,41 @@ DESCRIBE_INSTRUCTION = ( #: The full describe prompt: marker first (the mock's key), then the #: instruction — the single text part of the multimodal user message. +#: BYTE-IDENTICAL CONTRACT: the phase 122/126/137 suites pin this +#: string untouched — pictorial-page descriptions go through their +#: own marker (:data:`PICTORIAL_PAGE_MODE`), never a re-wrap of this +#: one. DESCRIBE_PROMPT = f"{IMAGE_DESCRIPTION_MODE}: {DESCRIBE_INSTRUCTION}" +#: Pictorial-page marker (phase 140, task 02, LOCKED A1/A2) — the +#: deterministic E2E mock LLM keys on it (same convention as +#: ``IMAGE_DESCRIPTION_MODE``, PLAN §6). It heads the text part of +#: the multimodal describe message, so it rides the user message, not +#: a system prompt. +PICTORIAL_PAGE_MODE = "PICTORIAL_PAGE_MODE" + +#: Locked instruction for the CHAT (vision) model (phase 140, LOCKED +#: A2): the description APPENDS to the page's indexed content block +#: (``[Page N illustration: …]``) — the embeddings then carry what +#: the thin text layer couldn't (the owner's L5 rationale) — so it +#: must be a faithful, retrieval-oriented account of what the page +#: SHOWS: the assembly steps, part names, arrows, tool icons, and +#: before/after states — ESPECIALLY the information the extracted +#: text does not contain. +PICTORIAL_INSTRUCTION = ( + "Describe in detail what this page of the PDF shows, in plain text, " + "for a search index. State the assembly steps, part names, arrows, " + "tool icons, and before/after states it illustrates — ESPECIALLY the " + "information the extracted text does not contain. Write 2-4 sentences " + "of substance. Do not use markdown. Do not invent anything that is " + "not visible on the page." +) + +#: The full pictorial-page prompt: marker first (the mock's key), then +#: the instruction — the head of the multimodal user message's text +#: part (the page block is appended by :func:`describe_pdf_page`). +PICTORIAL_PROMPT = f"{PICTORIAL_PAGE_MODE}: {PICTORIAL_INSTRUCTION}" + #: Extension → MIME type for the image family (phase 122). Dotted, #: lowercase keys — the ``image_extension_set`` shape. Task 03 uses it #: for the describe call's data-URL mime; task 04's serve route reuses @@ -266,11 +313,43 @@ async def describe_image( if context else "" ) + return await _one_shot_vision_reply( + llm, + data_url=data_url, + text=DESCRIBE_PROMPT + context_prefix, + settings=settings, + what="image description", + ) + + +async def _one_shot_vision_reply( + llm: SummaryLLM, + *, + data_url: str, + text: str, + settings: Settings | None, + what: str, +) -> str | None: + """The shared one-chat-call multimodal contract (phase 122, reused + by phase 140): ONE user message + ``[{type: "text", text: }, + {type: "image_url", image_url: {url: }}]`` against + ``llm.settings.llm_chat_model`` — no system prompt, no tools, no + app-level retry. + + Returns the stripped reply cut exactly at + ``(settings or llm.settings).summary_max_chars`` (the phase-30 cap); + ``None`` on any client error (the client raises :class:`LLMError` + for a failure, an empty reply, or a non-2xx) or an empty/blank + reply after trimming (the house one-shot empty-content policy — the + duck-typed fakes may return one even though the real client + rejects it). *what* names the caller for the warning log line. + """ messages: list[dict[str, Any]] = [ { "role": "user", "content": [ - {"type": "text", "text": DESCRIBE_PROMPT + context_prefix}, + {"type": "text", "text": text}, {"type": "image_url", "image_url": {"url": data_url}}, ], } @@ -279,12 +358,67 @@ async def describe_image( try: raw = await llm.chat(messages, model=model) except LLMError as e: - logger.warning("image description failed (model=%s): %s", model, e) + logger.warning("%s failed (model=%s): %s", what, model, e) return None - text = raw.strip() - if not text: + text_out = raw.strip() + if not text_out: # The client already rejects empty content; this is the # defensive re-assert (the duck-typed fakes may return it). return None limit = (settings or llm.settings).summary_max_chars - return text[:limit] + return text_out[:limit] + + +async def describe_pdf_page( + llm: SummaryLLM, + *, + data: bytes, + mime: str, + page: int, + page_text: str | None, + settings: Settings | None = None, +) -> str | None: + """One-shot CHAT-model (vision) description of ONE rendered PDF page + (phase 140, task 02, LOCKED A2) — the text the importer appends to + that page's indexed content block as + ``[Page N illustration: …]`` (the page's own text stays; the + embeddings then carry what the thin text layer couldn't — the + owner's L5 rationale). + + ONE chat-model call against ``llm.settings.llm_chat_model`` (the + vision model — the ``lite`` summary model is NOT assumed + vision-capable) with the same multimodal contract as + :func:`describe_image` (via :func:`_one_shot_vision_reply`): no + system prompt, no tools, no app-level retry. The message's text + part is :data:`PICTORIAL_PROMPT` (the ``PICTORIAL_PAGE_MODE`` + marker + the locked instruction) followed by the page block — the + page's OWN extracted text as context, so the model describes what + the text does NOT contain:: + + PICTORIAL_PROMPT + + "\\n\\nPage {page} of the PDF. Text extracted from this page:\n" + + (page_text.strip() or "(none)") + + ``data`` is the rendered page's PNG bytes (phase 133's + ``render_pdf_pages`` output), ``mime`` its MIME type (``image/png`` + for the rendered page). + + Returns the stripped reply cut exactly at + ``(settings or llm.settings).summary_max_chars`` (the phase-30 cap). + Returns ``None`` on any client error, empty reply, or non-2xx + (fail-soft — the importer skips THAT page with a warning and + continues: one dead page never stalls a sync, LOCKED A2). + """ + data_url = image_data_url(data, mime) + extracted = page_text.strip() if page_text is not None else "" + return await _one_shot_vision_reply( + llm, + data_url=data_url, + text=( + PICTORIAL_PROMPT + + f"\n\nPage {page} of the PDF. Text extracted from this page:\n" + + (extracted or "(none)") + ), + settings=settings, + what=f"pictorial page description (page={page})", + ) diff --git a/tests/conftest.py b/tests/conftest.py index c3d48ed..3a98af5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,6 +85,17 @@ os.environ["BOR_IMAGE_DIR"] = str(_Settings.model_fields["image_dir"].default) # ``answer_image`` done-frame key absent) must see the code default. os.environ["BOR_ANSWER_IMAGES"] = str(_Settings.model_fields["answer_images"].default) +# Phase 140: the same leak class for the pictorial-PDF toggle + +# threshold — an operator's shell may carry BOR_PDF_PICTORIAL / +# BOR_PDF_PICTORIAL_MIN_CHARS (the owner runs the feature on in +# production), and the task-01 default pins (LOCKED A3 default-on, +# LOCKED A1 default-100) and the task-03 pass tests must see the code +# defaults. +os.environ["BOR_PDF_PICTORIAL"] = str(_Settings.model_fields["pdf_pictorial"].default) +os.environ["BOR_PDF_PICTORIAL_MIN_CHARS"] = str( + _Settings.model_fields["pdf_pictorial_min_chars"].default +) + from app.db import SessionLocal, db_available # noqa: E402 from app.main import app as fastapi_app # noqa: E402 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index d8f7be1..0e4acdb 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -41,6 +41,17 @@ USE_REAL_LLM = os.environ.get("E2E_REAL_LLM") == "1" ADMIN_PASSWORD = "e2e-admin-password" SESSION_SECRET = "e2e-session-secret-0123456789abcdef0123456789abcdef" +# Phase 140 (task 03): the pictorial-PDF kill switch is pinned OFF for +# every E2E app. The code default is ON (LOCKED A3), and the pre-phase- +# 140 story suites pin the PRE-phase content shape byte-exactly (their +# synthetic PDFs carry sparse pages — under the 100-char bar — so a +# default-ON app would append ``[Page N illustration: …]`` blocks and +# move those pins). The line rides into every app's env through the +# house ``env = dict(os.environ)`` pattern; the phase-140 story suite +# (tests/e2e/test_pictorial_pdf.py) opts in explicitly +# (``env["BOR_PDF_PICTORIAL"] = "1"`` in its module app). +os.environ["BOR_PDF_PICTORIAL"] = "0" + def _wait_http(url: str, timeout: float = 40.0) -> None: deadline = time.monotonic() + timeout diff --git a/tests/e2e/mock_llm.py b/tests/e2e/mock_llm.py index 8e44c6c..4d5ccb7 100644 --- a/tests/e2e/mock_llm.py +++ b/tests/e2e/mock_llm.py @@ -34,6 +34,15 @@ Implements just enough of the aipi surface: the image document's whole content + summary, so it is byte- stable and token-dense (its cosine against the story suite's question clears the mock-calibrated gate) + - ``PICTORIAL_PAGE_MODE`` (phase 140, pictorial PDF pages) -> the + fixed ``PICTORIAL_PAGE_ANSWER`` description: the CHAT model's + (vision) reply to ``summarizer.describe_pdf_page``'s multimodal + user message (one rendered sparse PDF page — the marker is its + first text part; the call is non-streaming, so only this answer + path serves it). The reply is appended to the page's indexed + content block as ``[Page N illustration: …]`` (LOCKED A4), so it + is byte-stable: a re-sync with the same reply re-digests to the + same content and lands as ``unchanged`` - ``DEFLECT_MODE`` -> honest "I haven't done anything like that" answer - otherwise -> upbeat answer quoting the provided document context - user message containing ``pretend to think slowly`` -> 3s warm-up delay @@ -992,6 +1001,21 @@ IMAGE_DESCRIPTION_CONTEXT_ANSWER = ( "cable runs. Title: Homelab Network Map." ) +#: Phase 140 (pictorial PDF pages, LOCKED A2): the fixed description +#: the mock's vision (CHAT) model returns for a +#: ``PICTORIAL_PAGE_MODE`` request — the multimodal user message +#: ``[{type: "text", …PICTORIAL_PAGE_MODE…}, {type: "image_url", +#: …}]`` (``app.rag.summarizer.describe_pdf_page`` — one rendered +#: sparse PDF page, the page's own text as context). The reply is +#: appended to the page's indexed content block as +#: ``[Page N illustration: …]`` (LOCKED A4), so it is byte-stable and +#: retrieval-friendly (distinctive + token-dense on the assembly +#: vocabulary the story E2E's question targets). +PICTORIAL_PAGE_ANSWER = ( + "Pictorial page: the illustration shows two parts being joined " + "with an arrow." +) + TABLE_ANSWER = ( "Here's the shape, in a table:\n" "\n" @@ -2913,6 +2937,17 @@ def compose_answer(body: dict[str, Any]) -> str: if "Extracted from " in user else IMAGE_DESCRIPTION_ANSWER ) + elif "PICTORIAL_PAGE_MODE" in user: + # Phase 140 (pictorial PDF pages, LOCKED A2): the CHAT model's + # vision description of ONE rendered sparse PDF page — a + # NON-STREAMING multimodal user message, so only this path + # ever sees it. ``_content_text`` joins the text parts, so the + # marker (the first text part) lands in ``user``. The two + # markers are disjoint fixed app constants (neither contains + # the other), so the branch order vs ``IMAGE_DESCRIPTION_MODE`` + # cannot shadow either flow; like that marker, it is an + # app-to-app marker a real question is never expected to type. + answer = PICTORIAL_PAGE_ANSWER elif TABLE_TRIGGER in user.lower(): # Markdown tables (phase 44, TODO.md L6): the story E2E's # deterministic table answer — a 3-column table, the diff --git a/tests/e2e/test_pictorial_pdf.py b/tests/e2e/test_pictorial_pdf.py new file mode 100644 index 0000000..2b2a878 --- /dev/null +++ b/tests/e2e/test_pictorial_pdf.py @@ -0,0 +1,729 @@ +"""Phase 140 task 04 E2E (Playwright, mock-only): the pictorial PDF — +sparse pages rendered + vision-described into the indexed content. + +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_pictorial_pdf.py -v --no-cov + +The dedicated suite for ``140_pictorial_pdf_pages`` (A16 — one Playwright +file per phase, run in isolation): it proves the owner's TODO L5 scenario +end to end on a SYNTHETIC fixture (the owner's real +``ikea_instructions.tar`` is pinned at the integration level, task 03 — +the E2E keeps the deterministic canned-reply premise): a 2-page PDF +built with PyMuPDF (the ``test_pdf_support.py`` recipe) — + +* page 1 = a 700+ char prose paragraph (over the 100-char sparseness + bar — NOT a pictorial candidate), +* page 2 = the 14-char ``Illustration 2`` marker + a drawn VECTOR figure + (a rectangle + an arrow — the IKEA class: vector-drawn pictograms, + ZERO embedded raster images, so the phase-122/137 embedded-image + pipeline sees nothing; the thin text IS a candidate), + +tarred as a single-root-member ``.tar`` (the owner's fixture's exact +format — the upload gate admits it) and arriving through the REAL admin +upload flow → the RAG page's Sync button: + +* (a) the document appears on the Sources page and the document viewer + renders the stored content: page 1's prose WITHOUT an illustration + block, page 2 WITH ``[Page 2 illustration: ]`` (the task-02 ``PICTORIAL_PAGE_MODE`` branch + — deterministic; the stored content is asserted BYTE-EXACT against + the fixture's own extraction in the seed — the candidates are computed + from ``extract_pdf_page_texts`` + ``pictorial_pages`` IN the test, no + hardcoded page list); +* (b) a chat question that ONLY the illustration answers ("how are the + two parts joined on page two?" — the phrase lives NOWHERE in the text + layer) grounds (the described page's embedding retrieves — the mock's + deterministic token-overlap cosine clears the mock-calibrated 0.30 + gate; FTS corroborates) and, scripted through the house seeded-read + convention (phase 119 A1 — a zero-read grounded turn chips NOTHING, so + the PDF must be READ to land in ``done.sources``), the SSE completes + without deflecting: the ``done`` event's ``sources`` carries the PDF + and the answer echoes the read result VERBATIM — the + ``[Page 2 illustration: …]`` block and its phrase reach the model and + land in the answer; +* (c) the sync's ``detail`` JSON keeps its pre-phase keys EXACTLY (no + new field — the ``pdfs_pictorial`` / ``pdf_pictorial_pages`` counters + are log-only, PLAN §9, the phase-133 convention). + +House pattern: one module app over the shared E2E Postgres (its own +port, module env — the conftest session app is never started in this +isolated run, so no port clash). The three tests SHARE the module seed +(one source, one document), so the shared registry + KB is truncated +ONCE, after the last test (a module-granularity autouse — a per-test +truncate would wipe the seed from under its siblings), with a +no-running-sync wait so a leaked run cannot race it. + +App env: ``BOR_IMPORT_EXTENSIONS`` pinned WITH the ``pdf`` token (the +phase-126 convention — without the token the walk is blind to the file) +and ``BOR_PDF_PICTORIAL`` pinned EXPLICITLY to ``1``: the conftest pins +it OFF in ``os.environ`` for the pre-phase-140 story suites (their +synthetic PDFs carry sparse pages — a default-ON app would move their +byte-exact pins), and this suite is the documented opt-in (the +conftest's phase-140 comment). ``BOR_IMAGES`` is pinned ``false`` — the +embedded-image pipeline is out of scope (the fixture's figure is +vector-drawn; it yields no image doc either way). The byte-exact +content assertion in the seed is the toggle's premise witness: without +the opt-in, page 2 would lack its illustration block and the seed +fails loudly. + +MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the described +block is the mock's deterministic ``PICTORIAL_PAGE_MODE`` canned reply +(``PICTORIAL_PAGE_ANSWER``), so the byte-exact content + echo +assertions are exact only against it. + +Test → observable mapping (Playwright Mapping Rule): +1. ``test_sources_page_and_viewer_show_the_described_page`` — the + Sources page reads 1 doc / 3 chunks (2 page-anchored content chunks + + the summary chunk — no image doc), the drill-down table lists the PDF + as a plain text row (LOCKED A4 — a text document, no thumbnail), and + the same-page document viewer (the row's path link) renders the + stored content under the METADATA title: the page-1 prose (NO + ``[Page 1 illustration`` block), the page-2 marker + its + ``[Page 2 illustration: ]`` block, and the Summary panel + (a non-image doc with a non-empty summary). +2. ``test_chat_question_grounds_and_echoes_the_illustration`` — the raw + SSE over the scripted seeded-read question (httpx — the ``done`` + event is the wire lens the task pins; the UI flow's premise is the + module seed): the stream settles ``done`` with ``deflected`` false + and NO ``error`` frame, EXACTLY one ``tool`` frame (the scripted + ``read`` on the PDF; the doc is far under the read cap — no + truncation, so no ``tool_result`` frame), the ``done`` event's + ``sources`` is EXACTLY the PDF (the citation surface — read docs + only, phase 119 A1), and the joined deltas echo the read result + VERBATIM — the ``Document /:`` header and the + ``[Page 2 illustration: …]`` block with the canned phrase (the + described page's content reaches the model and lands in the answer). +3. ``test_sync_detail_keeps_the_pre_phase_keys`` — the module run's + terminal ``detail`` (``GET /api/sync/status`` — the same object the + UI's "1 added" line renders from) carries the pre-phase key set + EXACTLY: the ``pdfs_pictorial`` / ``pdf_pictorial_pages`` counters + never ride the wire (the stats log-only convention), and the run's + own file counters read 1 file / 1 added / 0 unchanged / 0 pruned / + 0 errors. +""" +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tarfile +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import httpx +import pymupdf as fitz # PyMuPDF — the synthetic-fixture builder (the house recipe) +import pytest +from playwright.sync_api import Browser, Page, expect +from sqlalchemy import select, text + +from app.config import Settings +from app.db import SessionLocal +from app.models import Document +from app.rag.pdf import ( + PAGE_SEPARATOR, + extract_pdf_page_texts, + pictorial_pages, +) +from e2e.auth_helpers import login +from e2e.conftest import ADMIN_PASSWORD, SESSION_SECRET, USE_REAL_LLM, _wait_http +from e2e.mock_llm import PICTORIAL_PAGE_ANSWER + +REPO = Path(__file__).resolve().parents[2] +SOURCE_NAME = "e2e-pictorial" # the archive stem (archive_source_name) +DOC_PATH = "instructions.pdf" +PDF_TITLE = "Assembly Instructions" # the metadata title (LOCKED A2) + +#: Page 1 — a 700+ char prose paragraph (over the 100-char sparseness +#: bar — NOT a candidate). Deliberately carries NO "two parts joined" +#: content: the phrase the question targets lives ONLY in the +#: description the vision pass appends to page 2 (the owner's L5 +#: scenario — the illustration answers what the text layer cannot). +PAGE_ONE = ( + "This manual covers the complete flat-pack furniture frame, its " + "shipped hardware kit, and the safety checks a first-time assembler " + "should run before starting. The box holds the two side panels, the " + "top board, the adjustable leg set, eight cam locks, fourteen wood " + "screws, two bolts, and the wall-anchoring strap. Work on a flat " + "surface with soft flooring, keep the hardware sorted by size, and " + "read every step through before you begin so you know which panel " + "is which. The frame stands roughly a metre tall once assembled, " + "weights under twelve kilograms, and needs no further tools beyond " + "the included Allen key. Keep the small parts away from children " + "until the final step is done and the cam locks are fully seated." +) +assert len(PAGE_ONE) > 500, "page 1 must be a 500+ char paragraph (the task's recipe)" + +#: Page 2 — the sparse page: the marker line is ALL the text layer +#: carries (14 stripped chars, under the 100-char bar — the candidate); +#: the figure is drawn vector (no raster image). +PAGE_TWO = "Illustration 2" + +#: The stored illustration block (LOCKED A4) — the mock's canned +#: ``PICTORIAL_PAGE_MODE`` reply, deterministic. +ILLUSTRATION_BLOCK = f"[Page 2 illustration: {PICTORIAL_PAGE_ANSWER}]" + +#: The grounded question (b): the house seeded-read call (phase 119 A1 — +#: a zero-read grounded turn chips NOTHING, so the PDF must be READ to +#: earn its ``done.sources`` citation) + the natural question, whose +#: tokens ("two parts joined page") overlap ONLY the illustrated page's +#: content (the mock's token-overlap cosine grounds; the text layer +#: alone would answer nothing). +QUESTION = ( + f"Read the suggested document: read {SOURCE_NAME}/{DOC_PATH} — " + "how are the two parts joined on page two?" +) + +#: The sync ``detail`` key set EXACTLY as pre-phase 140 (the +#: ``app/api/sync.py`` terminal detail — phase 126/133 shape). The +#: phase-140 counters are log-only and must NOT appear (c). +PRE_PHASE_DETAIL_KEYS = frozenset( + { + "files", + "added", + "updated", + "unchanged", + "pruned", + "errors", + "chunks", + "summaries", + "summary_errors", + "dates_updated", + "overview", + "sources_version", + } +) + +#: Phase 79 (task 04, full inventory): the conftest session app owns its +#: port in a combined run — this module app binds its own port instead +#: (a same-port second uvicorn dies on bind and would drive the wrong +#: server). Env-overridable. +APP_PORT = int(os.environ.get("E2E_APP_PORT_PICTPDF", "8155")) +APP_URL = f"http://127.0.0.1:{APP_PORT}" + +UPLOAD_TIMEOUT_MS = 30_000 +SYNC_TIMEOUT_MS = 60_000 +SYNCED_LABEL = re.compile(r"^Synced \d{1,2}:\d{2}$") + + +# --------------------------------------------------------------------------- +# Fixture builder (the ``test_pdf_support.py`` PyMuPDF recipe, the +# owner's class: a dense prose page + a sparse vector-illustrated page) +# --------------------------------------------------------------------------- + + +def _build_pdf(path: Path) -> None: + """ONE 2-page PDF (the task's recipe): page 1 = the 700+ char prose + paragraph (``insert_textbox`` — word-wrapped, no mid-word breaks in + the extraction), page 2 = the ``Illustration 2`` marker + a drawn + VECTOR figure (a rectangle + a line arrow with a two-stroke head — + the IKEA pictogram class: ``get_image_info()`` sees ZERO embedded + raster images, the phase-122/137 pipeline is blind to it).""" + doc = fitz.open() + page1 = doc.new_page() + rc = page1.insert_textbox(fitz.Rect(72, 72, 523, 772), PAGE_ONE, fontsize=11) + assert rc >= 0, "the page-1 paragraph must fit its text box" + page2 = doc.new_page() + page2.insert_text((72, 72), PAGE_TWO) + # The vector figure: a rectangle (a part) + an arrow (the join). + page2.draw_rect(fitz.Rect(100, 120, 300, 220)) + page2.draw_line(fitz.Point(320, 170), fitz.Point(470, 170)) + page2.draw_line(fitz.Point(470, 170), fitz.Point(450, 158)) + page2.draw_line(fitz.Point(470, 170), fitz.Point(450, 182)) + doc.set_metadata({"title": PDF_TITLE}) + doc.save(path) + doc.close() + + +@pytest.fixture(scope="module") +def pdf_path(tmp_path_factory: pytest.TempPathFactory) -> Path: + """The fixture PDF (module-scoped, built once) — the local copy the + byte-exact content assertion computes against (the app indexes the + unpacked copy of the SAME bytes).""" + root = tmp_path_factory.mktemp("bor_pict_pdf") + pdf = root / DOC_PATH + _build_pdf(pdf) + # The premise pins (deterministic — the pass's candidate set and the + # gate's own contract, computed from the fixture itself): page 1 is + # prose (no candidate), page 2 is sparse (the candidate). + page_texts = extract_pdf_page_texts(pdf) + candidates = pictorial_pages( + page_texts, + min_chars=Settings.model_fields["pdf_pictorial_min_chars"].default, + ) + assert candidates == [2], ( + f"the fixture's sparse-page premise broke — candidates={candidates}" + ) + return pdf + + +@pytest.fixture(scope="module") +def tar_path(pdf_path: Path) -> Path: + """The fixture archive: ONE PDF at the root (the ``.tar`` format — + the owner's ``ikea_instructions.tar`` class; the upload gate admits + it) — source ``e2e-pictorial``, document path ``instructions.pdf``.""" + archive = pdf_path.parent / f"{SOURCE_NAME}.tar" + with tarfile.open(archive, "w") as tf: + tf.add(pdf_path, arcname=DOC_PATH) + return archive + + +# --------------------------------------------------------------------------- +# Module app (the conftest env pattern + the phase-140 knobs) +# --------------------------------------------------------------------------- + + +def _app_env(mock_llm: int, app_port: int, scratch: Path) -> dict[str, str]: + """The conftest app env (leak guards included) with the phase-140 + knobs: ``BOR_IMPORT_EXTENSIONS`` pinned to a set carrying the + ``pdf`` token (the phase-126 gate — an operator's local gitignored + ``.env`` cannot leak the scope either way), ``BOR_PDF_PICTORIAL`` + pinned EXPLICITLY on (the conftest pins it off in ``os.environ`` + for the pre-phase suites — this suite is the documented opt-in; + without it the whole premise is void), and ``BOR_IMAGES`` pinned + off (out of scope — the vector figure yields no image doc anyway).""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = ( + "https://aipi.reeseapps.com/v1" + if USE_REAL_LLM + else f"http://127.0.0.1:{mock_llm}/v1" + ) + # Mock-calibrated gate (conftest pattern) — the story's question + # grounds on these values (see QUESTION). + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15" + env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15" + # Phase 67: instant retry waits + the code-default budget. + env["BOR_LLM_RETRY_DELAY"] = "0" + env["BOR_LLM_RETRIES"] = str(Settings.model_fields["llm_retries"].default) + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + # Leak guards (conftest pattern). + env["BOR_GIT_SOURCES"] = "" + env["BOR_DOCS_REPO"] = "" + env["BOR_SUGGESTIONS"] = json.dumps( + Settings.model_fields["suggestions"].default + ) + env["BOR_INPUT_PLACEHOLDER"] = Settings.model_fields["input_placeholder"].default + env["BOR_FOOTER_TEXT"] = Settings.model_fields["footer_text"].default + # Phase 140: the import scope WITH the ``pdf`` token + the toggle + # under test (the EXPLICIT opt-in — see the module docstring) + the + # suite-private homes. + env["BOR_IMPORT_EXTENSIONS"] = "md,markdown,txt,pdf" + env["BOR_PDF_PICTORIAL"] = "1" + env["BOR_IMAGES"] = "false" + env["BOR_UPLOAD_DIR"] = str(scratch / "uploads") + env["BOR_SOURCES_DIR"] = str(scratch / "checkouts") + env["BOR_IMAGE_DIR"] = str(scratch / "images") + return env + + +@pytest.fixture(scope="module") +def app_server( + mock_llm: int, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[str]: + """The app under test (all three tests): one module app with the + phase-140 env (see :func:`_app_env`).""" + scratch = tmp_path_factory.mktemp("bor_pict_scratch") + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], + cwd=REPO, + env=_app_env(mock_llm, APP_PORT, scratch=scratch), + ) + try: + _wait_http(f"{APP_URL}/api/health") + yield APP_URL + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +# --------------------------------------------------------------------------- +# DB + UI helpers +# --------------------------------------------------------------------------- + + +def _truncate_all() -> None: + """Fresh KB + registry (the E2E isolation pattern): the suites share + one Postgres, and a leftover row would corrupt the counts. + ``sources_meta`` (the KB generation counter) resets with the KB — + the sync re-creates the row.""" + with SessionLocal() as db: + db.execute( + text( + "TRUNCATE chunks, documents, query_log, steering_notes, " + "kb_overview, git_sources, folder_summaries, sources_meta" + ) + ) + db.commit() # without the commit the TRUNCATE rolls back (the house pattern) + + +def _health_db(app_url: str) -> bool: + try: + return httpx.get(f"{app_url}/api/health", timeout=5).json()["db"] == "up" + except Exception: # noqa: BLE001 — unreachable is the skip case + return False + + +def _wait_sync_settled(base_url: str, timeout_s: float = 60.0) -> None: + """No background sync may leak across tests (the run state lives in + the app's memory, and a still-running run would keep importing into + the NEXT test's truncated KB): wait for the app's sync status to be + non-running BEFORE the truncate (the phase-96 pattern).""" + with httpx.Client(base_url=base_url, timeout=5.0) as client: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, r.text + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + r = client.get("/api/sync/status") + assert r.status_code == 200, r.text + if r.json()["state"] != "running": + return + time.sleep(0.2) + raise AssertionError(f"a sync was still running at a test boundary: {base_url}") + + +@pytest.fixture(scope="module", autouse=True) +def _clean_kb(app_url: str) -> Iterator[None]: + """Module-end KB cleanup (the phase-96 pattern, module granularity): + the three tests SHARE the module seed's single source + document, + so there is NO per-test truncate (it would wipe the seed from under + the next test — the per-test pattern belongs to the suites where + every test imports its own source); instead the shared KB is + flushed ONCE, after the last test — no rows leak into the next + suite's run. The pre-wait keeps a leaked sync (the module app is + still up here) from racing the truncate.""" + yield + _wait_sync_settled(app_url) + _truncate_all() + + +def _upload_tar(page: Page, app_url: str, archive: Path) -> None: + """The real admin upload flow: form login → the git-sources page → + pick the archive → submit → the terminal result line (phase 90: + "Uploaded — press Sync sources to import it.").""" + login(page, app_url, next="/git-sources.html") + expect(page).to_have_url(app_url + "/git-sources.html", timeout=30_000) + expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000) + page.set_input_files("#archive-upload-file", str(archive)) + page.click("#archive-upload-btn") + result = page.locator("#archive-upload-result") + expect(result).to_be_visible(timeout=UPLOAD_TIMEOUT_MS) + expect(result).to_have_text( + f"Uploaded {SOURCE_NAME} — press Sync sources to import it.", + timeout=UPLOAD_TIMEOUT_MS, + ) + + +def _sync_via_ui(page: Page, app_url: str, expected_result: str) -> None: + """The RAG page's "Sync sources" button (the scan): click → + "Syncing…" → the terminal "Synced HH:MM" label + the fresh counts + in #sync-result (the never-stale contract).""" + page.goto(app_url + "/sources.html") + btn = page.locator("#sync-btn") + expect(btn).to_be_visible(timeout=30_000) + expect(page.locator("#sync-label")).to_have_text( + re.compile(r"^Sync sources$|^Synced \d{1,2}:\d{2}$") + ) + btn.click() + expect(btn).to_be_disabled() + expect(page.locator("#sync-label")).to_have_text("Syncing…") + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) + expect(btn).to_be_enabled() + expect(page.locator("#sync-result")).to_have_text(expected_result) + + +def _drill_to_source(page: Page, name: str) -> None: + """Top level → the source (the file table then holds its direct + files — ours is at the source root).""" + page.locator("#folders-tbody .folder-link").first.wait_for(state="visible") + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=30_000) + + +def _chat_frames(app_url: str, message: str) -> list[dict[str, Any]]: + """One ``POST /api/chat`` turn as RAW SSE frames (the wire lens — + the house pattern from ``test_honest_deflection.py``): the signed + admin client streams the turn and parses the ``data: `` + frames (PLAN §4: ``data: \\n\\n``).""" + with httpx.Client(base_url=app_url, timeout=120.0) as client: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, r.text + frames: list[dict[str, Any]] = [] + with client.stream( + "POST", "/api/chat", json={"message": message}, timeout=120.0 + ) as r: + assert r.status_code == 200, r.read().decode() + buf = "" + for part in r.iter_text(): + buf += part + while "\n\n" in buf: + frame, buf = buf.split("\n\n", 1) + if frame.strip().startswith("data:"): + frames.append( + json.loads(frame.strip().removeprefix("data:").strip()) + ) + assert buf.strip() == "" # the stream ends cleanly on a frame boundary + return frames + + +# --------------------------------------------------------------------------- +# The module seed: upload + sync the fixture PDF through the real UI, +# once (the story E2E truncate/re-import-per-module pattern) +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def seeded( + app_url: str, + browser: Browser, + tar_path: Path, + pdf_path: Path, +) -> Iterator[dict[str, str]]: + """Upload the single-PDF tar through the admin UI, then scan it with + the RAG page's Sync button (the mock's vision model describes the + sparse page in-process — the ``PICTORIAL_PAGE_MODE`` canned reply). + The seed asserts the stored document BYTE-EXACT: the fixture's own + extraction (page blocks, the page-2 candidate carrying the canned + illustration block) — the candidates computed from + ``extract_pdf_page_texts`` + ``pictorial_pages`` IN the fixture (no + hardcoded page list). This is the module app's toggle premise + witness too: without the ``BOR_PDF_PICTORIAL`` opt-in, page 2 would + lack its block and this fails loudly. Yields the doc's identity.""" + if not _health_db(app_url): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + + _truncate_all() + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + _upload_tar(page, app_url, tar_path) + # The PDF alone (no embedded raster images — the vector figure + # yields no image doc): one file, one added document. + _sync_via_ui(page, app_url, expected_result="1 added") + with SessionLocal() as db: + doc = db.scalar( + select(Document).where( + Document.source == SOURCE_NAME, Document.path == DOC_PATH + ) + ) + assert doc is not None, "the uploaded PDF must become a document" + # LOCKED A2: one document per PDF, the metadata title; a TEXT + # document (no image doc, no image path). + assert doc.title == PDF_TITLE + assert doc.is_image is False and doc.image_path is None + # The candidates (computed from the fixture itself — the + # deterministic gate, LOCKED A1) + the byte-exact stored + # content: the extraction's page blocks, the described + # candidate carrying ``[Page N illustration: ]`` + # (LOCKED A4). + page_texts = extract_pdf_page_texts(pdf_path) + candidates = pictorial_pages( + page_texts, + min_chars=Settings.model_fields["pdf_pictorial_min_chars"].default, + ) + assert candidates == [2] + parts = [] + for p, t in page_texts.items(): + block = t + if p in candidates: + block += f"\n\n[Page {p} illustration: {PICTORIAL_PAGE_ANSWER}]" + parts.append(f"{PAGE_SEPARATOR.format(page=p)}\n{block}") + expected = "\n\n".join(parts) + assert doc.content == expected, ( + "the stored content is not the extraction with the " + "described sparse page — the pictorial pass misfired" + ) + # The non-markdown doc got its lite summary (the mock's + # SUMMARY_MODE digest) — the summary chunk + the viewer's + # Summary panel exist. + assert doc.summary + finally: + page.close() + yield {"title": PDF_TITLE} + + +# --------------------------------------------------------------------------- +# 1. Sources page + document viewer (a): the described page renders +# --------------------------------------------------------------------------- + + +def test_sources_page_and_viewer_show_the_described_page( + page: Page, app_url: str, seeded: dict[str, str] +) -> None: + """The KB total is 1 doc / 3 chunks (the PDF: 2 page-anchored + content chunks + the phase-30 summary chunk — no image doc; the + vector figure is invisible to the embedded-image pipeline). The + drill-down table lists the PDF as a plain text row (LOCKED A4), and + the same-page document viewer (the row's path link, phase 26) + renders the stored content under the METADATA title: page 1's prose + WITHOUT an illustration block (not a candidate) and page 2 WITH + ``[Page 2 illustration: ]`` (the described sparse page — + LOCKED A4's user surface), plus the Summary panel (a non-image doc + with a non-empty summary).""" + login(page, app_url) # lands on /sources.html + expect(page.locator("#stat-docs")).to_have_text("1") + expect(page.locator("#stat-chunks")).to_have_text("3") + expect(page.locator("#sources-empty")).to_be_hidden() + + _drill_to_source(page, SOURCE_NAME) + expect(page.locator("#docs-tbody tr")).to_have_count(1) + row = page.locator("#docs-tbody tr").first + # A plain text row — no thumbnail box (it is a text document, A4). + expect(row.locator(".kb-doc-thumb")).to_have_count(0) + + page.locator(f'#docs-tbody a.doc-link:text-is("{DOC_PATH}")').click() + modal = page.locator("#doc-modal") + expect(modal).to_be_visible(timeout=30_000) + expect(page.locator("#doc-modal-title")).to_have_text(PDF_TITLE) + + raw = modal.locator("pre.doc-raw") + # Page 1 — the prose WITHOUT any illustration block. The viewer + # renders the content verbatim (``insert_textbox`` wraps mid- + # sentence, so compare whitespace-normalized). + expect(raw).to_contain_text("---PAGE-1---") + rendered = " ".join(raw.inner_text().split()) + assert " ".join(PAGE_ONE.split()) in rendered, ( + "the page-1 prose is not what the viewer renders" + ) + assert "[Page 1 illustration" not in rendered, ( + "page 1 (over the sparseness bar) must carry NO illustration block" + ) + # Page 2 — the sparse page WITH the described block (a) — the mock's + # canned reply, deterministic. + expect(raw).to_contain_text("---PAGE-2---") + expect(raw).to_contain_text(PAGE_TWO) + expect(raw).to_contain_text(ILLUSTRATION_BLOCK) + # The Summary panel renders (a non-image doc with a non-empty + # summary); no image block (the PDF is not an image document). + expect(modal.locator(".doc-summary")).to_have_count(1) + expect(modal.locator(".doc-image-img")).to_have_count(0) + + +# --------------------------------------------------------------------------- +# 2. Chat (b): the question only the illustration answers grounds, and +# the answer echoes the described phrase +# --------------------------------------------------------------------------- + + +def test_chat_question_grounds_and_echoes_the_illustration( + app_url: str, seeded: dict[str, str] +) -> None: + """The raw SSE over the scripted seeded-read question (phase 119 A1 + — a zero-read grounded turn chips NOTHING, so the PDF must be READ + to earn its ``done.sources`` citation): the turn GROUNDS (no + deflection — the described page's embedding retrieves: the mock's + deterministic token-overlap cosine clears the mock-calibrated 0.30 + gate, FTS corroborates), the mock's scripted ``read`` runs (EXACTLY + one ``tool`` frame — the doc is far under the read cap, so no + truncation and no ``tool_result`` frame), and the answer ECHOES the + read result verbatim — the ``[Page 2 illustration: …]`` block and + its phrase reach the model and land in the rendered answer (the + described page's content is what a chat turn can ground on — the + owner's L5 rationale end to end).""" + frames = _chat_frames(app_url, QUESTION) + assert frames, "the SSE stream settled with no frames at all" + assert not any(f.get("type") == "error" for f in frames), ( + f"the turn errored: {[f for f in frames if f.get('type') == 'error']}" + ) + + # The scripted read — exactly one tool call, on the PDF. + tools = [f for f in frames if f.get("type") == "tool"] + assert len(tools) == 1, f"expected exactly one tool frame: {tools}" + assert tools[0]["name"] == "read" + assert tools[0]["argument"] == f"{SOURCE_NAME}/{DOC_PATH}" + # The doc is far under the read cap — no truncation frame (the echo + # below therefore carries the FULL content, the block included). + assert not any(f.get("type") == "tool_result" for f in frames) + + # The terminal frame: ``done``, NOT deflected. + done = [f for f in frames if f.get("type") == "done"] + assert len(done) == 1 and frames[-1] is done[0], ( + f"the stream must settle with exactly one terminal done frame: " + f"{[f.get('type') for f in frames]}" + ) + assert done[0]["deflected"] is False, ( + "the question grounded on the described page — a deflection means " + "the illustrated content did not retrieve" + ) + # The ``done`` event's ``sources`` — the citation surface (read docs + # only, phase 119 A1) — carries EXACTLY the PDF. + refs = done[0]["sources"] + assert [(r["source"], r["path"]) for r in refs] == [(SOURCE_NAME, DOC_PATH)], ( + f"the done event's sources must be exactly the read PDF: {refs}" + ) + assert refs[0]["title"] == PDF_TITLE + + # The answer (the joined deltas) echoes the read result VERBATIM — + # the header line and the described block with the canned phrase. + answer = "".join(f["text"] for f in frames if f.get("type") == "delta") + assert "Here's what the read returned:" in answer + assert f"Document {SOURCE_NAME}/{DOC_PATH}:" in answer + assert ILLUSTRATION_BLOCK in answer, ( + "the answer does not echo the described illustration block — " + "the vision pass's content never reached the model" + ) + assert PICTORIAL_PAGE_ANSWER in answer # the phrase itself (the task's pin) + + +# --------------------------------------------------------------------------- +# 3. Sync detail (c): the pre-phase key set is untouched (stats are +# log-only) +# --------------------------------------------------------------------------- + + +def test_sync_detail_keeps_the_pre_phase_keys( + app_url: str, seeded: dict[str, str] +) -> None: + """The module run's terminal ``detail`` (``GET /api/sync/status`` — + the same object the UI's "1 added" line renders from) keeps the + pre-phase key set EXACTLY: the phase-140 counters + (``pdfs_pictorial`` / ``pdf_pictorial_pages``) are log-only (PLAN + §9, the phase-133 convention) and must NOT ride the wire, and the + run's own file counters read 1 file / 1 added / 0 unchanged / + 0 pruned / 0 errors (the single PDF, a fresh KB).""" + with httpx.Client(base_url=app_url, timeout=30.0) as client: + r = client.post("/api/login", json={"password": ADMIN_PASSWORD}) + assert r.status_code == 204, r.text + r = client.get("/api/sync/status") + assert r.status_code == 200, r.text + body = r.json() + assert body["state"] == "success", body + detail = body["detail"] + # The stats log-only convention — the pictorial counters never + # reach the wire. + assert "pdfs_pictorial" not in detail, ( + "the log-only pdfs_pictorial counter leaked into the sync detail" + ) + assert "pdf_pictorial_pages" not in detail, ( + "the log-only pdf_pictorial_pages counter leaked into the sync detail" + ) + # The shape itself — the pre-phase key set EXACTLY (no new field). + assert set(detail) == set(PRE_PHASE_DETAIL_KEYS), ( + f"the sync detail shape changed — got {sorted(detail)}" + ) + # The run's own file counters (a fresh KB: one file, one added). + assert detail["files"] == 1, detail + assert detail["added"] == 1, detail + assert detail["unchanged"] == 0, detail + assert detail["pruned"] == 0, detail + assert detail["errors"] == 0, detail diff --git a/tests/integration/test_importer.py b/tests/integration/test_importer.py index 6624693..1e04c88 100644 --- a/tests/integration/test_importer.py +++ b/tests/integration/test_importer.py @@ -34,16 +34,26 @@ from __future__ import annotations import asyncio import base64 import hashlib +import logging +import tarfile +from collections.abc import Iterator from pathlib import Path import pymupdf as fitz # PyMuPDF — the temp-PDF fixture builder (phase 126) +import pytest from sqlalchemy import select from app.config import Settings from app.models import Chunk, Document from app.rag.importer import import_sources from app.rag.llm import EmbeddingError -from app.rag.summarizer import DESCRIBE_PROMPT +from app.rag.pdf import ( + PAGE_SEPARATOR, + extract_pdf_page_texts, + extract_pdf_text, + pictorial_pages, +) +from app.rag.summarizer import DESCRIBE_PROMPT, PICTORIAL_PAGE_MODE from tests.fakes import FakeEmbedder #: A real 1×1 transparent PNG (the phase-122 fixture — the importer is @@ -99,10 +109,17 @@ def _image_llm[ ](tmp_path, llm_cls: type[LLM] = _MockVisionLLM, **kwargs) -> LLM: """A fake LLM with the phase-122 image knobs (toggle ON; the image dir defaults under *tmp_path*). *llm_cls* may be the mock vision - client (the default — ``text_parts``/``chat_models`` state).""" + client (the default — ``text_parts``/``chat_models`` state). + Phase 140: ``pdf_pictorial`` defaults to OFF here — the pre-phase- + 140 PDF suites in this file pin the pre-phase content shape (their + fixture pages are sparse — under the 100-char bar — so the + default-ON pictorial pass would append illustration blocks and + move the byte-exact pins); the phase-140 suites pass + ``pdf_pictorial=True`` explicitly.""" kwargs.setdefault("_env_file", None) kwargs.setdefault("images", True) kwargs.setdefault("image_dir", str(tmp_path / "images")) + kwargs.setdefault("pdf_pictorial", False) llm = llm_cls() llm.settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return llm @@ -1506,3 +1523,314 @@ def test_backfill_embed_failure_rolls_back_the_image_and_retries_next_sync( assert img2.image_context.startswith("Extracted from ") finally: _cleanup_source(db, source_root.name) + + +# --------------------------------------------------------------------------- +# Phase 140 — Pictorial PDFs: sparse pages rendered + vision-described +# into the indexed content (the owner's IKEA repro, TODO L5; the kill +# switch; the phase-133 precedence pin). +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parents[2] + +#: The owner's untracked fixture (TODO L5 — the stable reference: the +#: TARBALL, never the extracted PDF; the phase-133 real-fixture +#: convention — ``test_sources/`` is gitignored, the test extracts it +#: into ``tmp_path`` and never touches the repo, and the owner's +#: machine always has it: a missing tar skips LOUDLY with the named +#: reason, it is never silently green). +IKEA_TARBALL = REPO_ROOT / "test_sources" / "ikea_instructions.tar" + +#: The temp source dir's basename is the source name (phase 38). +IKEA_SOURCE = "ikea" + +#: The fixture's embedded metadata title (Adobe InDesign's +#: "Assembly Instructions" — the phase-140 context). +IKEA_TITLE = "Assembly Instructions" + +#: The fixture's text-heavy pages (pages 2–5 — the multilingual +#: warnings, 3,091–7,288 chars — never candidates; the sparse pages +#: run 4–73 chars, so the 100-char bar sits between the classes). +IKEA_TEXT_HEAVY_PAGES = (2, 3, 4, 5) + + +class _PictorialVisionLLM(FakeEmbedder): + """The capturing multimodal fake (task 03): a vision call whose + text part carries ``PICTORIAL_PAGE_MODE`` (the pictorial pass's + seam — ``describe_pdf_page``) gets the fixed per-candidate canned + reply and is recorded in ``pictorial_texts``; a call carrying + ``IMAGE_DESCRIPTION_MODE`` (the phase-133 fallback's / phase-122 + images' seam — ``describe_image``) gets a DISTINCT canned reply + and is recorded in ``image_texts``. The fixed replies make the + stored content a PURE FUNCTION of the fixture (the digest-stable + premise of the re-sync assertion), and the two recordings witness + WHICH seam fired (the precedence pin). String (``lite``) calls + keep the ``FakeEmbedder`` behaviour. The REAL + ``import_sources`` → ``_pictorial_pdf_pass`` → + ``describe_pdf_page`` chain runs against it (no seam patch).""" + + PICTORIAL_ANSWER = ( + "IKEA pictorial page: the illustration shows two parts being " + "joined with a blue arrow; a cam lock is tightened with the " + "included hex key." + ) + IMAGE_ANSWER = ( + "Fallback screenshot: the page shows an assembly pictogram." + ) + + def __init__(self) -> None: + super().__init__() + self.pictorial_texts: list[str] = [] + self.image_texts: list[str] = [] + + async def chat(self, messages, model=None): + self.chat_calls.append(list(messages)) + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + if isinstance(user, list): + text = next((p["text"] for p in user if p.get("type") == "text"), "") + if PICTORIAL_PAGE_MODE in text: + self.pictorial_texts.append(text) + return self.PICTORIAL_ANSWER + self.image_texts.append(text) + return self.IMAGE_ANSWER + first = user.split() + return "Summary of " + (first[0] if first else "") + + +@pytest.fixture() +def ikea_pdf(db, tmp_path) -> Iterator[Path]: + """The owner's IKEA tarball extracted into a temp ``ikea/`` source + dir (the phase-133 real-fixture convention — ``tmp_path`` only, + never the repo). Any leftover ``ikea`` source rows from a crashed + earlier run are cleaned before AND after (the shared dev DB is + reused across runs). Skips LOUDLY with the named reason when the + untracked owner fixture is missing.""" + if not IKEA_TARBALL.is_file(): + pytest.skip( + "test_sources/ikea_instructions.tar is missing — the untracked " + "owner fixture (one 24-page vector-drawn assembly-instruction " + "PDF; the owner's machine always has it)" + ) + _cleanup_source(db, IKEA_SOURCE) + root = tmp_path / IKEA_SOURCE + root.mkdir() + with tarfile.open(IKEA_TARBALL) as tar: + tar.extractall(root, filter="data") + pdfs = sorted(root.rglob("*.pdf")) + assert len(pdfs) == 1, f"the IKEA tar carries exactly one PDF, found: {pdfs}" + yield pdfs[0] + _cleanup_source(db, IKEA_SOURCE) + + +def test_ikea_tar_sparse_pages_are_described_and_rerun_unchanged( + db, tmp_path, ikea_pdf: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The owner's EXACT file (TODO L5) through the REAL importer: + the deterministic gate's candidates — computed IN the test from + ``extract_pdf_page_texts`` + ``pictorial_pages`` (no hardcoded + page list — the fixture is stable but the test must not rot) — + every one carries ``[Page N illustration: ]`` AFTER its + own text (LOCKED A2 — no LLM judge: every described page joins + the content; A4 — the page's text stays), the text-heavy pages + (2–5) carry NO block, the chunks carry the canned description + (the owner's rationale — more information is better for the + embeddings), the title is the metadata title, the digest is over + the FINAL content, and a re-sync with the same fake lands + ``unchanged`` (digest stable, byte-stable row, NO re-embed; the + vision calls are re-issued before the unchanged check can skip + them — the documented no-cache cost, the phase-133 convention, + A21 endorses it).""" + pdf_path = ikea_pdf + llm = _image_llm( + tmp_path, + llm_cls=_PictorialVisionLLM, + pdf_pictorial=True, + import_extensions="md,txt,pdf", + ) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + s1 = asyncio.run(import_sources([pdf_path.parent], llm, session=db)) + assert (s1.files, s1.added, s1.errors) == (1, 1, 0) + assert s1.formats == {"pdf": 1} + + # The candidates — computed IN the test (no hardcoded page + # list). The fixture class: the sparse pages are 4–73 chars, + # the text-heavy pages 3,091+ — the 100-char bar separates. + page_texts = extract_pdf_page_texts(pdf_path) + candidates = pictorial_pages( + page_texts, min_chars=llm.settings.pdf_pictorial_min_chars + ) + assert candidates, "the fixture must carry sparse (pictorial) pages" + for n in IKEA_TEXT_HEAVY_PAGES: + assert n not in candidates, f"page {n} is text-heavy, never a candidate" + + doc = db.scalar( + select(Document).where( + Document.source == IKEA_SOURCE, Document.path == pdf_path.name + ) + ) + assert doc is not None, "the IKEA PDF did not import" + assert doc.title == IKEA_TITLE # the PDF's metadata title + assert doc.is_image is False + + content = doc.content + # Every page's block is byte-faithful: the page text, the + # described candidates carrying their illustration block + # AFTER the text (A4), and NO block on the non-candidates. + for n, text in page_texts.items(): + block = f"{PAGE_SEPARATOR.format(page=n)}\n{text}" + if n in candidates: + block += ( + f"\n\n[Page {n} illustration: " + f"{_PictorialVisionLLM.PICTORIAL_ANSWER}]" + ) + assert block in content, f"page {n}'s block is not byte-faithful" + if n not in candidates: + assert f"[Page {n} illustration" not in content + assert content.count(" illustration: ") == len(candidates) + # The digest runs over the FINAL content (the phase-126/133 + # convention — A4). + assert doc.content_hash == hashlib.sha256( + content.encode("utf-8") + ).hexdigest() + + # The chunks carry the canned description (the owner's L5 + # rationale — the embeddings hit the illustrations), and the + # phase-30/118 summary ran on the final content. + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + content_chunks = [c for c in chunks if not c.is_summary] + assert any( + _PictorialVisionLLM.PICTORIAL_ANSWER in c.content for c in content_chunks + ), "no content chunk carries the canned description" + for c in chunks: + assert c.embedding is not None and len(c.embedding) == 768 + assert doc.summary is not None + + # The counters (log-only — the sync detail shape is untouched) + # + the ONE loud log line (PLAN §9), with the computed N. + assert s1.pdfs_pictorial == 1 + assert s1.pdf_pictorial_pages == len(candidates) + assert s1.pdfs_screenshotted == 0 # the layer is USABLE — no fallback + assert llm.image_texts == [] # the fallback seam never fired + min_chars = llm.settings.pdf_pictorial_min_chars + lines = [ + r.getMessage() for r in caplog.records if "pictorial pass" in r.getMessage() + ] + assert lines == [ + f"pdf: pictorial pass source={IKEA_SOURCE} path={pdf_path.name} — " + f"{len(candidates)} candidate pages, {len(candidates)} described " + f"(min_chars={min_chars})" + ] + + # The re-sync with the SAME fake: ``unchanged`` — the final + # content is a pure function of the fixture + the canned + # replies, so the digest matches and the row is skipped (no + # re-chunk, no re-embed, no re-summary, no KB change). The + # vision calls ARE re-issued (the documented no-cache cost — + # the phase-133 convention; a non-deterministic vision model + # re-describes the row each sync: a cost, not a bug). + embeds1 = len(llm.calls) + pictorial1 = len(llm.pictorial_texts) + content1, digest1 = doc.content, doc.content_hash + s2 = asyncio.run(import_sources([pdf_path.parent], llm, session=db)) + assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 1) + assert s2.pdfs_pictorial == 1 # the pass re-fired + assert len(llm.pictorial_texts) == 2 * pictorial1 # re-issued (cost) + assert len(llm.calls) == embeds1, "unchanged: no re-embed" + assert s2.changed_paths == [], "unchanged: no KB change recorded" + doc2 = db.scalar( + select(Document).where( + Document.source == IKEA_SOURCE, Document.path == pdf_path.name + ) + ) + assert doc2 is not None + assert doc2.content == content1 # byte-stable content + assert doc2.content_hash == digest1 # byte-stable digest + finally: + _cleanup_source(db, IKEA_SOURCE) + + +def test_ikea_tar_kill_switch_is_byte_identical_to_extraction( + db, tmp_path, ikea_pdf: Path +) -> None: + """``BOR_PDF_PICTORIAL=0`` (the house kill switch — LOCKED A3): + the pass never runs on the owner's file — no vision call, no + counters, no log line — and the stored content is + ``extract_pdf_text``'s output BYTE-IDENTICAL to the pre-phase-140 + path.""" + pdf_path = ikea_pdf + llm = _image_llm( + tmp_path, + llm_cls=_PictorialVisionLLM, + pdf_pictorial=False, + import_extensions="md,txt,pdf", + ) + try: + s1 = asyncio.run(import_sources([pdf_path.parent], llm, session=db)) + assert (s1.files, s1.added, s1.errors) == (1, 1, 0) + doc = db.scalar( + select(Document).where( + Document.source == IKEA_SOURCE, Document.path == pdf_path.name + ) + ) + assert doc is not None + assert doc.content == extract_pdf_text(pdf_path), ( + "the kill switch must keep the pre-phase-140 content " + "byte-identical" + ) + assert llm.pictorial_texts == [] # zero vision calls + assert (s1.pdfs_pictorial, s1.pdf_pictorial_pages) == (0, 0) + finally: + _cleanup_source(db, IKEA_SOURCE) + + +def test_garbage_pdf_with_pictorial_on_takes_the_fallback_not_the_pass( + db, tmp_path +) -> None: + """The precedence pin (LOCKED): an UNUSABLE text layer takes the + phase-133 full-page screenshot fallback — the pictorial pass + NEVER runs and never double-visions (the gate is the only gate): + with ``pdf_pictorial=True`` (the code default), a garbage-text + PDF (the phase-133 synthetic recipe — XML-ish lines) lands with + the fallback's full-page canned descriptions, and the fake + records ZERO ``PICTORIAL_PAGE_MODE`` calls.""" + source_root = tmp_path / "garbpike" + source_root.mkdir() + doc = fitz.open() + for p in (1, 2): + page = doc.new_page() + for i in range(10): + page.insert_text((72, 72 + i * 14), f"{i}") + doc.save(source_root / "garbage.pdf") + doc.close() + llm = _image_llm( + tmp_path, + llm_cls=_PictorialVisionLLM, + pdf_pictorial=True, + import_extensions="md,txt,pdf", + ) + try: + summary = asyncio.run(import_sources([source_root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + assert summary.pdfs_screenshotted == 1 # the gate fired → the fallback + assert summary.pdfs_pictorial == 0 + assert summary.pdf_pictorial_pages == 0 + assert llm.pictorial_texts == [], ( + "the pictorial pass must never run on an unusable layer " + "(no double vision)" + ) + assert len(llm.image_texts) == 2 # one fallback describe per page + row = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == "garbage.pdf" + ) + ) + assert row is not None + assert row.content == ( + f"---PAGE-1---\n{_PictorialVisionLLM.IMAGE_ANSWER}\n\n" + f"---PAGE-2---\n{_PictorialVisionLLM.IMAGE_ANSWER}" + ) + finally: + _cleanup_source(db, source_root.name) diff --git a/tests/integration/test_pdf_hnb_guide_regression.py b/tests/integration/test_pdf_hnb_guide_regression.py index 9b3a66d..c8d86bb 100644 --- a/tests/integration/test_pdf_hnb_guide_regression.py +++ b/tests/integration/test_pdf_hnb_guide_regression.py @@ -360,7 +360,16 @@ def test_hnb_guide_imports_clean_through_the_real_pipeline( ) ] assert len(multimodal_calls) >= 1, "the multimodal chat was never called" - assert len(llm.vision_texts) == len(image_docs), ( + # Phase 140 (task 03): the default-ON pictorial pass ALSO runs on + # this fixture's sparse pages (the fake is seam-agnostic — the + # same canned reply) — so the pin counts only the EMBEDDED-IMAGE + # describe calls (the ``DESCRIBE_PROMPT``-headed text parts; the + # ``PICTORIAL_PAGE_MODE`` parts are the pass's own calls, covered + # by the phase-140 suites). + image_vision_texts = [ + t for t in llm.vision_texts if t.startswith(DESCRIBE_PROMPT) + ] + assert len(image_vision_texts) == len(image_docs), ( "one vision call per embedded image doc (no failures)" ) # The locked prompt + the generalized prefix + the document context @@ -372,7 +381,7 @@ def test_hnb_guide_imports_clean_through_the_real_pipeline( + _expected_context_block(PDF_REL, _page_of(img.path), page_texts) for img in image_docs ) - assert Counter(llm.vision_texts) == expected_calls + assert Counter(image_vision_texts) == expected_calls assert summary.images_failed == 0 # The added accounting (see the module docstring): the walked file diff --git a/tests/unit/test_importer.py b/tests/unit/test_importer.py index a7b8f40..3c8a01d 100644 --- a/tests/unit/test_importer.py +++ b/tests/unit/test_importer.py @@ -989,8 +989,10 @@ def test_import_summary_log_line_includes_summary_counters( sits between ``summary_errors`` and ``dates_updated``; the phase-106 date-refresh counter, the phase-122 ``images_failed`` counter, the phase-137 ``image_context_backfilled`` counter (next - to ``images_failed``), and the phase-133 ``pdfs_screenshotted`` - counter sit after ``dates_updated``, before ``formats``.""" + to ``images_failed``), the phase-133 ``pdfs_screenshotted`` + counter, and the phase-140 ``pdfs_pictorial`` / + ``pdf_pictorial_pages`` counters (next to ``pdfs_screenshotted``) + sit after ``dates_updated``, before ``formats``.""" s = ImportSummary() s.files, s.added, s.chunks, s.embed_batches = 3, 3, 5, 4 s.summaries, s.summary_errors = 2, 1 @@ -999,6 +1001,8 @@ def test_import_summary_log_line_includes_summary_counters( s.images_failed = 0 s.image_context_backfilled = 2 s.pdfs_screenshotted = 1 + s.pdfs_pictorial = 2 + s.pdf_pictorial_pages = 5 s.formats = {"md": 1, "yaml": 2} with caplog.at_level(logging.INFO, logger="app.importer"): s.log() @@ -1007,7 +1011,8 @@ def test_import_summary_log_line_includes_summary_counters( "import: summary files=3 added=3 updated=0 unchanged=0 pruned=0 errors=0 " "chunks=5 embed_batches=4 summaries=2 summary_errors=1 summary_backfilled=1 " "dates_updated=0 images_failed=0 image_context_backfilled=2 " - "pdfs_screenshotted=1 formats=yaml:2,md:1" + "pdfs_screenshotted=1 pdfs_pictorial=2 pdf_pictorial_pages=5 " + "formats=yaml:2,md:1" ) @@ -1636,6 +1641,11 @@ def test_on_chunk_progress_pdf_branch_reports_content_chunks( images=True, image_dir=str(tmp_path / "images"), import_extensions="md,txt,pdf", + # Phase 140: the pre-phase-140 shape pin — this fixture's pages + # are sparse (the default-ON pictorial pass would append + # illustration blocks and add vision calls); the pass is + # covered in tests/unit/test_pictorial_import.py. + pdf_pictorial=False, ) try: summary = asyncio.run( @@ -1742,12 +1752,18 @@ def _pdf_settings(tmp_path: Path, images: bool = False) -> Settings: silently skip the PDF) with the image knobs at their test values (``images`` off unless pinned otherwise — the fallback's own vision calls need no image copy, only the phase-122 embedded-image pass - does).""" + does). Phase 140: ``pdf_pictorial`` is pinned OFF — these suites + pin the PRE-phase-140 content shape (their fixture pages are + sparse — under the 100-char bar — so the default-ON pictorial + pass would append illustration blocks and move the byte-exact + pins; ``tests/unit/test_pictorial_import.py`` covers the pass + itself, with the kill switch in both states).""" return Settings( _env_file=None, # pyright: ignore[reportCallIssue] (kwarg exists at runtime) import_extensions="md,txt,pdf", images=images, image_dir=str(tmp_path / "images"), + pdf_pictorial=False, ) diff --git a/tests/unit/test_pdf.py b/tests/unit/test_pdf.py index 90323ee..825e48e 100644 --- a/tests/unit/test_pdf.py +++ b/tests/unit/test_pdf.py @@ -155,11 +155,17 @@ def _pdf_llm[ """A fake LLM whose settings admit the ``pdf`` import token and carry the phase-122 image knobs (the image dir under *tmp_path*); *llm_cls* may be a mock vision client (the ``FakeEmbedder`` - subclass carrying extra state such as ``chat_models``).""" + subclass carrying extra state such as ``chat_models``). Phase 140: + ``pdf_pictorial`` is pinned OFF — this suite pins the PRE-phase-140 + content shape (its fixture pages are sparse — under the 100-char + bar — so the default-ON pictorial pass would append illustration + blocks and move the byte-exact pins; the pass is covered in + tests/unit/test_pictorial_import.py).""" kwargs.setdefault("_env_file", None) kwargs.setdefault("images", images) kwargs.setdefault("image_dir", str(tmp_path / "images")) kwargs.setdefault("import_extensions", "md,txt,pdf") + kwargs.setdefault("pdf_pictorial", False) llm = llm_cls() # pyright: ignore[reportCallIssue] llm.settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return llm diff --git a/tests/unit/test_pictorial_gate.py b/tests/unit/test_pictorial_gate.py new file mode 100644 index 0000000..db3f281 --- /dev/null +++ b/tests/unit/test_pictorial_gate.py @@ -0,0 +1,196 @@ +"""Unit tests: the pictorial PDF gate (phase 140, task 01, LOCKED A1/A3). + +Covers the two new settings (``pdf_pictorial`` — the house kill-switch, +default ON — and ``pdf_pictorial_min_chars`` — the sparseness bar with +its fail-loud validator) and the pure, deterministic sparse-page gate +:func:`app.rag.pdf.pictorial_pages`: the ``<``/``=``/``>`` boundary, +whitespace-only pages, the empty-dict no-op, ascending order, and a +large dict. The IKEA docstring example is pinned against the named +threshold. +""" +from __future__ import annotations + +from typing import Any + +import pytest +from pydantic import ValidationError + +from app.config import Settings +from app.rag.pdf import pictorial_pages + + +def _settings(**kwargs: Any) -> Settings: + """Build Settings without reading a .env file (deterministic tests).""" + kwargs.setdefault("_env_file", None) + return Settings(**kwargs) # pyright: ignore[reportCallIssue] (kwarg exists at runtime) + + +# --------------------------------------------------------------------------- +# The settings — defaults + env overrides +# --------------------------------------------------------------------------- + + +def test_pdf_pictorial_defaults_on_and_env_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """LOCKED A3: the pass is ON by default (the owner-confirmed + 2026-09-30 default); ``BOR_PDF_PICTORIAL`` is the house kill-switch + (``0``/``false`` disables the pass entirely — the ``BOR_IMAGES`` + naming pattern).""" + monkeypatch.delenv("BOR_PDF_PICTORIAL", raising=False) + assert _settings().pdf_pictorial is True + monkeypatch.setenv("BOR_PDF_PICTORIAL", "0") + assert _settings().pdf_pictorial is False + monkeypatch.setenv("BOR_PDF_PICTORIAL", "false") + assert _settings().pdf_pictorial is False + monkeypatch.setenv("BOR_PDF_PICTORIAL", "1") + assert _settings().pdf_pictorial is True + + +def test_pdf_pictorial_min_chars_defaults_to_100_and_env_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """LOCKED A1: the default sparseness bar is 100 stripped chars + (the IKEA fixture's sparse pages run 4–73, its text-heavy pages + 3,091+ — 100 sits between the classes); ``BOR_PDF_PICTORIAL_MIN_CHARS`` + overrides it.""" + monkeypatch.delenv("BOR_PDF_PICTORIAL_MIN_CHARS", raising=False) + assert _settings().pdf_pictorial_min_chars == 100 + monkeypatch.setenv("BOR_PDF_PICTORIAL_MIN_CHARS", "50") + assert _settings().pdf_pictorial_min_chars == 50 + monkeypatch.setenv("BOR_PDF_PICTORIAL_MIN_CHARS", "0") + assert _settings().pdf_pictorial_min_chars == 0 + + +# --------------------------------------------------------------------------- +# The validators — the fail-loud pattern +# --------------------------------------------------------------------------- + + +def test_pdf_pictorial_min_chars_zero_is_legal() -> None: + """PLAN A21: ``0`` is the every-page-is-a-candidate mode (the LLM + calls are free) — it must NOT raise.""" + assert _settings(pdf_pictorial_min_chars=0).pdf_pictorial_min_chars == 0 + + +def test_pdf_pictorial_min_chars_default_is_legal() -> None: + """The default (100) is trivially legal — the constructor must + accept it (guards a validator regression that would block startup).""" + assert _settings().pdf_pictorial_min_chars == 100 + + +def test_pdf_pictorial_min_chars_negative_fails_loud_naming_the_field() -> None: + """A negative threshold is a typo — the validator fails startup + loudly (the ``agent_max_rounds`` pattern), naming the field.""" + with pytest.raises(ValidationError, match="pdf_pictorial_min_chars"): + _settings(pdf_pictorial_min_chars=-1) + + +# --------------------------------------------------------------------------- +# pictorial_pages — the pure, deterministic gate +# --------------------------------------------------------------------------- + + +def test_pictorial_pages_strict_less_than_boundary() -> None: + """The ASSUMPTION's boundary, pinned from all three sides at + ``min_chars=5``: 4 stripped chars IS a candidate (``<``), EXACTLY 5 + is NOT (``=`` — a page with exactly the bar's stripped chars is not + sparse), and 6 is NOT (``>``).""" + page_texts = { + 1: "abcd", # 4 < 5 → candidate + 2: "abcde", # 5 == 5 → NOT a candidate (the pinned boundary) + 3: "abcdef", # 6 > 5 → NOT a candidate + } + assert pictorial_pages(page_texts, min_chars=5) == [1] + + +def test_pictorial_pages_min_chars_zero_marks_every_page() -> None: + """PLAN A21's special case: ``min_chars=0`` is the "describe + everything" mode — EVERY page in the map is a candidate (the strict + ``<`` test cannot express this: no length is below 0, so 0 is not a + stricter bar); an empty map still yields ``[]``.""" + assert pictorial_pages({1: "a", 2: "x" * 7000}, min_chars=0) == [1, 2] + # A whitespace-only entry (defensive — the extractor's map never + # holds one) is still a candidate: the 0 branch is ALL KEYS. + assert pictorial_pages({1: " "}, min_chars=0) == [1] + assert pictorial_pages({}, min_chars=0) == [] + + +def test_pictorial_pages_whitespace_only_page_is_a_candidate() -> None: + """A whitespace-only page (strips to nothing — 0 < any positive bar) + is a candidate: the gate counts STRIPPED chars, so padding cannot + mask sparseness. (A truly text-LESS page has no key in the map at + all — the importer's pass owns that case.)""" + assert pictorial_pages({3: " \n\t \r\n "}, min_chars=100) == [3] + + +def test_pictorial_pages_empty_dict_yields_empty_list() -> None: + """An empty map (no readable pages at all) yields ``[]`` — the + clean-text PDF's zero-candidates no-op (no render, no vision + call).""" + assert pictorial_pages({}, min_chars=100) == [] + + +def test_pictorial_pages_result_is_ascending_even_when_input_isnt() -> None: + """The result is ASCENDING (the render/describe order) regardless of + the dict's insertion order — the unsorted candidate set + {9, 2, 50, 5} comes back as [2, 5, 9, 50], and the one non-candidate + (the 210-char page 12) stays out.""" + page_texts = { + 9: "tail marker", + 2: "head marker", + 12: "not sparse " + "x" * 200, # 12+? — long enough, excluded + 50: "Illustration 50", + 5: "mid marker", + } + assert pictorial_pages(page_texts, min_chars=100) == [2, 5, 9, 50] + + +def test_pictorial_pages_large_dict() -> None: + """A 500-page map with every 10th page sparse (an IKEA-class shape + at scale): exactly the sparse pages come back, ascending, and the + 450 text-heavy pages are excluded — the pure function handles a + whole book in one call (no cap, LOCKED A3/A21).""" + page_texts: dict[int, str] = {} + expected: list[int] = [] + for page in range(1, 501): + if page % 10 == 0: + page_texts[page] = "Illustration " + str(page) # 11-13 chars + expected.append(page) + else: + page_texts[page] = "w" * 3000 # text-heavy, never a candidate + assert len(page_texts) == 500 and len(expected) == 50 + assert pictorial_pages(page_texts, min_chars=100) == expected + + +def test_pictorial_pages_ikea_example_from_the_docstring() -> None: + """The docstring's IKEA example, pinned against the named threshold + (LOCKED A1, default 100): the title page ("BESTÅ", 5 chars) and the + "Illustration N" marker pages ARE candidates; the 7 KB of + multilingual warnings is NOT. A title page is a candidate — + describing it is useful (the logo + product name) and never + harmful.""" + page_texts = { + 1: "BESTÅ", # 5 chars → candidate + 2: "w" * 7000, # 7 KB warnings → NOT + 6: "Illustration 6", # 14 chars → candidate + } + assert pictorial_pages(page_texts, min_chars=100) == [1, 6] + + +def test_pictorial_pages_is_pure_and_does_not_mutate_input() -> None: + """Pure + deterministic: repeated calls agree, and the input map + survives untouched (the gate reads, never rewrites).""" + page_texts = {3: "marker", 1: "x" * 500, 2: " "} + original = dict(page_texts) + first = pictorial_pages(page_texts, min_chars=100) + assert pictorial_pages(page_texts, min_chars=100) == first + assert page_texts == original + + +def test_pictorial_pages_uses_stripped_length_defensively() -> None: + """Defensive strip: an UNSTRIPPED map entry (a caller that did not + pre-strip) is judged on its stripped length — padding on either + side cannot push a sparse page past the bar.""" + assert pictorial_pages({1: " " + "y" * 98 + " "}, min_chars=100) == [1] + assert pictorial_pages({1: " " + "y" * 100 + " "}, min_chars=100) == [] diff --git a/tests/unit/test_pictorial_import.py b/tests/unit/test_pictorial_import.py new file mode 100644 index 0000000..4ee007d --- /dev/null +++ b/tests/unit/test_pictorial_import.py @@ -0,0 +1,607 @@ +"""Phase 140, task 03 — the importer's pictorial pass (unit). + +The pass itself (``app.rag.importer._pictorial_pdf_pass`` — the +IKEA assembly-instruction class, TODO L5): a PDF whose text layer +PASSES the phase-133 garbage gate but is SPARSE (mostly pictorial +pages) gets every candidate page (``pictorial_pages`` — the +deterministic gate, LOCKED A1) rendered and vision-described, and the +description appends to the page's content block as +``[Page N illustration: …]`` (LOCKED A2/A4 — no LLM judge; the digest +runs over the FINAL content). + +Pinned here (the synthetic IKEA-class recipe — PyMuPDF, no binary +fixtures; the real tar is pinned in +``tests/integration/test_importer.py``): + +* the happy path — every candidate page (computed IN the test from + ``extract_pdf_page_texts`` + ``pictorial_pages`` — no hardcoded page + list) carries its illustration block, the text-heavy pages do NOT, + the chunks carry the canned description, the digest is over the + FINAL content, the counters + the ONE loud log line fire (PLAN §9). +* the assembly BYTE-IDENTITY pin — with NO descriptions (every vision + call fails) the stored content is ``extract_pdf_text``'s output + byte-for-byte (the page's text is never lost, never reworded). +* the branch matrix: the kill switch off (``pdf_pictorial=False`` → + byte-identical to pre-phase, zero vision calls), zero candidates + (a clean-text PDF — no render, no vision call, no log line), the + fail-soft ``None`` page (one dead page keeps its text only, the + others are described), the unreadable-page placeholder skip (the + zero-byte ``render_pdf_pages`` slot burns no vision call, the page + keeps its text only). +* the digest-stability rerun — a stable file + a stable vision reply + re-syncs as ``unchanged`` (no re-embed, byte-stable content + + digest); the vision calls are re-issued before the unchanged check + can skip them — the documented no-cache cost (the phase-133 + convention, A21 endorses it). + +The E2E surface (the real UI + the mock LLM's ``PICTORIAL_PAGE_MODE`` +branch) is pinned in ``tests/e2e/test_pictorial_pdf.py`` (task 04). +""" +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import logging +import textwrap +from pathlib import Path +from typing import Any + +import pymupdf as fitz +import pytest +from sqlalchemy import select + +from app.config import Settings +from app.models import Chunk, Document +from app.rag import importer +from app.rag.importer import import_sources +from app.rag.llm import LLMError +from app.rag.pdf import ( + PAGE_SEPARATOR, + extract_pdf_page_texts, + extract_pdf_text, + pictorial_pages, +) +from app.rag.summarizer import PICTORIAL_PAGE_MODE, PICTORIAL_PROMPT +from tests.fakes import FakeEmbedder + +#: The IKEA fixture's metadata title (the phase-140 fixture, TODO L5 — +#: Adobe InDesign's "Assembly Instructions"). +TITLE = "Assembly Instructions" + +#: The text-heavy pages (IKEA pages 2–5 — the multilingual warnings, +#: 3,091–7,288 chars in the real fixture): a fixed warning block well +#: over the 100-char sparseness bar, so it is NEVER a candidate. +DENSE_TEXT = ( + "WARNING: Adult assembly required. Read all instructions before " + "starting. Keep this package away from children. The furniture " + "must be secured to the wall using the supplied anti-tip device." +) +assert len(DENSE_TEXT.strip()) >= 100 # the class property, not a typo + + +def _insert_dense(page) -> None: + """One dense-warning page's text, inserted in 60-char wrapped + lines: a single 188-char line would be CLIPPED at the page edge + on extraction (verified — only ~104 chars survive), while the + multi-line read round-trips byte-exactly (the fixture's class + property — over the 100-char bar — must hold on the EXTRACTED + text, which the gate sees).""" + for i, line in enumerate(textwrap.wrap(DENSE_TEXT, 60)): + page.insert_text((72, 72 + i * 14), line) + + +def _build_ikea_like_pdf(path: Path) -> None: + """A synthetic IKEA-class PDF (TODO L5 — the vector-drawn + assembly-instruction class): page 1 is the title page (``BESTA`` + — 5 chars, as in the real fixture's ``BESTÅ``), pages 2–5 the + text-heavy warnings (≥100 chars — never candidates), pages 6–10 + sparse ``Illustration N`` markers (15–16 chars — candidates) + standing in for the vector pictograms (no embedded raster images + — the phase-122 image pipeline sees nothing, as on the real + fixture). The metadata title is the fixture's.""" + doc = fitz.open() + doc.new_page().insert_text((72, 72), "BESTA") + for _ in range(4): + _insert_dense(doc.new_page()) + for n in range(6, 11): + doc.new_page().insert_text((72, 72), f"Illustration {n}") + doc.set_metadata({"title": TITLE}) + doc.save(path) + doc.close() + + +def _build_dense_pdf(path: Path) -> None: + """A clean-text PDF: every page over the 100-char bar — ZERO + pictorial candidates (the pass's no-op branch).""" + doc = fitz.open() + for _ in range(2): + _insert_dense(doc.new_page()) + doc.set_metadata({"title": "Clean Reference"}) + doc.save(path) + doc.close() + + +class _PictorialLLM(FakeEmbedder): + """The capturing fake LLM (task 03): the MULTIMODAL describe call + (``user`` content = a part list) gets a canned reply DETERMINED BY + THE RENDERED PAGE'S BYTES (a short sha256 of the data-URL's + base64 — the same rendered page always gets the same reply, + ACROSS RUNS: the digest-stability premise holds), and the TEXT + part is recorded (``pictorial_texts`` — the ``PICTORIAL_PROMPT`` + head + the ``Page N of the PDF`` block are the per-page context + witness). String (``lite``) calls keep the ``FakeEmbedder`` + behaviour. The REAL ``import_sources`` → ``_pictorial_pdf_pass`` + → ``describe_pdf_page`` chain runs against it (no seam patch).""" + + def __init__(self) -> None: + super().__init__() + self.pictorial_texts: list[str] = [] + self.pictorial_calls = 0 + + @staticmethod + def reply_for(png: bytes) -> str: + """The canned reply a vision call for *png* gets — + deterministic in the bytes (the tests build the expected + content with the same helper).""" + return ( + f"Page {hashlib.sha256(png).hexdigest()[:8]} illustrated: " + "two parts are joined with a blue arrow." + ) + + async def chat( + self, messages: list[dict[str, Any]], model: str | None = None + ) -> str: + self.chat_calls.append(list(messages)) + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + if isinstance(user, list): + text = next((p["text"] for p in user if p.get("type") == "text"), "") + assert PICTORIAL_PAGE_MODE in text, ( + "a multimodal call reached the fake that is not the " + f"pictorial pass's seam: {text[:80]!r}" + ) + self.pictorial_calls += 1 + self.pictorial_texts.append(text) + b64 = next(p for p in user if p.get("type") == "image_url")[ + "image_url" + ]["url"].rsplit(",", 1)[1] + return self.reply_for(base64.b64decode(b64)) + first = user.split() + return "Summary of " + (first[0] if first else "") + + +class _FailPagePictorialLLM(_PictorialLLM): + """Rejects the vision call for page *fail_page* (parsed from the + text part's ``Page N of the PDF.`` block) and describes the rest — + drives the per-page fail-soft (the dead page keeps its text only, + the other candidates still get their descriptions).""" + + def __init__(self, fail_page: int) -> None: + super().__init__() + self.fail_page = fail_page + + async def chat( + self, messages: list[dict[str, Any]], model: str | None = None + ) -> str: + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + if isinstance(user, list): + text = next((p["text"] for p in user if p.get("type") == "text"), "") + if f"Page {self.fail_page} of the PDF." in text: + self.pictorial_calls += 1 + self.pictorial_texts.append(text) + raise LLMError( + f"simulated dead vision call for page {self.fail_page} " + "(test sentinel)" + ) + return await super().chat(messages, model=model) + + +class _DeadVisionLLM(FakeEmbedder): + """A vision client that REJECTS every multimodal call (the dead + vision endpoint — the fail-soft's total-failure case: EVERY + candidate page keeps its text only, and the stored content is the + extraction byte-identical).""" + + def __init__(self) -> None: + super().__init__() + self.pictorial_calls = 0 + + async def chat( + self, messages: list[dict[str, Any]], model: str | None = None + ) -> str: + self.chat_calls.append(list(messages)) + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + if isinstance(user, list): + self.pictorial_calls += 1 + raise LLMError("simulated dead vision endpoint (test sentinel)") + first = user.split() + return "Summary of " + (first[0] if first else "") + + +def _settings(tmp_path: Path, **kwargs: Any) -> Settings: + """Settings admitting the ``pdf`` import token with the phase-140 + knobs at their CODE DEFAULTS (``pdf_pictorial`` ON — LOCKED A3 — + and ``pdf_pictorial_min_chars`` 100 — LOCKED A1) unless pinned + otherwise; ``images`` off (the embedded-image pass is not in play + — the fixture class is vector-drawn).""" + kwargs.setdefault("_env_file", None) + kwargs.setdefault("import_extensions", "md,txt,pdf") + kwargs.setdefault("images", False) + kwargs.setdefault("image_dir", str(tmp_path / "images")) + return Settings(**kwargs) # pyright: ignore[reportCallIssue] + + +def _pdf_doc(db, source: str, rel: str) -> Document: + doc = db.scalar( + select(Document).where(Document.source == source, Document.path == rel) + ) + assert doc is not None, f"no documents row for ({source!r}, {rel!r})" + return doc + + +def _cleanup_source(db, source: str) -> None: + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + db.delete(doc) + db.commit() + + +def _candidates_and_pngs(pdf_path: Path) -> tuple[list[int], list[bytes]]: + """The pass's own inputs, computed IN the test (no hardcoded page + list — the fixture is stable but the test must not rot): the + deterministic gate over the per-page text map, and the candidate + render (aligned with the candidates — ascending, one slot each).""" + page_texts = extract_pdf_page_texts(pdf_path) + candidates = pictorial_pages(page_texts, min_chars=100) + pngs = importer.render_pdf_pages(pdf_path, pages=candidates) + return candidates, pngs + + +def _expected_content( + pdf_path: Path, fail_pages: frozenset[int] = frozenset() +) -> tuple[str, list[int]]: + """The FINAL content the pass produces for *pdf_path* under the + stable fake — derived from the same inputs the pass uses: every + page of the extraction map as its ``PAGE_SEPARATOR`` + the page + text, the described candidates (the candidates MINUS + *fail_pages*) carrying ``[Page N illustration: ]`` + AFTER their text (LOCKED A4 — the text stays).""" + page_texts = extract_pdf_page_texts(pdf_path) + candidates, pngs = _candidates_and_pngs(pdf_path) + parts: list[str] = [] + for page, text in page_texts.items(): + block = text + if page in candidates and page not in fail_pages: + png = pngs[candidates.index(page)] + block += f"\n\n[Page {page} illustration: {_PictorialLLM.reply_for(png)}]" + parts.append(f"{PAGE_SEPARATOR.format(page=page)}\n{block}") + return "\n\n".join(parts), candidates + + +def test_sparse_pages_get_their_illustration_blocks( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The happy path end to end (the owner's L5 rationale): every + candidate page (computed from the gate — the fixture's title page + + its six illustration pages) carries its illustration block, the + text-heavy pages (2–5) do NOT, the chunks carry the canned + description (the embeddings hit the illustrations), the title is + the metadata title, the digest is over the FINAL content, and the + counters + the ONE loud log line fire (PLAN §9).""" + root = tmp_path / "pik" + root.mkdir() + pdf = root / "ikea.pdf" + _build_ikea_like_pdf(pdf) + llm = _PictorialLLM() + llm.settings = _settings(tmp_path) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + + content, candidates = _expected_content(pdf) + assert len(candidates) == 6 # the title page + the six markers + assert set(candidates) == {1, 6, 7, 8, 9, 10} + + doc = _pdf_doc(db, "pik", "ikea.pdf") + assert doc.title == TITLE # the metadata title (the fixture's) + assert doc.is_image is False + # The assembly is byte-exact: the page text STAYS, the + # description appends after it (LOCKED A4). + assert doc.content == content + # The text-heavy pages (2–5 — the multilingual warnings) carry + # NO illustration block (the gate is the ONLY selector). + for n in (2, 3, 4, 5): + assert f"[Page {n} illustration" not in doc.content + # The digest runs over the FINAL content (the phase-126/133 + # convention — A4). + assert doc.content_hash == hashlib.sha256( + doc.content.encode("utf-8") + ).hexdigest() + + # The counters (log-only — the sync detail shape is untouched): + # one PDF with six described sparse pages. + assert summary.pdfs_pictorial == 1 + assert summary.pdf_pictorial_pages == 6 + assert summary.pdfs_screenshotted == 0 # the gate passed (usable) + + # The chunks carry the canned description — the joined chunk + # text hits every candidate's reply (each page is its own + # ``Page N:`` chunk, the page-aware chunker). + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id) + .order_by(Chunk.position) + ).all() + content_chunks = [c for c in chunks if not c.is_summary] + joined = "\n".join(c.content for c in content_chunks) + _, pngs = _candidates_and_pngs(pdf) + for page in candidates: + reply = _PictorialLLM.reply_for(pngs[candidates.index(page)]) + assert reply in joined, f"page {page}'s description is not indexed" + for c in chunks: + assert c.embedding is not None and len(c.embedding) == 768 + summary_chunks = [c for c in chunks if c.is_summary] + assert len(summary_chunks) == 1 and summary_chunks[0].position == -1 + assert doc.summary is not None + + # ONE vision call per candidate, in candidate order — the text + # part heads with the locked prompt and carries the page's OWN + # text as context (the page block, task 02's contract). + assert llm.pictorial_calls == len(candidates) + assert len(llm.pictorial_texts) == len(candidates) + page_texts = extract_pdf_page_texts(pdf) + for page, text_part in zip(candidates, llm.pictorial_texts, strict=True): + assert text_part.startswith(PICTORIAL_PROMPT) + assert f"Page {page} of the PDF." in text_part + assert page_texts[page] in text_part # the page's own text + + # The ONE loud log line (PLAN §9) — exact, with the computed N. + lines = [ + r.getMessage() + for r in caplog.records + if "pictorial pass" in r.getMessage() + ] + assert lines == [ + "pdf: pictorial pass source=pik path=ikea.pdf — " + "6 candidate pages, 6 described (min_chars=100)" + ] + finally: + _cleanup_source(db, "pik") + + +def test_assembly_byte_identity_without_descriptions( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """THE byte-identity pin: with NO descriptions (the dead vision + endpoint — every candidate's call fails) the stored content is + ``extract_pdf_text``'s output BYTE-FOR-BYTE (the page's text is + never lost, never reworded, the separators are untouched), the + pass still counted its candidates in the log line (0 described), + and no PDF is counted pictorial (a description is what counts).""" + root = tmp_path / "dead" + root.mkdir() + pdf = root / "ikea.pdf" + _build_ikea_like_pdf(pdf) + llm = _DeadVisionLLM() + llm.settings = _settings(tmp_path) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + assert llm.pictorial_calls == 6 # every candidate was attempted + + doc = _pdf_doc(db, "dead", "ikea.pdf") + assert doc.content == extract_pdf_text(pdf), ( + "no descriptions → the stored content is the extraction " + "byte-identical (the assembly pin)" + ) + assert summary.pdfs_pictorial == 0 # nothing was described + assert summary.pdf_pictorial_pages == 0 + lines = [ + r.getMessage() + for r in caplog.records + if "pictorial pass" in r.getMessage() + ] + assert lines == [ + "pdf: pictorial pass source=dead path=ikea.pdf — " + "6 candidate pages, 0 described (min_chars=100)" + ] + # The dead pages' warnings (the fail-soft's signal, per page). + assert [ + r + for r in caplog.records + if r.levelno == logging.WARNING + and "has no description" in r.getMessage() + ] + finally: + _cleanup_source(db, "dead") + + +def test_feature_off_is_byte_identical_to_pre_phase(db, tmp_path: Path) -> None: + """The kill switch (LOCKED A3, ``BOR_PDF_PICTORIAL=0``): the pass + never runs — no render, no vision call, no log line, no counters — + and the stored content is ``extract_pdf_text``'s output + byte-identical to the pre-phase-140 path.""" + root = tmp_path / "off" + root.mkdir() + pdf = root / "ikea.pdf" + _build_ikea_like_pdf(pdf) + llm = _PictorialLLM() + llm.settings = _settings(tmp_path, pdf_pictorial=False) + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + doc = _pdf_doc(db, "off", "ikea.pdf") + assert doc.content == extract_pdf_text(pdf) + assert llm.pictorial_calls == 0 + assert (summary.pdfs_pictorial, summary.pdf_pictorial_pages) == (0, 0) + finally: + _cleanup_source(db, "off") + + +def test_zero_candidates_is_a_noop( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A clean-text PDF (every page over the 100-char bar) has ZERO + candidates — the pass is a no-op: no render, NO vision call + (the pin — a clean-text PDF never burns a vision call), no log + line, no counters, and the content is the extraction untouched.""" + root = tmp_path / "dense" + root.mkdir() + pdf = root / "clean.pdf" + _build_dense_pdf(pdf) + llm = _PictorialLLM() + llm.settings = _settings(tmp_path) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + doc = _pdf_doc(db, "dense", "clean.pdf") + assert doc.content == extract_pdf_text(pdf) + assert llm.pictorial_calls == 0 + assert (summary.pdfs_pictorial, summary.pdf_pictorial_pages) == (0, 0) + assert not [ + r for r in caplog.records if "pictorial pass" in r.getMessage() + ] + finally: + _cleanup_source(db, "dense") + + +def test_fail_soft_page_keeps_its_text( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """One dead candidate page (the vision call rejects it) keeps its + text only — a warning is the signal, the OTHER candidates still + get their descriptions, one dead page never kills the pass + (LOCKED A2 fail-soft). The described count is five of six; the + PDF still counts pictorial (≥1 described page).""" + root = tmp_path / "partfail" + root.mkdir() + pdf = root / "ikea.pdf" + _build_ikea_like_pdf(pdf) + llm = _FailPagePictorialLLM(fail_page=7) + llm.settings = _settings(tmp_path) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + assert llm.pictorial_calls == 6 # every candidate was attempted + + content, candidates = _expected_content(pdf, fail_pages=frozenset({7})) + doc = _pdf_doc(db, "partfail", "ikea.pdf") + assert doc.content == content + assert "[Page 7 illustration" not in doc.content # the dead page + assert "---PAGE-7---\nIllustration 7" in doc.content # its text stays + assert summary.pdfs_pictorial == 1 + assert summary.pdf_pictorial_pages == 5 + lines = [ + r.getMessage() + for r in caplog.records + if "pictorial pass" in r.getMessage() + ] + assert lines == [ + "pdf: pictorial pass source=partfail path=ikea.pdf — " + "6 candidate pages, 5 described (min_chars=100)" + ] + # The importer's per-page warning (the dead page's signal). + warnings = [ + r + for r in caplog.records + if r.levelno == logging.WARNING + and "pictorial page 7" in r.getMessage() + ] + assert len(warnings) == 1 + finally: + _cleanup_source(db, "partfail") + + +def test_unreadable_page_placeholder_skips_the_vision_call( + db, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """A candidate page that could not be RENDERED (the zero-byte + ``render_pdf_pages`` placeholder — the phase-133 handling) is + skipped with a warning: it burns NO vision call, keeps its text + only, and the OTHER candidates keep their positions + + descriptions (the result's order never shifts).""" + root = tmp_path / "halfblind" + root.mkdir() + pdf = root / "ikea.pdf" + _build_ikea_like_pdf(pdf) + + real_render = importer.render_pdf_pages + + def _render(path: Path, dpi: int = 150, pages: Any = None) -> list[bytes]: + assert pages is not None # only the pass calls it (with a selector) + pngs = real_render(path, dpi=dpi, pages=pages) + return [ + b"" if n == 6 else png + for n, png in zip(pages, pngs, strict=True) + ] + + monkeypatch.setattr(importer, "render_pdf_pages", _render) + llm = _PictorialLLM() + llm.settings = _settings(tmp_path) + try: + with caplog.at_level(logging.WARNING, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + assert llm.pictorial_calls == 5 # the blank page burned no call + + content, candidates = _expected_content(pdf, fail_pages=frozenset({6})) + doc = _pdf_doc(db, "halfblind", "ikea.pdf") + assert doc.content == content + assert "[Page 6 illustration" not in doc.content + assert "---PAGE-6---\nIllustration 6" in doc.content + assert summary.pdfs_pictorial == 1 + assert summary.pdf_pictorial_pages == 5 + warnings = [ + r + for r in caplog.records + if r.levelno == logging.WARNING + and "skipping pictorial page 6" in r.getMessage() + ] + assert len(warnings) == 1 + finally: + _cleanup_source(db, "halfblind") + + +def test_rerun_lands_unchanged_with_byte_stable_content( + db, tmp_path: Path +) -> None: + """Digest stability (A4 — the phase-126/133 convention): the + digest runs over the FINAL content, so a re-sync of an unchanged + IKEA-class PDF (same pages + the stable fake's same replies) + re-imports as UNCHANGED: no re-chunk, no re-embed, no re-summary, + no KB change (not ``updated``, no changed path), and the content + + digest are byte-stable. The vision calls ARE re-issued before the + unchanged check can skip them — the documented no-cache cost (the + phase-133 convention; a non-deterministic vision model re-describes + the row each sync — a cost, not a bug; A21 endorses it).""" + root = tmp_path / "stable" + root.mkdir() + pdf = root / "ikea.pdf" + _build_ikea_like_pdf(pdf) + llm = _PictorialLLM() + llm.settings = _settings(tmp_path) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert (s1.added, s1.pdfs_pictorial, s1.pdf_pictorial_pages) == (1, 1, 6) + assert llm.pictorial_calls == 6 + embeds1 = len(llm.calls) + doc1 = _pdf_doc(db, "stable", "ikea.pdf") + content1, digest1 = doc1.content, doc1.content_hash + + s2 = asyncio.run(import_sources([root], llm, session=db)) + # The pass re-ran (gate re-fired, vision re-issued — the + # documented cost) — but the row itself re-imported unchanged. + assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 1) + assert s2.pdfs_pictorial == 1 # the pass re-fired + assert llm.pictorial_calls == 12 # the vision calls were re-issued + assert len(llm.calls) == embeds1, "unchanged: no re-embed" + assert s2.changed_paths == [], "unchanged: no KB change recorded" + + doc2 = _pdf_doc(db, "stable", "ikea.pdf") + assert doc2.content == content1 # byte-stable content + assert doc2.content_hash == digest1 # byte-stable digest + finally: + _cleanup_source(db, "stable") diff --git a/tests/unit/test_pictorial_seam.py b/tests/unit/test_pictorial_seam.py new file mode 100644 index 0000000..c6d1e42 --- /dev/null +++ b/tests/unit/test_pictorial_seam.py @@ -0,0 +1,450 @@ +"""Phase 140, task 02 — the ``PICTORIAL_PAGE_MODE`` vision seam + the +``render_pdf_pages`` page selector (unit). + +Pins the task-02 contracts: + +* ``PICTORIAL_PAGE_MODE`` / ``PICTORIAL_INSTRUCTION`` / + ``PICTORIAL_PROMPT`` — the marker convention next to + ``IMAGE_DESCRIPTION_MODE`` (marker first, the deterministic E2E mock + LLM keys on it, PLAN §6) with the LOCKED wording (phase 140 design: + describe in detail what the page shows — the assembly steps, part + names, arrows, tool icons, before/after states — ESPECIALLY the + information the extracted text does not contain). +* ``DESCRIBE_PROMPT`` byte-identity (regression pin — the phase 122/126/ + 137 suites pass unedited; the pictorial pass has its OWN marker, it + never re-wraps the image prompt). +* :func:`describe_pdf_page` — the same one-chat-call multimodal + contract as :func:`describe_image`: a capturing fake LLM pins the + message shape (the text part starts with ``PICTORIAL_PROMPT``, then + the page block — the page's own text as context, the + ``"(none)`` case; the ``image_url`` part is the data URL from the + PNG bytes + mime), the CHAT (vision) model (never ``lite``), the + phase-30 cap, and ``None`` on an empty reply or a client error + (fail-soft — one dead page never stalls a sync). +* :func:`render_pdf_pages`' optional 1-based *pages* selector — the + subset's count + ascending order + PNG magic bytes (a 3-page + fixture, the phase-126/133 recipe), ``None`` = every page (the + phase-133 caller stays byte-identical), an unreadable/out-of-range + named page → the zero-byte placeholder at its position, an + unopenable file → ``[]``. +""" +from __future__ import annotations + +import asyncio +import base64 +import logging +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pymupdf as fitz +import pytest + +from app.config import Settings +from app.rag import pdf as pdf_mod +from app.rag.llm import LLMError +from app.rag.pdf import render_pdf_pages +from app.rag.summarizer import ( + DESCRIBE_PROMPT, + IMAGE_DESCRIPTION_MODE, + PICTORIAL_INSTRUCTION, + PICTORIAL_PAGE_MODE, + PICTORIAL_PROMPT, + describe_pdf_page, +) + +PNG_MAGIC = b"\x89PNG" + +#: A tiny non-empty PNG-ish payload for the describe-call's ``data`` +#: (the seam is content-agnostic — it never parses the image; a +#: well-formed magic keeps the data-URL assertion honest). +PNG_1X1 = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGP4DwQACfsD/fteaysAAAAASUVORK5CYII=" +) + + +def _settings(**kwargs: Any) -> Settings: + """Build Settings without reading a .env file (deterministic tests).""" + kwargs.setdefault("_env_file", None) + return Settings(**kwargs) # pyright: ignore[reportCallIssue] (kwarg exists at runtime) + + +class _VisionMock: + """Duck-typed chat-model mock for :func:`describe_pdf_page` (task + 02): records the message + model and returns a canned reply (or + raises it). Satisfies the summarizer's ``SummaryLLM`` protocol. + """ + + def __init__( + self, reply: str | Exception = "Two parts joined with an arrow." + ) -> None: + self.settings = _settings() + self.reply = reply + self.messages: list[Any] = [] + self.model: str | None = None + + async def chat( + self, messages: list[dict[str, Any]], model: str | None = None + ) -> str: + self.messages = list(messages) + self.model = model + if isinstance(self.reply, Exception): + raise self.reply + return self.reply + + +def _message(mock: _VisionMock) -> list[dict[str, Any]]: + """The one user message's multimodal ``content`` list (asserted to + be a list of two parts by the shape test).""" + assert len(mock.messages) == 1 + message = mock.messages[0] + assert message["role"] == "user" + content = message["content"] + assert isinstance(content, list) + return content + + +# --------------------------------------------------------------------------- +# The marker + the locked instruction (the mock's key, PLAN §6) +# --------------------------------------------------------------------------- + + +def test_pictorial_prompt_carries_the_mode_marker_and_locked_wording() -> None: + """The ``PICTORIAL_PAGE_MODE`` marker heads the pictorial prompt + (the ``SUMMARY_MODE``/``IMAGE_DESCRIPTION_MODE`` convention — the + deterministic E2E mock LLM keys on it), and the LOCKED wording's + load-bearing fragments are in the instruction: a re-wrap of the + prompt cannot silently re-route the mock, and the "describe what + the text does NOT contain" mandate stays in the constant.""" + assert PICTORIAL_PAGE_MODE == "PICTORIAL_PAGE_MODE" + assert PICTORIAL_PROMPT.startswith(PICTORIAL_PAGE_MODE + ": ") + assert ( + PICTORIAL_PROMPT == PICTORIAL_PAGE_MODE + ": " + PICTORIAL_INSTRUCTION + ) + for fragment in ( + "assembly steps", + "part names", + "arrows", + "tool icons", + "before/after states", + "ESPECIALLY", + "extracted text", + "plain text", + ): + assert fragment in PICTORIAL_PROMPT + # The markers are disjoint fixed app constants (neither contains + # the other) — the mock's two vision branches cannot shadow each + # other regardless of branch order. + assert IMAGE_DESCRIPTION_MODE not in PICTORIAL_PROMPT + assert PICTORIAL_PAGE_MODE not in DESCRIBE_PROMPT + + +def test_describe_prompt_byte_identity_regression_pin() -> None: + """REGRESSION PIN: the phase-122 ``DESCRIBE_PROMPT`` is untouched + by the pictorial seam (the phase 122/126/137 suites pass unedited — + the image marker's mock branch serves the exact pre-phase-140 + prompt). The full value is pinned, not just its shape: any + re-wrap of the locked instruction fails here, not in the story + suites.""" + assert DESCRIBE_PROMPT == ( + "IMAGE_DESCRIPTION_MODE: Describe this image faithfully, in " + "plain text, for a search index. State what is depicted, " + "transcribe any visible text, labels, or titles, describe the " + "structure of any diagram, table, or layout, and call out the " + "most salient details. Write 2-4 sentences of substance. Do not " + "use markdown. Do not invent anything that is not visible in " + "the image. Your description is the ONLY text that will ever be " + "retrieved for this image — it must carry the image's full " + "meaning." + ) + + +# --------------------------------------------------------------------------- +# describe_pdf_page — message shape, model, cap, fail-soft +# --------------------------------------------------------------------------- + + +def test_describe_pdf_page_message_shape_multimodal_and_chat_model() -> None: + """ONE chat-model call (LOCKED A2 — the vision ``llm_chat_model``, + NOT the ``lite`` summary model) with the OpenAI-compatible + multimodal user message: the text part is exactly + ``PICTORIAL_PROMPT + "\\n\\nPage {page} of the PDF. Text extracted + from this page:\\n" + page_text.strip()`` (the page's OWN text as + context, so the model describes what the text does NOT carry) and + the ``image_url`` part is the data URL ``data:;base64,`` — no system prompt, no tools.""" + mock = _VisionMock("Two parts joined with an arrow.") + result = asyncio.run( + describe_pdf_page( + mock, + data=PNG_1X1, + mime="image/png", + page=7, + page_text="Illustration 7", + ) + ) + assert result == "Two parts joined with an arrow." + content = _message(mock) + assert len(content) == 2 + assert content[0] == { + "type": "text", + "text": ( + PICTORIAL_PROMPT + + "\n\nPage 7 of the PDF. Text extracted from this page:\n" + + "Illustration 7" + ), + } + b64 = base64.b64encode(PNG_1X1).decode("ascii") + assert content[1] == { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + } + # LOCKED A2: the CHAT (vision) model — never the ``lite`` summary + # model (it is not assumed vision-capable). + assert mock.model == mock.settings.llm_chat_model + assert mock.model != mock.settings.llm_summary_model + + +@pytest.mark.parametrize( + ("page_text", "suffix"), + [ + (None, "(none)"), + ("", "(none)"), + (" \t\n ", "(none)"), + (" Illustration 7 ", "Illustration 7"), + ], +) +def test_describe_pdf_page_page_block_none_and_strip_cases( + page_text: str | None, suffix: str, +) -> None: + """The page block's tail is ``page_text.strip() or "(none)"`` — a + page with no extracted text (``None``/empty/blank) sends the + explicit ``(none)`` sentinel (the vision model knows there is + nothing to restate), and a text-carrying page sends its STRIPPED + text (the leading/trailing whitespace of the extraction never + reaches the model).""" + mock = _VisionMock("ok") + asyncio.run( + describe_pdf_page( + mock, data=PNG_1X1, mime="image/png", page=3, page_text=page_text + ) + ) + content = _message(mock) + assert content[0]["text"] == ( + PICTORIAL_PROMPT + + f"\n\nPage 3 of the PDF. Text extracted from this page:\n{suffix}" + ) + + +def test_describe_pdf_page_strips_and_caps_at_summary_max_chars() -> None: + """The reply is whitespace-stripped and cut exactly at + ``summary_max_chars`` (the phase-30 cap — the same uniform ceiling + as :func:`describe_image`). The explicit ``settings`` parameter + wins; ``settings=None`` falls back to the LLM's own settings.""" + long_reply = "x" * 500 + mock = _VisionMock(long_reply) + capped = asyncio.run( + describe_pdf_page( + mock, + data=PNG_1X1, + mime="image/png", + page=2, + page_text=None, + settings=_settings(summary_max_chars=100), + ) + ) + assert capped == "x" * 100 + mock2 = _VisionMock(" " + "y" * 50 + "\n") + mock2.settings = _settings(summary_max_chars=42) + result = asyncio.run( + describe_pdf_page(mock2, data=PNG_1X1, mime="image/png", page=2, page_text=None) + ) + assert result == ("y" * 42) + + +def test_describe_pdf_page_returns_none_on_failure_classes( + caplog: pytest.LogCaptureFixture, +) -> None: + """Fail-soft (LOCKED A2 — one dead page never stalls a sync): + ``None`` on a client error (with a warning log naming the page — + the house one-shot style) and on an empty or whitespace-only reply + (the house one-shot empty-content policy).""" + failing = _VisionMock(LLMError("simulated vision-model failure (test sentinel)")) + with caplog.at_level(logging.WARNING, logger="app.summarizer"): + assert ( + asyncio.run( + describe_pdf_page( + failing, data=PNG_1X1, mime="image/png", page=9, page_text=None + ) + ) + is None + ) + warnings = [ + r + for r in caplog.records + if r.levelno == logging.WARNING and "pictorial page description" in r.getMessage() + ] + assert len(warnings) == 1 and "page=9" in warnings[0].getMessage() + for empty in ("", " \n"): + blank = _VisionMock(empty) + assert ( + asyncio.run( + describe_pdf_page( + blank, data=PNG_1X1, mime="image/png", page=9, page_text=None + ) + ) + is None + ) + + +# --------------------------------------------------------------------------- +# render_pdf_pages — the optional 1-based page selector +# --------------------------------------------------------------------------- + + +def _three_page_text_pdf(path: Path) -> Path: + """A 3-page temp PDF with EXPLICIT letter pages (612x792pt — + deterministic render sizes) and page-specific text (the render must + be page-specific, so the pages' PNG bytes must differ — the + phase-126/133 recipe, extended to three pages for the selector's + count + order assertions).""" + doc = fitz.open() + for marker in ("Alpha page text.", "Beta page text.", "Gamma page text."): + doc.new_page(width=612, height=792).insert_text((72, 72), marker) + doc.save(path) + doc.close() + return path + + +def test_render_selector_none_renders_every_page(tmp_path: Path) -> None: + """``pages=None`` (the default) = every page in page order — the + phase-133 caller's result, byte-identical: a 3-page PDF → 3 PNGs, + each a genuine decodable PNG, page-specific (the bytes differ).""" + path = _three_page_text_pdf(tmp_path / "three.pdf") + pages = render_pdf_pages(path) + assert len(pages) == 3 + for png in pages: + assert png.startswith(PNG_MAGIC) + pix = fitz.Pixmap(png) + assert (pix.width, pix.height) == (1275, 1650) # 150-DPI letter + assert pages[0] != pages[1] != pages[2] and pages[0] != pages[2] + + +def test_render_selector_subset_count_ascending_order_and_pngs( + tmp_path: Path, +) -> None: + """A named subset renders ONLY the named pages (3-page PDF, + ``pages=[3, 1]`` → 2 PNGs, not 3), in ASCENDING order regardless of + the input order (list index *i* is the *i*-th smallest NAMED page — + index 0 is page 1, index 1 is page 3), each a genuine PNG whose + bytes equal the corresponding page's render in the full + (``pages=None``) pass — the selector changes the set, never the + per-page output.""" + path = _three_page_text_pdf(tmp_path / "three.pdf") + full = render_pdf_pages(path) + selected = render_pdf_pages(path, pages=[3, 1]) + assert len(selected) == 2 + for png in selected: + assert png.startswith(PNG_MAGIC) + pix = fitz.Pixmap(png) + assert (pix.width, pix.height) == (1275, 1650) # 150-DPI letter + assert selected[0] == full[0] # the smallest named page (1) first + assert selected[1] == full[2] # page 3 second, page 2 rendered by no one + single = render_pdf_pages(path, pages=[2]) + assert single == [full[1]] + + +def test_render_selector_out_of_range_named_page_is_zero_bytes( + tmp_path: Path, caplog: pytest.LogCaptureFixture, +) -> None: + """A named page outside the document's page range yields ``b""`` at + ITS position (the caller's placeholder — the result's order never + shifts, so it stays aligned with the requested pages) + a warning; + the in-range page keeps its PNG.""" + path = _three_page_text_pdf(tmp_path / "three.pdf") + full = render_pdf_pages(path) + with caplog.at_level(logging.WARNING, logger="app.rag.pdf"): + selected = render_pdf_pages(path, pages=[2, 99]) + assert selected == [full[1], b""] + warnings = [ + r + for r in caplog.records + if r.levelno == logging.WARNING and "could not render page 99" in r.getMessage() + ] + assert len(warnings) == 1 + + +def test_render_selector_empty_list_renders_nothing(tmp_path: Path) -> None: + """``pages=[]`` (no candidates) renders nothing — the zero- + candidates no-op the importer relies on (a clean-text PDF's path + never burns a render or a vision call).""" + path = _three_page_text_pdf(tmp_path / "three.pdf") + assert render_pdf_pages(path, pages=[]) == [] + + +def test_render_selector_unopenable_file_returns_empty( + tmp_path: Path, +) -> None: + """A file that cannot be OPENED yields ``[]`` even with a selector + (the caller's existing empty-content path — unchanged by the + phase-140 parameter).""" + bad = tmp_path / "bad.pdf" + bad.write_bytes(b"this is not a pdf at all") + assert render_pdf_pages(bad, pages=[1, 2]) == [] + assert render_pdf_pages(tmp_path / "missing.pdf", pages=[1]) == [] + + +def test_render_selector_broken_named_page_is_zero_bytes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + """A named page whose ``get_pixmap`` raises → ``b""`` at ITS + position (the caller's placeholder slot — the page index NEVER + shifts, so the result stays aligned with the requested pages); the + good named pages keep their PNGs, and the failure is logged (the + :func:`extract_pdf_text` fail-soft style).""" + path = tmp_path / "fake.pdf" + path.write_bytes(b"%PDF-1.4 fake") + + class _RenderPage: + def __init__(self, data: bytes | None) -> None: + self._data = data + + def get_pixmap(self, dpi: int) -> Any: + if self._data is None: + raise RuntimeError("boom: page render failed") + return SimpleNamespace(tobytes=lambda _fmt: self._data) + + class _FakeDoc: + def __init__(self, pages: list[Any]) -> None: + self._pages = pages + + def __len__(self) -> int: + return len(self._pages) + + def __getitem__(self, index: int) -> Any: + return self._pages[index] + + def __enter__(self) -> _FakeDoc: # noqa: F821 (nested class, future annotations) + return self + + def __exit__(self, *args: object) -> bool: + return False + + doc = _FakeDoc( + [ + _RenderPage(b"\x89PNG-fake-page-1"), + _RenderPage(None), + _RenderPage(b"\x89PNG-fake-page-3"), + ] + ) + monkeypatch.setattr(pdf_mod.fitz, "open", lambda _p: doc) + # pages=[3, 1, 2] renders ascending 1, 2, 3 — the broken page 2 + # holds ITS position (index 1), not the tail. + assert render_pdf_pages(path, pages=[3, 1, 2]) == [ + b"\x89PNG-fake-page-1", + b"", + b"\x89PNG-fake-page-3", + ] + # A repeated page number renders ONCE (the selector is a set of + # pages, ascending). + assert render_pdf_pages(path, pages=[1, 1]) == [b"\x89PNG-fake-page-1"]