diff --git a/.agents/phases/todo/126_pdf_support/00_phase.md b/.agents/phases/complete/126_pdf_support/00_phase.md similarity index 100% rename from .agents/phases/todo/126_pdf_support/00_phase.md rename to .agents/phases/complete/126_pdf_support/00_phase.md diff --git a/.agents/phases/todo/126_pdf_support/01_pdf_dependency.md b/.agents/phases/complete/126_pdf_support/01_pdf_dependency.md similarity index 100% rename from .agents/phases/todo/126_pdf_support/01_pdf_dependency.md rename to .agents/phases/complete/126_pdf_support/01_pdf_dependency.md diff --git a/.agents/phases/todo/126_pdf_support/02_pdf_parser.md b/.agents/phases/complete/126_pdf_support/02_pdf_parser.md similarity index 100% rename from .agents/phases/todo/126_pdf_support/02_pdf_parser.md rename to .agents/phases/complete/126_pdf_support/02_pdf_parser.md diff --git a/.agents/phases/todo/126_pdf_support/03_pdf_chunking.md b/.agents/phases/complete/126_pdf_support/03_pdf_chunking.md similarity index 100% rename from .agents/phases/todo/126_pdf_support/03_pdf_chunking.md rename to .agents/phases/complete/126_pdf_support/03_pdf_chunking.md diff --git a/.agents/phases/todo/126_pdf_support/04_pdf_indexing.md b/.agents/phases/complete/126_pdf_support/04_pdf_indexing.md similarity index 100% rename from .agents/phases/todo/126_pdf_support/04_pdf_indexing.md rename to .agents/phases/complete/126_pdf_support/04_pdf_indexing.md diff --git a/.agents/phases/todo/126_pdf_support/05_pdf_archive_and_tests.md b/.agents/phases/complete/126_pdf_support/05_pdf_archive_and_tests.md similarity index 100% rename from .agents/phases/todo/126_pdf_support/05_pdf_archive_and_tests.md rename to .agents/phases/complete/126_pdf_support/05_pdf_archive_and_tests.md diff --git a/.agents/reports/126_pdf_support/126_pdf_support__00_phase.a1.err b/.agents/reports/126_pdf_support/126_pdf_support__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/126_pdf_support/126_pdf_support__00_phase.a1.md b/.agents/reports/126_pdf_support/126_pdf_support__00_phase.a1.md new file mode 100644 index 0000000..aee0b51 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__00_phase.a1.md @@ -0,0 +1,25 @@ +All gates verified green — no code changes were needed on this pass. Final report: + +## Phase 126 — PDF support: final verification pass + +**Verified (all tasks 01–05 already complete in working tree):** +- `pymupdf` dep in `pyproject.toml`/`uv.lock`; `.env.example` documents the `pdf` token +- `app/rag/pdf.py` (text + image extraction, MIME mapping), page-aware `_chunk_pdf`, importer `_index_pdf_file` + embedded-image pass, archives covered via extension walk + +**Test / lint / coverage results:** +- `uv run pytest` → 2997 passed +- `uv run pytest tests/e2e/test_pdf_support.py -v --no-cov` (isolation) → 6 passed +- `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90% gate), 2997 passed +- `uv run ruff check . && uv run pyright` → clean (0 errors) + +**Completion criteria:** +1. PDF → doc with text/chunks/embedding/summary — ✅ (integration + E2E Sources-page tests) +2. `BOR_IMAGES=true`: embedded images → standalone image docs — ✅ (integration + E2E chat-chip test) +3. `BOR_IMAGES=false`: images skipped, text indexed — ✅ (both suites) +4. PDF renders as extracted text in viewer — ✅ (E2E) +5. Test/coverage/lint gates — ✅ (above) +6. Commit + phase move — harness handles (per protocol, no `git` on this pass) + +**Decisions:** none new; inline `[Embedded image: …]` placeholders were a documented task-04 deviation (simplified spec wins) — not a defect. No defects found; repo left functional with changes in the working tree. + +**Next pending phase:** none — `todo/` is empty once 126 is moved. diff --git a/.agents/reports/126_pdf_support/126_pdf_support__00_phase.a1.validate b/.agents/reports/126_pdf_support/126_pdf_support__00_phase.a1.validate new file mode 100644 index 0000000..f0dddec --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__00_phase.a1.validate @@ -0,0 +1,113 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 72%] +........................................................................ [ 74%] +........................................................................ [ 76%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 96%] +........................................................................ [ 98%] +............................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 262 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 241 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 256 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 132 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 374 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 38 0 100% +app/rag/importer.py 563 9 98% +app/rag/llm.py 244 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 81 0 100% +app/rag/prompts.py 144 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 400 0 100% +-------------------------------------------------- +TOTAL 5114 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/126_pdf_support/126_pdf_support__01_pdf_dependency.a1.err b/.agents/reports/126_pdf_support/126_pdf_support__01_pdf_dependency.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/126_pdf_support/126_pdf_support__01_pdf_dependency.a1.md b/.agents/reports/126_pdf_support/126_pdf_support__01_pdf_dependency.a1.md new file mode 100644 index 0000000..78747fb --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__01_pdf_dependency.a1.md @@ -0,0 +1,19 @@ +Task complete — all criteria green. Report: + +**Implemented** +- Added `pymupdf>=1.24,<2.0` to `pyproject.toml` dependencies (after `python-multipart`, before dev group) with a Phase 126 provenance comment, per locked decision A1. +- `uv sync` installed `pymupdf==1.28.2` (satisfies the pin; `uv.lock` updated accordingly). + +**Verification** +- `uv sync` — completed, `+ pymupdf==1.28.2` +- `uv run python -c "import fitz; print(fitz.VersionBind)"` — OK, `VersionBind: 1.28.2` +- `uv run pytest --cov=app --cov-report=term` — 2926 passed, TOTAL coverage **99%** (>90% gate) +- `uv run ruff check .` — all checks passed; `uv run pyright` — 0 errors, 0 warnings + +**Notable** +- `import fitz` emits a deprecation warning in pymupdf 1.28 ("use `import pymupdf` instead"). The phase file locks `fitz` as the import name (A1), so I kept it; task 02 can equally use `import pymupdf as fitz` to silence the warning without deviating from the API. + +**Next pending task** +- `.agents/phases/todo/126_pdf_support/02_pdf_parser.md` — `app/rag/pdf.py`: text extraction, image extraction, MIME mapping. + +No commit made (harness commits at phase completion); working tree left with `pyproject.toml` + `uv.lock` modified. diff --git a/.agents/reports/126_pdf_support/126_pdf_support__01_pdf_dependency.a1.validate b/.agents/reports/126_pdf_support/126_pdf_support__01_pdf_dependency.a1.validate new file mode 100644 index 0000000..fec8448 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__01_pdf_dependency.a1.validate @@ -0,0 +1,111 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 17%] +........................................................................ [ 19%] +........................................................................ [ 22%] +........................................................................ [ 24%] +........................................................................ [ 27%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 34%] +........................................................................ [ 36%] +........................................................................ [ 39%] +........................................................................ [ 41%] +........................................................................ [ 44%] +........................................................................ [ 46%] +........................................................................ [ 49%] +........................................................................ [ 51%] +........................................................................ [ 54%] +........................................................................ [ 56%] +........................................................................ [ 59%] +........................................................................ [ 61%] +........................................................................ [ 63%] +........................................................................ [ 66%] +........................................................................ [ 68%] +........................................................................ [ 71%] +........................................................................ [ 73%] +........................................................................ [ 76%] +........................................................................ [ 78%] +........................................................................ [ 81%] +........................................................................ [ 83%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 95%] +........................................................................ [ 98%] +.............................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 262 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 241 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 256 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 132 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 374 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 38 0 100% +app/rag/importer.py 432 10 98% +app/rag/llm.py 244 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/prompts.py 144 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 400 0 100% +-------------------------------------------------- +TOTAL 4888 23 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/126_pdf_support/126_pdf_support__02_pdf_parser.a1.err b/.agents/reports/126_pdf_support/126_pdf_support__02_pdf_parser.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/126_pdf_support/126_pdf_support__02_pdf_parser.a1.md b/.agents/reports/126_pdf_support/126_pdf_support__02_pdf_parser.a1.md new file mode 100644 index 0000000..0b68ea6 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__02_pdf_parser.a1.md @@ -0,0 +1,16 @@ +Task 02 complete — all criteria green. + +**Implemented** +- `app/rag/pdf.py`: `extract_pdf_text` (`---PAGE-N---` 1-indexed separators, A2), `extract_pdf_images` (original encoded bytes via `doc.extract_image`, per-occurrence page/index), `PdfImage` dataclass, `_mime_from_image_info` (ext first → type code → `application/octet-stream`) +- `tests/unit/test_pdf.py`: 39 tests — multi/single-page text, image-only → `""`, blank middle page, embedded/duplicate images, missing/garbage files (no raise), broken-page/image skip branches, MIME mapping (22 cases), old-pymupdf `xrefs` fallback + +**Validation** +- `uv run pytest --cov=app --cov-report=term-missing` → **2965 passed**, TOTAL **99%** (gate >90%); `app/rag/pdf.py` **100%** +- `uv run ruff check .` → all checks passed; `uv run pyright` → 0 errors, 0 warnings + +**Notable decisions** +- `import pymupdf as fitz` (per task-01 report) — avoids the 1.28 deprecation warning, same API +- Images use `extract_image` original bytes (no lossy Pixmap re-encode); `get_image_info()`'s `ext` is `None` and `type` key absent in PyMuPDF 1.28, so `extract_image`'s reported `ext` drives the MIME +- Pages without text get no separator (satisfies "image-only → empty string" spec); task-file type codes (8=tiff, 9=webp) kept despite `00_phase.md` having 8/9 swapped + +**Next pending task:** `.agents/phases/todo/126_pdf_support/03_pdf_chunking.md` diff --git a/.agents/reports/126_pdf_support/126_pdf_support__02_pdf_parser.a1.validate b/.agents/reports/126_pdf_support/126_pdf_support__02_pdf_parser.a1.validate new file mode 100644 index 0000000..c5c4712 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__02_pdf_parser.a1.validate @@ -0,0 +1,113 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 97%] +........................................................................ [ 99%] +............. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 262 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 241 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 256 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 132 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 374 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 38 0 100% +app/rag/importer.py 432 10 98% +app/rag/llm.py 244 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 81 0 100% +app/rag/prompts.py 144 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 400 0 100% +-------------------------------------------------- +TOTAL 4969 23 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a1.err b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a1.md b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a1.md new file mode 100644 index 0000000..a529ea0 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a1.md @@ -0,0 +1 @@ +Now let me look at the current chunker, existing tests, and completed task files for this phase. diff --git a/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a1.validate b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a1.validate new file mode 100644 index 0000000..c5c4712 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a1.validate @@ -0,0 +1,113 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 97%] +........................................................................ [ 99%] +............. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 262 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 241 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 256 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 132 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 374 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 38 0 100% +app/rag/importer.py 432 10 98% +app/rag/llm.py 244 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 81 0 100% +app/rag/prompts.py 144 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 400 0 100% +-------------------------------------------------- +TOTAL 4969 23 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a2.err b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a2.md b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a2.md new file mode 100644 index 0000000..a529ea0 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a2.md @@ -0,0 +1 @@ +Now let me look at the current chunker, existing tests, and completed task files for this phase. diff --git a/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a2.validate b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a2.validate new file mode 100644 index 0000000..c5c4712 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__03_pdf_chunking.a2.validate @@ -0,0 +1,113 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 29%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 41%] +........................................................................ [ 43%] +........................................................................ [ 46%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 55%] +........................................................................ [ 58%] +........................................................................ [ 60%] +........................................................................ [ 63%] +........................................................................ [ 65%] +........................................................................ [ 67%] +........................................................................ [ 70%] +........................................................................ [ 72%] +........................................................................ [ 75%] +........................................................................ [ 77%] +........................................................................ [ 80%] +........................................................................ [ 82%] +........................................................................ [ 84%] +........................................................................ [ 87%] +........................................................................ [ 89%] +........................................................................ [ 92%] +........................................................................ [ 94%] +........................................................................ [ 97%] +........................................................................ [ 99%] +............. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 262 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 241 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 256 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 132 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 374 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 206 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 38 0 100% +app/rag/importer.py 432 10 98% +app/rag/llm.py 244 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 81 0 100% +app/rag/prompts.py 144 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 400 0 100% +-------------------------------------------------- +TOTAL 4969 23 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a1.err b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a1.md b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a1.md new file mode 100644 index 0000000..55ac868 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a1.md @@ -0,0 +1,12 @@ +Everything is verified. Final state: + +- **Implemented** + - `app/rag/chunker.py` — `_chunk_pdf` page-aware chunker (`---PAGE-N---` markers as primary boundary, `Page N:` anchor on every chunk, anchor-budgeted to the hard cap) + `.pdf` dispatch. **This backfills task 03, whose task file was marked complete but whose code was never written** (truncated executor report; the chunker had no PDF branch). + - `app/rag/importer.py` — PDF branch in `_index_file` (after image, before text); `_index_pdf_file` (extract → text-digest upsert with the full unchanged/D1/D4 dance → shared chunk+embed → lite summary → image pass); `_extract_pdf_title` (metadata title, 500-char cap, stem fallback); `_index_pdf_embedded_images`/`_index_one_embedded_image` (A3: `BOR_IMAGES`-gated, the single `_describe_or_skip` seam, `image_dir` copy written only after a successful description, stale-copy delete on change, no-orphan cleanup on embed failure, synthetic `#page=N#img=I` paths, added/updated counts). + - Tests: +17 unit in `test_pdf.py` (title ×6, index paths, images-off, unchanged+backfill+manual-date, corrupt/empty PDF, unsupported mime, describe- and embed-failure, changed-image copy replacement), +7 chunker unit, +3 integration (real seam). + +- **Results**: `uv run pytest --cov=app` → **2993 passed**, TOTAL **99%** (gate >90%; touched modules: chunker 98%, importer 98%, new code fully covered); `uv run ruff check .` → all checks passed; `uv run pyright` → 0 errors. + +- **Notable decisions**: task-04 file's simplified spec wins over the 00_phase sketch — PDF `content` is the raw extracted text (no inline `[Embedded image: …]` placeholders); unchanged PDF (same text digest) skips the image pass entirely, so a transient embedded-image describe failure is retried on the next PDF text change (documented in docstrings); per-image embed failure is caught (rollback + copy delete + `errors`), not propagated. A chunk-order flake was fixed with `order_by(Chunk.position)` in the new tests. + +- **Next pending task**: `.agents/phases/todo/126_pdf_support/05_pdf_archive_and_tests.md` diff --git a/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a1.validate b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a1.validate new file mode 100644 index 0000000..a363a01 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a1.validate @@ -0,0 +1,113 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 72%] +........................................................................ [ 74%] +........................................................................ [ 76%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 96%] +........................................................................ [ 98%] +......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 262 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 241 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 256 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 132 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 374 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 38 0 100% +app/rag/importer.py 555 9 98% +app/rag/llm.py 244 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 81 0 100% +app/rag/prompts.py 144 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 400 0 100% +-------------------------------------------------- +TOTAL 5106 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a2.err b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a2.md b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a2.md new file mode 100644 index 0000000..98212e3 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a2.md @@ -0,0 +1,19 @@ +All green — the previous rejection was a harness issue (missing final report), not a code failure. All changes from the prior attempt remain intact in the working tree and have been fully re-verified. + +**Report** + +**Implemented** +- `app/rag/chunker.py` — `_chunk_pdf` page-aware chunker (`---PAGE-N---` as primary boundary, `Page N:` anchor on every chunk, anchor-budgeted to the hard cap) + `.pdf` dispatch (backfills task 03, whose code was never written despite the file being marked complete) +- `app/rag/importer.py` — PDF branch in `_index_file` (after image, before text); `_index_pdf_file` (extract → text-digest upsert with the unchanged/D1/D4 dance → shared chunk+embed → lite summary → image pass); `_extract_pdf_title` (metadata title, 500-char cap, stem fallback); `_index_pdf_embedded_images`/`_index_one_embedded_image` (A3: `BOR_IMAGES`-gated, single `_describe_or_skip` seam, `image_dir` copy only after successful description, stale-copy delete, no-orphan cleanup on embed failure, `#page=N#img=I` paths) +- Tests: +17 unit in `test_pdf.py`, +7 chunker unit, +3 integration (real describe seam) + +**Results** +- `uv run pytest --cov=app` → **2993 passed**, TOTAL **99%** (gate >90%; touched modules: chunker 98%, importer 98%, new code fully covered) +- `uv run ruff check .` → all checks passed; `uv run pyright` → 0 errors, 0 warnings + +**Notable decisions** +- Task-04's simplified spec wins over the 00_phase sketch: PDF content is raw extracted text (no inline image placeholders); unchanged PDF (same text digest) skips the image pass — transient describe failures retry on the next text change (documented); per-image embed failure is caught (rollback + copy delete + `errors`) +- Chunk-order flake fixed with `order_by(Chunk.position)` in new tests + +**Next pending task** +- `.agents/phases/todo/126_pdf_support/05_pdf_archive_and_tests.md` diff --git a/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a2.validate b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a2.validate new file mode 100644 index 0000000..a363a01 --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__04_pdf_indexing.a2.validate @@ -0,0 +1,113 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 72%] +........................................................................ [ 74%] +........................................................................ [ 76%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 89%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 96%] +........................................................................ [ 98%] +......................................... [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 262 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 241 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 256 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 132 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 374 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 38 0 100% +app/rag/importer.py 555 9 98% +app/rag/llm.py 244 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 81 0 100% +app/rag/prompts.py 144 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 400 0 100% +-------------------------------------------------- +TOTAL 5106 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.agents/reports/126_pdf_support/126_pdf_support__05_pdf_archive_and_tests.a1.err b/.agents/reports/126_pdf_support/126_pdf_support__05_pdf_archive_and_tests.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/126_pdf_support/126_pdf_support__05_pdf_archive_and_tests.a1.md b/.agents/reports/126_pdf_support/126_pdf_support__05_pdf_archive_and_tests.a1.md new file mode 100644 index 0000000..a26899f --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__05_pdf_archive_and_tests.a1.md @@ -0,0 +1,21 @@ +All green in the final state. Summary report: + +**Phase 126 task 05 — PDFs in archives + full test suite: complete** + +Implemented: +- Verified archive path needs no code: `unpack_archive` extracts all files; the `pdf` token in `BOR_IMPORT_EXTENSIONS` admits PDFs (proven end-to-end via the E2E upload→sync flow) +- `.env.example`: documented the `pdf` token under `BOR_IMPORT_EXTENSIONS` +- **Defect fix (task 04 work):** `_prune` destroyed PDF-embedded image docs in the same sync that indexed them (synthetic `#page=N#img=I` paths never enter the walk's `seen`); added the parent-PDF rule (skip while the parent rel is walked; prune with the parent when it leaves) +- **Parity fix:** `_index_one_embedded_image` now runs the phase-125 task-03 markdown context scan (embedded images referenced by a same-source markdown are enriched in both walk orderings, `image_context` stored, context → vision prompt + content/summary) +- `tests/integration/test_importer.py` +4 tests: no-text PDF (empty doc, no content chunks), prune parent rule, markdown-first (record stored + enrichment), PDF-first (ordering-independent enrichment) +- `tests/e2e/test_pdf_support.py` (new, 6 tests): module apps images ON/OFF, fitz-built PDF (2 text pages + embedded PNG) via the real admin upload → Sources rows (text row + thumbnail) → viewer renders extracted text (A4) + image doc → chat grounds on the PDF (scripted read echo) with the embedded image's auto-cited chip + figure (ON) / no image chip (OFF) + +Results: +- `uv run pytest` → 2997 passed; `uv run pytest --cov=app --cov-report=term-missing` → TOTAL 99% (gate >90%) +- `uv run pytest tests/e2e/test_pdf_support.py -v --no-cov` → 6 passed (isolation) +- `uv run ruff check . && uv run pyright` → clean (0 errors) +- Regression E2E: `test_image_documents.py` + `test_image_anchor_context.py` → 8 passed; `test_archive_upload_sources.py` → 5 passed + +Notable: the two importer fixes above were required for the phase-126 completion criteria ("images ON → embedded images become image documents") to hold under a real `prune=True` sync. + +Next pending task: none — task 05 was this phase's final task (phase dir move + atomic commit are harness-gated). diff --git a/.agents/reports/126_pdf_support/126_pdf_support__05_pdf_archive_and_tests.a1.validate b/.agents/reports/126_pdf_support/126_pdf_support__05_pdf_archive_and_tests.a1.validate new file mode 100644 index 0000000..f0dddec --- /dev/null +++ b/.agents/reports/126_pdf_support/126_pdf_support__05_pdf_archive_and_tests.a1.validate @@ -0,0 +1,113 @@ +........................................................................ [ 2%] +........................................................................ [ 4%] +........................................................................ [ 7%] +........................................................................ [ 9%] +........................................................................ [ 12%] +........................................................................ [ 14%] +........................................................................ [ 16%] +........................................................................ [ 19%] +........................................................................ [ 21%] +........................................................................ [ 24%] +........................................................................ [ 26%] +........................................................................ [ 28%] +........................................................................ [ 31%] +........................................................................ [ 33%] +........................................................................ [ 36%] +........................................................................ [ 38%] +........................................................................ [ 40%] +........................................................................ [ 43%] +........................................................................ [ 45%] +........................................................................ [ 48%] +........................................................................ [ 50%] +........................................................................ [ 52%] +........................................................................ [ 55%] +........................................................................ [ 57%] +........................................................................ [ 60%] +........................................................................ [ 62%] +........................................................................ [ 64%] +........................................................................ [ 67%] +........................................................................ [ 69%] +........................................................................ [ 72%] +........................................................................ [ 74%] +........................................................................ [ 76%] +........................................................................ [ 79%] +........................................................................ [ 81%] +........................................................................ [ 84%] +........................................................................ [ 86%] +........................................................................ [ 88%] +........................................................................ [ 91%] +........................................................................ [ 93%] +........................................................................ [ 96%] +........................................................................ [ 98%] +............................................. [100%] +=============================== warnings summary =============================== +.venv/lib/python3.13/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/brain_of_reese/.venv/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.13.13-final-0 _______________ + +Name Stmts Miss Cover +-------------------------------------------------- +app/__init__.py 1 0 100% +app/api/__init__.py 0 0 100% +app/api/auth.py 52 0 100% +app/api/chat.py 262 1 99% +app/api/chat_images.py 50 0 100% +app/api/chats.py 110 0 100% +app/api/config.py 13 0 100% +app/api/doc_drafts.py 99 0 100% +app/api/docs.py 179 1 99% +app/api/git_sources.py 241 0 100% +app/api/health.py 10 0 100% +app/api/steering.py 42 0 100% +app/api/suggestions.py 33 0 100% +app/api/sync.py 139 0 100% +app/api/tokens.py 40 0 100% +app/api/ui_settings.py 55 0 100% +app/config.py 256 0 100% +app/core/__init__.py 0 0 100% +app/core/auth.py 45 0 100% +app/core/caching.py 124 0 100% +app/core/debugging.py 29 2 93% +app/core/docs_push.py 39 0 100% +app/core/errors.py 5 0 100% +app/core/logging.py 13 0 100% +app/core/rate_limit.py 44 0 100% +app/core/security_headers.py 20 0 100% +app/core/theming.py 38 0 100% +app/core/tokens.py 44 0 100% +app/db.py 22 0 100% +app/main.py 68 0 100% +app/models.py 132 0 100% +app/rag/__init__.py 0 0 100% +app/rag/agent.py 374 1 99% +app/rag/archive_upload.py 134 0 100% +app/rag/chunker.py 220 4 98% +app/rag/doc_dates.py 18 0 100% +app/rag/folder_summaries.py 123 0 100% +app/rag/git_sources.py 38 0 100% +app/rag/importer.py 563 9 98% +app/rag/llm.py 244 1 99% +app/rag/md_images.py 63 0 100% +app/rag/overview.py 71 0 100% +app/rag/pdf.py 81 0 100% +app/rag/prompts.py 144 0 100% +app/rag/retriever.py 248 3 99% +app/rag/scaffolding.py 55 0 100% +app/rag/source_removal.py 41 0 100% +app/rag/sources_meta.py 16 0 100% +app/rag/suggestions.py 27 0 100% +app/rag/summarizer.py 49 0 100% +app/schemas.py 400 0 100% +-------------------------------------------------- +TOTAL 5114 22 99% +coverage gate: app/ 99% (>90%) OK +All checks passed! +0 errors, 0 warnings, 0 informations +WARNING: there is a new pyright version available (v1.1.411 -> v1.1.414). +Please install the new version or set PYRIGHT_PYTHON_FORCE_VERSION to `latest` + +validation OK diff --git a/.env.example b/.env.example index 71f57cb..b5423fd 100644 --- a/.env.example +++ b/.env.example @@ -77,6 +77,10 @@ BOR_NAME_HIT_BONUS=0.005 # selection-time bonus a name-hit doc's best f # insensitive, no partial names — phase 102). A blank list or a # malformed token (e.g. md,sh!) fails startup loudly, naming the value. BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py,container,network,volume,image,pod,kube,swap,os,endpoint,j2 +# Phase 126: add the ``pdf`` token to make PDFs first-class, retrievable +# documents (page-aware text extraction, embedded images through the +# image pipeline, chunking + embedding + summary): +# BOR_IMPORT_EXTENSIONS=md,markdown,txt,yaml,yml,json,py,pdf # BOR_SUGGESTIONS=["How is my Kubernetes cluster set up?"] # JSON seed chips — shown only before any question has been saved (phase 80); after that the chips are the 3 newest chats' opening questions (phase 103) # --- Import sources (git; phase 28, admin-managed since phase 35) --- diff --git a/app/rag/chunker.py b/app/rag/chunker.py index 6eb9c93..af74d11 100644 --- a/app/rag/chunker.py +++ b/app/rag/chunker.py @@ -26,6 +26,11 @@ per-format policies: endpoint / j2** (A9 revised 2026-08-27) — quadlet unit files (TOML) and Jinja templates; plain-text paragraph packing (``chunk_text``) — no format-specific splitter (owner decision). +* **pdf** (phase 126, LOCKED A2) — page-aware: the extracted text's + ``---PAGE-N---`` markers are the primary split boundary (one logical + unit per text-bearing page; a long page sub-splits through the + paragraph packer), every chunk keeps its ``Page N:`` anchor, and the + markers themselves never appear in a chunk. Every format honors :data:`HARD_MAX_CHARS` (1200 — the aipi ~1024-token request cap) and the target/overlap settings; oversized blocks are split @@ -324,6 +329,47 @@ def chunk_text(content: str, target_chars: int = 2000, overlap_chars: int = 200) return _pack_blocks(_paragraph_blocks(lines, [False] * len(lines)), target, overlap) +#: The page marker of the extracted PDF text (phase 126, A2 — +#: ``app.rag.pdf.extract_pdf_text`` prefixes every text-bearing page +#: block with it; the number is the 1-indexed page number). +_PDF_PAGE_MARKER_RE = re.compile(r"---PAGE-(\d+)---") + + +def _chunk_pdf(content: str, target_chars: int = 2000, overlap_chars: int = 200) -> list[str]: + """Page-aware chunking of one PDF's extracted text (phase 126, A2). + + A page is the primary split boundary: each ``---PAGE-N---`` block is + packed on its own (the plain-text paragraph packer — PDFs have no + code fences, so paragraph boundaries = blank lines) into chunks of + at most ``target_chars`` with the ``overlap_chars`` carry-over, and + every chunk of the page is prefixed with the ``Page N:`` anchor so a + hit is readable in context. The marker number is the page number + (blank pages carry no marker — the numbering follows the markers, + not the chunk count). The markers themselves never appear in a + chunk, and the anchor-aware budget keeps the anchored chunk under + the target — :data:`HARD_MAX_CHARS` included. + """ + target, overlap = _normalize_target_overlap(target_chars, overlap_chars) + # re.split with a capturing group keeps the markers: the pieces are + # [before-first, page, text, page, text, …]. + pieces = _PDF_PAGE_MARKER_RE.split(content) + chunks: list[str] = [] + for i in range(1, len(pieces) - 1, 2): + page = int(pieces[i]) + text = pieces[i + 1].strip() + if not text: + continue + anchor = f"Page {page}:" + # Budget the anchor out of the page's packing target so the + # anchored chunk cannot exceed ``target`` (the markdown + # section-anchor precedent, the ``budget`` rule). + budget = max(1, target - len(anchor) - 1) + chunks.extend( + f"{anchor}\n{part}" for part in chunk_text(text, budget, overlap) + ) + return chunks + + def _normalize_target_overlap(target_chars: int, overlap_chars: int) -> tuple[int, int]: """Validate + clamp the size policy (shared by every format).""" if target_chars <= 0: @@ -358,6 +404,9 @@ _FORMAT_CHUNKERS = { ".os": chunk_text, ".endpoint": chunk_text, ".j2": chunk_text, + # Phase 126 (LOCKED A2): the page-aware PDF chunker — the extracted + # text's ``---PAGE-N---`` markers are the primary split boundary. + ".pdf": _chunk_pdf, } diff --git a/app/rag/importer.py b/app/rag/importer.py index b66b135..376677d 100644 --- a/app/rag/importer.py +++ b/app/rag/importer.py @@ -85,6 +85,30 @@ summary (stored verbatim, no ``lite`` call, no pointer line), so the ``is_summary`` position −1 chunk mirrors ``Document.summary``, which equals ``Document.content``. +PDF files (phase 126, LOCKED A1/A2/A3/A4): a ``.pdf`` file (the +``pdf`` token of ``BOR_IMPORT_EXTENSIONS``) takes the PDF index path +(:func:`_index_pdf_file` — the branch sits in :func:`_index_file` +AFTER the image branch, BEFORE any text read — a PDF is never +``read_text``-ed; the binary is owned by the phase-126 parser). The +document's ``content`` is the extracted text (every text-bearing page +prefixed with the ``---PAGE-N---`` marker — A2, one document per PDF), +the digest is sha256 over that text (the digest rule is unchanged — +content identity), the title is the PDF's embedded metadata title or +the file stem, and the page-aware chunker (task 03) chunks it with a +``Page N:`` anchor. With the phase-122 ``images`` toggle ON, the +embedded images join the image pipeline (A3, :func: +`_index_pdf_embedded_images`): each is described through the SAME +``_describe_or_skip`` seam (the bytes as a data URL), copied to +``settings.image_dir`` (the copy is written only AFTER a successful +description — a failure never leaves an orphan), and indexed as a +standalone ``is_image`` document under the synthetic path +``#page=#img=.`` (the description IS the +content and summary — the phase-122 convention). Toggle OFF → the +embedded images are skipped (debug-logged) and the prune guard keeps +any existing image docs; an unsupported image mime (outside +``BOR_IMAGE_EXTENSIONS``) is skipped with a warning. A PDF's viewer +presentation is its extracted text (A4 — nothing binary is served). + Image-anchor context (phase 125, task 03, LOCKED A1/A2/A3): BEFORE the description, the same source's importable markdown files are scanned for ``![alt](path)`` references that name the image (the task-02 @@ -135,6 +159,7 @@ from datetime import UTC, datetime from pathlib import Path, PurePosixPath from typing import Any, Protocol +import pymupdf as fitz # PyMuPDF (phase 126, locked A1); `fitz` = the stable API name from sqlalchemy import select from sqlalchemy.orm import Session @@ -145,6 +170,7 @@ from app.rag.chunker import chunk_document, extract_title from app.rag.doc_dates import file_mtime_datetime, normalize_doc_date from app.rag.llm import EmbeddingError, LLMError from app.rag.md_images import ImageRef, extract_image_refs +from app.rag.pdf import PdfImage, extract_pdf_images, extract_pdf_text from app.rag.summarizer import ( IMAGE_FALLBACK_MIME, IMAGE_MIMES, @@ -652,6 +678,14 @@ async def _index_file( session, source=source, rel=rel, full_path=full_path, llm=llm, summary=summary, raw_date=raw_date, ) + # Phase 126: PDF files take the PDF index path — AFTER the image + # branch, BEFORE any text read (a PDF is binary; ``extract_pdf_text`` + # owns it and ``read_text`` is never called for one). + if full_path.suffix.lower() == ".pdf": + return await _index_pdf_file( + session, source=source, rel=rel, full_path=full_path, llm=llm, + summary=summary, raw_date=raw_date, + ) content = full_path.read_text(encoding="utf-8", errors="replace").replace("\x00", "") digest = hashlib.sha256(content.encode("utf-8")).hexdigest() doc = session.scalar(select(Document).where(Document.source == source, Document.path == rel)) @@ -855,6 +889,461 @@ async def _store_summary( logger.error("import: summary failed source=%s path=%s — %s", source, rel, e) +# --------------------------------------------------------------------------- +# PDF indexing (phase 126, LOCKED A1/A2/A3/A4). +# --------------------------------------------------------------------------- + +#: The ``Document.title`` column width (``app.models``) — the metadata +#: title is capped to it (an over-long PDF title must not blow up the +#: upsert). +_PDF_TITLE_MAX_CHARS = 500 + +#: MIME → extension for the phase-122 image family (phase 126, task 04) +#: — the reverse of the ``app.rag.summarizer.IMAGE_MIMES`` values for the +#: formats PyMuPDF reports. An unlisted mime takes ``.bin`` (never a +#: member of ``BOR_IMAGE_EXTENSIONS`` — the image is skipped with a +#: warning, the honest fail-soft outcome). +_MIME_TO_EXT: dict[str, str] = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/gif": ".gif", + "image/webp": ".webp", + "image/bmp": ".bmp", + "image/tiff": ".tiff", +} + + +def _extract_pdf_title(full_path: Path, fallback: str) -> str: + """The title of one PDF document (phase 126, task 04). + + The embedded metadata title (``fitz``'s ``doc.metadata["title"]``) + — trimmed and capped at the :data:`_PDF_TITLE_MAX_CHARS` column + width. An unreadable PDF, an absent metadata dict, or an + empty/whitespace-only title falls back to *fallback* (the file stem + — the non-markdown title rule; the metadata title is a convenience, + never a requirement). + """ + try: + doc = fitz.open(str(full_path)) + except Exception: + return fallback + with doc: + meta = doc.metadata + title = str(meta.get("title") or "").strip() if meta else "" + if not title: + return fallback + return title[:_PDF_TITLE_MAX_CHARS] + + +async def _index_pdf_file( + session: Session, + *, + source: str, + rel: str, + full_path: Path, + llm: Embedder, + summary: ImportSummary, + raw_date: datetime | None = None, +) -> None: + """The phase-126 PDF branch of :func:`_index_file` — a PDF is + indexed from its EXTRACTED TEXT (LOCKED A1/A2/A4), one transaction + for the document, then the embedded images (A3): + + * ``content`` is the parser's text — every text-bearing page + prefixed with the ``---PAGE-N---`` marker (A2 — one document per + PDF, the markers are the page anchors the page-aware chunker + (task 03) splits on); an unopenable/empty PDF yields ``""`` + (still a document — it may carry embedded images — with no + content chunks; a warning is the signal). + * the sha256 digest is over the extracted text (the digest rule is + content identity — the same text is the same document version; + the empty case digests the empty byte string). + * the title is the metadata title or the file stem + (:func:`_extract_pdf_title`); the date follows the text path + exactly (the phase-106 D2 mtime fallback — PDFs carry no + filesystem-visible creation date — plus the D1 manual-date lock + and the D4 refresh on the unchanged path). + * chunk + embed runs through the shared + :func:`_replace_content_chunks` helper (the endpoint token-cap + retry lives there), and the phase-30/118 (A2) ``lite`` summary + path runs on the text — PDFs are no more excluded than any other + non-markdown format. + * the unchanged path (same text digest) skips everything — no + re-extract, no re-embed, no image pass — and runs the same + backfill/manual-date/date-refresh dance as the text path. + * THEN the embedded images: :func:`_index_pdf_embedded_images` + (A3 — the phase-122 pipeline, gated by the ``images`` toggle). + + ``raw_date`` follows the text path: the caller's git last-commit + date or ``None`` (the mtime fallback, read before the unchanged + early-return — the unchanged path refreshes from the same source). + """ + # Extract the text (fail-soft: an unopenable PDF yields "" — the + # document may still carry embedded images worth indexing). + content = extract_pdf_text(full_path) + if not content: + logger.warning( + "import: PDF has no extractable text source=%s path=%s", source, rel + ) + # Content identity: the digest is over the extracted text (the + # empty case is the empty byte string's digest). + digest = ( + hashlib.sha256(content.encode("utf-8")).hexdigest() + if content + else hashlib.sha256(b"").hexdigest() + ) + title = _extract_pdf_title(full_path, fallback=full_path.stem) + if raw_date is None: + # D2 fallback (same as the text path): no source date in the + # map → the file's mtime (one stat). Read before the unchanged + # early-return — the unchanged path refreshes from it. + raw_date = file_mtime_datetime(full_path) + + doc = session.scalar(select(Document).where(Document.source == source, Document.path == rel)) + if doc is not None and doc.content_hash == digest: + # Unchanged PDF (text digest) — the text path's unchanged branch, + # unmodified in shape: no re-extract, no re-embed, no image pass. + summary.unchanged += 1 + logger.info("import: unchanged source=%s path=%s", source, rel) + # Phase 118 (A2): an unchanged doc whose summary is still NULL + # gets the same best-effort summary pass (see the text path). + if doc.summary is None: + await _store_summary( + session, doc=doc, source=source, rel=rel, content=content, + llm=llm, summary=summary, backfill=True, + ) + if doc.created_at_manual: + # D1/D4: the owner's correction survives the sync — no write + # at all (the text path's manual-date early-return). + return + # D4: the date refreshes on every sync and may go OLDER (no + # monotonic guard); a date-only refresh counts unchanged. + target = normalize_doc_date(raw_date) + if target != doc.created_at: + doc.created_at = target + session.commit() + summary.dates_updated += 1 + logger.info( + "import: date-refreshed source=%s path=%s date=%s", + source, rel, doc.created_at.isoformat(), + ) + return + + verb = "updated" if doc is not None else "added" + if doc is None: + doc = Document( + source=source, + path=rel, + full_path=str(full_path), + title=title, + content=content, + content_hash=digest, + indexed_at=datetime.now(UTC), + created_at=normalize_doc_date(raw_date), + ) + session.add(doc) + else: + doc.full_path = str(full_path) + doc.title = title + doc.content = content + doc.content_hash = digest + doc.indexed_at = datetime.now(UTC) + # Phase 106 (D4): a content change is a new document version — + # the date is re-sourced and a manual correction reset. + doc.created_at = normalize_doc_date(raw_date) + doc.created_at_manual = False + + session.flush() # guarantees doc.id even for brand-new rows + + # Phase 1+2 — the shared chunk/replace/embed pipeline (the page- + # aware chunker dispatches on the ``.pdf`` suffix; empty content + # clears the chunk rows and returns 0), committed atomically with + # the document. + chunks_n = await _replace_content_chunks( + session, doc=doc, llm=llm, content=content, rel=rel + ) + session.commit() + + if verb == "added": + summary.added += 1 + else: + summary.updated += 1 + summary.chunks += chunks_n + logger.info("import: %s source=%s path=%s chunks=%d", verb, source, rel, chunks_n) + + # Phase 30; phase 118 (A2): the ``lite`` summary of the extracted + # text (best-effort — a failure leaves the PDF indexed without a + # summary, counted in ``summary_errors``). + await _store_summary( + session, doc=doc, source=source, rel=rel, content=content, llm=llm, summary=summary + ) + + # Phase 126 (A3): the embedded images join the phase-122 pipeline + # (gated by the ``images`` toggle — skipped, debug-logged, while it + # is off). Runs after the document commit, so a per-image failure + # never loses the PDF's text index. + await _index_pdf_embedded_images( + session, source=source, rel=rel, full_path=full_path, + llm=llm, summary=summary, raw_date=raw_date, + ) + + +async def _index_pdf_embedded_images( + session: Session, + *, + source: str, + rel: str, + full_path: Path, + llm: Embedder, + summary: ImportSummary, + raw_date: datetime, +) -> None: + """The phase-126 (A3) embedded-image pass — one call per (re)indexed + PDF, after its document commit. + + Each embedded image (page order, 0-based index on the page — the + parser's order) is indexed as a STANDALONE image document through + the phase-122 mechanics, inline (no temp file, no + ``_index_image_file`` refactor): + + * gated by the phase-122 ``images`` toggle (the single knob, LOCKED + A3): while it is OFF the whole pass is skipped (one debug line — + the extraction itself is not worth running) and the prune guard + keeps any existing image docs; + * an image whose mime maps to no ``BOR_IMAGE_EXTENSIONS`` member + (:data:`_MIME_TO_EXT`) is skipped with a warning (the unsupported- + extension rule, same as a standalone image's); + * the document's synthetic ``path`` is + ``#page=#img=.`` — unique per image + occurrence, stable across re-indexes (the upsert key); the title + is ``Embedded image from page ``; + * the digest is sha256 over the image BYTES (the digest rule is + content identity); an image doc whose digest matches is skipped + (no re-describe, no re-embed — unchanged); + * the description comes through the SAME single seam + (:func:`_describe_or_skip` — the bytes as a data URL whose mime + comes from the copy path's extension); a failed/empty description + SKIPS the image entirely (no row, no copy) — ``images_failed`` + + a warning, the sync continues (LOCKED A3 fail-soft). The + ``image_dir`` copy (``.`` — a new image's id is the + uuid4 chosen here; a changed image keeps its id and its stale + copy is deleted before replacement) is written only AFTER a + successful description (a failure never leaves an orphan); + * phase 125 parity (the standalone image's task-03 scan): a same- + source markdown file that references the image by its synthetic + path (``![alt](#page=#img=.)``) enriches + the description — the combined context goes to the vision call + and is stored on the doc's ``image_context``, and ``content`` + becomes ``\\n\\n`` (as does ``summary`` — + the image convention); no reference keeps the bare shape + (byte-identical to the un-enriched case). The markdown-FIRST case + (a markdown indexed before the PDF recorded its reference on its + own row) is closed by the shared post-index pass + (:func:`_enrich_pending_images`), exactly as for a standalone + image; + * the description IS the content and the summary (the phase-122 + image convention — :func:`_store_summary` stores it verbatim), + chunked + embedded through :func:`_replace_content_chunks` (one + transaction per image); a per-image embed failure rolls back that + image only (``errors`` + the error line) and the pass moves on — + the PDF's text index and the other images are already committed. + * a new/updated image doc counts ``added``/``updated`` (it really + changed the KB — the sources-version bump fires on it, like a + standalone image's). + + ``raw_date`` is the PDF's resolved source date (the pass runs only + for (re)indexed PDFs, so it is always concrete) — the image docs + inherit the PDF's date (they are part of it). + """ + settings = llm.settings + if not settings.images: + # Phase-122 toggle off: embedded images are skipped (debug- + # logged) — and the prune guard keeps any existing image docs + # (they are invisible to the walk, not deleted files). + logger.debug( + "import: PDF embedded images skipped (images toggle off) " + "source=%s path=%s", + source, rel, + ) + return + images = extract_pdf_images(full_path) + for img in images: + await _index_one_embedded_image( + session, img, source=source, rel=rel, full_path=full_path, + llm=llm, summary=summary, raw_date=raw_date, + ) + + +async def _index_one_embedded_image( + session: Session, + img: PdfImage, + *, + source: str, + rel: str, + full_path: Path, + llm: Embedder, + summary: ImportSummary, + raw_date: datetime, +) -> None: + """One embedded image of one PDF → the phase-122 image pipeline, + with the phase-125 task-03 markdown context scan (parity with the + standalone image's :func:`_index_image_file` — see + :func:`_index_pdf_embedded_images` for the full contract).""" + settings = llm.settings + ext = _MIME_TO_EXT.get(img.mime, ".bin") + if ext not in settings.image_extension_set: + logger.warning( + "import: PDF embedded image skipped (unsupported extension) " + "source=%s path=%s page=%d index=%d mime=%s", + source, rel, img.page, img.index, img.mime, + ) + return + image_rel = f"{rel}#page={img.page}#img={img.index}{ext}" + digest = hashlib.sha256(img.data).hexdigest() + image_doc = session.scalar( + select(Document).where(Document.source == source, Document.path == image_rel) + ) + if image_doc is not None and image_doc.content_hash == digest: + # Unchanged embedded image (byte digest) — no re-describe, no + # re-embed (the standalone image's unchanged rule). + logger.debug( + "import: PDF embedded image unchanged source=%s path=%s", + source, image_rel, + ) + return + verb = "updated" if image_doc is not None else "added" + # Named by the doc id (the phase-122 copy rule): a new image's id is + # the uuid4 chosen here (row and copy agree); a changed image keeps + # its id (the copy path is stable, the stale copy is deleted below). + doc_id = image_doc.id if image_doc is not None else uuid.uuid4() + image_dir = Path(settings.image_dir).expanduser() + copy_path = image_dir / f"{doc_id}{ext}" + # Phase 125 (task 03, LOCKED A1/A2) — the embedded image's parity + # with the standalone image's path (phase 126): BEFORE the + # description, the markdown reference context for THIS image — a + # ``![alt](#page=#img=.)`` reference in a + # same-source markdown file (the synthetic path is the doc's + # ``path``, and the reference resolves to it through the shared + # :func:`_resolve_ref_path` rule) — or ``None`` (no reference — + # the description call and the row stay byte-identical to the + # un-enriched shape). It goes to the vision model (a context-aware + # description, LOCKED A3) AND is stored on the doc (``image_context`` + # — the retrieval display's context line, task 05). A re-indexed + # image re-scans: the context follows the current source state + # (a deleted referencing markdown drops it back to ``None``). + image_context = _markdown_image_context( + _source_root_for(full_path, rel), llm, source=source, image_rel=image_rel + ) + # The description through the single seam (the mime resolves from + # the copy path's extension — the extension was derived from the + # mime, so the round-trip is lossless for the default family). The + # description is generated BEFORE anything is written, so a failure + # skips the image with no row and no copy. + description = await _describe_or_skip( + llm, data=img.data, source=source, rel=image_rel, full_path=copy_path, + context=image_context, + ) + if description is None: + # LOCKED A3 fail-soft: an undescribed image is unsearchable + # noise — skip it (no row, no copy). + summary.images_failed += 1 + logger.warning( + "import: PDF embedded image description failed source=%s path=%s " + "page=%d index=%d", + source, image_rel, img.page, img.index, + ) + return + if image_doc is not None: + # A CHANGED image: the stale copy is deleted before replacement. + _delete_image_copy(image_doc.image_path) + image_dir.mkdir(parents=True, exist_ok=True) + copy_path.write_bytes(img.data) + + # Phase 125 (task 03): the enriched content — the reference context + # FIRST (for retrieval), then the description (the standalone image's + # rule; ``summary = content`` below keeps the image convention). + content = ( + f"{image_context}\n\n{description}" if image_context is not None else description + ) + + title = f"Embedded image from {full_path.stem} page {img.page}" + if image_doc is None: + image_doc = Document( + id=doc_id, + source=source, + path=image_rel, + full_path=str(copy_path), + title=title, + content=content, + content_hash=digest, + indexed_at=datetime.now(UTC), + created_at=normalize_doc_date(raw_date), + is_image=True, + image_path=str(copy_path), + image_context=image_context, + ) + session.add(image_doc) + else: + image_doc.full_path = str(copy_path) + image_doc.title = title + image_doc.content = content + image_doc.content_hash = digest + image_doc.indexed_at = datetime.now(UTC) + # Phase 106 (D4): a content change is a new document version — + # the date is re-sourced and a manual correction reset. + image_doc.created_at = normalize_doc_date(raw_date) + image_doc.created_at_manual = False + image_doc.is_image = True + image_doc.image_path = str(copy_path) + # Phase 125 (task 03): a re-indexed image re-scans its markdown + # references — the context follows the current source state. + image_doc.image_context = image_context + + session.flush() # guarantees doc.id even for brand-new rows + + # Phase 1+2 — the shared pipeline on the description (one + # transaction per image; a per-image embed failure rolls back THIS + # image only — the PDF doc and the other images are already + # committed — and the pass moves on). + try: + chunks_n = await _replace_content_chunks( + session, doc=image_doc, llm=llm, content=content, rel=image_rel + ) + session.commit() + except EmbeddingError as e: + session.rollback() + # The description succeeded (the copy was written), but the + # embed gave up: delete the copy so a failure never leaves an + # orphan (the phase-122 no-orphan rule). + _delete_image_copy(str(copy_path)) + summary.errors += 1 + logger.error( + "import: PDF embedded image embed failed source=%s path=%s " + "page=%d index=%d — %s", + source, image_rel, img.page, img.index, e, + ) + return + + if verb == "added": + summary.added += 1 + else: + summary.updated += 1 + summary.chunks += chunks_n + logger.info( + "import: %s source=%s path=%s chunks=%d", verb, source, image_rel, chunks_n + ) + # Phase 122 (task 03) shape on the description — image-aware: the + # description IS the summary (stored verbatim, no ``lite`` call), so + # ``doc.summary`` == ``doc.content`` and the ``is_summary`` position + # −1 chunk mirrors it. With a markdown reference (phase 125) the + # summary is the ENRICHED content — the same value the standalone + # image's path stores. + await _store_summary( + session, doc=image_doc, source=source, rel=image_rel, + content=content, llm=llm, summary=summary, + ) + + def _source_root_for(full_path: Path, rel: str) -> Path: """The source root of one walked file (phase 125, task 03). @@ -1517,6 +2006,12 @@ async def _index_image_file( ) +#: The synthetic path marker of a PDF-embedded image doc (phase 126, +#: LOCKED A3 — ``#page=#img=.``): +#: :data:`_EMBEDDED_IMAGE_RE` is the prune's parent rule below. +_EMBEDDED_IMAGE_RE = re.compile(r"#page=\d+#img=\d+") + + def _prune( session: Session, source_names: set[str], @@ -1532,6 +2027,19 @@ def _prune( first images-off sync. Toggle ON → normal semantics: a deleted image file prunes its doc, and the pruned image's ``image_dir`` copy is deleted with it. + + Phase 126 (LOCKED A3) — the PDF-embedded image's parent rule: an + embedded image doc's synthetic path (``#page=#img=``) + NEVER appears in the walk's *seen* — the image is derived from its + parent PDF, whose on-disk file is the walked file. Pruning it on the + plain rule would destroy it in the very sync that (re)indexes it, + so it is skipped while its parent PDF's rel is still in *seen* (the + image is invisible to the walk, not deleted). A pruned/absent parent + PDF prunes its embedded image docs with it (row + ``image_dir`` copy + — no orphans when the PDF leaves the source). The rule holds in BOTH + toggle states (toggle off, the phase-122 guard already skips every + ``is_image`` doc; this one is the toggle-on half of the same + "invisible, not deleted" contract). """ if not source_names: return 0 @@ -1542,6 +2050,14 @@ def _prune( if doc.is_image and not images: # Prune guard: invisible to the walk, not deleted. continue + if doc.is_image and _EMBEDDED_IMAGE_RE.search(doc.path): + # Phase 126 (A3): the parent PDF's rel is the path minus + # its ``#page=…#img=…`` fragment — the walked file. + parent_rel = doc.path.split("#", 1)[0] + if (doc.source, parent_rel) in seen: + # Parent PDF still in the source — the image is + # invisible to the walk, not deleted. + continue _delete_image_copy(doc.image_path) session.delete(doc) pruned += 1 diff --git a/app/rag/pdf.py b/app/rag/pdf.py new file mode 100644 index 0000000..73c95f5 --- /dev/null +++ b/app/rag/pdf.py @@ -0,0 +1,210 @@ +"""PDF parsing utilities (phase 126 — PDF support, locked decision A1). + +Pure functions (I/O = reading the PDF file only) for extracting text +and embedded images from PDF files using PyMuPDF (the ``pymupdf`` +package, imported as ``fitz`` — its stable API name). + +* :func:`extract_pdf_text` — the text of every page that has text, + each prefixed with :data:`PAGE_SEPARATOR` (``---PAGE-N---``, N is the + 1-indexed page number). The PDF chunker (phase 126 task 03) splits on + these markers so every chunk keeps its page anchor. Pages without + text (image-only pages) contribute nothing, so a PDF with no + extractable text returns ``""``. +* :func:`extract_pdf_images` — one :class:`PdfImage` per embedded-image + occurrence, in page order then 0-based index on the page. ``data`` is + the image's original encoded bytes as stored in the PDF (no + re-encoding), ``mime`` is derived via :func:`_mime_from_image_info`. +* :func:`_mime_from_image_info` — maps a PyMuPDF image-info dict to a + MIME type: the embedded file extension (``ext``) first, then the + image type code (``type``: 2=JPEG, 3=PNG, 4=GIF, 8=TIFF, 9=WebP), + else the ``application/octet-stream`` fallback. + +Every failure mode is fail-soft: an unreadable file returns an empty +result, an unreadable page or image is logged and skipped — this module +never raises for malformed input (the importer owns the loud failures). +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pymupdf as fitz # PyMuPDF (locked A1); `fitz` is the stable API name + +logger = logging.getLogger("app.rag.pdf") + +#: Page separator used in extracted text (the chunker splits on this). +#: ``{page}`` is the 1-indexed page number. +PAGE_SEPARATOR = "---PAGE-{page}---" + +#: Mapping from PyMuPDF image type codes to MIME types +#: (2=JPEG, 3=PNG, 4=GIF, 8=TIFF, 9=WebP — phase-126 design). +_TYPE_TO_MIME: dict[int, str] = { + 2: "image/jpeg", + 3: "image/png", + 4: "image/gif", + 8: "image/tiff", + 9: "image/webp", +} + +#: Mapping from embedded-image file extensions to MIME types. +_EXT_TO_MIME: dict[str, str] = { + "png": "image/png", + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "gif": "image/gif", + "webp": "image/webp", + "tif": "image/tiff", + "tiff": "image/tiff", +} + + +@dataclass +class PdfImage: + """One embedded image from a PDF page.""" + + page: int #: 1-indexed page number + index: int #: image index on the page (0-based, occurrence order) + width: int #: native pixel width + height: int #: native pixel height + data: bytes #: original encoded bytes as stored in the PDF + mime: str #: MIME type (e.g. ``image/png``) + + +def _mime_from_image_info(info: dict[str, Any]) -> str: + """Map a PyMuPDF image-info dict to a MIME type. + + The embedded file extension (``ext``) wins over the image type code + (``type``); unknown/missing both fall back to + ``application/octet-stream``. + """ + ext = str(info.get("ext") or "").lower() + if ext in _EXT_TO_MIME: + return _EXT_TO_MIME[ext] + type_code = info.get("type") + if isinstance(type_code, int): + return _TYPE_TO_MIME.get(type_code, "application/octet-stream") + return "application/octet-stream" + + +def _page_image_entries(page: Any) -> list[dict[str, Any]]: + """One info dict per embedded-image occurrence on *page*. + + Prefers ``get_image_info(xrefs=True)`` (one entry per occurrence, + with the ``xref`` object number); older PyMuPDF releases without the + ``xrefs`` keyword are served from ``get_images(full=True)`` (one + entry per unique image). + """ + try: + infos: list[dict[str, Any]] = page.get_image_info(xrefs=True) + except TypeError: + infos = [ + {"xref": entry[0], "width": entry[2], "height": entry[3]} + for entry in page.get_images(full=True) + ] + return [dict(info) for info in infos] + + +def extract_pdf_text(pdf_path: Path) -> str: + """Extract the text of all pages of *pdf_path*. + + Every page that has text is prefixed with :data:`PAGE_SEPARATOR` + (``---PAGE-N---``, N 1-indexed) and the page blocks are joined with + a blank line. Pages without text contribute nothing, so a PDF with + no extractable text (or a zero-page file) returns ``""``. + + Fail-soft: an unreadable file returns ``""``; a page that cannot be + read is logged and skipped. + """ + try: + doc = fitz.open(str(pdf_path)) + except Exception as exc: + logger.warning("pdf: could not open %s — %s", pdf_path, exc) + return "" + + parts: list[str] = [] + with doc: + for page_num in range(len(doc)): + try: + raw = doc[page_num].get_text("text") + except Exception as exc: + logger.warning( + "pdf: could not read text of page %d of %s — %s", + page_num + 1, pdf_path, exc, + ) + continue + # get_text("text") always returns a str; the union return type + # comes from pymupdf's other flavors (dict/words/blocks). + text = raw.strip() if isinstance(raw, str) else "" + if text: + parts.append(f"{PAGE_SEPARATOR.format(page=page_num + 1)}\n{text}") + return "\n\n".join(parts) + + +def extract_pdf_images(pdf_path: Path) -> list[PdfImage]: + """Extract all embedded images from *pdf_path*. + + Returns one :class:`PdfImage` per image occurrence, in page order + then 0-based index on the page. ``data`` is the original encoded + bytes as stored in the PDF; ``mime`` is derived from the image's + extension/type via :func:`_mime_from_image_info`. + + Fail-soft: an unreadable file returns ``[]``; an image without an + ``xref`` or that cannot be extracted is logged and skipped. + """ + try: + doc = fitz.open(str(pdf_path)) + except Exception as exc: + logger.warning("pdf: could not open %s for image extraction — %s", pdf_path, exc) + return [] + + images: list[PdfImage] = [] + with doc: + for page_num in range(len(doc)): + try: + entries = _page_image_entries(doc[page_num]) + except Exception as exc: + logger.warning( + "pdf: could not list images on page %d of %s — %s", + page_num + 1, pdf_path, exc, + ) + continue + for index, info in enumerate(entries): + xref = int(info.get("xref") or 0) + if not xref: + logger.warning( + "pdf: skipping image page=%d index=%d of %s — no xref", + page_num + 1, index, pdf_path, + ) + continue + try: + extracted = doc.extract_image(xref) + except Exception as exc: + logger.warning( + "pdf: could not extract image page=%d index=%d of %s — %s", + page_num + 1, index, pdf_path, exc, + ) + continue + data = extracted.get("image") or b"" + if not data: + logger.warning( + "pdf: skipping image page=%d index=%d of %s — empty data", + page_num + 1, index, pdf_path, + ) + continue + # The image info's own `ext` is unreliable in some + # PyMuPDF versions (None); `extract_image` reports the + # actual format of the returned bytes, so it wins. + mime = _mime_from_image_info({**info, "ext": extracted.get("ext") or ""}) + images.append( + PdfImage( + page=page_num + 1, + index=index, + width=int(extracted.get("width") or info.get("width") or 0), + height=int(extracted.get("height") or info.get("height") or 0), + data=data, + mime=mime, + ) + ) + return images diff --git a/pyproject.toml b/pyproject.toml index 0012fde..9020b0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,10 @@ dependencies = [ # (POST /api/git-sources/upload) — an A2 FastAPI implementation detail, # not a new architectural anchor (phase 49 locked decisions). "python-multipart>=0.0.9,<0.1", + # Phase 126: PDF support (locked A1) — PyMuPDF (import name `fitz`) is the + # PDF parser: page-by-page text extraction AND embedded image extraction + # in a single API. Chosen over pdfplumber (text-only) per phase 126 design. + "pymupdf>=1.24,<2.0", ] [dependency-groups] diff --git a/tests/e2e/test_pdf_support.py b/tests/e2e/test_pdf_support.py new file mode 100644 index 0000000..2458adc --- /dev/null +++ b/tests/e2e/test_pdf_support.py @@ -0,0 +1,644 @@ +"""Phase 126 E2E (Playwright): PDF support — the user path. + +Run in isolation (DB must be up: ``podman compose up -d db``): + + uv run pytest tests/e2e/test_pdf_support.py -v --no-cov + +PDFs are first-class, retrievable documents behind the ``pdf`` token of +``BOR_IMPORT_EXTENSIONS`` (the two module apps pin the token EXPLICITLY +— an operator's local gitignored ``.env`` cannot leak the scope in +either direction, and without the token the walk is blind to the file): + +* the story app (``BOR_IMAGES=true``): the fixture PDF (two text pages + + one embedded PNG on page 2, built with PyMuPDF in the fixture — + LOCKED A1) arrives through the REAL admin upload flow (a zip with a + single PDF member — the archive unpack is content-agnostic, the + extension walk admits it once ``pdf`` is in the token set) and is + first-class at every surface: + - the Sources page lists it (the extracted text's document — a plain + text row, no thumbnail) plus the embedded image's document (the + phase-122 pipeline, LOCKED A3 — the 48px thumbnail from the bytes + route, alt = the mock vision description); + - the document viewer renders the EXTRACTED TEXT (LOCKED A4 — the + ``---PAGE-N---`` page markers and the page text, never binary + bytes) with the metadata title (LOCKED A2 — one document per PDF); + - a grounded chat question (the mock's token-overlap cosine against + the summary chunk ≈0.61, over the mock-calibrated gate 0.30 — + FTS-corroborated anyway) scripted to READ it echoes the read result + verbatim (the extracted text reaches the model AND the answer), and + the sources block carries the PDF's citation chip PLUS the embedded + image's auto-cited chip + compact inline figure (phase 124 — the + suggested-tier image doc, best chunk ≈0.22 over the usefulness + floor); +* the default app (``BOR_IMAGES`` forced ``false``): the SAME upload + + sync indexes the PDF's text normally — the embedded image is skipped + (no image document, no copy, the source is a 1-document source) and + the chat turn grounds with the PDF's chip only (no image chip, no + figure — LOCKED A3's toggle gate end-to-end). +""" +from __future__ import annotations + +import base64 +import json +import os +import re +import subprocess +import sys +import zipfile +from collections.abc import Iterator +from pathlib import Path + +import httpx +import pymupdf as fitz # PyMuPDF — the fixture-PDF builder (LOCKED A1) +import pytest +from playwright.sync_api import Browser, Page, expect +from sqlalchemy import select, text + +from app.config import Settings +from app.db import SessionLocal +from app.models import Document +from e2e.auth_helpers import login +from e2e.conftest import ADMIN_PASSWORD, SESSION_SECRET, USE_REAL_LLM, _wait_http +from e2e.mock_llm import IMAGE_DESCRIPTION_ANSWER + +REPO = Path(__file__).resolve().parents[2] +SOURCE_NAME = "e2e-pdf" # the archive stem (archive_source_name) +DOC_PATH = "report.pdf" +IMAGE_REL = "report.pdf#page=2#img=0.png" # the synthetic embedded-image path +PDF_TITLE = "Quarterly Homelab Report" # the metadata title (LOCKED A2) + +#: The fixture PDF's page texts (the chunker's ``Page N:`` anchors make +#: the chunks ``Page 1:\n`` / ``Page 2:\n``). The question +#: below shares its tokens with them (the mock's token-overlap cosine — +#: the summary chunk ≈0.61 — clears the mock-calibrated gate 0.30). +PAGE_ONE = "The quarterly homelab report covers the core switch firmware upgrade." +PAGE_TWO = "The homelab backup schedule runs nightly at 02:00 and keeps seven rotations." + +#: The grounded question, carrying the house scripted-read call +#: (``SUMMARY_SEED_READ_TRIGGER`` — the phase-119 A1 convention: a +#: zero-read grounded turn chips NOTHING, so the PDF doc must be READ to +#: earn its citation chip): the mock emits the scripted +#: ``read e2e-pdf/report.pdf``, then echoes the tool result VERBATIM — +#: the read result carries the document's FULL extracted text, so the +#: echo in the answer is the E2E lens on LOCKED A4 (the PDF served as +#: extracted text, page markers and all). +QUESTION = ( + "Read the suggested document: read e2e-pdf/report.pdf — " + "what does the homelab report say about the backup schedule?" +) + +#: A real 1×1 transparent PNG (the unit/integration suites' fixture — +#: the pipeline is content-agnostic; the well-formed bytes keep the +#: extraction + copy + serve + render path honest). +PNG_1X1 = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ" + "AAAAC0lEQVR4nGP4DwQACfsD/fteaysAAAAASUVORK5CYII=" +) + +UPLOAD_TIMEOUT_MS = 30_000 +SYNC_TIMEOUT_MS = 60_000 +SYNCED_LABEL = re.compile(r"^Synced \d{1,2}:\d{2}$") + + +# --------------------------------------------------------------------------- +# Module apps: images ON (the story) and images forced OFF (the default +# contract). Each owns its port + its scratch upload/source/image dirs. +# Both pin BOR_IMPORT_EXTENSIONS WITH the ``pdf`` token explicitly +# (phase 126 — the feature lives behind the token, and process env ranks +# above an operator's local gitignored ``.env``). +# --------------------------------------------------------------------------- + + +def _app_env(mock_llm: int, app_port: int, *, images: bool, scratch: Path) -> dict[str, str]: + """The conftest app env (leak guards included) with the phase-126 + knobs: ``BOR_IMPORT_EXTENSIONS`` pinned to a set carrying the + ``pdf`` token, ``BOR_IMAGES`` pinned EXPLICITLY (true for the story + app, false for the default app — the contract under test cannot + leak in either direction), and the image/upload homes in the suite's + scratch dir (the app under test must not write copies into the + owner's real ``~/bor-sources``).""" + env = dict(os.environ) + env.pop("DEBUGPY", None) + env["BOR_ENVIRONMENT"] = "e2e" + env["BOR_STATIC_DIR"] = str(REPO / "frontend") + env["BOR_LLM_BASE_URL"] = ( + "https://aipi.reeseapps.com/v1" + if USE_REAL_LLM + else f"http://127.0.0.1:{mock_llm}/v1" + ) + # Mock-calibrated gate (conftest pattern) — the story's question + # grounds on these values (see QUESTION). + env["BOR_RELEVANCE_THRESHOLD"] = "0.30" + env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15" + env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15" + # Phase 67: instant retry waits + the code-default budget. + env["BOR_LLM_RETRY_DELAY"] = "0" + env["BOR_LLM_RETRIES"] = str(Settings.model_fields["llm_retries"].default) + env.setdefault( + "BOR_DATABASE_URL", + "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", + ) + env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD + env["BOR_SESSION_SECRET"] = SESSION_SECRET + # Leak guards (conftest pattern). + env["BOR_GIT_SOURCES"] = "" + env["BOR_DOCS_REPO"] = "" + env["BOR_SUGGESTIONS"] = json.dumps( + Settings.model_fields["suggestions"].default + ) + env["BOR_INPUT_PLACEHOLDER"] = Settings.model_fields["input_placeholder"].default + env["BOR_FOOTER_TEXT"] = Settings.model_fields["footer_text"].default + # Phase 126: the import scope WITH the ``pdf`` token (the feature is + # gated by the token — the explicit pin keeps a local ``.env`` from + # changing the premise) + the toggle under test + the suite-private + # homes. + env["BOR_IMPORT_EXTENSIONS"] = "md,markdown,txt,pdf" + env["BOR_IMAGES"] = "true" if images else "false" + env["BOR_UPLOAD_DIR"] = str(scratch / "uploads") + env["BOR_SOURCES_DIR"] = str(scratch / "checkouts") + env["BOR_IMAGE_DIR"] = str(scratch / "images") + return env + + +@pytest.fixture(scope="module") +def app_server(mock_llm: int, tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: + """The story app — ``BOR_IMAGES=true`` + the ``pdf`` token (the + module env override; the conftest session app is never started in + this isolated run, so no port clash).""" + scratch = tmp_path_factory.mktemp("bor_pdf_on") + port = int(os.environ.get("E2E_APP_PORT_PDF", "8152")) + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], + cwd=REPO, + env=_app_env(mock_llm, port, images=True, scratch=scratch), + ) + try: + _wait_http(f"http://127.0.0.1:{port}/api/health") + yield f"http://127.0.0.1:{port}" + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def app_url(app_server: str) -> str: + return app_server + + +@pytest.fixture(scope="module") +def default_app_server( + mock_llm: int, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[str]: + """The default-contract app — ``BOR_IMAGES`` forced ``false`` (the + toggle gate for the embedded images; only the off-app tests start + it).""" + scratch = tmp_path_factory.mktemp("bor_pdf_off") + port = int(os.environ.get("E2E_APP_PORT_PDF_OFF", "8153")) + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "app.main:app", + "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], + cwd=REPO, + env=_app_env(mock_llm, port, images=False, scratch=scratch), + ) + try: + _wait_http(f"http://127.0.0.1:{port}/api/health") + yield f"http://127.0.0.1:{port}" + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def default_app_url(default_app_server: str) -> str: + return default_app_server + + +@pytest.fixture(scope="module") +def zip_path(tmp_path_factory: pytest.TempPathFactory) -> Path: + """The fixture archive: ONE PDF at the root (two text pages + one + embedded PNG on page 2, built with PyMuPDF — LOCKED A1) — source + ``e2e-pdf``, document path ``report.pdf``.""" + root = tmp_path_factory.mktemp("bor_pdf_zip") + pdf_path = root / DOC_PATH + doc = fitz.open() + doc.new_page().insert_text((72, 72), PAGE_ONE) + page2 = doc.new_page() + page2.insert_text((72, 72), PAGE_TWO) + page2.insert_image(fitz.Rect(72, 150, 73, 151), stream=PNG_1X1) + doc.set_metadata({"title": PDF_TITLE}) + doc.save(pdf_path) + doc.close() + archive = root / f"{SOURCE_NAME}.zip" + with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf: + zf.write(pdf_path, arcname=DOC_PATH) + return archive + + +# --------------------------------------------------------------------------- +# DB + UI helpers +# --------------------------------------------------------------------------- + + +def _truncate_all() -> None: + """Fresh KB + registry per test group (the E2E isolation pattern): + the suites share one Postgres, and a leftover row would corrupt the + counts. ``sources_meta`` (the KB generation counter) resets with + the KB — the sync re-creates the row.""" + with SessionLocal() as db: + db.execute(text( + "TRUNCATE chunks, documents, query_log, kb_overview, " + "git_sources, sources_meta, folder_summaries" + )) + db.commit() + + +def _health_db(app_url: str) -> bool: + try: + return httpx.get(f"{app_url}/api/health", timeout=5).json()["db"] == "up" + except Exception: # noqa: BLE001 — unreachable is the skip case + return False + + +def _upload_zip(page: Page, app_url: str, archive: Path) -> None: + """The real admin upload flow: form login → the git-sources page → + pick the archive → submit → the terminal result line (phase 90: + "Uploaded — press Sync sources to import it.").""" + login(page, app_url, next="/git-sources.html") + expect(page).to_have_url(app_url + "/git-sources.html", timeout=30_000) + expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000) + page.set_input_files("#archive-upload-file", str(archive)) + page.click("#archive-upload-btn") + result = page.locator("#archive-upload-result") + expect(result).to_be_visible(timeout=UPLOAD_TIMEOUT_MS) + expect(result).to_have_text( + f"Uploaded {SOURCE_NAME} — press Sync sources to import it.", + timeout=UPLOAD_TIMEOUT_MS, + ) + + +def _sync_via_ui(page: Page, app_url: str, expected_result: str) -> None: + """The RAG page's "Sync sources" button (the scan): click → + "Syncing…" → the terminal "Synced HH:MM" label + the fresh counts + in #sync-result (the never-stale contract).""" + page.goto(app_url + "/sources.html") + btn = page.locator("#sync-btn") + expect(btn).to_be_visible(timeout=30_000) + expect(page.locator("#sync-label")).to_have_text( + re.compile(r"^Sync sources$|^Synced \d{1,2}:\d{2}$") + ) + btn.click() + expect(btn).to_be_disabled() + expect(page.locator("#sync-label")).to_have_text("Syncing…") + expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) + expect(btn).to_be_enabled() + expect(page.locator("#sync-result")).to_have_text(expected_result) + + +def _drill_to_source(page: Page, name: str) -> None: + """Top level → the source (the file table then holds its direct + files — ours are at the source root).""" + page.locator("#folders-tbody .folder-link").first.wait_for(state="visible") + page.click(f'#folders-tbody a.folder-link:text-is("{name}")') + expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=30_000) + + +# --------------------------------------------------------------------------- +# The module seeds: upload + sync the fixture PDF through the real UI, +# once per app (the story E2E truncate/re-import-per-module pattern). +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def seeded( + app_url: str, + browser: Browser, + zip_path: Path, +) -> Iterator[dict[str, str]]: + """Upload the single-PDF zip through the admin UI on the story app, + then scan it with the RAG page's Sync button (the mock's vision + model describes the embedded image in-process). Yields the embedded + image doc's identity (the bytes route + the description) for the + render assertions. The module app's ``/api/config`` flag is + asserted first — the env override must have reached the app under + test, or the whole premise of the suite is void.""" + if not _health_db(app_url): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + cfg = httpx.get(f"{app_url}/api/config", timeout=5).json() + assert cfg["images"] is True, "the module app must run with BOR_IMAGES=true" + + _truncate_all() + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + _upload_zip(page, app_url, zip_path) + # The PDF + its one embedded image (A3): both count added. + _sync_via_ui(page, app_url, expected_result="2 added") + with SessionLocal() as db: + pdf = db.scalar( + select(Document).where( + Document.source == SOURCE_NAME, Document.path == DOC_PATH + ) + ) + assert pdf is not None, "the uploaded PDF must become a document" + # LOCKED A2: one document per PDF, the extracted text with + # the page markers; the title is the metadata title. + assert pdf.is_image is False and pdf.image_path is None + assert pdf.title == PDF_TITLE + assert "---PAGE-1---" in pdf.content and PAGE_ONE in pdf.content + assert "---PAGE-2---" in pdf.content and PAGE_TWO in pdf.content + assert pdf.summary is not None + img = db.scalar( + select(Document).where( + Document.source == SOURCE_NAME, Document.path == IMAGE_REL + ) + ) + assert img is not None, "the embedded image must become an image doc" + assert img.is_image is True and img.image_path is not None + # content == summary == the mock's description (LOCKED A3 + # fail-soft shape; no markdown reference exists — the bare + # prompt's answer). + assert img.content == IMAGE_DESCRIPTION_ANSWER + assert img.summary == IMAGE_DESCRIPTION_ANSWER + finally: + page.close() + yield { + "image_url": f"/api/documents/{img.id}/image", + "description": IMAGE_DESCRIPTION_ANSWER, + } + + +@pytest.fixture(scope="module") +def seeded_off( + default_app_url: str, + browser: Browser, + zip_path: Path, +) -> Iterator[dict[str, str]]: + """The SAME fixture zip through the default app (images off): the + PDF's text indexes normally, the embedded image is skipped — the + source is a 1-document source. Yields the PDF doc's identity.""" + if not _health_db(default_app_url): + pytest.skip("Postgres not reachable — run `podman compose up -d db` first") + cfg = httpx.get(f"{default_app_url}/api/config", timeout=5).json() + assert cfg["images"] is False, "the default app must run with images off" + + _truncate_all() + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + _upload_zip(page, default_app_url, zip_path) + _sync_via_ui(page, default_app_url, expected_result="1 added") + with SessionLocal() as db: + pdf = db.scalar( + select(Document).where( + Document.source == SOURCE_NAME, Document.path == DOC_PATH + ) + ) + assert pdf is not None, "the uploaded PDF must become a document" + assert PAGE_ONE in pdf.content and PAGE_TWO in pdf.content + assert ( + db.scalar(select(Document).where(Document.is_image.is_(True))) + is None + ), "images off: the embedded image is skipped (no image doc)" + finally: + page.close() + yield {"title": PDF_TITLE} + + +# --------------------------------------------------------------------------- +# 1. Sources page (images ON): the PDF's text row + the image's thumbnail +# --------------------------------------------------------------------------- + + +def test_sources_page_lists_the_pdf_and_image_docs( + page: Page, app_url: str, seeded: dict[str, str] +) -> None: + """The KB total is 2 docs / 5 chunks (the PDF: 2 page-anchored + content chunks + the phase-30 summary chunk; the image: the + description chunk + its summary chunk). The drill-down table lists + both documents: the PDF as a PLAIN text row (LOCKED A4 — it is a + text document, no thumbnail box) and the embedded image with the + FIXED 48px thumbnail (the bytes route, alt = the description).""" + login(page, app_url) # lands on /sources.html + expect(page.locator("#stat-docs")).to_have_text("2") + expect(page.locator("#stat-chunks")).to_have_text("5") + expect(page.locator("#sources-empty")).to_be_hidden() + + _drill_to_source(page, SOURCE_NAME) + expect(page.locator("#docs-tbody tr")).to_have_count(2) + + # The PDF — a plain text row (no thumbnail box at all). The row is + # pinned by the EXACT doc-link text (the image row's path carries + # ``report.pdf`` as a prefix — a substring match would hit both). + pdf_row = page.locator( + "#docs-tbody tr", + has=page.locator(f'a.doc-link:text-is("{DOC_PATH}")'), + ) + expect(pdf_row).to_have_count(1) + expect(pdf_row.locator(".kb-doc-thumb")).to_have_count(0) + + # The embedded image — the thumbnail row (the bytes route's lazy + # , alt = the description — the WCAG contract). + img_row = page.locator( + "#docs-tbody tr", + has=page.locator(f'a.doc-link:text-is("{IMAGE_REL}")'), + ) + expect(img_row).to_have_count(1) + img = img_row.locator(".kb-doc-thumb-img") + expect(img).to_have_count(1) + expect(img).to_be_visible(timeout=15_000) # the fetch resolves — no glyph + expect(img).to_have_attribute("src", seeded["image_url"]) + expect(img).to_have_attribute("loading", "lazy") + expect(img).to_have_attribute("alt", seeded["description"]) + expect(img_row.locator(".kb-doc-thumb-glyph")).to_have_count(0) + + +# --------------------------------------------------------------------------- +# 2. Document viewer (images ON): the extracted text renders (A4) +# --------------------------------------------------------------------------- + + +def test_document_viewer_renders_the_extracted_text( + page: Page, app_url: str, seeded: dict[str, str] +) -> None: + """The Sources row's path link opens the same-page document modal + (phase 26): the PDF renders its EXTRACTED TEXT (LOCKED A4 — the + ``---PAGE-N---`` page markers + the page texts in the plain-content + slot, never binary bytes), under the METADATA title (LOCKED A2), + with the phase-30 Summary panel above (a non-image doc with a + non-empty summary — the panel renders).""" + login(page, app_url) + _drill_to_source(page, SOURCE_NAME) + page.locator(f'#docs-tbody a.doc-link:text-is("{DOC_PATH}")').click() + + modal = page.locator("#doc-modal") + expect(modal).to_be_visible(timeout=30_000) + expect(page.locator("#doc-modal-title")).to_have_text(PDF_TITLE) + + # The extracted text (A4) — both page markers and both page texts. + raw = modal.locator("pre.doc-raw") + expect(raw).to_contain_text("---PAGE-1---") + expect(raw).to_contain_text(PAGE_ONE) + expect(raw).to_contain_text("---PAGE-2---") + expect(raw).to_contain_text(PAGE_TWO) + # The summary panel (a non-image doc — the panel renders; the + # verbatim-description suppression applies to image docs only). + expect(modal.locator(".doc-summary")).to_have_count(1) + # No image block — the PDF is not an image document. + expect(modal.locator(".doc-image-img")).to_have_count(0) + + +# --------------------------------------------------------------------------- +# 3. Document viewer (images ON): the embedded image doc renders +# --------------------------------------------------------------------------- + + +def test_document_viewer_renders_the_embedded_image_doc( + page: Page, app_url: str, seeded: dict[str, str] +) -> None: + """The embedded image's row (the synthetic ``#page=2#img=0.png`` + path) opens the modal: the PERSISTENT copy renders first (the bytes + route, alt = the description), the description follows in the + content slot, and the labeled Summary panel is SUPPRESSED for the + verbatim-description case (summary === content — the importer + invariant; the panel would duplicate the text right below the + image).""" + login(page, app_url) + _drill_to_source(page, SOURCE_NAME) + page.locator(f'#docs-tbody a.doc-link:text-is("{IMAGE_REL}")').click() + + modal = page.locator("#doc-modal") + expect(modal).to_be_visible(timeout=30_000) + + img = modal.locator(".doc-image-img") + expect(img).to_have_count(1) + expect(img).to_be_visible(timeout=15_000) # the bytes route serves the PNG + expect(img).to_have_attribute("src", seeded["image_url"]) + expect(img).to_have_attribute("alt", seeded["description"]) + + expect(modal.locator("pre.doc-raw")).to_have_text(seeded["description"]) + # The verbatim case: no duplicate Summary panel. + expect(modal.locator(".doc-summary")).to_have_count(0) + # No "Image unavailable" note — the copy is intact. + expect(modal.locator(".doc-image-unavailable")).to_have_count(0) + + +# --------------------------------------------------------------------------- +# 4. Chat (images ON): the answer grounds on the PDF; the embedded +# image is auto-cited with its figure +# --------------------------------------------------------------------------- + + +def test_chat_grounds_on_the_pdf_and_chips_the_embedded_image( + page: Page, app_url: str, seeded: dict[str, str] +) -> None: + """A question the mock grounds on (the summary chunk's cosine + ≈0.61 over the 0.30 gate; FTS corroborates), scripted to READ the + PDF (phase 119 A1): the echo carries the read result VERBATIM — + the document's FULL extracted text (LOCKED A4 — the page markers + and both page texts reach the model and land in the answer), and + the sources block carries the PDF's citation chip (the READ doc) + PLUS the embedded image's AUTO-cited chip (phase 124 — the + suggested-tier image doc) with its compact inline figure (the + bytes route, caption/alt settling to the description).""" + login(page, app_url, next="/") + expect(page.locator("#kb-banner")).to_be_hidden() + + page.fill("#message-input", QUESTION) + page.click("#send-btn") + + bubble = page.locator(".msg.brain .bubble").last + # The scripted read's echoed result — the read header and the FULL + # extracted text (page markers and all). + expect(bubble).to_contain_text("Here's what the read returned:", timeout=30_000) + expect(bubble).to_contain_text(f"Document {SOURCE_NAME}/{DOC_PATH}:") + expect(bubble).to_contain_text("---PAGE-1---") + expect(bubble).to_contain_text(PAGE_ONE) + expect(bubble).to_contain_text("---PAGE-2---") + expect(bubble).to_contain_text(PAGE_TWO) + expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000) + expect(page.locator("#send-label")).to_have_text("Send") + + # The citation chips: the READ PDF first (read order), then the + # suggested image doc's auto-cite (phase 124). + chip = page.locator(".msg.brain .source-chip") + expect(chip).to_have_count(2) + expect(chip.nth(0)).to_have_text(f"{SOURCE_NAME}/{DOC_PATH}") + expect(chip.nth(1)).to_have_text(f"{SOURCE_NAME}/{IMAGE_REL}") + + # The inline figure on the image chip: image + caption, both + # settling to the description (the content fetch resolves the + # summary after the title). + fig = page.locator(".msg.brain .source-image") + expect(fig).to_have_count(1) + img = fig.locator(".source-image-img") + expect(img).to_be_visible(timeout=15_000) + expect(img).to_have_attribute("src", seeded["image_url"]) + expect(img).to_have_attribute("alt", seeded["description"], timeout=15_000) + caption = fig.locator(".source-image-caption") + expect(caption).to_have_text(seeded["description"], timeout=15_000) + + +# --------------------------------------------------------------------------- +# 5. Default app (images off): the same upload + sync indexes the PDF +# text normally and skips the embedded image +# --------------------------------------------------------------------------- + + +def test_images_off_upload_indexes_the_pdf_and_skips_the_image( + page: Page, default_app_url: str, seeded_off: dict[str, str] +) -> None: + """The toggle gate end-to-end (LOCKED A3): with ``BOR_IMAGES`` + false the PDF's text indexes normally (1 added — the PDF only; the + embedded image is skipped, not a failure) while NO image document + exists (no row, no copy). The Sources page reads 1 doc / 3 chunks + and the drill-down table lists the PDF alone.""" + login(page, default_app_url) # lands on /sources.html (admin) + expect(page.locator("#stat-docs")).to_have_text("1") + expect(page.locator("#stat-chunks")).to_have_text("3") + + _drill_to_source(page, SOURCE_NAME) + row = page.locator("#docs-tbody tr", has_text=DOC_PATH) + expect(row).to_have_count(1) + expect(row.locator(".kb-doc-thumb")).to_have_count(0) + expect(page.locator("#docs-tbody tr", has_text=IMAGE_REL)).to_have_count(0) + + +# --------------------------------------------------------------------------- +# 6. Chat (images off): the answer grounds on the PDF; NO image chip +# --------------------------------------------------------------------------- + + +def test_images_off_chat_grounds_with_no_image_chip( + page: Page, default_app_url: str, seeded_off: dict[str, str] +) -> None: + """The SAME question on the off app: the turn grounds on the PDF's + extracted text (the read echo carries the page markers + texts — + the PDF text is indexed normally, LOCKED A3's off-contract) and the + sources block carries EXACTLY the PDF's chip — no image chip, no + figure (the embedded image was never indexed).""" + try: + login(page, default_app_url, next="/") + expect(page.locator("#kb-banner")).to_be_hidden() + + page.fill("#message-input", QUESTION) + page.click("#send-btn") + + bubble = page.locator(".msg.brain .bubble").last + expect(bubble).to_contain_text("Here's what the read returned:", timeout=30_000) + expect(bubble).to_contain_text(f"Document {SOURCE_NAME}/{DOC_PATH}:") + expect(bubble).to_contain_text(PAGE_ONE) + expect(bubble).to_contain_text(PAGE_TWO) + expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000) + + chip = page.locator(".msg.brain .source-chip") + expect(chip).to_have_count(1) + expect(chip.nth(0)).to_have_text(f"{SOURCE_NAME}/{DOC_PATH}") + # No image affordance at all (no chip, no figure). + expect(page.locator(".msg.brain .source-image")).to_have_count(0) + finally: + _truncate_all() diff --git a/tests/integration/test_importer.py b/tests/integration/test_importer.py index 8714e41..7137b61 100644 --- a/tests/integration/test_importer.py +++ b/tests/integration/test_importer.py @@ -33,7 +33,10 @@ from __future__ import annotations import asyncio import base64 +import hashlib +from pathlib import Path +import pymupdf as fitz # PyMuPDF — the temp-PDF fixture builder (phase 126) from sqlalchemy import select from app.config import Settings @@ -444,3 +447,469 @@ def test_markdown_referencing_existing_image_keeps_null_record(db, tmp_path) -> assert md3 is not None and md3.image_context is None finally: _cleanup_source(db, source_root.name) + + +# --------------------------------------------------------------------------- +# Phase 126 — PDF indexing through the REAL import pipeline (the real +# ``app.rag.pdf`` extractor, the page-aware chunker, the REAL +# ``_describe_or_skip`` → ``describe_image`` seam — NO seam patch). +# --------------------------------------------------------------------------- + +PDF_TITLE = "Quarterly Homelab Report" + +PAGE_ONE = "The core switch firmware was upgraded to the stable channel." +PAGE_TWO = "The backup schedule runs nightly at 02:00 and keeps seven rotations." + + +def _png_bytes(width: int, height: int) -> bytes: + pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height), 0) + pix.clear_with(200) + return pix.tobytes("png") + + +def _build_pdf( + path, pages: list[tuple[str, list[bytes]]], title: str | None = None +) -> None: + """One temp PDF (built with PyMuPDF — no binary fixtures): *pages* is + (page text, embedded-image bytes) per page; *title* sets the metadata + title (``None`` = leave it empty).""" + doc = fitz.open() + for text, imgs in pages: + page = doc.new_page() + if text: + page.insert_text((72, 72), text) + y = 150 + for data in imgs: + pix = fitz.Pixmap(data) + page.insert_image( + fitz.Rect(72, y, 72 + pix.width, y + pix.height), stream=data + ) + y += pix.height + 10 + if title is not None: + doc.set_metadata({"title": title}) + doc.save(path) + doc.close() + + +def test_pdf_becomes_document_with_text_chunks_embedding_and_summary( + db, tmp_path +) -> None: + """The phase-126 primary scenario: a PDF in a source (the ``pdf`` + import token) → sync → the document carries the extracted text + (the ``---PAGE-N---`` markers, A2 — the title is the metadata + title), page-anchored chunks with 768-dim embeddings, and the + phase-30/118 lite summary. With ``images=True`` (A3) the embedded + image joins the pipeline through the REAL seam — one vision call on + the CHAT model with the bare locked prompt (no markdown reference + exists — ``image_context`` stays NULL), the copy in ``image_dir``, + and the description as the image doc's content AND summary.""" + source_root = tmp_path / "pdfkb" + source_root.mkdir() + png = _png_bytes(40, 30) + _build_pdf( + source_root / "report.pdf", + [(PAGE_ONE, []), (PAGE_TWO, [png])], + title=PDF_TITLE, + ) + llm = _image_llm(tmp_path, import_extensions="md,txt,pdf") + try: + summary = asyncio.run(import_sources([source_root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0) + assert summary.formats == {"pdf": 1} + + doc = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == "report.pdf" + ) + ) + assert doc is not None and doc.is_image is False + assert doc.title == PDF_TITLE + assert doc.content == f"---PAGE-1---\n{PAGE_ONE}\n\n---PAGE-2---\n{PAGE_TWO}" + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + content_chunks = [c for c in chunks if not c.is_summary] + assert [c.content for c in content_chunks] == [ + f"Page 1:\n{PAGE_ONE}", + f"Page 2:\n{PAGE_TWO}", + ] + for c in chunks: + assert c.embedding is not None and len(c.embedding) == 768 + summary_chunks = [c for c in chunks if c.is_summary] + assert len(summary_chunks) == 1 and summary_chunks[0].position == -1 + assert doc.summary is not None + assert doc.summary.endswith(f"Source: {source_root.name}/report.pdf") + + # The embedded image (A3) — through the REAL describe seam. + image_rel = "report.pdf#page=2#img=0.png" + img = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == image_rel + ) + ) + assert img is not None and img.is_image is True + assert img.image_context is None, "no markdown reference → bare prompt" + assert img.content == DESCRIPTION and img.summary == DESCRIPTION + copy = Path(img.image_path) + assert copy.parent == Path(llm.settings.image_dir).expanduser() + assert copy.name == f"{img.id}.png" + assert copy.read_bytes() == png + # The vision call went to the CHAT model with the BARE locked + # prompt (the PDF's lite summary is the other chat call). + assert llm.chat_models == [ + llm.settings.llm_summary_model, llm.settings.llm_chat_model + ] + assert llm.text_parts == [DESCRIBE_PROMPT] + finally: + _cleanup_source(db, source_root.name) + + +def test_pdf_embedded_images_skipped_when_images_off(db, tmp_path) -> None: + """With ``images=False`` (the DEFAULT): the PDF text is indexed + normally and the embedded images are skipped — no image documents, + no ``images_failed`` (a skip is not a failure), no copies, no vision + calls.""" + source_root = tmp_path / "pdfoffkb" + source_root.mkdir() + _build_pdf( + source_root / "report.pdf", + [(PAGE_ONE, [_png_bytes(40, 30)]), (PAGE_TWO, [])], + ) + llm = _image_llm(tmp_path, images=False, import_extensions="md,txt,pdf") + try: + summary = asyncio.run(import_sources([source_root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0) + doc = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == "report.pdf" + ) + ) + assert doc is not None and PAGE_ONE in doc.content + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + assert any(c.embedding is not None for c in chunks) + assert ( + db.scalar( + select(Document).where( + Document.source == source_root.name, + Document.is_image.is_(True), + ) + ) + is None + ), "images off: no embedded-image documents" + # No vision call at all — only the PDF's lite summary. + assert llm.chat_models == [llm.settings.llm_summary_model] + finally: + _cleanup_source(db, source_root.name) + + +def test_pdf_without_embedded_images_and_unchanged_rerun(db, tmp_path) -> None: + """A PDF without embedded images → no image documents; an unchanged + re-sync → ``unchanged`` (no re-extract, no re-embed, no vision or + embed calls), and a second run never creates image documents out of + nothing.""" + source_root = tmp_path / "pdfnonekb" + source_root.mkdir() + _build_pdf( + source_root / "notes.pdf", + [(PAGE_ONE, []), (PAGE_TWO, [])], + ) + llm = _image_llm(tmp_path, import_extensions="md,txt,pdf") + try: + s1 = asyncio.run(import_sources([source_root], llm, session=db)) + assert (s1.files, s1.added, s1.unchanged) == (1, 1, 0) + assert ( + db.scalar( + select(Document).where( + Document.source == source_root.name, + Document.is_image.is_(True), + ) + ) + is None + ) + calls1, chat1 = len(llm.calls), len(llm.chat_models) + s2 = asyncio.run(import_sources([source_root], llm, session=db)) + assert (s2.files, s2.added, s2.updated, s2.unchanged) == (1, 0, 0, 1) + assert len(llm.calls) == calls1 and len(llm.chat_models) == chat1 + finally: + _cleanup_source(db, source_root.name) + + +def test_pdf_without_text_becomes_an_empty_document(db, tmp_path) -> None: + """A PDF with no extractable text (image-only, ``images`` OFF — the + embedded image is invisible) → the document is still created, with + EMPTY content (the empty byte string's digest), NO content chunks + and NO content embedding (there is no text to embed — the only + chunk is the best-effort ``lite`` summary's ``is_summary`` row), and + the warning line is the signal.""" + source_root = tmp_path / "pdfemptykb" + source_root.mkdir() + _build_pdf(source_root / "pics.pdf", [("", [_png_bytes(40, 30)])]) + llm = _image_llm(tmp_path, images=False, import_extensions="md,txt,pdf") + try: + summary = asyncio.run(import_sources([source_root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0) + + doc = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == "pics.pdf" + ) + ) + assert doc is not None and doc.is_image is False + assert doc.content == "" + assert doc.content_hash == hashlib.sha256(b"").hexdigest() + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + assert [c for c in chunks if not c.is_summary] == [], ( + "no text → no content chunks (and no content embedding)" + ) + # The phase-30/118 path still runs best-effort on the empty + # content — the summary is the only chunk, and it is embedded. + assert len(chunks) == 1 and chunks[0].is_summary + assert chunks[0].embedding is not None and len(chunks[0].embedding) == 768 + assert doc.summary is not None + finally: + _cleanup_source(db, source_root.name) + + +def test_pdf_embedded_image_survives_prune_and_leaves_with_the_pdf( + db, tmp_path +) -> None: + """Phase 126 (A3) prune parent rule — the defect the E2E surfaced: + the embedded image doc's synthetic path NEVER appears in the walk's + ``seen``, so the plain prune rule destroyed it in the very sync that + (re)indexed it (the toggle-on half of the phase-122 "invisible, not + deleted" guard was missing). Now: with the parent PDF's rel still in + the walk — even UNCHANGED, so no image pass runs at all — the image + doc survives (0 pruned); when the PDF LEAVES the source, the image + doc is pruned with it (row + ``image_dir`` copy — no orphans).""" + source_root = tmp_path / "pdfprune" + source_root.mkdir() + png = _png_bytes(40, 30) + _build_pdf( + source_root / "report.pdf", + [(PAGE_ONE, []), (PAGE_TWO, [png])], + ) + image_rel = "report.pdf#page=2#img=0.png" + llm = _image_llm(tmp_path, import_extensions="md,txt,pdf") + try: + s1 = asyncio.run( + import_sources([source_root], llm, session=db, prune=True) + ) + assert (s1.added, s1.pruned) == (2, 0) + img1 = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == image_rel + ) + ) + assert img1 is not None and img1.image_path is not None + copy = Path(img1.image_path) + + # Unchanged re-sync (prune ON): the PDF's text digest matches → + # no image pass at all — the image doc survives anyway (the + # parent's rel is in the walk's seen set). + s2 = asyncio.run( + import_sources([source_root], llm, session=db, prune=True) + ) + assert (s2.unchanged, s2.pruned) == (1, 0) + assert db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == image_rel + ) + ) is not None, "parent PDF present → the image doc survives the prune" + + # The PDF LEAVES the source: the image doc is pruned WITH it + # (row + copy — no orphan in image_dir). + (source_root / "report.pdf").unlink() + s3 = asyncio.run( + import_sources([source_root], llm, session=db, prune=True) + ) + assert (s3.pruned, s3.unchanged) == (2, 0) + assert db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == image_rel + ) + ) is None, "parent PDF gone → the image doc is pruned with it" + assert not copy.exists(), "the pruned image's copy leaves image_dir" + finally: + _cleanup_source(db, source_root.name) + + +# --- Phase 125 parity: a markdown file referencing a PDF-embedded image --- + +#: A markdown file referencing the PDF's embedded image by its SYNTHETIC +#: path (``#page=#img=.`` — the doc's ``path`` +#: is the upsert key, and the reference resolves to it through the shared +#: ``_resolve_ref_path`` rule) — one context paragraph before, one after. +PDF_GUIDE_MD = ( + "The VLAN map diagram in the report shows the subnet layout.\n" + "\n" + "![VLAN map](report.pdf#page=2#img=0.png)\n" + "\n" + "Only expand the lab subnet after the VLAN map is reviewed.\n" +) + +#: The same reference against ``alpha.pdf`` (the PDF-FIRST ordering's +#: fixture — ``alpha.pdf`` sorts before ``zguide.md`` in the walk). +PDF_ZGUIDE_MD = PDF_GUIDE_MD.replace("report.pdf", "alpha.pdf") + + +def _pdf_expected_block(source: str, md_name: str, pdf_name: str) -> str: + """The task-03-shape combined string for the PDF reference (the disk + scan builds exactly this — the standalone image's shape, the + synthetic path as the image identity).""" + return ( + f"Referenced in {source}/{md_name}:\n" + '- Alt text: "VLAN map"\n' + " Context: The VLAN map diagram in the report shows the subnet layout.\n" + "\n" + "Only expand the lab subnet after the VLAN map is reviewed." + ) + + +def test_pdf_embedded_image_referenced_by_markdown_markdown_first( + db, tmp_path +) -> None: + """The MARKDOWN-FIRST ordering (``guide.md`` sorts before + ``report.pdf``): the markdown is indexed while its referenced image + has no document yet → it records the reference on its OWN row (the + ``IMAGE_CONTEXT:`` record — phase 125, task 04, work 1, never + cleared); the PDF is indexed next and the task-03 disk scan (the + standalone image's parity path — the markdown is still on disk) finds + the reference, so the image doc is enriched AT INDEX TIME: the vision + call carries the context after the locked prompt, ``image_context`` + is set, ``content``/``summary`` are ``\\n\\n``. + The post-index pass then skips the already-enriched doc (idempotent). + Final state: the record AND the enrichment coexist.""" + source_root = tmp_path / "pdfmdfirst" + source_root.mkdir() + (source_root / "guide.md").write_text(PDF_GUIDE_MD, encoding="utf-8") + _build_pdf( + source_root / "report.pdf", + [(PAGE_ONE, []), (PAGE_TWO, [_png_bytes(40, 30)])], + ) + llm = _image_llm(tmp_path, import_extensions="md,txt,pdf") + try: + summary = asyncio.run(import_sources([source_root], llm, session=db)) + # guide.md + report.pdf + the embedded image doc: all added. + assert (summary.files, summary.added, summary.images_failed) == (2, 3, 0) + + expected_block = _pdf_expected_block(source_root.name, "guide.md", "report.pdf") + + # Work 1 — the markdown doc STORES the reference (the record — + # line 3 is the reference, 2 context paragraphs). It is the + # post-index pass's reference index and the retrieval display's + # record; the pass does not clear it on success. + md = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == "guide.md" + ) + ) + assert md is not None and md.is_image is False + assert md.image_context == ( + "IMAGE_CONTEXT:\n" + '- [report.pdf#page=2#img=0.png] (line 3): "VLAN map"' + " — 2 paragraphs of context" + ) + + # The enrichment (the disk scan at PDF index time) — the image + # doc's shape is the standalone image's enriched shape. + image_rel = "report.pdf#page=2#img=0.png" + img = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == image_rel + ) + ) + assert img is not None and img.is_image is True + assert img.image_context == expected_block + expected_content = f"{expected_block}\n\n{DESCRIPTION}" + assert img.content == expected_content + assert img.summary == expected_content + # The vision call carried the context AFTER the locked prompt + # (the multimodal part list's text part — the describe seam's + # phase-125 extension). + assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_block}"] + # Chat call order (walk order): guide.md's ``lite`` summary, + # the PDF's ``lite`` summary, then the image's CHAT-model + # description (the ONLY multimodal call). + assert llm.chat_models == [ + llm.settings.llm_summary_model, + llm.settings.llm_summary_model, + llm.settings.llm_chat_model, + ] + # The chunks are embedded from the ENRICHED content (the + # plain-text packer joins the enriched paragraphs with single + # newlines; the summary chunk mirrors the raw content). + packed = "\n".join(expected_content.split("\n\n")) + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == img.id).order_by(Chunk.position) + ).all() + assert [c.content for c in chunks if not c.is_summary] == [packed] + summary_chunks = [c for c in chunks if c.is_summary] + assert len(summary_chunks) == 1 and summary_chunks[0].content == expected_content + for c in chunks: + assert c.embedding is not None and len(c.embedding) == 768 + finally: + _cleanup_source(db, source_root.name) + + +def test_pdf_embedded_image_referenced_by_markdown_pdf_first( + db, tmp_path +) -> None: + """The PDF-FIRST ordering (``alpha.pdf`` sorts before ``zguide.md`` + in the walk): the PDF (and its image doc) is indexed while the + markdown is still un-indexed — the task-03 disk scan sees the + reference on disk and enriches the image AT INDEX TIME (the ordering + that, without the parity scan, would ship an un-enriched image — the + markdown's record stays NULL: at markdown index time the image doc + ALREADY exists, so the task-03 path owns the case and nothing is + recorded). Same final state as the markdown-first test — the + enrichment never depends on the walk order.""" + source_root = tmp_path / "pdfpdffirst" + source_root.mkdir() + (source_root / "zguide.md").write_text(PDF_ZGUIDE_MD, encoding="utf-8") + _build_pdf( + source_root / "alpha.pdf", + [(PAGE_ONE, []), (PAGE_TWO, [_png_bytes(40, 30)])], + ) + llm = _image_llm(tmp_path, import_extensions="md,txt,pdf") + try: + summary = asyncio.run(import_sources([source_root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (2, 3, 0) + + expected_block = _pdf_expected_block(source_root.name, "zguide.md", "alpha.pdf") + + # The task-03 path owns this case: the image doc existed when + # the markdown was indexed → NO record (the field is the + # pending-enrichment index, not a reference dump). + md = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == "zguide.md" + ) + ) + assert md is not None and md.image_context is None + + # The image doc is enriched (the disk scan at PDF index time) — + # byte-identical in shape to the markdown-first final state. + image_rel = "alpha.pdf#page=2#img=0.png" + img = db.scalar( + select(Document).where( + Document.source == source_root.name, Document.path == image_rel + ) + ) + assert img is not None and img.image_context == expected_block + expected_content = f"{expected_block}\n\n{DESCRIPTION}" + assert img.content == expected_content + assert img.summary == expected_content + assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_block}"] + # Chat call order (walk order): the PDF's ``lite`` summary, the + # image's CHAT-model description, then zguide.md's summary. + assert llm.chat_models == [ + llm.settings.llm_summary_model, + llm.settings.llm_chat_model, + llm.settings.llm_summary_model, + ] + finally: + _cleanup_source(db, source_root.name) diff --git a/tests/unit/test_chunker.py b/tests/unit/test_chunker.py index 54cbd34..1ef02df 100644 --- a/tests/unit/test_chunker.py +++ b/tests/unit/test_chunker.py @@ -419,6 +419,93 @@ def test_dispatch_still_falls_back_for_unknown_suffix() -> None: assert chunk_document(content, "x/notes.whatever") == chunk_text(content) +# --------------------------------------------------------------------------- +# pdf (phase 126, task 03, LOCKED A2) — the page-aware chunker +# --------------------------------------------------------------------------- + + +def test_pdf_single_page_chunks_with_page_anchor() -> None: + content = "---PAGE-1---\nHello single page.\n" + chunks = chunk_document(content, "report.pdf") + assert chunks == ["Page 1:\nHello single page."] + assert "---PAGE-" not in "\n".join(chunks) + + +def test_pdf_multi_page_keeps_the_marker_page_number() -> None: + """Blank pages carry no marker — the anchor follows the MARKER's + number, not the chunk count (page 3 stays page 3 when page 2 was + blank).""" + content = ( + "---PAGE-1---\nAlpha text one.\n\n" + "---PAGE-3---\nGamma text three.\n" + ) + chunks = chunk_document(content, "report.pdf") + assert chunks == ["Page 1:\nAlpha text one.", "Page 3:\nGamma text three."] + assert "Page 2:" not in "\n".join(chunks) + + +def test_pdf_markers_never_appear_in_chunks() -> None: + content = ( + "---PAGE-1---\nAlpha text one.\n\n" + "---PAGE-2---\nBeta text two.\n\n" + "---PAGE-3---\nGamma text three.\n" + ) + chunks = chunk_document(content, "deep/dir/report.pdf") + assert len(chunks) == 3 + for chunk in chunks: + assert "---PAGE-" not in chunk + assert chunks[0].startswith("Page 1:") + assert chunks[1].startswith("Page 2:") + assert chunks[2].startswith("Page 3:") + + +def test_pdf_empty_and_markerless_content_yield_no_chunks() -> None: + assert chunk_document("", "report.pdf") == [] + # No marker at all (the parser always emits one for text — belt and + # braces): nothing to anchor, no chunks (never plain-text fallback — + # a PDF chunk must carry its page). + assert chunk_document("just words, no marker", "report.pdf") == [] + assert chunk_document("---PAGE-1---\n\n---PAGE-2---\n ", "report.pdf") == [] + + +def test_pdf_long_page_sub_splits_and_every_chunk_keeps_the_anchor() -> None: + """A page larger than the target sub-splits through the paragraph + packer — EVERY chunk of the page keeps the ``Page N:`` anchor and + the anchored chunk never exceeds the target.""" + page = "\n\n".join(f"paragraph {i} " + "l" * 300 for i in range(6)) + content = f"---PAGE-1---\n{page}\n\n---PAGE-2---\nshort second page\n" + chunks = chunk_document(content, "report.pdf", target_chars=800, overlap_chars=100) + assert len(chunks) >= 3 # the long page sub-splits + the short page + page1 = [c for c in chunks if c.startswith("Page 1:")] + page2 = [c for c in chunks if c.startswith("Page 2:")] + assert len(page1) >= 2 and page2 == ["Page 2:\nshort second page"] + for chunk in chunks: + assert len(chunk) <= 800, f"target exceeded: {len(chunk)}" + # All six paragraphs survive the split. + joined = "\n".join(chunks) + for i in range(6): + assert f"paragraph {i}" in joined + + +def test_pdf_overlap_carryover_stays_within_the_page() -> None: + """The tail carry-over packs paragraphs INSIDE one page; a new page + never inherits the previous page's tail (the page is the boundary).""" + a = "\n\n".join(f"para-a-{i} " + "l" * 250 for i in range(3)) + b = "\n\n".join(f"para-b-{i} " + "l" * 250 for i in range(3)) + content = f"---PAGE-1---\n{a}\n\n---PAGE-2---\n{b}\n" + chunks = chunk_document(content, "r.pdf", target_chars=700, overlap_chars=120) + first_of_page2 = next(c for c in chunks if c.startswith("Page 2:")) + assert "para-a-" not in first_of_page2, "page 2 must not inherit page 1's tail" + assert all(len(c) <= 700 for c in chunks) + + +def test_pdf_dispatch_matches_direct_call() -> None: + content = "---PAGE-1---\nalpha\n\n---PAGE-2---\nbeta\n" + from app.rag.chunker import _chunk_pdf + + assert chunk_document(content, "x/report.PDF") == _chunk_pdf(content) + + # --------------------------------------------------------------------------- # Hard cap across every format (aipi ~1024-token request cap) # --------------------------------------------------------------------------- @@ -434,6 +521,8 @@ def test_dispatch_still_falls_back_for_unknown_suffix() -> None: ("line of text\n\n" * 800, "big.txt"), ("[Unit]\n" + "key=" + "v" * 5000 + "\n", "big.container"), # phase 47 ("{% for x in y %}" + "{{ x }} " * 400 + "{% endfor %}", "big.j2"), # phase 47 + # phase 126: one long page (a single 12 500-char line) + one short. + ("---PAGE-1---\n" + "word " * 2500 + "\n\n---PAGE-2---\nshort\n", "big.pdf"), ], ) def test_hard_cap_holds_for_every_format(content: str, path: str) -> None: diff --git a/tests/unit/test_pdf.py b/tests/unit/test_pdf.py new file mode 100644 index 0000000..6349dce --- /dev/null +++ b/tests/unit/test_pdf.py @@ -0,0 +1,1071 @@ +"""Unit tests: PDF parser module (phase 126 task 02, locked A1/A2). + +Covers ``app/rag/pdf.py``: text extraction with ``---PAGE-N---`` +separators (A2), embedded image extraction (original encoded bytes), +MIME mapping (extension first, then image type code, else the +``application/octet-stream`` fallback), and the fail-soft contract — +unreadable files and broken pages/images return empty results, never +raise. Temp PDFs are built with PyMuPDF itself (no binary fixtures). +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +from collections.abc import Sequence +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pymupdf as fitz +import pytest +from sqlalchemy import select + +from app.config import Settings +from app.models import Chunk, Document +from app.rag import importer +from app.rag import pdf as pdf_mod +from app.rag.importer import import_sources +from app.rag.llm import EmbeddingError, LLMError +from app.rag.pdf import ( + PAGE_SEPARATOR, + PdfImage, + _mime_from_image_info, + _page_image_entries, + extract_pdf_images, + extract_pdf_text, +) +from tests.fakes import FakeEmbedder + +PNG_MAGIC = b"\x89PNG" +JPEG_MAGIC = b"\xff\xd8\xff" + + +# --------------------------------------------------------------------------- +# Phase 126, task 04 — PDF indexing in the importer +# --------------------------------------------------------------------------- + +DESCRIPTION = ( + "A network diagram of the homelab VLANs: the core switch, the " + "router, and three labeled subnets." +) + + +class _VisionLLM(FakeEmbedder): + """The phase-122 mock VISION client (task 03's pattern): the CHAT + model answers the multimodal describe call with the fixed + :data:`DESCRIPTION`; text (``lite``) calls keep the ``FakeEmbedder`` + behaviour. Every chat call's model is recorded (``chat_models``).""" + + def __init__(self) -> None: + super().__init__() + self.chat_models: list[str | None] = [] + + async def chat(self, messages, model=None): + self.chat_calls.append(list(messages)) + self.chat_models.append(model) + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + if isinstance(user, list): + return DESCRIPTION + first = user.split() + return "Summary of " + (first[0] if first else "") + + +class _VisionFailsLLM(FakeEmbedder): + """A chat model that REJECTS the multimodal describe call (the + non-vision model of the LOCKED A3 note). Text calls keep the + ``FakeEmbedder`` behaviour.""" + + def __init__(self) -> None: + super().__init__() + self.chat_models: list[str | None] = [] + + async def chat(self, messages, model=None): + self.chat_calls.append(list(messages)) + self.chat_models.append(model) + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + if isinstance(user, list): + raise LLMError("simulated non-vision chat model (test sentinel)") + first = user.split() + return "Summary of " + (first[0] if first else "") + + +class _FirstImageEmbedFailsLLM(FakeEmbedder): + """Describes two images (the first with the :data:`EMBED_FAIL` + sentinel) and FAILS the embed of any text carrying the sentinel — + drives the per-image embed-failure branch (the pass moves on; the + second image is still indexed).""" + + EMBED_FAIL = "EMBED-FAIL-SENTINEL" + + def __init__(self) -> None: + super().__init__() + self.chat_models: list[str | None] = [] + self._describe_calls = 0 + + async def chat(self, messages, model=None): + self.chat_calls.append(list(messages)) + self.chat_models.append(model) + user = next((m["content"] for m in messages if m.get("role") == "user"), "") + if isinstance(user, list): + self._describe_calls += 1 + if self._describe_calls == 1: + return f"First image. {self.EMBED_FAIL} details." + return "Second image, a clean description." + first = user.split() + return "Summary of " + (first[0] if first else "") + + async def embed(self, texts: list[str]) -> list[list[float]]: + if any(self.EMBED_FAIL in t for t in texts): + raise EmbeddingError("simulated embed failure (test sentinel)") + return await super().embed(texts) + + +def _pdf_llm[ + LLM: FakeEmbedder +]( + tmp_path: Path, + llm_cls: type[LLM] = _VisionLLM, + images: bool = True, + **kwargs: Any, +) -> LLM: + """A fake LLM whose settings admit the ``pdf`` import token and + carry the phase-122 image knobs (the image dir under *tmp_path*); + *llm_cls* may be a mock vision client (the ``FakeEmbedder`` + subclass carrying extra state such as ``chat_models``).""" + kwargs.setdefault("_env_file", None) + kwargs.setdefault("images", images) + kwargs.setdefault("image_dir", str(tmp_path / "images")) + kwargs.setdefault("import_extensions", "md,txt,pdf") + llm = llm_cls() # pyright: ignore[reportCallIssue] + llm.settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] + return llm + + +def _build_pdf( + path: Path, + pages: Sequence[tuple[str, Sequence[bytes]]], + title: str | None = None, +) -> Path: + """One temp PDF: *pages* is (page text, embedded-image bytes) per + page; *title* sets the metadata title (``None`` = leave it empty). + Built with PyMuPDF itself (the phase-126 convention — no binary + fixtures).""" + doc = fitz.open() + for text, imgs in pages: + page = doc.new_page() + if text: + page.insert_text((72, 72), text) + y = 150 + for data in imgs: + pix = fitz.Pixmap(data) + page.insert_image( + fitz.Rect(72, y, 72 + pix.width, y + pix.height), stream=data + ) + y += pix.height + 10 + if title is not None: + doc.set_metadata({"title": title}) + doc.save(path) + doc.close() + return path + + +def _doc(db, source: str, rel: str) -> Document: + doc = db.scalar( + select(Document).where(Document.source == source, Document.path == rel) + ) + assert doc is not None, f"no documents row for ({source!r}, {rel!r})" + return doc + + +def _cleanup_source(db, source: str) -> None: + for doc in db.scalars(select(Document).where(Document.source == source)).all(): + db.delete(doc) + db.commit() + + +# --- _extract_pdf_title ---------------------------------------------------- + + +def test_extract_pdf_title_prefers_metadata_title(tmp_path: Path) -> None: + path = _build_pdf(tmp_path / "t.pdf", [("text", [])], title=" My Report ") + assert importer._extract_pdf_title(path, fallback="t") == "My Report" + + +def test_extract_pdf_title_without_metadata_uses_fallback(tmp_path: Path) -> None: + path = _build_pdf(tmp_path / "report.pdf", [("text", [])]) + assert importer._extract_pdf_title(path, fallback="report") == "report" + + +def test_extract_pdf_title_empty_or_blank_metadata_uses_fallback(tmp_path: Path) -> None: + empty = _build_pdf(tmp_path / "a.pdf", [("text", [])], title="") + assert importer._extract_pdf_title(empty, fallback="a") == "a" + blank = _build_pdf(tmp_path / "b.pdf", [("text", [])], title=" ") + assert importer._extract_pdf_title(blank, fallback="b") == "b" + + +def test_extract_pdf_title_is_capped_to_the_column_width(tmp_path: Path) -> None: + long_title = "x" * 600 + path = _build_pdf(tmp_path / "long.pdf", [("text", [])], title=long_title) + title = importer._extract_pdf_title(path, fallback="long") + assert title == "x" * 500 and len(title) == 500 + + +def test_extract_pdf_title_unreadable_file_uses_fallback(tmp_path: Path) -> None: + bad = tmp_path / "bad.pdf" + bad.write_bytes(b"not a pdf at all") + assert importer._extract_pdf_title(bad, fallback="bad") == "bad" + + +# --- the PDF index path (import_sources, real pipeline, fake LLM) ---------- + + +def test_pdf_becomes_a_document_with_text_chunks_and_summary( + db, tmp_path: Path +) -> None: + """The primary path: a PDF in a source → sync → the document's + ``content`` is the extracted text (the ``---PAGE-N---`` markers, + A2), the page-aware chunks carry the ``Page N:`` anchor, every + chunk (content + summary) is embedded, the title is the metadata + title, and the phase-30/118 summary is stored.""" + root = tmp_path / "pdfsrc" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("First page text about networking.", []), + ("Second page text about storage.", [])], + title="Quarterly Report", + ) + llm = _pdf_llm(tmp_path) + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added) == (1, 1) + assert summary.formats == {"pdf": 1} + + doc = _doc(db, "pdfsrc", "report.pdf") + assert doc.title == "Quarterly Report" + assert doc.is_image is False and doc.image_path is None + assert doc.content == ( + f"{PAGE_SEPARATOR.format(page=1)}\nFirst page text about networking.\n\n" + f"{PAGE_SEPARATOR.format(page=2)}\nSecond page text about storage." + ) + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + content_chunks = [c for c in chunks if not c.is_summary] + assert [c.content for c in content_chunks] == [ + "Page 1:\nFirst page text about networking.", + "Page 2:\nSecond page text about storage.", + ] + for c in chunks: + assert c.embedding is not None and len(c.embedding) == 768 + summary_chunks = [c for c in chunks if c.is_summary] + assert len(summary_chunks) == 1 and summary_chunks[0].position == -1 + assert doc.summary is not None + assert doc.summary.endswith("Source: pdfsrc/report.pdf") + finally: + _cleanup_source(db, "pdfsrc") + + +def test_pdf_embedded_images_become_image_documents(db, tmp_path: Path) -> None: + """A3 with the toggle ON: each embedded image joins the phase-122 + pipeline — the single seam describes it (the CHAT model), the copy + lands in ``image_dir`` named by the doc id, and the doc row carries + the synthetic ``#page=#img=`` path, ``is_image`` + + ``image_path``, and the description as content AND summary (the + phase-122 convention — the summary chunk mirrors it).""" + root = tmp_path / "pdfimg" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [png])]) + llm = _pdf_llm(tmp_path) + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + # The PDF + its one embedded image: both count added. + assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0) + + image_rel = "report.pdf#page=2#img=0.png" + img = _doc(db, "pdfimg", image_rel) + assert img.is_image is True and img.image_path is not None + assert img.title == "Embedded image from report page 2" + assert img.content == DESCRIPTION and img.summary == DESCRIPTION + copy = Path(img.image_path) + assert copy.parent == Path(llm.settings.image_dir).expanduser() + assert copy.name == f"{img.id}.png" # row and copy agree on the id + assert copy.read_bytes() == png + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == img.id).order_by(Chunk.position) + ).all() + assert all(c.embedding is not None and len(c.embedding) == 768 for c in chunks) + summary_chunks = [c for c in chunks if c.is_summary] + assert len(summary_chunks) == 1 and summary_chunks[0].content == DESCRIPTION + + # The vision describe call went to the CHAT model (the PDF's own + # lite summary is the first chat call — the phase-30 path). + assert llm.chat_models == [ + llm.settings.llm_summary_model, llm.settings.llm_chat_model, + ] + finally: + _cleanup_source(db, "pdfimg") + + +def test_pdf_embedded_images_skipped_when_images_off( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The toggle OFF: the PDF text indexes normally, the embedded + images are skipped (one debug line, NO warning, NO + ``images_failed`` — a skip is not a failure), no image doc, no + copy — and not even the image dir.""" + root = tmp_path / "pdfoff" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", [_png_bytes(40, 30)]), ("Text page two.", [])], + ) + llm = _pdf_llm(tmp_path, images=False) + try: + with caplog.at_level(logging.DEBUG, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0) + doc = _doc(db, "pdfoff", "report.pdf") + assert "Text page one." in doc.content and doc.summary is not None + assert ( + db.scalar( + select(Document).where( + Document.source == "pdfoff", Document.is_image.is_(True) + ) + ) + is None + ), "toggle off: no image documents" + assert not Path(llm.settings.image_dir).expanduser().exists(), ( + "no copy — and not even the image dir — may be left behind" + ) + # The skip is debug-logged (not a warning, not a failure). + debugs = [ + r for r in caplog.records if "images toggle off" in r.getMessage() + ] + assert len(debugs) == 1 and debugs[0].levelno == logging.DEBUG + assert not [ + r for r in caplog.records if "description failed" in r.getMessage() + ] + # No vision call at all — only the PDF's lite summary. + assert llm.chat_models == [llm.settings.llm_summary_model] + finally: + _cleanup_source(db, "pdfoff") + + +def test_pdf_unchanged_rerun_skips_everything(db, tmp_path: Path) -> None: + """The unchanged path (same text digest) is the text path's rule: + no re-extract, no re-embed, NO image pass — the second sync costs + no vision and no embed calls at all.""" + root = tmp_path / "pdfonce" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", [_png_bytes(40, 30)]), ("Text page two.", [])], + ) + llm = _pdf_llm(tmp_path) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert (s1.added, s1.unchanged) == (2, 0) + calls1, chat1 = len(llm.calls), len(llm.chat_models) + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.files, s2.added, s2.updated, s2.unchanged) == (1, 0, 0, 1) + assert len(llm.calls) == calls1, "unchanged: no re-embed" + assert len(llm.chat_models) == chat1, "unchanged: no vision/summary calls" + finally: + _cleanup_source(db, "pdfonce") + + +def test_pdf_unchanged_path_backfills_and_honors_manual_date( + db, tmp_path: Path +) -> None: + """The unchanged path's phase-118/D1/D4 dance, PDF flavour: a + NULL-summary unchanged doc gets the summary backfill (counted + ``summary_backfilled``, never ``summaries``); a manual-date row + (D1) is not touched at all (the backfill runs BEFORE the + early-return — the flag protects the DATE only); and with the flag + off the stored date REFRESHES from the file's mtime (D4 — counted + ``dates_updated``, still ``unchanged``).""" + root = tmp_path / "pdfdate" + root.mkdir() + _build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [])]) + llm = _pdf_llm(tmp_path, images=False) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert s1.added == 1 + doc = _doc(db, "pdfdate", "report.pdf") + + # Run 2 — simulate a pre-phase-30 row: NULL summary + a manual + # date correction. + doc.summary = None + doc.created_at_manual = True + doc.created_at = doc.created_at - timedelta(days=30) + db.commit() + manual_date = doc.created_at + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.unchanged, s2.summary_backfilled, s2.summaries) == (1, 1, 0) + assert s2.dates_updated == 0, "the manual date must survive the sync" + doc2 = _doc(db, "pdfdate", "report.pdf") + assert doc2.created_at_manual is True and doc2.created_at == manual_date + assert doc2.summary is not None, "the backfill restored the summary" + + # Run 3 — the manual flag off: the date REFRESHES from the + # file's mtime (D4) — and a date-only refresh is still + # unchanged (no KB-change count). + doc2.created_at_manual = False + doc2.created_at = doc2.created_at - timedelta(days=60) + db.commit() + s3 = asyncio.run(import_sources([root], llm, session=db)) + assert (s3.unchanged, s3.dates_updated, s3.added, s3.updated) == (1, 1, 0, 0) + doc3 = _doc(db, "pdfdate", "report.pdf") + assert doc3.created_at != manual_date + finally: + _cleanup_source(db, "pdfdate") + + +def test_pdf_text_change_updates_doc_and_keeps_unchanged_images( + db, tmp_path: Path +) -> None: + """A text change re-indexes the PDF (updated) and re-runs the image + pass — but an embedded image whose BYTES are unchanged (same digest) + is skipped: no re-describe (no new vision call for it), the doc id + and the copy path are stable.""" + root = tmp_path / "pdfchg" + root.mkdir() + png = _png_bytes(40, 30) + _build_pdf( + root / "report.pdf", + [("Original first page.", []), ("Second page.", [png])], + ) + llm = _pdf_llm(tmp_path) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert s1.added == 2 + image_rel = "report.pdf#page=2#img=0.png" + img1 = _doc(db, "pdfchg", image_rel) + + # Same image bytes, changed page-1 text. + (root / "report.pdf").unlink() + _build_pdf( + root / "report.pdf", + [("Revised first page.", []), ("Second page.", [png])], + ) + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.added, s2.updated, s2.unchanged) == (0, 1, 0) + + img2 = _doc(db, "pdfchg", image_rel) + assert img2.id == img1.id and img2.image_path == img1.image_path + assert img2.image_path is not None + assert Path(img2.image_path).read_bytes() == png + # The lite summary of the changed PDF fired again — but NO new + # describe call (the image bytes are unchanged). + assert llm.chat_models == [ + llm.settings.llm_summary_model, # run 1: PDF summary + llm.settings.llm_chat_model, # run 1: the image description + llm.settings.llm_summary_model, # run 2: the revised PDF summary + ] + finally: + _cleanup_source(db, "pdfchg") + + +def test_pdf_without_text_still_becomes_a_document( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An image-only PDF (no extractable text) still becomes a document + — with empty content (the empty byte string's digest), no content + chunks, and a warning (the signal). The summary path still runs + (best-effort on the empty content).""" + root = tmp_path / "pdfempty" + root.mkdir() + _build_pdf(root / "pics.pdf", [("", [_png_bytes(40, 30)])]) + llm = _pdf_llm(tmp_path, images=False) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added) == (1, 1) + doc = _doc(db, "pdfempty", "pics.pdf") + assert doc.content == "" + assert doc.content_hash == hashlib.sha256(b"").hexdigest() + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + assert [c for c in chunks if not c.is_summary] == [] + assert doc.summary is not None # best-effort on the empty content + warnings = [ + r + for r in caplog.records + if "no extractable text" in r.getMessage() + ] + assert len(warnings) == 1 and warnings[0].levelno == logging.WARNING + finally: + _cleanup_source(db, "pdfempty") + + +def test_corrupt_pdf_file_becomes_an_empty_document(db, tmp_path: Path) -> None: + """A .pdf file that is not a PDF at all: the parser's fail-soft + contract (no text, no images) — the document lands with empty + content and no chunks, and the sync stays green.""" + root = tmp_path / "pdfbad" + root.mkdir() + (root / "broken.pdf").write_bytes(b"this is not a pdf at all") + llm = _pdf_llm(tmp_path) + try: + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.errors) == (1, 1, 0) + doc = _doc(db, "pdfbad", "broken.pdf") + assert doc.content == "" and doc.title == "broken" + chunks = db.scalars( + select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position) + ).all() + assert [c for c in chunks if not c.is_summary] == [] + finally: + _cleanup_source(db, "pdfbad") + + +def test_pdf_embedded_image_unsupported_mime_is_skipped( + db, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """An embedded image whose mime maps to no ``BOR_IMAGE_EXTENSIONS`` + member (a TIFF under the default set) is skipped with a warning — + NOT a failure (``images_failed`` stays 0), no row, no copy.""" + root = tmp_path / "pdfmime" + root.mkdir() + _build_pdf(root / "report.pdf", [("Text page.", [])]) + monkeypatch.setattr( + importer, + "extract_pdf_images", + lambda _path: [ + PdfImage(page=1, index=0, width=5, height=5, data=b"fakedata", mime="image/tiff") + ], + ) + llm = _pdf_llm(tmp_path) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0) + assert ( + db.scalar( + select(Document).where( + Document.source == "pdfmime", Document.is_image.is_(True) + ) + ) + is None + ) + assert not Path(llm.settings.image_dir).expanduser().exists() + warnings = [ + r + for r in caplog.records + if "unsupported extension" in r.getMessage() + ] + assert len(warnings) == 1 and "image/tiff" in warnings[0].getMessage() + finally: + _cleanup_source(db, "pdfmime") + + +def test_pdf_embedded_image_failed_description_skips_image_keeps_pdf( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """LOCKED A3 fail-soft: a failed description skips the IMAGE + entirely (no row, no copy — and not even the image dir), counts + ``images_failed`` + a warning, and the PDF's text index is + unaffected (already committed).""" + root = tmp_path / "pdffail" + root.mkdir() + _build_pdf( + root / "report.pdf", + [("Text page one.", []), ("Text page two.", [_png_bytes(40, 30)])], + ) + llm = _pdf_llm(tmp_path, _VisionFailsLLM) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + assert (summary.files, summary.added, summary.images_failed) == (1, 1, 1) + doc = _doc(db, "pdffail", "report.pdf") + assert "Text page one." in doc.content and doc.summary is not None + assert ( + db.scalar( + select(Document).where( + Document.source == "pdffail", Document.is_image.is_(True) + ) + ) + is None + ), "a failed description must leave NO image row" + assert not Path(llm.settings.image_dir).expanduser().exists(), ( + "no copy — and not even the image dir — may be left behind" + ) + warnings = [ + r + for r in caplog.records + if "embedded image description failed" in r.getMessage() + ] + assert len(warnings) == 1 and warnings[0].levelno == logging.WARNING + finally: + _cleanup_source(db, "pdffail") + + +def test_pdf_embedded_image_embed_failure_keeps_the_others( + db, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A per-image embed failure rolls back THAT image only (no row, + no orphan copy), counts ``errors`` + the error line, and the pass + moves on — the second image is still indexed, and the PDF's text + index (committed earlier) is untouched.""" + root = tmp_path / "pdfemb" + root.mkdir() + _build_pdf( + root / "report.pdf", + [ + ("Text page one.", []), + ("Text page two.", [_png_bytes(40, 30)]), + ("Text page three.", [_png_bytes(30, 20)]), + ], + ) + llm = _pdf_llm(tmp_path, _FirstImageEmbedFailsLLM) + try: + with caplog.at_level(logging.INFO, logger="app.importer"): + summary = asyncio.run(import_sources([root], llm, session=db)) + # PDF + the surviving second image; the first image's embed failed. + assert (summary.added, summary.errors, summary.images_failed) == (2, 1, 0) + errors = [ + r for r in caplog.records if "embed failed" in r.getMessage() + ] + assert len(errors) == 1 and errors[0].levelno == logging.ERROR + + # Only the second image (page 3) landed. + img = _doc(db, "pdfemb", "report.pdf#page=3#img=0.png") + assert img.is_image is True + assert ( + db.scalar( + select(Document).where( + Document.source == "pdfemb", + Document.path == "report.pdf#page=2#img=0.png", + ) + ) + is None + ) + # No orphan copy of the failed image: exactly one file in the dir. + copies = list(Path(llm.settings.image_dir).expanduser().iterdir()) + assert len(copies) == 1 and copies[0].name == f"{img.id}.png" + finally: + _cleanup_source(db, "pdfemb") + + +def test_pdf_changed_embedded_image_replaces_the_copy(db, tmp_path: Path) -> None: + """A PDF whose text AND an embedded image's bytes changed: the PDF + is ``updated`` and the image doc is ``updated`` in place — same id, + the stale copy is deleted before the new one is written (one file, + the new bytes), and the image is re-described (a second vision + call).""" + root = tmp_path / "pdfrep" + root.mkdir() + png_a = _png_bytes(40, 30) + png_b = _png_bytes(25, 18) # different size → different bytes + _build_pdf( + root / "report.pdf", + [("Original first page.", []), ("Second page.", [png_a])], + ) + llm = _pdf_llm(tmp_path) + try: + s1 = asyncio.run(import_sources([root], llm, session=db)) + assert s1.added == 2 + image_rel = "report.pdf#page=2#img=0.png" + img1 = _doc(db, "pdfrep", image_rel) + assert img1.image_path is not None + copy1 = Path(img1.image_path) + + # Changed page-1 text AND changed image bytes. + (root / "report.pdf").unlink() + _build_pdf( + root / "report.pdf", + [("Revised first page.", []), ("Second page.", [png_b])], + ) + s2 = asyncio.run(import_sources([root], llm, session=db)) + assert (s2.added, s2.updated, s2.unchanged) == (0, 2, 0) + + img2 = _doc(db, "pdfrep", image_rel) + assert img2.id == img1.id, "the changed image keeps its doc id" + assert img2.image_path == img1.image_path, "the copy path is stable" + assert img2.image_path is not None + assert Path(img2.image_path).read_bytes() == png_b + # Exactly one copy in the dir — the stale one was deleted. + copies = list(Path(llm.settings.image_dir).expanduser().iterdir()) + assert len(copies) == 1 and copies[0] == copy1 + # The re-describe fired (a second vision call after run 2's lite + # summary of the revised PDF). + assert llm.chat_models == [ + llm.settings.llm_summary_model, # run 1: PDF summary + llm.settings.llm_chat_model, # run 1: the image description + llm.settings.llm_summary_model, # run 2: the revised PDF summary + llm.settings.llm_chat_model, # run 2: the re-description + ] + finally: + _cleanup_source(db, "pdfrep") + + +def _png_bytes(width: int, height: int, alpha: bool = False) -> bytes: + pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height), 1 if alpha else 0) + pix.clear_with(200) + return pix.tobytes("png") + + +def _jpeg_bytes(width: int, height: int) -> bytes: + pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height)) + pix.clear_with(200) + return pix.tobytes("jpeg") + + +@pytest.fixture() +def single_page_pdf(tmp_path: Path) -> Path: + path = tmp_path / "single.pdf" + doc = fitz.open() + doc.new_page().insert_text((72, 72), "Hello single page.") + doc.save(path) + doc.close() + return path + + +@pytest.fixture() +def multi_page_pdf(tmp_path: Path) -> Path: + """3 pages: p1 text, p2 text + PNG, p3 text + JPEG twice + gray PNG.""" + path = tmp_path / "multi.pdf" + doc = fitz.open() + doc.new_page().insert_text((72, 72), "Alpha text one.") + p2 = doc.new_page() + p2.insert_text((72, 72), "Beta text two.") + p2.insert_image(fitz.Rect(72, 150, 172, 180), stream=_png_bytes(40, 30)) + p3 = doc.new_page() + p3.insert_text((72, 72), "Gamma text three.") + p3.insert_image(fitz.Rect(72, 150, 172, 180), stream=_jpeg_bytes(40, 30)) + p3.insert_image(fitz.Rect(200, 150, 260, 172), stream=_jpeg_bytes(40, 30)) + p3.insert_image(fitz.Rect(72, 200, 102, 230), stream=_png_bytes(15, 15)) + doc.save(path) + doc.close() + return path + + +@pytest.fixture() +def image_only_pdf(tmp_path: Path) -> Path: + path = tmp_path / "image_only.pdf" + doc = fitz.open() + doc.new_page().insert_image(fitz.Rect(72, 72, 200, 150), stream=_png_bytes(80, 60)) + doc.save(path) + doc.close() + return path + + +@pytest.fixture() +def blank_middle_page_pdf(tmp_path: Path) -> Path: + """p1 text, p2 blank (no text), p3 text.""" + path = tmp_path / "blank_middle.pdf" + doc = fitz.open() + doc.new_page().insert_text((72, 72), "First text.") + doc.new_page() + doc.new_page().insert_text((72, 72), "Third text.") + doc.save(path) + doc.close() + return path + + +# --------------------------------------------------------------------------- +# extract_pdf_text +# --------------------------------------------------------------------------- + + +def test_single_page_text_has_exactly_one_separator(single_page_pdf: Path) -> None: + text = extract_pdf_text(single_page_pdf) + assert text == f"{PAGE_SEPARATOR.format(page=1)}\nHello single page." + assert text.count("---PAGE-") == 1 + + +def test_multi_page_text_separators_and_order(multi_page_pdf: Path) -> None: + text = extract_pdf_text(multi_page_pdf) + assert text == ( + f"{PAGE_SEPARATOR.format(page=1)}\nAlpha text one.\n\n" + f"{PAGE_SEPARATOR.format(page=2)}\nBeta text two.\n\n" + f"{PAGE_SEPARATOR.format(page=3)}\nGamma text three." + ) + assert [f"---PAGE-{n}---" for n in (1, 2, 3)] == [ + line for line in text.splitlines() if line.startswith("---PAGE-") + ] + + +def test_blank_middle_page_gets_no_separator(blank_middle_page_pdf: Path) -> None: + text = extract_pdf_text(blank_middle_page_pdf) + expected = ( + f"{PAGE_SEPARATOR.format(page=1)}\nFirst text.\n\n" + f"{PAGE_SEPARATOR.format(page=3)}\nThird text." + ) + assert text == expected + assert "---PAGE-2---" not in text + + +def test_image_only_pdf_returns_empty_string(image_only_pdf: Path) -> None: + assert extract_pdf_text(image_only_pdf) == "" + + +def test_missing_file_text_returns_empty_no_raise(tmp_path: Path) -> None: + assert extract_pdf_text(tmp_path / "nope.pdf") == "" + + +def test_garbage_file_text_returns_empty_no_raise(tmp_path: Path) -> None: + bad = tmp_path / "garbage.pdf" + bad.write_bytes(b"this is not a pdf at all") + assert extract_pdf_text(bad) == "" + + +def test_broken_page_is_skipped(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """A page whose get_text raises is logged/skipped; other pages stay.""" + path = tmp_path / "fake.pdf" + path.write_bytes(b"%PDF-1.4 fake") + + class _Page: + def __init__(self, text: str | None) -> None: + self._text = text + + def get_text(self, flavor: str) -> str: + if self._text is None: + raise RuntimeError("boom: page unreadable") + return self._text + + monkeypatch.setattr( + pdf_mod.fitz, "open", + lambda _p: _FakeDoc([_Page(None), _Page("Fine text.\n")]), + ) + assert extract_pdf_text(path) == f"{PAGE_SEPARATOR.format(page=2)}\nFine text." + + +# --------------------------------------------------------------------------- +# extract_pdf_images +# --------------------------------------------------------------------------- + + +def test_pdf_without_images_returns_empty_list(single_page_pdf: Path) -> None: + assert extract_pdf_images(single_page_pdf) == [] + + +def test_embedded_images_extracted_in_order(multi_page_pdf: Path) -> None: + images = extract_pdf_images(multi_page_pdf) + assert [(img.page, img.index) for img in images] == [ + (2, 0), (3, 0), (3, 1), (3, 2), + ] + # page 2: 40x30 PNG + img = images[0] + assert (img.width, img.height) == (40, 30) + assert img.mime == "image/png" + assert img.data.startswith(PNG_MAGIC) + # page 3: 40x30 JPEG, then the same JPEG again (distinct index) + for img in images[1:3]: + assert (img.width, img.height) == (40, 30) + assert img.mime == "image/jpeg" + assert img.data.startswith(JPEG_MAGIC) + assert images[1].data == images[2].data # same embedded xref + # page 3: 15x15 PNG + img = images[3] + assert (img.width, img.height) == (15, 15) + assert img.mime == "image/png" + assert img.data.startswith(PNG_MAGIC) + # every entry is a well-formed PdfImage of the right shape + for img in images: + assert isinstance(img, PdfImage) + assert isinstance(img.page, int) and img.page >= 1 + assert isinstance(img.index, int) and img.index >= 0 + assert isinstance(img.data, bytes) and img.data + assert isinstance(img.mime, str) and img.mime.startswith("image/") + + +def test_image_data_round_trips_through_fitz(multi_page_pdf: Path) -> None: + """Extracted bytes are genuine decodable image bytes (not garbage).""" + images = extract_pdf_images(multi_page_pdf) + assert images + for img in images: + pix = fitz.Pixmap(img.data) + assert (pix.width, pix.height) == (img.width, img.height) + + +def test_missing_file_images_returns_empty_no_raise(tmp_path: Path) -> None: + assert extract_pdf_images(tmp_path / "nope.pdf") == [] + + +def test_garbage_file_images_returns_empty_no_raise(tmp_path: Path) -> None: + bad = tmp_path / "garbage.pdf" + bad.write_bytes(b"definitely not a pdf") + assert extract_pdf_images(bad) == [] + + +def _fake_image_page(page_entries: list[dict[str, Any]]) -> SimpleNamespace: + return SimpleNamespace(get_image_info=lambda xrefs: [dict(e) for e in page_entries]) + + +class _FakeDoc: + """A stand-in fitz Document: given pages, stubbed extract_image.""" + + def __init__(self, pages: list[Any], extract: Any = None) -> None: + self._pages = pages + self._extract = extract + + def __len__(self) -> int: + return len(self._pages) + + def __getitem__(self, index: int) -> Any: + return self._pages[index] + + def extract_image(self, xref: int) -> Any: + if self._extract is None: + pytest.fail("extract_image() called unexpectedly") + return self._extract(xref) + + def __enter__(self) -> _FakeDoc: + return self + + def __exit__(self, *args: object) -> bool: + return False + + +def test_image_without_xref_is_skipped( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + path = tmp_path / "fake.pdf" + path.write_bytes(b"%PDF-1.4 fake") + doc = _FakeDoc( + [_fake_image_page([{"xref": 0, "width": 5, "height": 5}])], + lambda x: pytest.fail("no call"), + ) + monkeypatch.setattr(pdf_mod.fitz, "open", lambda _p: doc) + assert extract_pdf_images(path) == [] + + +def test_image_extraction_failure_is_skipped( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + path = tmp_path / "fake.pdf" + path.write_bytes(b"%PDF-1.4 fake") + + def _boom(xref: int) -> None: + raise RuntimeError("corrupt xref") + + doc = _FakeDoc([_fake_image_page([{"xref": 7, "width": 5, "height": 5}])], _boom) + monkeypatch.setattr(pdf_mod.fitz, "open", lambda _p: doc) + assert extract_pdf_images(path) == [] + + +def test_image_with_empty_data_is_skipped( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + path = tmp_path / "fake.pdf" + path.write_bytes(b"%PDF-1.4 fake") + doc = _FakeDoc( + [_fake_image_page([{"xref": 7, "width": 5, "height": 5}])], + lambda xref: {"image": b"", "ext": "png", "width": 5, "height": 5}, + ) + monkeypatch.setattr(pdf_mod.fitz, "open", lambda _p: doc) + assert extract_pdf_images(path) == [] + + +def test_image_listing_failure_is_skipped( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A page whose image listing raises (beyond the version fallback) is skipped.""" + path = tmp_path / "fake.pdf" + path.write_bytes(b"%PDF-1.4 fake") + + def _boom(xrefs: bool) -> None: + raise RuntimeError("page image listing failed") + + page = SimpleNamespace( + get_image_info=_boom, + get_images=lambda full: pytest.fail("should not be called"), + ) + doc = _FakeDoc([page]) + monkeypatch.setattr(pdf_mod.fitz, "open", lambda _p: doc) + assert extract_pdf_images(path) == [] + + +def test_extracted_ext_wins_over_info_ext( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The image info's `ext` is unreliable (None) in PyMuPDF; the ext of + the actually returned bytes must drive the MIME type.""" + path = tmp_path / "fake.pdf" + path.write_bytes(b"%PDF-1.4 fake") + doc = _FakeDoc( + [_fake_image_page([{"xref": 7, "width": 5, "height": 5, "ext": None, "type": 3}])], + lambda xref: {"image": b"x" * 10, "ext": "jpeg", "width": 5, "height": 5}, + ) + monkeypatch.setattr(pdf_mod.fitz, "open", lambda _p: doc) + images = extract_pdf_images(path) + assert len(images) == 1 + assert images[0].mime == "image/jpeg" # ext beats the PNG type code + + +# --------------------------------------------------------------------------- +# _mime_from_image_info +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("info", "expected"), + [ + ({"ext": "png"}, "image/png"), + ({"ext": "PNG"}, "image/png"), + ({"ext": "jpg"}, "image/jpeg"), + ({"ext": "jpeg"}, "image/jpeg"), + ({"ext": "gif"}, "image/gif"), + ({"ext": "webp"}, "image/webp"), + ({"ext": "tif"}, "image/tiff"), + ({"ext": "tiff"}, "image/tiff"), + # extension wins over the type code + ({"ext": "png", "type": 2}, "image/png"), + # known type codes (phase-126 design) + ({"type": 2}, "image/jpeg"), + ({"type": 3}, "image/png"), + ({"type": 4}, "image/gif"), + ({"type": 8}, "image/tiff"), + ({"type": 9}, "image/webp"), + # unknown / missing → octet-stream fallback + ({"type": 7}, "application/octet-stream"), + ({"type": 42}, "application/octet-stream"), + ({"type": None, "ext": None}, "application/octet-stream"), + ({"ext": "unknown-ext"}, "application/octet-stream"), + ({"type": "not-an-int"}, "application/octet-stream"), + ({}, "application/octet-stream"), + ], +) +def test_mime_from_image_info(info: dict[str, Any], expected: str) -> None: + assert _mime_from_image_info(info) == expected + + +# --------------------------------------------------------------------------- +# _page_image_entries (PyMuPDF version fallback) +# --------------------------------------------------------------------------- + + +def test_page_image_entries_prefers_get_image_info() -> None: + page = SimpleNamespace( + get_image_info=lambda xrefs: [ + {"xref": 1, "width": 4, "height": 5, "ext": None}, + ], + get_images=lambda full: pytest.fail("should not be called"), + ) + entries = _page_image_entries(page) + assert entries == [{"xref": 1, "width": 4, "height": 5, "ext": None}] + # returned dicts are copies — mutating them cannot touch the page + entries[0]["xref"] = 999 + assert page.get_image_info(xrefs=True)[0]["xref"] == 1 + + +def test_page_image_entries_falls_back_on_old_pymupdf() -> None: + def _no_xrefs_kwarg(xrefs: bool) -> None: + raise TypeError("get_image_info() got an unexpected keyword argument 'xrefs'") + + page = SimpleNamespace( + get_image_info=_no_xrefs_kwarg, + # (xref, smask, width, height, bpc, colorspace, alt_cs, name, filter) + get_images=lambda full: [(7, 0, 40, 30, 8, "DeviceRGB", "", "Im0", "DCTDecode", 0)], + ) + entries = _page_image_entries(page) + assert entries == [{"xref": 7, "width": 40, "height": 30}] diff --git a/uv.lock b/uv.lock index 2294ce1..0f71865 100644 --- a/uv.lock +++ b/uv.lock @@ -61,6 +61,7 @@ dependencies = [ { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "pydantic-settings" }, + { name = "pymupdf" }, { name = "python-dotenv" }, { name = "python-multipart" }, { name = "sqlalchemy" }, @@ -88,6 +89,7 @@ requires-dist = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.1,<4.0" }, { name = "pydantic", specifier = ">=2.7,<3.0" }, { name = "pydantic-settings", specifier = ">=2.3,<3.0" }, + { name = "pymupdf", specifier = ">=1.24,<2.0" }, { name = "python-dotenv", specifier = ">=1.0,<2.0" }, { name = "python-multipart", specifier = ">=0.0.9,<0.1" }, { name = "sqlalchemy", specifier = ">=2.0,<2.1" }, @@ -846,6 +848,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] +[[package]] +name = "pymupdf" +version = "1.28.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/fb/b6761fa2d5266f2cdb24c3b91f4023070ab7848381417678e7a289a1d52a/pymupdf-1.28.2.tar.gz", hash = "sha256:5e0be7908a715aa20333caddd73f1d6f01e4cd0c26e869fa2dd0b7f344da2249", size = 87903557, upload-time = "2026-08-06T21:43:23.321Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/51/550c9a75c4ff3245cb4ecb7bb95cbe2ab7374230b8e2b7a1f7259444150b/pymupdf-1.28.2-cp310-abi3-macosx_10_15_x86_64.whl", hash = "sha256:5fc315b425ff1f7afdd1ea2f348205cb19b806767daae7ce4d64115799c2bae1", size = 24645079, upload-time = "2026-08-06T21:37:25.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/3591f781b417b382a8487a2356e927acfe858b1043bab0ec47f6805bb109/pymupdf-1.28.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7113846b35dbf0a033f088e4f4fb543dabeb4b0b12c112966a1ca1ee2d5eacae", size = 23875605, upload-time = "2026-08-06T21:37:40.369Z" }, + { url = "https://files.pythonhosted.org/packages/d2/86/4a68f080b71b46802178346af46486e1697508e760855ff5f3b218a6dff7/pymupdf-1.28.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3050a233dde1211efe89ada74e2add6238436434159f46097a1423aad2842545", size = 25095554, upload-time = "2026-08-06T21:37:58.485Z" }, + { url = "https://files.pythonhosted.org/packages/c7/06/dace3e27af26690cb20bead80dbac42941b0841eb689b8aabbd67dde16f0/pymupdf-1.28.2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:397d6715c1f0df7548a92d0afd8ce370fc48fa47aeefac16be2bc04a16a8227f", size = 25762500, upload-time = "2026-08-06T21:38:17.438Z" }, + { url = "https://files.pythonhosted.org/packages/e5/61/4146dfa1d8172a1ce8d59f0eed94896ddefb8deb2274534d0522fbb8abf5/pymupdf-1.28.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f89fb2d86d07d643a269f17a093105057e20c79c1d06c103b53600067b6d2b01", size = 25986309, upload-time = "2026-08-06T21:38:35.472Z" }, + { url = "https://files.pythonhosted.org/packages/52/60/1fb6e64676f7500ebe89054b9e5bbbe14d3101c92d5f1a40ac9a35227673/pymupdf-1.28.2-cp310-abi3-win32.whl", hash = "sha256:530ef543a3885b3b81cb72a854e7c5a625a9233201221132bb6c31698c6a2bdb", size = 18525353, upload-time = "2026-08-06T21:38:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/4a/61/d563bbccba262f9dd6d2d35ccb72593648184d886188efb12d9ce8f34dd6/pymupdf-1.28.2-cp310-abi3-win_amd64.whl", hash = "sha256:ebd244918798502d7b4504c90410d1711a4d7675a32584ca30f1bab419ecbffe", size = 19826532, upload-time = "2026-08-06T21:39:00.213Z" }, + { url = "https://files.pythonhosted.org/packages/e2/93/08f404a1f0155fe24137cf2d3aabd3e2b4b08c62053ed89c60f2611be3e9/pymupdf-1.28.2-cp310-abi3-win_arm64.whl", hash = "sha256:ffe91a24edc75c80da2a4b62f50fc0f54632d34fc8fe4cbc48e5c7ff07cf8fb4", size = 19759252, upload-time = "2026-08-06T21:39:12.937Z" }, + { url = "https://files.pythonhosted.org/packages/58/8c/d897dcd32a25b58186c968b15ce4324ca029e9d96460de12325314e390be/pymupdf-1.28.2-cp313-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2e1b574c0fd2cb238021033fd3c0f9c4388816638df064e4bfb56d9d81736dc8", size = 18399403, upload-time = "2026-08-06T21:39:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f1/de34a1c53fe2bf8c6e71db84b0ced782d408970c9810d2b456a2ae96814c/pymupdf-1.28.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:fd481ed48bef56305c41fb7e05a055c03345c899c7b101dad086258b438f8168", size = 25802333, upload-time = "2026-08-06T21:39:41.426Z" }, +] + [[package]] name = "pyright" version = "1.1.411"