phase: 140_pictorial_pdf_pages

Everything is verified green. Final report:

## Phase 140 final verification pass — report

- Verified all 4 tasks already complete (code in working tree, task files in `complete/`): `pictorial_pages` gate + 2 settings (`BOR_PDF_PICTORIAL`, `BOR_PDF_PICTORIAL_MIN_CHARS` + validator), `PICTORIAL_PAGE_MODE`/`describe_pdf_page` seam + `render_pdf_pages` page selector + mock branch, `_pictorial_pdf_pass` in `_index_pdf_file` with log-only counters, dedicated E2E. No code changes needed.
- Test results (exact commands):
  - `uv run pytest --cov=app --cov-report=term-missing` → **3320 passed, 11 skipped, 0 failed; TOTAL 99%** (>90% gate)
  - `uv run pytest tests/e2e/test_pictorial_pdf.py -v --no-cov` → **3 passed in isolation** (Playwright, mock-only)
  - Pre-existing suites unedited: `test_pdf_support.py` 6 passed; `test_garbage_pdf.py` 2 passed + 1 skipped (owner fixture); `test_pdf_image_context.py` 4 passed
  - `uv run ruff check .` → All checks passed; `uv run pyright` → 0 errors
- Completion criteria:
  1. IKEA tar import repro — pinned by 2 integration tests; **skipped loudly here** (`test_sources/ikea_instructions.tar` absent on this machine — verified repo-wide; same house convention as completed phase-133 HNB tests; runs real on the owner's machine). Equivalent logic fully covered by unit pins on synthetic IKEA-class (vector-drawn, zero-raster) PDFs.
  2. Chat grounded on described page — **green** (E2E test b: phrase-only-in-illustration question, PDF in `done.sources`, answer echoes the block)
  3. Clean-text no-op (zero vision calls) + garbage precedence pin — **green** (`test_zero_candidates_is_a_noop` unit; `test_garbage_pdf_with_pictorial_on_takes_the_fallback_not_the_pass` integration, verified alone)
  4. `BOR_PDF_PICTORIAL=0` byte-identical — **green** (unit pin on synthetic PDF; IKEA variant runs on owner's machine)
  5. Full gate + E2E isolation + pre-existing suites — **green** (above)
  6. ruff + pyright — **clean**
  7. Commit/phase-move — delegated to harness per run rules (left in working tree: 16 modified + 8 new files, uncommitted)
- Deviations: none beyond the expected owner-fixture skips (11, all pre-existing fixture-missing skips, including 2 IKEA).
- Next pending phase: **none** — `todo/` holds only 140; 139 is already complete.

tests/integration/test_importer.py

@@ -34,16 +34,26 @@ from __future__ import annotations
 import asyncio
 import base64
 import hashlib
+import logging
+import tarfile
+from collections.abc import Iterator
 from pathlib import Path
 
 import pymupdf as fitz  # PyMuPDF — the temp-PDF fixture builder (phase 126)
+import pytest
 from sqlalchemy import select
 
 from app.config import Settings
 from app.models import Chunk, Document
 from app.rag.importer import import_sources
 from app.rag.llm import EmbeddingError
-from app.rag.summarizer import DESCRIBE_PROMPT
+from app.rag.pdf import (
+    PAGE_SEPARATOR,
+    extract_pdf_page_texts,
+    extract_pdf_text,
+    pictorial_pages,
+)
+from app.rag.summarizer import DESCRIBE_PROMPT, PICTORIAL_PAGE_MODE
 from tests.fakes import FakeEmbedder
 
 #: A real 1×1 transparent PNG (the phase-122 fixture — the importer is
@@ -99,10 +109,17 @@ def _image_llm[
 ](tmp_path, llm_cls: type[LLM] = _MockVisionLLM, **kwargs) -> LLM:
     """A fake LLM with the phase-122 image knobs (toggle ON; the image
     dir defaults under *tmp_path*). *llm_cls* may be the mock vision
-    client (the default — ``text_parts``/``chat_models`` state)."""
+    client (the default — ``text_parts``/``chat_models`` state).
+    Phase 140: ``pdf_pictorial`` defaults to OFF here — the pre-phase-
+    140 PDF suites in this file pin the pre-phase content shape (their
+    fixture pages are sparse — under the 100-char bar — so the
+    default-ON pictorial pass would append illustration blocks and
+    move the byte-exact pins); the phase-140 suites pass
+    ``pdf_pictorial=True`` explicitly."""
     kwargs.setdefault("_env_file", None)
     kwargs.setdefault("images", True)
     kwargs.setdefault("image_dir", str(tmp_path / "images"))
+    kwargs.setdefault("pdf_pictorial", False)
     llm = llm_cls()
     llm.settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
     return llm
@@ -1506,3 +1523,314 @@ def test_backfill_embed_failure_rolls_back_the_image_and_retries_next_sync(
         assert img2.image_context.startswith("Extracted from ")
     finally:
         _cleanup_source(db, source_root.name)
+
+
+# ---------------------------------------------------------------------------
+# Phase 140 — Pictorial PDFs: sparse pages rendered + vision-described
+# into the indexed content (the owner's IKEA repro, TODO L5; the kill
+# switch; the phase-133 precedence pin).
+# ---------------------------------------------------------------------------
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+#: The owner's untracked fixture (TODO L5 — the stable reference: the
+#: TARBALL, never the extracted PDF; the phase-133 real-fixture
+#: convention — ``test_sources/`` is gitignored, the test extracts it
+#: into ``tmp_path`` and never touches the repo, and the owner's
+#: machine always has it: a missing tar skips LOUDLY with the named
+#: reason, it is never silently green).
+IKEA_TARBALL = REPO_ROOT / "test_sources" / "ikea_instructions.tar"
+
+#: The temp source dir's basename is the source name (phase 38).
+IKEA_SOURCE = "ikea"
+
+#: The fixture's embedded metadata title (Adobe InDesign's
+#: "Assembly Instructions" — the phase-140 context).
+IKEA_TITLE = "Assembly Instructions"
+
+#: The fixture's text-heavy pages (pages 2–5 — the multilingual
+#: warnings, 3,091–7,288 chars — never candidates; the sparse pages
+#: run 4–73 chars, so the 100-char bar sits between the classes).
+IKEA_TEXT_HEAVY_PAGES = (2, 3, 4, 5)
+
+
+class _PictorialVisionLLM(FakeEmbedder):
+    """The capturing multimodal fake (task 03): a vision call whose
+    text part carries ``PICTORIAL_PAGE_MODE`` (the pictorial pass's
+    seam — ``describe_pdf_page``) gets the fixed per-candidate canned
+    reply and is recorded in ``pictorial_texts``; a call carrying
+    ``IMAGE_DESCRIPTION_MODE`` (the phase-133 fallback's / phase-122
+    images' seam — ``describe_image``) gets a DISTINCT canned reply
+    and is recorded in ``image_texts``. The fixed replies make the
+    stored content a PURE FUNCTION of the fixture (the digest-stable
+    premise of the re-sync assertion), and the two recordings witness
+    WHICH seam fired (the precedence pin). String (``lite``) calls
+    keep the ``FakeEmbedder`` behaviour. The REAL
+    ``import_sources`` → ``_pictorial_pdf_pass`` →
+    ``describe_pdf_page`` chain runs against it (no seam patch)."""
+
+    PICTORIAL_ANSWER = (
+        "IKEA pictorial page: the illustration shows two parts being "
+        "joined with a blue arrow; a cam lock is tightened with the "
+        "included hex key."
+    )
+    IMAGE_ANSWER = (
+        "Fallback screenshot: the page shows an assembly pictogram."
+    )
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.pictorial_texts: list[str] = []
+        self.image_texts: list[str] = []
+
+    async def chat(self, messages, model=None):
+        self.chat_calls.append(list(messages))
+        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
+        if isinstance(user, list):
+            text = next((p["text"] for p in user if p.get("type") == "text"), "")
+            if PICTORIAL_PAGE_MODE in text:
+                self.pictorial_texts.append(text)
+                return self.PICTORIAL_ANSWER
+            self.image_texts.append(text)
+            return self.IMAGE_ANSWER
+        first = user.split()
+        return "Summary of " + (first[0] if first else "<empty>")
+
+
+@pytest.fixture()
+def ikea_pdf(db, tmp_path) -> Iterator[Path]:
+    """The owner's IKEA tarball extracted into a temp ``ikea/`` source
+    dir (the phase-133 real-fixture convention — ``tmp_path`` only,
+    never the repo). Any leftover ``ikea`` source rows from a crashed
+    earlier run are cleaned before AND after (the shared dev DB is
+    reused across runs). Skips LOUDLY with the named reason when the
+    untracked owner fixture is missing."""
+    if not IKEA_TARBALL.is_file():
+        pytest.skip(
+            "test_sources/ikea_instructions.tar is missing — the untracked "
+            "owner fixture (one 24-page vector-drawn assembly-instruction "
+            "PDF; the owner's machine always has it)"
+        )
+    _cleanup_source(db, IKEA_SOURCE)
+    root = tmp_path / IKEA_SOURCE
+    root.mkdir()
+    with tarfile.open(IKEA_TARBALL) as tar:
+        tar.extractall(root, filter="data")
+    pdfs = sorted(root.rglob("*.pdf"))
+    assert len(pdfs) == 1, f"the IKEA tar carries exactly one PDF, found: {pdfs}"
+    yield pdfs[0]
+    _cleanup_source(db, IKEA_SOURCE)
+
+
+def test_ikea_tar_sparse_pages_are_described_and_rerun_unchanged(
+    db, tmp_path, ikea_pdf: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    """The owner's EXACT file (TODO L5) through the REAL importer:
+    the deterministic gate's candidates — computed IN the test from
+    ``extract_pdf_page_texts`` + ``pictorial_pages`` (no hardcoded
+    page list — the fixture is stable but the test must not rot) —
+    every one carries ``[Page N illustration: <canned>]`` AFTER its
+    own text (LOCKED A2 — no LLM judge: every described page joins
+    the content; A4 — the page's text stays), the text-heavy pages
+    (2–5) carry NO block, the chunks carry the canned description
+    (the owner's rationale — more information is better for the
+    embeddings), the title is the metadata title, the digest is over
+    the FINAL content, and a re-sync with the same fake lands
+    ``unchanged`` (digest stable, byte-stable row, NO re-embed; the
+    vision calls are re-issued before the unchanged check can skip
+    them — the documented no-cache cost, the phase-133 convention,
+    A21 endorses it)."""
+    pdf_path = ikea_pdf
+    llm = _image_llm(
+        tmp_path,
+        llm_cls=_PictorialVisionLLM,
+        pdf_pictorial=True,
+        import_extensions="md,txt,pdf",
+    )
+    try:
+        with caplog.at_level(logging.INFO, logger="app.importer"):
+            s1 = asyncio.run(import_sources([pdf_path.parent], llm, session=db))
+        assert (s1.files, s1.added, s1.errors) == (1, 1, 0)
+        assert s1.formats == {"pdf": 1}
+
+        # The candidates — computed IN the test (no hardcoded page
+        # list). The fixture class: the sparse pages are 4–73 chars,
+        # the text-heavy pages 3,091+ — the 100-char bar separates.
+        page_texts = extract_pdf_page_texts(pdf_path)
+        candidates = pictorial_pages(
+            page_texts, min_chars=llm.settings.pdf_pictorial_min_chars
+        )
+        assert candidates, "the fixture must carry sparse (pictorial) pages"
+        for n in IKEA_TEXT_HEAVY_PAGES:
+            assert n not in candidates, f"page {n} is text-heavy, never a candidate"
+
+        doc = db.scalar(
+            select(Document).where(
+                Document.source == IKEA_SOURCE, Document.path == pdf_path.name
+            )
+        )
+        assert doc is not None, "the IKEA PDF did not import"
+        assert doc.title == IKEA_TITLE  # the PDF's metadata title
+        assert doc.is_image is False
+
+        content = doc.content
+        # Every page's block is byte-faithful: the page text, the
+        # described candidates carrying their illustration block
+        # AFTER the text (A4), and NO block on the non-candidates.
+        for n, text in page_texts.items():
+            block = f"{PAGE_SEPARATOR.format(page=n)}\n{text}"
+            if n in candidates:
+                block += (
+                    f"\n\n[Page {n} illustration: "
+                    f"{_PictorialVisionLLM.PICTORIAL_ANSWER}]"
+                )
+            assert block in content, f"page {n}'s block is not byte-faithful"
+            if n not in candidates:
+                assert f"[Page {n} illustration" not in content
+        assert content.count(" illustration: ") == len(candidates)
+        # The digest runs over the FINAL content (the phase-126/133
+        # convention — A4).
+        assert doc.content_hash == hashlib.sha256(
+            content.encode("utf-8")
+        ).hexdigest()
+
+        # The chunks carry the canned description (the owner's L5
+        # rationale — the embeddings hit the illustrations), and the
+        # phase-30/118 summary ran on the final content.
+        chunks = db.scalars(
+            select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
+        ).all()
+        content_chunks = [c for c in chunks if not c.is_summary]
+        assert any(
+            _PictorialVisionLLM.PICTORIAL_ANSWER in c.content for c in content_chunks
+        ), "no content chunk carries the canned description"
+        for c in chunks:
+            assert c.embedding is not None and len(c.embedding) == 768
+        assert doc.summary is not None
+
+        # The counters (log-only — the sync detail shape is untouched)
+        # + the ONE loud log line (PLAN §9), with the computed N.
+        assert s1.pdfs_pictorial == 1
+        assert s1.pdf_pictorial_pages == len(candidates)
+        assert s1.pdfs_screenshotted == 0  # the layer is USABLE — no fallback
+        assert llm.image_texts == []  # the fallback seam never fired
+        min_chars = llm.settings.pdf_pictorial_min_chars
+        lines = [
+            r.getMessage() for r in caplog.records if "pictorial pass" in r.getMessage()
+        ]
+        assert lines == [
+            f"pdf: pictorial pass source={IKEA_SOURCE} path={pdf_path.name} — "
+            f"{len(candidates)} candidate pages, {len(candidates)} described "
+            f"(min_chars={min_chars})"
+        ]
+
+        # The re-sync with the SAME fake: ``unchanged`` — the final
+        # content is a pure function of the fixture + the canned
+        # replies, so the digest matches and the row is skipped (no
+        # re-chunk, no re-embed, no re-summary, no KB change). The
+        # vision calls ARE re-issued (the documented no-cache cost —
+        # the phase-133 convention; a non-deterministic vision model
+        # re-describes the row each sync: a cost, not a bug).
+        embeds1 = len(llm.calls)
+        pictorial1 = len(llm.pictorial_texts)
+        content1, digest1 = doc.content, doc.content_hash
+        s2 = asyncio.run(import_sources([pdf_path.parent], llm, session=db))
+        assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 1)
+        assert s2.pdfs_pictorial == 1  # the pass re-fired
+        assert len(llm.pictorial_texts) == 2 * pictorial1  # re-issued (cost)
+        assert len(llm.calls) == embeds1, "unchanged: no re-embed"
+        assert s2.changed_paths == [], "unchanged: no KB change recorded"
+        doc2 = db.scalar(
+            select(Document).where(
+                Document.source == IKEA_SOURCE, Document.path == pdf_path.name
+            )
+        )
+        assert doc2 is not None
+        assert doc2.content == content1  # byte-stable content
+        assert doc2.content_hash == digest1  # byte-stable digest
+    finally:
+        _cleanup_source(db, IKEA_SOURCE)
+
+
+def test_ikea_tar_kill_switch_is_byte_identical_to_extraction(
+    db, tmp_path, ikea_pdf: Path
+) -> None:
+    """``BOR_PDF_PICTORIAL=0`` (the house kill switch — LOCKED A3):
+    the pass never runs on the owner's file — no vision call, no
+    counters, no log line — and the stored content is
+    ``extract_pdf_text``'s output BYTE-IDENTICAL to the pre-phase-140
+    path."""
+    pdf_path = ikea_pdf
+    llm = _image_llm(
+        tmp_path,
+        llm_cls=_PictorialVisionLLM,
+        pdf_pictorial=False,
+        import_extensions="md,txt,pdf",
+    )
+    try:
+        s1 = asyncio.run(import_sources([pdf_path.parent], llm, session=db))
+        assert (s1.files, s1.added, s1.errors) == (1, 1, 0)
+        doc = db.scalar(
+            select(Document).where(
+                Document.source == IKEA_SOURCE, Document.path == pdf_path.name
+            )
+        )
+        assert doc is not None
+        assert doc.content == extract_pdf_text(pdf_path), (
+            "the kill switch must keep the pre-phase-140 content "
+            "byte-identical"
+        )
+        assert llm.pictorial_texts == []  # zero vision calls
+        assert (s1.pdfs_pictorial, s1.pdf_pictorial_pages) == (0, 0)
+    finally:
+        _cleanup_source(db, IKEA_SOURCE)
+
+
+def test_garbage_pdf_with_pictorial_on_takes_the_fallback_not_the_pass(
+    db, tmp_path
+) -> None:
+    """The precedence pin (LOCKED): an UNUSABLE text layer takes the
+    phase-133 full-page screenshot fallback — the pictorial pass
+    NEVER runs and never double-visions (the gate is the only gate):
+    with ``pdf_pictorial=True`` (the code default), a garbage-text
+    PDF (the phase-133 synthetic recipe — XML-ish lines) lands with
+    the fallback's full-page canned descriptions, and the fake
+    records ZERO ``PICTORIAL_PAGE_MODE`` calls."""
+    source_root = tmp_path / "garbpike"
+    source_root.mkdir()
+    doc = fitz.open()
+    for p in (1, 2):
+        page = doc.new_page()
+        for i in range(10):
+            page.insert_text((72, 72 + i * 14), f"<foo:p{p}>{i}</foo:p{p}>")
+    doc.save(source_root / "garbage.pdf")
+    doc.close()
+    llm = _image_llm(
+        tmp_path,
+        llm_cls=_PictorialVisionLLM,
+        pdf_pictorial=True,
+        import_extensions="md,txt,pdf",
+    )
+    try:
+        summary = asyncio.run(import_sources([source_root], llm, session=db))
+        assert (summary.files, summary.added, summary.errors) == (1, 1, 0)
+        assert summary.pdfs_screenshotted == 1  # the gate fired → the fallback
+        assert summary.pdfs_pictorial == 0
+        assert summary.pdf_pictorial_pages == 0
+        assert llm.pictorial_texts == [], (
+            "the pictorial pass must never run on an unusable layer "
+            "(no double vision)"
+        )
+        assert len(llm.image_texts) == 2  # one fallback describe per page
+        row = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == "garbage.pdf"
+            )
+        )
+        assert row is not None
+        assert row.content == (
+            f"---PAGE-1---\n{_PictorialVisionLLM.IMAGE_ANSWER}\n\n"
+            f"---PAGE-2---\n{_PictorialVisionLLM.IMAGE_ANSWER}"
+        )
+    finally:
+        _cleanup_source(db, source_root.name)