phase: 126_pdf_support

All gates verified green — no code changes were needed on this pass. Final report:

## Phase 126 — PDF support: final verification pass

**Verified (all tasks 01–05 already complete in working tree):**
- `pymupdf` dep in `pyproject.toml`/`uv.lock`; `.env.example` documents the `pdf` token
- `app/rag/pdf.py` (text + image extraction, MIME mapping), page-aware `_chunk_pdf`, importer `_index_pdf_file` + embedded-image pass, archives covered via extension walk

**Test / lint / coverage results:**
- `uv run pytest` → 2997 passed
- `uv run pytest tests/e2e/test_pdf_support.py -v --no-cov` (isolation) → 6 passed
- `uv run pytest --cov=app --cov-report=term-missing` → TOTAL **99%** (>90% gate), 2997 passed
- `uv run ruff check . && uv run pyright` → clean (0 errors)

**Completion criteria:**
1. PDF → doc with text/chunks/embedding/summary — ✅ (integration + E2E Sources-page tests)
2. `BOR_IMAGES=true`: embedded images → standalone image docs — ✅ (integration + E2E chat-chip test)
3. `BOR_IMAGES=false`: images skipped, text indexed — ✅ (both suites)
4. PDF renders as extracted text in viewer — ✅ (E2E)
5. Test/coverage/lint gates — ✅ (above)
6. Commit + phase move — harness handles (per protocol, no `git` on this pass)

**Decisions:** none new; inline `[Embedded image: …]` placeholders were a documented task-04 deviation (simplified spec wins) — not a defect. No defects found; repo left functional with changes in the working tree.

**Next pending phase:** none — `todo/` is empty once 126 is moved.

tests/integration/test_importer.py

@@ -33,7 +33,10 @@ from __future__ import annotations
 
 import asyncio
 import base64
+import hashlib
+from pathlib import Path
 
+import pymupdf as fitz  # PyMuPDF — the temp-PDF fixture builder (phase 126)
 from sqlalchemy import select
 
 from app.config import Settings
@@ -444,3 +447,469 @@ def test_markdown_referencing_existing_image_keeps_null_record(db, tmp_path) ->
         assert md3 is not None and md3.image_context is None
     finally:
         _cleanup_source(db, source_root.name)
+
+
+# ---------------------------------------------------------------------------
+# Phase 126 — PDF indexing through the REAL import pipeline (the real
+# ``app.rag.pdf`` extractor, the page-aware chunker, the REAL
+# ``_describe_or_skip`` → ``describe_image`` seam — NO seam patch).
+# ---------------------------------------------------------------------------
+
+PDF_TITLE = "Quarterly Homelab Report"
+
+PAGE_ONE = "The core switch firmware was upgraded to the stable channel."
+PAGE_TWO = "The backup schedule runs nightly at 02:00 and keeps seven rotations."
+
+
+def _png_bytes(width: int, height: int) -> bytes:
+    pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height), 0)
+    pix.clear_with(200)
+    return pix.tobytes("png")
+
+
+def _build_pdf(
+    path, pages: list[tuple[str, list[bytes]]], title: str | None = None
+) -> None:
+    """One temp PDF (built with PyMuPDF — no binary fixtures): *pages* is
+    (page text, embedded-image bytes) per page; *title* sets the metadata
+    title (``None`` = leave it empty)."""
+    doc = fitz.open()
+    for text, imgs in pages:
+        page = doc.new_page()
+        if text:
+            page.insert_text((72, 72), text)
+        y = 150
+        for data in imgs:
+            pix = fitz.Pixmap(data)
+            page.insert_image(
+                fitz.Rect(72, y, 72 + pix.width, y + pix.height), stream=data
+            )
+            y += pix.height + 10
+    if title is not None:
+        doc.set_metadata({"title": title})
+    doc.save(path)
+    doc.close()
+
+
+def test_pdf_becomes_document_with_text_chunks_embedding_and_summary(
+    db, tmp_path
+) -> None:
+    """The phase-126 primary scenario: a PDF in a source (the ``pdf``
+    import token) → sync → the document carries the extracted text
+    (the ``---PAGE-N---`` markers, A2 — the title is the metadata
+    title), page-anchored chunks with 768-dim embeddings, and the
+    phase-30/118 lite summary. With ``images=True`` (A3) the embedded
+    image joins the pipeline through the REAL seam — one vision call on
+    the CHAT model with the bare locked prompt (no markdown reference
+    exists — ``image_context`` stays NULL), the copy in ``image_dir``,
+    and the description as the image doc's content AND summary."""
+    source_root = tmp_path / "pdfkb"
+    source_root.mkdir()
+    png = _png_bytes(40, 30)
+    _build_pdf(
+        source_root / "report.pdf",
+        [(PAGE_ONE, []), (PAGE_TWO, [png])],
+        title=PDF_TITLE,
+    )
+    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
+    try:
+        summary = asyncio.run(import_sources([source_root], llm, session=db))
+        assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0)
+        assert summary.formats == {"pdf": 1}
+
+        doc = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == "report.pdf"
+            )
+        )
+        assert doc is not None and doc.is_image is False
+        assert doc.title == PDF_TITLE
+        assert doc.content == f"---PAGE-1---\n{PAGE_ONE}\n\n---PAGE-2---\n{PAGE_TWO}"
+        chunks = db.scalars(
+            select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
+        ).all()
+        content_chunks = [c for c in chunks if not c.is_summary]
+        assert [c.content for c in content_chunks] == [
+            f"Page 1:\n{PAGE_ONE}",
+            f"Page 2:\n{PAGE_TWO}",
+        ]
+        for c in chunks:
+            assert c.embedding is not None and len(c.embedding) == 768
+        summary_chunks = [c for c in chunks if c.is_summary]
+        assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
+        assert doc.summary is not None
+        assert doc.summary.endswith(f"Source: {source_root.name}/report.pdf")
+
+        # The embedded image (A3) — through the REAL describe seam.
+        image_rel = "report.pdf#page=2#img=0.png"
+        img = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == image_rel
+            )
+        )
+        assert img is not None and img.is_image is True
+        assert img.image_context is None, "no markdown reference → bare prompt"
+        assert img.content == DESCRIPTION and img.summary == DESCRIPTION
+        copy = Path(img.image_path)
+        assert copy.parent == Path(llm.settings.image_dir).expanduser()
+        assert copy.name == f"{img.id}.png"
+        assert copy.read_bytes() == png
+        # The vision call went to the CHAT model with the BARE locked
+        # prompt (the PDF's lite summary is the other chat call).
+        assert llm.chat_models == [
+            llm.settings.llm_summary_model, llm.settings.llm_chat_model
+        ]
+        assert llm.text_parts == [DESCRIBE_PROMPT]
+    finally:
+        _cleanup_source(db, source_root.name)
+
+
+def test_pdf_embedded_images_skipped_when_images_off(db, tmp_path) -> None:
+    """With ``images=False`` (the DEFAULT): the PDF text is indexed
+    normally and the embedded images are skipped — no image documents,
+    no ``images_failed`` (a skip is not a failure), no copies, no vision
+    calls."""
+    source_root = tmp_path / "pdfoffkb"
+    source_root.mkdir()
+    _build_pdf(
+        source_root / "report.pdf",
+        [(PAGE_ONE, [_png_bytes(40, 30)]), (PAGE_TWO, [])],
+    )
+    llm = _image_llm(tmp_path, images=False, import_extensions="md,txt,pdf")
+    try:
+        summary = asyncio.run(import_sources([source_root], llm, session=db))
+        assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0)
+        doc = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == "report.pdf"
+            )
+        )
+        assert doc is not None and PAGE_ONE in doc.content
+        chunks = db.scalars(
+            select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
+        ).all()
+        assert any(c.embedding is not None for c in chunks)
+        assert (
+            db.scalar(
+                select(Document).where(
+                    Document.source == source_root.name,
+                    Document.is_image.is_(True),
+                )
+            )
+            is None
+        ), "images off: no embedded-image documents"
+        # No vision call at all — only the PDF's lite summary.
+        assert llm.chat_models == [llm.settings.llm_summary_model]
+    finally:
+        _cleanup_source(db, source_root.name)
+
+
+def test_pdf_without_embedded_images_and_unchanged_rerun(db, tmp_path) -> None:
+    """A PDF without embedded images → no image documents; an unchanged
+    re-sync → ``unchanged`` (no re-extract, no re-embed, no vision or
+    embed calls), and a second run never creates image documents out of
+    nothing."""
+    source_root = tmp_path / "pdfnonekb"
+    source_root.mkdir()
+    _build_pdf(
+        source_root / "notes.pdf",
+        [(PAGE_ONE, []), (PAGE_TWO, [])],
+    )
+    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
+    try:
+        s1 = asyncio.run(import_sources([source_root], llm, session=db))
+        assert (s1.files, s1.added, s1.unchanged) == (1, 1, 0)
+        assert (
+            db.scalar(
+                select(Document).where(
+                    Document.source == source_root.name,
+                    Document.is_image.is_(True),
+                )
+            )
+            is None
+        )
+        calls1, chat1 = len(llm.calls), len(llm.chat_models)
+        s2 = asyncio.run(import_sources([source_root], llm, session=db))
+        assert (s2.files, s2.added, s2.updated, s2.unchanged) == (1, 0, 0, 1)
+        assert len(llm.calls) == calls1 and len(llm.chat_models) == chat1
+    finally:
+        _cleanup_source(db, source_root.name)
+
+
+def test_pdf_without_text_becomes_an_empty_document(db, tmp_path) -> None:
+    """A PDF with no extractable text (image-only, ``images`` OFF — the
+    embedded image is invisible) → the document is still created, with
+    EMPTY content (the empty byte string's digest), NO content chunks
+    and NO content embedding (there is no text to embed — the only
+    chunk is the best-effort ``lite`` summary's ``is_summary`` row), and
+    the warning line is the signal."""
+    source_root = tmp_path / "pdfemptykb"
+    source_root.mkdir()
+    _build_pdf(source_root / "pics.pdf", [("", [_png_bytes(40, 30)])])
+    llm = _image_llm(tmp_path, images=False, import_extensions="md,txt,pdf")
+    try:
+        summary = asyncio.run(import_sources([source_root], llm, session=db))
+        assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0)
+
+        doc = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == "pics.pdf"
+            )
+        )
+        assert doc is not None and doc.is_image is False
+        assert doc.content == ""
+        assert doc.content_hash == hashlib.sha256(b"").hexdigest()
+        chunks = db.scalars(
+            select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
+        ).all()
+        assert [c for c in chunks if not c.is_summary] == [], (
+            "no text → no content chunks (and no content embedding)"
+        )
+        # The phase-30/118 path still runs best-effort on the empty
+        # content — the summary is the only chunk, and it is embedded.
+        assert len(chunks) == 1 and chunks[0].is_summary
+        assert chunks[0].embedding is not None and len(chunks[0].embedding) == 768
+        assert doc.summary is not None
+    finally:
+        _cleanup_source(db, source_root.name)
+
+
+def test_pdf_embedded_image_survives_prune_and_leaves_with_the_pdf(
+    db, tmp_path
+) -> None:
+    """Phase 126 (A3) prune parent rule — the defect the E2E surfaced:
+    the embedded image doc's synthetic path NEVER appears in the walk's
+    ``seen``, so the plain prune rule destroyed it in the very sync that
+    (re)indexed it (the toggle-on half of the phase-122 "invisible, not
+    deleted" guard was missing). Now: with the parent PDF's rel still in
+    the walk — even UNCHANGED, so no image pass runs at all — the image
+    doc survives (0 pruned); when the PDF LEAVES the source, the image
+    doc is pruned with it (row + ``image_dir`` copy — no orphans)."""
+    source_root = tmp_path / "pdfprune"
+    source_root.mkdir()
+    png = _png_bytes(40, 30)
+    _build_pdf(
+        source_root / "report.pdf",
+        [(PAGE_ONE, []), (PAGE_TWO, [png])],
+    )
+    image_rel = "report.pdf#page=2#img=0.png"
+    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
+    try:
+        s1 = asyncio.run(
+            import_sources([source_root], llm, session=db, prune=True)
+        )
+        assert (s1.added, s1.pruned) == (2, 0)
+        img1 = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == image_rel
+            )
+        )
+        assert img1 is not None and img1.image_path is not None
+        copy = Path(img1.image_path)
+
+        # Unchanged re-sync (prune ON): the PDF's text digest matches →
+        # no image pass at all — the image doc survives anyway (the
+        # parent's rel is in the walk's seen set).
+        s2 = asyncio.run(
+            import_sources([source_root], llm, session=db, prune=True)
+        )
+        assert (s2.unchanged, s2.pruned) == (1, 0)
+        assert db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == image_rel
+            )
+        ) is not None, "parent PDF present → the image doc survives the prune"
+
+        # The PDF LEAVES the source: the image doc is pruned WITH it
+        # (row + copy — no orphan in image_dir).
+        (source_root / "report.pdf").unlink()
+        s3 = asyncio.run(
+            import_sources([source_root], llm, session=db, prune=True)
+        )
+        assert (s3.pruned, s3.unchanged) == (2, 0)
+        assert db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == image_rel
+            )
+        ) is None, "parent PDF gone → the image doc is pruned with it"
+        assert not copy.exists(), "the pruned image's copy leaves image_dir"
+    finally:
+        _cleanup_source(db, source_root.name)
+
+
+# --- Phase 125 parity: a markdown file referencing a PDF-embedded image ---
+
+#: A markdown file referencing the PDF's embedded image by its SYNTHETIC
+#: path (``<pdf path>#page=<N>#img=<index>.<ext>`` — the doc's ``path``
+#: is the upsert key, and the reference resolves to it through the shared
+#: ``_resolve_ref_path`` rule) — one context paragraph before, one after.
+PDF_GUIDE_MD = (
+    "The VLAN map diagram in the report shows the subnet layout.\n"
+    "\n"
+    "![VLAN map](report.pdf#page=2#img=0.png)\n"
+    "\n"
+    "Only expand the lab subnet after the VLAN map is reviewed.\n"
+)
+
+#: The same reference against ``alpha.pdf`` (the PDF-FIRST ordering's
+#: fixture — ``alpha.pdf`` sorts before ``zguide.md`` in the walk).
+PDF_ZGUIDE_MD = PDF_GUIDE_MD.replace("report.pdf", "alpha.pdf")
+
+
+def _pdf_expected_block(source: str, md_name: str, pdf_name: str) -> str:
+    """The task-03-shape combined string for the PDF reference (the disk
+    scan builds exactly this — the standalone image's shape, the
+    synthetic path as the image identity)."""
+    return (
+        f"Referenced in {source}/{md_name}:\n"
+        '- Alt text: "VLAN map"\n'
+        "  Context: The VLAN map diagram in the report shows the subnet layout.\n"
+        "\n"
+        "Only expand the lab subnet after the VLAN map is reviewed."
+    )
+
+
+def test_pdf_embedded_image_referenced_by_markdown_markdown_first(
+    db, tmp_path
+) -> None:
+    """The MARKDOWN-FIRST ordering (``guide.md`` sorts before
+    ``report.pdf``): the markdown is indexed while its referenced image
+    has no document yet → it records the reference on its OWN row (the
+    ``IMAGE_CONTEXT:`` record — phase 125, task 04, work 1, never
+    cleared); the PDF is indexed next and the task-03 disk scan (the
+    standalone image's parity path — the markdown is still on disk) finds
+    the reference, so the image doc is enriched AT INDEX TIME: the vision
+    call carries the context after the locked prompt, ``image_context``
+    is set, ``content``/``summary`` are ``<context>\\n\\n<description>``.
+    The post-index pass then skips the already-enriched doc (idempotent).
+    Final state: the record AND the enrichment coexist."""
+    source_root = tmp_path / "pdfmdfirst"
+    source_root.mkdir()
+    (source_root / "guide.md").write_text(PDF_GUIDE_MD, encoding="utf-8")
+    _build_pdf(
+        source_root / "report.pdf",
+        [(PAGE_ONE, []), (PAGE_TWO, [_png_bytes(40, 30)])],
+    )
+    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
+    try:
+        summary = asyncio.run(import_sources([source_root], llm, session=db))
+        # guide.md + report.pdf + the embedded image doc: all added.
+        assert (summary.files, summary.added, summary.images_failed) == (2, 3, 0)
+
+        expected_block = _pdf_expected_block(source_root.name, "guide.md", "report.pdf")
+
+        # Work 1 — the markdown doc STORES the reference (the record —
+        # line 3 is the reference, 2 context paragraphs). It is the
+        # post-index pass's reference index and the retrieval display's
+        # record; the pass does not clear it on success.
+        md = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == "guide.md"
+            )
+        )
+        assert md is not None and md.is_image is False
+        assert md.image_context == (
+            "IMAGE_CONTEXT:\n"
+            '- [report.pdf#page=2#img=0.png] (line 3): "VLAN map"'
+            " — 2 paragraphs of context"
+        )
+
+        # The enrichment (the disk scan at PDF index time) — the image
+        # doc's shape is the standalone image's enriched shape.
+        image_rel = "report.pdf#page=2#img=0.png"
+        img = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == image_rel
+            )
+        )
+        assert img is not None and img.is_image is True
+        assert img.image_context == expected_block
+        expected_content = f"{expected_block}\n\n{DESCRIPTION}"
+        assert img.content == expected_content
+        assert img.summary == expected_content
+        # The vision call carried the context AFTER the locked prompt
+        # (the multimodal part list's text part — the describe seam's
+        # phase-125 extension).
+        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_block}"]
+        # Chat call order (walk order): guide.md's ``lite`` summary,
+        # the PDF's ``lite`` summary, then the image's CHAT-model
+        # description (the ONLY multimodal call).
+        assert llm.chat_models == [
+            llm.settings.llm_summary_model,
+            llm.settings.llm_summary_model,
+            llm.settings.llm_chat_model,
+        ]
+        # The chunks are embedded from the ENRICHED content (the
+        # plain-text packer joins the enriched paragraphs with single
+        # newlines; the summary chunk mirrors the raw content).
+        packed = "\n".join(expected_content.split("\n\n"))
+        chunks = db.scalars(
+            select(Chunk).where(Chunk.document_id == img.id).order_by(Chunk.position)
+        ).all()
+        assert [c.content for c in chunks if not c.is_summary] == [packed]
+        summary_chunks = [c for c in chunks if c.is_summary]
+        assert len(summary_chunks) == 1 and summary_chunks[0].content == expected_content
+        for c in chunks:
+            assert c.embedding is not None and len(c.embedding) == 768
+    finally:
+        _cleanup_source(db, source_root.name)
+
+
+def test_pdf_embedded_image_referenced_by_markdown_pdf_first(
+    db, tmp_path
+) -> None:
+    """The PDF-FIRST ordering (``alpha.pdf`` sorts before ``zguide.md``
+    in the walk): the PDF (and its image doc) is indexed while the
+    markdown is still un-indexed — the task-03 disk scan sees the
+    reference on disk and enriches the image AT INDEX TIME (the ordering
+    that, without the parity scan, would ship an un-enriched image — the
+    markdown's record stays NULL: at markdown index time the image doc
+    ALREADY exists, so the task-03 path owns the case and nothing is
+    recorded). Same final state as the markdown-first test — the
+    enrichment never depends on the walk order."""
+    source_root = tmp_path / "pdfpdffirst"
+    source_root.mkdir()
+    (source_root / "zguide.md").write_text(PDF_ZGUIDE_MD, encoding="utf-8")
+    _build_pdf(
+        source_root / "alpha.pdf",
+        [(PAGE_ONE, []), (PAGE_TWO, [_png_bytes(40, 30)])],
+    )
+    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
+    try:
+        summary = asyncio.run(import_sources([source_root], llm, session=db))
+        assert (summary.files, summary.added, summary.images_failed) == (2, 3, 0)
+
+        expected_block = _pdf_expected_block(source_root.name, "zguide.md", "alpha.pdf")
+
+        # The task-03 path owns this case: the image doc existed when
+        # the markdown was indexed → NO record (the field is the
+        # pending-enrichment index, not a reference dump).
+        md = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == "zguide.md"
+            )
+        )
+        assert md is not None and md.image_context is None
+
+        # The image doc is enriched (the disk scan at PDF index time) —
+        # byte-identical in shape to the markdown-first final state.
+        image_rel = "alpha.pdf#page=2#img=0.png"
+        img = db.scalar(
+            select(Document).where(
+                Document.source == source_root.name, Document.path == image_rel
+            )
+        )
+        assert img is not None and img.image_context == expected_block
+        expected_content = f"{expected_block}\n\n{DESCRIPTION}"
+        assert img.content == expected_content
+        assert img.summary == expected_content
+        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_block}"]
+        # Chat call order (walk order): the PDF's ``lite`` summary, the
+        # image's CHAT-model description, then zguide.md's summary.
+        assert llm.chat_models == [
+            llm.settings.llm_summary_model,
+            llm.settings.llm_chat_model,
+            llm.settings.llm_summary_model,
+        ]
+    finally:
+        _cleanup_source(db, source_root.name)