brain-of-reese/tests/integration/test_importer.py

"""Integration: phase 125, task 03 — the image-anchor enrichment through
the REAL import pipeline (real compose Postgres, the real
``_describe_or_skip`` → ``describe_image`` seam, the mock vision
client — NO seam patch).

A markdown file referencing an image in the SAME source → the vision
call receives the markdown context (appended after the locked
``DESCRIBE_PROMPT``), the image doc's ``image_context`` is set, its
``content``/``summary`` is ``<context>\\n\\n<description>``, and its
chunks are embedded from the enriched content. A standalone image (no
markdown reference) → ``image_context`` NULL, ``content`` the bare
description, the bare locked prompt — byte-identical to pre-phase-125.

Phase 125, task 04 (the markdown-FIRST case) — a markdown indexed while
its referenced image has no document yet stores the reference on the
markdown doc's ``image_context`` (the ``IMAGE_CONTEXT:`` record); the
image that appears later picks the context up at index time (the
task-03 disk scan) or, when the markdown file is gone from disk by
then, through the post-index enrichment pass (the stored doc rows —
record as the index, stored content as the source). Pinned here: the
record-then-image sync pair (the record stored, not cleared), the
pass-only path (markdown file deleted from disk, ``prune=False``), and
the existing-image case (the markdown's record stays NULL — the
task-03 path owns that case).

The unit-level mechanics (the prefix shape, the scan's combined string,
the seam forwarding, the changed-image re-scan, the record formatter +
parser, the pass's idempotency/fail-soft) are pinned in
``tests/unit/test_image_anchor_context.py``; the task-06 story suite
pins the E2E surface.
"""
from __future__ import annotations

import asyncio
import base64
import hashlib
from pathlib import Path

import pymupdf as fitz  # PyMuPDF — the temp-PDF fixture builder (phase 126)
from sqlalchemy import select

from app.config import Settings
from app.models import Chunk, Document
from app.rag.importer import import_sources
from app.rag.summarizer import DESCRIBE_PROMPT
from tests.fakes import FakeEmbedder

#: A real 1×1 transparent PNG (the phase-122 fixture — the importer is
#: content-agnostic, the bytes just need to be plausible image bytes).
PNG_1X1 = base64.b64decode(
    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGP4DwQACfsD/fteaysAAAAASUVORK5CYII="
)

DESCRIPTION = (
    "A network diagram of the homelab VLANs: the core switch, the "
    "router, and three labeled subnets."
)

#: The exact context prefix the task-03 extension appends to the locked
#: prompt (``app.rag.summarizer.describe_image``).
CONTEXT_PREFIX = "\n\nThis image is referenced in documentation with the following context:\n"


class _MockVisionLLM(FakeEmbedder):
    """The phase-122 mock VISION client: the CHAT model answers the
    multimodal describe call with the fixed :data:`DESCRIPTION` (the
    text part is recorded — ``text_parts``); text (``lite``) calls keep
    the ``FakeEmbedder`` behaviour. The REAL
    ``import_sources`` → ``_describe_or_skip`` → ``describe_image``
    chain runs against it (no seam patch)."""

    DESCRIPTION = DESCRIPTION

    def __init__(self) -> None:
        super().__init__()
        self.chat_models: list[str | None] = []
        self.text_parts: list[str] = []

    async def chat(self, messages, model=None):
        self.chat_calls.append(list(messages))
        self.chat_models.append(model)
        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
        if isinstance(user, list):
            self.text_parts.append(user[0]["text"])
            return self.DESCRIPTION
        first = user.split()
        return "Summary of " + (first[0] if first else "<empty>")


def _image_llm[
    LLM: FakeEmbedder
](tmp_path, llm_cls: type[LLM] = _MockVisionLLM, **kwargs) -> LLM:
    """A fake LLM with the phase-122 image knobs (toggle ON; the image
    dir defaults under *tmp_path*). *llm_cls* may be the mock vision
    client (the default — ``text_parts``/``chat_models`` state)."""
    kwargs.setdefault("_env_file", None)
    kwargs.setdefault("images", True)
    kwargs.setdefault("image_dir", str(tmp_path / "images"))
    llm = llm_cls()
    llm.settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return llm


def _cleanup_source(db, source: str) -> None:
    for doc in db.scalars(select(Document).where(Document.source == source)).all():
        db.delete(doc)
    db.commit()


#: A markdown file that references ``pic.png`` on line 3 — one context
#: paragraph before and one after (the task-02 window).
GUIDE_MD = (
    "The restore order diagram explains the backup procedure.\n"
    "\n"
    "![Restore order](pic.png)\n"
    "\n"
    "Only run the restore script after the checks pass.\n"
)


def _expected_block(source: str) -> str:
    """The task-03-shape combined string *GUIDE_MD* yields for
    ``<source>/guide.md`` (the disk scan AND the task-04 pass build
    exactly this)."""
    return (
        f"Referenced in {source}/guide.md:\n"
        '- Alt text: "Restore order"\n'
        "  Context: The restore order diagram explains the backup procedure.\n"
        "\n"
        "Only run the restore script after the checks pass."
    )


#: The ``IMAGE_CONTEXT:`` record the markdown-side bookkeeping (phase
#: 125, task 04, work 1) stores for *GUIDE_MD* — the reference on line
#: 3, 2 context paragraphs.
RECORD = (
    "IMAGE_CONTEXT:\n"
    '- [pic.png] (line 3): "Restore order" — 2 paragraphs of context'
)


def test_markdown_referenced_image_gets_context_and_enriched_chunks(
    db, tmp_path
) -> None:
    """A markdown file referencing an image in the same source → sync
    with ``images=True`` (the REAL seam — no patch) → the vision call
    carries the context after the locked prompt, the image doc's
    ``image_context`` is set, its ``content`` includes the context
    (context first, then the description), and its chunks are embedded
    from the enriched content (the ``is_summary`` chunk mirrors it)."""
    source_root = tmp_path / "anchoredkb"
    source_root.mkdir()
    (source_root / "guide.md").write_text(
        "The backup procedure diagram below shows the restore order.\n"
        "\n"
        "![Backup procedure diagram](diagram.png)\n"
        "\n"
        "Run the restore script only after the diagram's checks pass.\n",
        encoding="utf-8",
    )
    (source_root / "diagram.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path)
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)

        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "diagram.png"
            )
        )
        assert doc is not None and doc.is_image is True and doc.image_path is not None

        expected_context = (
            f"Referenced in {source_root.name}/guide.md:\n"
            '- Alt text: "Backup procedure diagram"\n'
            "  Context: The backup procedure diagram below shows the restore order.\n"
            "\n"
            "Run the restore script only after the diagram's checks pass."
        )
        # The vision model received the context — appended AFTER the
        # locked prompt (the ``IMAGE_DESCRIPTION_MODE`` marker is
        # untouched). The describe call (the ONLY multimodal one) went
        # to the CHAT model (``chat_models`` also carries the markdown
        # doc's ``lite`` summary call — the phase-30 path).
        assert llm.chat_models[0] == llm.settings.llm_chat_model
        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_context}"]

        # ``image_context`` stores the combined context (task 05's
        # retrieval display reads from here).
        assert doc.image_context == expected_context
        # The enriched content — context FIRST, then the description —
        # IS the summary (the phase-122 convention) and the chunks are
        # embedded from it (768 dims, the summary chunk mirrors it). The
        # plain-text packer joins the enriched paragraphs with single
        # newlines (the chunker's normalization, PLAN §5), so the content
        # chunk carries the packed form and the summary chunk the raw
        # text.
        expected_content = f"{expected_context}\n\n{DESCRIPTION}"
        assert doc.content == expected_content
        assert doc.summary == expected_content
        packed_content = "\n".join(expected_content.split("\n\n"))
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
        content_chunks = [c for c in chunks if not c.is_summary]
        assert [c.content for c in content_chunks] == [packed_content]
        summary_chunks = [c for c in chunks if c.is_summary]
        assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
        assert summary_chunks[0].content == expected_content
        # Both enriched forms reached the embedder (the markdown doc's
        # two texts complete the four embedded strings).
        embedded = [t for batch in llm.calls for t in batch]
        assert len(embedded) == 4
        assert packed_content in embedded and expected_content in embedded
        for c in chunks:
            assert c.embedding is not None and len(c.embedding) == 768

        # The referencing markdown stays a plain text doc (the
        # ``image_context`` column is an image-doc field — NULL).
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.is_image is False and md.image_context is None
    finally:
        _cleanup_source(db, source_root.name)


def test_standalone_image_keeps_null_context_and_bare_description(db, tmp_path) -> None:
    """A standalone image (no markdown reference in the source — a
    sibling markdown exists but references nothing) → ``image_context``
    NULL, ``content`` the bare description, the vision message the bare
    locked prompt — byte-identical to pre-phase-125 (the chunk set and
    the embedded text are exactly the phase-122 shape)."""
    source_root = tmp_path / "standalonekb"
    source_root.mkdir()
    (source_root / "notes.md").write_text("# Notes\n\nBody text.\n", encoding="utf-8")
    (source_root / "diagram.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path)
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)

        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "diagram.png"
            )
        )
        assert doc is not None and doc.is_image is True
        assert doc.image_context is None, "standalone: the context stays NULL"
        assert doc.content == DESCRIPTION, "standalone: the bare description"
        assert doc.summary == DESCRIPTION
        assert llm.text_parts == [DESCRIBE_PROMPT], "standalone: the bare prompt"
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
        assert all(c.content == DESCRIPTION for c in chunks)
        for c in chunks:
            assert c.embedding is not None and len(c.embedding) == 768
    finally:
        _cleanup_source(db, source_root.name)


# ---------------------------------------------------------------------------
# Phase 125, task 04 — the markdown-FIRST case: the ``IMAGE_CONTEXT:``
# record on the markdown doc + the post-index enrichment pass.
# ---------------------------------------------------------------------------


def test_markdown_indexed_before_image_stores_record_and_image_picks_it_up(
    db, tmp_path
) -> None:
    """Task 04's primary scenario — a markdown is indexed while the
    image it references does not yet exist: the reference is stored on
    the markdown doc's ``image_context`` (the ``IMAGE_CONTEXT:``
    record). The image file then appears and is synced: the image doc
    is enriched with the markdown's context (the task-03 disk scan —
    the markdown is still on disk), and the markdown's record is NOT
    cleared (ASSUMPTION 4)."""
    source_root = tmp_path / "beforekb"
    source_root.mkdir()
    (source_root / "guide.md").write_text(GUIDE_MD, encoding="utf-8")
    llm = _image_llm(tmp_path)
    try:
        # Run 1 — the markdown alone: the record is stored, no image
        # doc exists.
        s1 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s1.files, s1.added) == (1, 1)
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.image_context == RECORD
        assert (
            db.scalar(
                select(Document).where(
                    Document.source == source_root.name, Document.path == "pic.png"
                )
            )
            is None
        )

        # Run 2 — the image appears: enriched at index time (the
        # task-03 disk scan sees the on-disk markdown), the record
        # survives.
        (source_root / "pic.png").write_bytes(PNG_1X1)
        s2 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s2.added, s2.unchanged) == (1, 1)
        block = _expected_block(source_root.name)
        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pic.png"
            )
        )
        assert doc is not None and doc.is_image is True
        assert doc.image_context == block
        expected_content = f"{block}\n\n{DESCRIPTION}"
        assert doc.content == expected_content
        assert doc.summary == expected_content
        # The describe call carried the context after the locked prompt.
        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{block}"]
        md2 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md2 is not None and md2.image_context == RECORD, "the record is NOT cleared"
    finally:
        _cleanup_source(db, source_root.name)


def test_enrichment_pass_enriches_when_markdown_file_is_gone_from_disk(
    db, tmp_path
) -> None:
    """The post-index enrichment pass (task 04, work 3) — the case the
    task-03 disk scan CANNOT cover: the markdown is indexed first (the
    record lands on its doc row), its file is then removed from disk
    (``prune=False`` keeps the row), and only then does the image
    appear. The image is indexed BARE (the disk scan finds no
    reference), and the pass enriches it from the markdown doc's
    STORED rows: the record identifies the referencing markdown, the
    reference is re-extracted from the stored content, and the
    re-describe goes out WITH the context (a second vision call — the
    first, at index time, was bare)."""
    source_root = tmp_path / "passkb"
    source_root.mkdir()
    (source_root / "guide.md").write_text(GUIDE_MD, encoding="utf-8")
    llm = _image_llm(tmp_path)
    try:
        s1 = asyncio.run(import_sources([source_root], llm, session=db))
        assert s1.added == 1
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.image_context == RECORD

        # The markdown file disappears from disk (``prune=False`` keeps
        # the row); the image appears.
        (source_root / "guide.md").unlink()
        (source_root / "pic.png").write_bytes(PNG_1X1)
        s2 = asyncio.run(import_sources([source_root], llm, session=db))
        # The image is added (bare) and then enriched by the pass
        # (updated) — both in this run.
        assert (s2.files, s2.added, s2.updated) == (1, 1, 1)
        block = _expected_block(source_root.name)
        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pic.png"
            )
        )
        assert doc is not None and doc.is_image is True
        assert doc.image_context == block
        expected_content = f"{block}\n\n{DESCRIPTION}"
        assert doc.content == expected_content
        assert doc.summary == expected_content
        # The re-embedded pipeline (the pass replaced the bare chunks).
        packed = "\n".join(expected_content.split("\n\n"))
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
        assert [c.content for c in chunks if not c.is_summary] == [packed]
        assert [c.content for c in chunks if c.is_summary] == [expected_content]
        # Two vision calls: the bare index-time describe (the disk scan
        # saw no reference) and the pass's context-carrying re-describe.
        assert llm.text_parts == [DESCRIBE_PROMPT, f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{block}"]
        # The markdown doc's record survives (ASSUMPTION 4).
        md2 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md2 is not None and md2.image_context == RECORD
    finally:
        _cleanup_source(db, source_root.name)


def test_markdown_referencing_existing_image_keeps_null_record(db, tmp_path) -> None:
    """An EXISTING image doc (indexed before the markdown ever
    existed): the markdown's record stays NULL when it is indexed (the
    task-03 path owns that case — enrichment at IMAGE-index time), the
    unchanged image is not re-scanned in that run, and the image picks
    up the context on its NEXT re-index (bytes changed) via the
    task-03 disk scan — the markdown's record is NULL throughout."""
    source_root = tmp_path / "existingkb"
    source_root.mkdir()
    (source_root / "pic.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path)
    try:
        s1 = asyncio.run(import_sources([source_root], llm, session=db))
        assert s1.added == 1

        # Run 2 — the markdown appears (the image is unchanged).
        (source_root / "guide.md").write_text(GUIDE_MD, encoding="utf-8")
        s2 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s2.added, s2.unchanged) == (1, 1)
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.image_context is None
        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pic.png"
            )
        )
        assert doc is not None and doc.image_context is None and doc.content == DESCRIPTION

        # Run 3 — the image changes: the re-index re-scans the on-disk
        # markdown (task 03) and enriches; the record is still NULL.
        (source_root / "pic.png").write_bytes(PNG_1X1 + b"changed")
        s3 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s3.added, s3.updated, s3.unchanged) == (0, 1, 1)
        block = _expected_block(source_root.name)
        doc3 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pic.png"
            )
        )
        assert doc3 is not None and doc3.image_context == block
        assert doc3.content == f"{block}\n\n{DESCRIPTION}"
        md3 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md3 is not None and md3.image_context is None
    finally:
        _cleanup_source(db, source_root.name)


# ---------------------------------------------------------------------------
# Phase 126 — PDF indexing through the REAL import pipeline (the real
# ``app.rag.pdf`` extractor, the page-aware chunker, the REAL
# ``_describe_or_skip`` → ``describe_image`` seam — NO seam patch).
# ---------------------------------------------------------------------------

PDF_TITLE = "Quarterly Homelab Report"

PAGE_ONE = "The core switch firmware was upgraded to the stable channel."
PAGE_TWO = "The backup schedule runs nightly at 02:00 and keeps seven rotations."


def _png_bytes(width: int, height: int) -> bytes:
    pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height), 0)
    pix.clear_with(200)
    return pix.tobytes("png")


def _build_pdf(
    path, pages: list[tuple[str, list[bytes]]], title: str | None = None
) -> None:
    """One temp PDF (built with PyMuPDF — no binary fixtures): *pages* is
    (page text, embedded-image bytes) per page; *title* sets the metadata
    title (``None`` = leave it empty)."""
    doc = fitz.open()
    for text, imgs in pages:
        page = doc.new_page()
        if text:
            page.insert_text((72, 72), text)
        y = 150
        for data in imgs:
            pix = fitz.Pixmap(data)
            page.insert_image(
                fitz.Rect(72, y, 72 + pix.width, y + pix.height), stream=data
            )
            y += pix.height + 10
    if title is not None:
        doc.set_metadata({"title": title})
    doc.save(path)
    doc.close()


def test_pdf_becomes_document_with_text_chunks_embedding_and_summary(
    db, tmp_path
) -> None:
    """The phase-126 primary scenario: a PDF in a source (the ``pdf``
    import token) → sync → the document carries the extracted text
    (the ``---PAGE-N---`` markers, A2 — the title is the metadata
    title), page-anchored chunks with 768-dim embeddings, and the
    phase-30/118 lite summary. With ``images=True`` (A3) the embedded
    image joins the pipeline through the REAL seam — one vision call on
    the CHAT model with the bare locked prompt (no markdown reference
    exists — ``image_context`` stays NULL), the copy in ``image_dir``,
    and the description as the image doc's content AND summary."""
    source_root = tmp_path / "pdfkb"
    source_root.mkdir()
    png = _png_bytes(40, 30)
    _build_pdf(
        source_root / "report.pdf",
        [(PAGE_ONE, []), (PAGE_TWO, [png])],
        title=PDF_TITLE,
    )
    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0)
        assert summary.formats == {"pdf": 1}

        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "report.pdf"
            )
        )
        assert doc is not None and doc.is_image is False
        assert doc.title == PDF_TITLE
        assert doc.content == f"---PAGE-1---\n{PAGE_ONE}\n\n---PAGE-2---\n{PAGE_TWO}"
        chunks = db.scalars(
            select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
        ).all()
        content_chunks = [c for c in chunks if not c.is_summary]
        assert [c.content for c in content_chunks] == [
            f"Page 1:\n{PAGE_ONE}",
            f"Page 2:\n{PAGE_TWO}",
        ]
        for c in chunks:
            assert c.embedding is not None and len(c.embedding) == 768
        summary_chunks = [c for c in chunks if c.is_summary]
        assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
        assert doc.summary is not None
        assert doc.summary.endswith(f"Source: {source_root.name}/report.pdf")

        # The embedded image (A3) — through the REAL describe seam.
        image_rel = "report.pdf#page=2#img=0.png"
        img = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == image_rel
            )
        )
        assert img is not None and img.is_image is True
        assert img.image_context is None, "no markdown reference → bare prompt"
        assert img.content == DESCRIPTION and img.summary == DESCRIPTION
        copy = Path(img.image_path)
        assert copy.parent == Path(llm.settings.image_dir).expanduser()
        assert copy.name == f"{img.id}.png"
        assert copy.read_bytes() == png
        # The vision call went to the CHAT model with the BARE locked
        # prompt (the PDF's lite summary is the other chat call).
        assert llm.chat_models == [
            llm.settings.llm_summary_model, llm.settings.llm_chat_model
        ]
        assert llm.text_parts == [DESCRIBE_PROMPT]
    finally:
        _cleanup_source(db, source_root.name)


def test_pdf_embedded_images_skipped_when_images_off(db, tmp_path) -> None:
    """With ``images=False`` (the DEFAULT): the PDF text is indexed
    normally and the embedded images are skipped — no image documents,
    no ``images_failed`` (a skip is not a failure), no copies, no vision
    calls."""
    source_root = tmp_path / "pdfoffkb"
    source_root.mkdir()
    _build_pdf(
        source_root / "report.pdf",
        [(PAGE_ONE, [_png_bytes(40, 30)]), (PAGE_TWO, [])],
    )
    llm = _image_llm(tmp_path, images=False, import_extensions="md,txt,pdf")
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0)
        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "report.pdf"
            )
        )
        assert doc is not None and PAGE_ONE in doc.content
        chunks = db.scalars(
            select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
        ).all()
        assert any(c.embedding is not None for c in chunks)
        assert (
            db.scalar(
                select(Document).where(
                    Document.source == source_root.name,
                    Document.is_image.is_(True),
                )
            )
            is None
        ), "images off: no embedded-image documents"
        # No vision call at all — only the PDF's lite summary.
        assert llm.chat_models == [llm.settings.llm_summary_model]
    finally:
        _cleanup_source(db, source_root.name)


def test_pdf_without_embedded_images_and_unchanged_rerun(db, tmp_path) -> None:
    """A PDF without embedded images → no image documents; an unchanged
    re-sync → ``unchanged`` (no re-extract, no re-embed, no vision or
    embed calls), and a second run never creates image documents out of
    nothing."""
    source_root = tmp_path / "pdfnonekb"
    source_root.mkdir()
    _build_pdf(
        source_root / "notes.pdf",
        [(PAGE_ONE, []), (PAGE_TWO, [])],
    )
    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
    try:
        s1 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s1.files, s1.added, s1.unchanged) == (1, 1, 0)
        assert (
            db.scalar(
                select(Document).where(
                    Document.source == source_root.name,
                    Document.is_image.is_(True),
                )
            )
            is None
        )
        calls1, chat1 = len(llm.calls), len(llm.chat_models)
        s2 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s2.files, s2.added, s2.updated, s2.unchanged) == (1, 0, 0, 1)
        assert len(llm.calls) == calls1 and len(llm.chat_models) == chat1
    finally:
        _cleanup_source(db, source_root.name)


def test_pdf_without_text_becomes_an_empty_document(db, tmp_path) -> None:
    """A PDF with no extractable text (image-only, ``images`` OFF — the
    embedded image is invisible) → the document is still created, with
    EMPTY content (the empty byte string's digest), NO content chunks
    and NO content embedding (there is no text to embed — the only
    chunk is the best-effort ``lite`` summary's ``is_summary`` row), and
    the warning line is the signal."""
    source_root = tmp_path / "pdfemptykb"
    source_root.mkdir()
    _build_pdf(source_root / "pics.pdf", [("", [_png_bytes(40, 30)])])
    llm = _image_llm(tmp_path, images=False, import_extensions="md,txt,pdf")
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0)

        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pics.pdf"
            )
        )
        assert doc is not None and doc.is_image is False
        assert doc.content == ""
        assert doc.content_hash == hashlib.sha256(b"").hexdigest()
        chunks = db.scalars(
            select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
        ).all()
        assert [c for c in chunks if not c.is_summary] == [], (
            "no text → no content chunks (and no content embedding)"
        )
        # The phase-30/118 path still runs best-effort on the empty
        # content — the summary is the only chunk, and it is embedded.
        assert len(chunks) == 1 and chunks[0].is_summary
        assert chunks[0].embedding is not None and len(chunks[0].embedding) == 768
        assert doc.summary is not None
    finally:
        _cleanup_source(db, source_root.name)


def test_pdf_embedded_image_survives_prune_and_leaves_with_the_pdf(
    db, tmp_path
) -> None:
    """Phase 126 (A3) prune parent rule — the defect the E2E surfaced:
    the embedded image doc's synthetic path NEVER appears in the walk's
    ``seen``, so the plain prune rule destroyed it in the very sync that
    (re)indexed it (the toggle-on half of the phase-122 "invisible, not
    deleted" guard was missing). Now: with the parent PDF's rel still in
    the walk — even UNCHANGED, so no image pass runs at all — the image
    doc survives (0 pruned); when the PDF LEAVES the source, the image
    doc is pruned with it (row + ``image_dir`` copy — no orphans)."""
    source_root = tmp_path / "pdfprune"
    source_root.mkdir()
    png = _png_bytes(40, 30)
    _build_pdf(
        source_root / "report.pdf",
        [(PAGE_ONE, []), (PAGE_TWO, [png])],
    )
    image_rel = "report.pdf#page=2#img=0.png"
    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
    try:
        s1 = asyncio.run(
            import_sources([source_root], llm, session=db, prune=True)
        )
        assert (s1.added, s1.pruned) == (2, 0)
        img1 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == image_rel
            )
        )
        assert img1 is not None and img1.image_path is not None
        copy = Path(img1.image_path)

        # Unchanged re-sync (prune ON): the PDF's text digest matches →
        # no image pass at all — the image doc survives anyway (the
        # parent's rel is in the walk's seen set).
        s2 = asyncio.run(
            import_sources([source_root], llm, session=db, prune=True)
        )
        assert (s2.unchanged, s2.pruned) == (1, 0)
        assert db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == image_rel
            )
        ) is not None, "parent PDF present → the image doc survives the prune"

        # The PDF LEAVES the source: the image doc is pruned WITH it
        # (row + copy — no orphan in image_dir).
        (source_root / "report.pdf").unlink()
        s3 = asyncio.run(
            import_sources([source_root], llm, session=db, prune=True)
        )
        assert (s3.pruned, s3.unchanged) == (2, 0)
        assert db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == image_rel
            )
        ) is None, "parent PDF gone → the image doc is pruned with it"
        assert not copy.exists(), "the pruned image's copy leaves image_dir"
    finally:
        _cleanup_source(db, source_root.name)


# --- Phase 125 parity: a markdown file referencing a PDF-embedded image ---

#: A markdown file referencing the PDF's embedded image by its SYNTHETIC
#: path (``<pdf path>#page=<N>#img=<index>.<ext>`` — the doc's ``path``
#: is the upsert key, and the reference resolves to it through the shared
#: ``_resolve_ref_path`` rule) — one context paragraph before, one after.
PDF_GUIDE_MD = (
    "The VLAN map diagram in the report shows the subnet layout.\n"
    "\n"
    "![VLAN map](report.pdf#page=2#img=0.png)\n"
    "\n"
    "Only expand the lab subnet after the VLAN map is reviewed.\n"
)

#: The same reference against ``alpha.pdf`` (the PDF-FIRST ordering's
#: fixture — ``alpha.pdf`` sorts before ``zguide.md`` in the walk).
PDF_ZGUIDE_MD = PDF_GUIDE_MD.replace("report.pdf", "alpha.pdf")


def _pdf_expected_block(source: str, md_name: str, pdf_name: str) -> str:
    """The task-03-shape combined string for the PDF reference (the disk
    scan builds exactly this — the standalone image's shape, the
    synthetic path as the image identity)."""
    return (
        f"Referenced in {source}/{md_name}:\n"
        '- Alt text: "VLAN map"\n'
        "  Context: The VLAN map diagram in the report shows the subnet layout.\n"
        "\n"
        "Only expand the lab subnet after the VLAN map is reviewed."
    )


def test_pdf_embedded_image_referenced_by_markdown_markdown_first(
    db, tmp_path
) -> None:
    """The MARKDOWN-FIRST ordering (``guide.md`` sorts before
    ``report.pdf``): the markdown is indexed while its referenced image
    has no document yet → it records the reference on its OWN row (the
    ``IMAGE_CONTEXT:`` record — phase 125, task 04, work 1, never
    cleared); the PDF is indexed next and the task-03 disk scan (the
    standalone image's parity path — the markdown is still on disk) finds
    the reference, so the image doc is enriched AT INDEX TIME: the vision
    call carries the context after the locked prompt, ``image_context``
    is set, ``content``/``summary`` are ``<context>\\n\\n<description>``.
    The post-index pass then skips the already-enriched doc (idempotent).
    Final state: the record AND the enrichment coexist."""
    source_root = tmp_path / "pdfmdfirst"
    source_root.mkdir()
    (source_root / "guide.md").write_text(PDF_GUIDE_MD, encoding="utf-8")
    _build_pdf(
        source_root / "report.pdf",
        [(PAGE_ONE, []), (PAGE_TWO, [_png_bytes(40, 30)])],
    )
    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        # guide.md + report.pdf + the embedded image doc: all added.
        assert (summary.files, summary.added, summary.images_failed) == (2, 3, 0)

        expected_block = _pdf_expected_block(source_root.name, "guide.md", "report.pdf")

        # Work 1 — the markdown doc STORES the reference (the record —
        # line 3 is the reference, 2 context paragraphs). It is the
        # post-index pass's reference index and the retrieval display's
        # record; the pass does not clear it on success.
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.is_image is False
        assert md.image_context == (
            "IMAGE_CONTEXT:\n"
            '- [report.pdf#page=2#img=0.png] (line 3): "VLAN map"'
            " — 2 paragraphs of context"
        )

        # The enrichment (the disk scan at PDF index time) — the image
        # doc's shape is the standalone image's enriched shape.
        image_rel = "report.pdf#page=2#img=0.png"
        img = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == image_rel
            )
        )
        assert img is not None and img.is_image is True
        assert img.image_context == expected_block
        expected_content = f"{expected_block}\n\n{DESCRIPTION}"
        assert img.content == expected_content
        assert img.summary == expected_content
        # The vision call carried the context AFTER the locked prompt
        # (the multimodal part list's text part — the describe seam's
        # phase-125 extension).
        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_block}"]
        # Chat call order (walk order): guide.md's ``lite`` summary,
        # the PDF's ``lite`` summary, then the image's CHAT-model
        # description (the ONLY multimodal call).
        assert llm.chat_models == [
            llm.settings.llm_summary_model,
            llm.settings.llm_summary_model,
            llm.settings.llm_chat_model,
        ]
        # The chunks are embedded from the ENRICHED content (the
        # plain-text packer joins the enriched paragraphs with single
        # newlines; the summary chunk mirrors the raw content).
        packed = "\n".join(expected_content.split("\n\n"))
        chunks = db.scalars(
            select(Chunk).where(Chunk.document_id == img.id).order_by(Chunk.position)
        ).all()
        assert [c.content for c in chunks if not c.is_summary] == [packed]
        summary_chunks = [c for c in chunks if c.is_summary]
        assert len(summary_chunks) == 1 and summary_chunks[0].content == expected_content
        for c in chunks:
            assert c.embedding is not None and len(c.embedding) == 768
    finally:
        _cleanup_source(db, source_root.name)


def test_pdf_embedded_image_referenced_by_markdown_pdf_first(
    db, tmp_path
) -> None:
    """The PDF-FIRST ordering (``alpha.pdf`` sorts before ``zguide.md``
    in the walk): the PDF (and its image doc) is indexed while the
    markdown is still un-indexed — the task-03 disk scan sees the
    reference on disk and enriches the image AT INDEX TIME (the ordering
    that, without the parity scan, would ship an un-enriched image — the
    markdown's record stays NULL: at markdown index time the image doc
    ALREADY exists, so the task-03 path owns the case and nothing is
    recorded). Same final state as the markdown-first test — the
    enrichment never depends on the walk order."""
    source_root = tmp_path / "pdfpdffirst"
    source_root.mkdir()
    (source_root / "zguide.md").write_text(PDF_ZGUIDE_MD, encoding="utf-8")
    _build_pdf(
        source_root / "alpha.pdf",
        [(PAGE_ONE, []), (PAGE_TWO, [_png_bytes(40, 30)])],
    )
    llm = _image_llm(tmp_path, import_extensions="md,txt,pdf")
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (2, 3, 0)

        expected_block = _pdf_expected_block(source_root.name, "zguide.md", "alpha.pdf")

        # The task-03 path owns this case: the image doc existed when
        # the markdown was indexed → NO record (the field is the
        # pending-enrichment index, not a reference dump).
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "zguide.md"
            )
        )
        assert md is not None and md.image_context is None

        # The image doc is enriched (the disk scan at PDF index time) —
        # byte-identical in shape to the markdown-first final state.
        image_rel = "alpha.pdf#page=2#img=0.png"
        img = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == image_rel
            )
        )
        assert img is not None and img.image_context == expected_block
        expected_content = f"{expected_block}\n\n{DESCRIPTION}"
        assert img.content == expected_content
        assert img.summary == expected_content
        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_block}"]
        # Chat call order (walk order): the PDF's ``lite`` summary, the
        # image's CHAT-model description, then zguide.md's summary.
        assert llm.chat_models == [
            llm.settings.llm_summary_model,
            llm.settings.llm_chat_model,
            llm.settings.llm_summary_model,
        ]
    finally:
        _cleanup_source(db, source_root.name)