brain-of-reese/tests/integration/test_importer.py

"""Integration: phase 125, task 03 — the image-anchor enrichment through
the REAL import pipeline (real compose Postgres, the real
``_describe_or_skip`` → ``describe_image`` seam, the mock vision
client — NO seam patch).

A markdown file referencing an image in the SAME source → the vision
call receives the markdown context (appended after the locked
``DESCRIBE_PROMPT``), the image doc's ``image_context`` is set, its
``content``/``summary`` is ``<context>\\n\\n<description>``, and its
chunks are embedded from the enriched content. A standalone image (no
markdown reference) → ``image_context`` NULL, ``content`` the bare
description, the bare locked prompt — byte-identical to pre-phase-125.

Phase 125, task 04 (the markdown-FIRST case) — a markdown indexed while
its referenced image has no document yet stores the reference on the
markdown doc's ``image_context`` (the ``IMAGE_CONTEXT:`` record); the
image that appears later picks the context up at index time (the
task-03 disk scan) or, when the markdown file is gone from disk by
then, through the post-index enrichment pass (the stored doc rows —
record as the index, stored content as the source). Pinned here: the
record-then-image sync pair (the record stored, not cleared), the
pass-only path (markdown file deleted from disk, ``prune=False``), and
the existing-image case (the markdown's record stays NULL — the
task-03 path owns that case).

The unit-level mechanics (the prefix shape, the scan's combined string,
the seam forwarding, the changed-image re-scan, the record formatter +
parser, the pass's idempotency/fail-soft) are pinned in
``tests/unit/test_image_anchor_context.py``; the task-06 story suite
pins the E2E surface.
"""
from __future__ import annotations

import asyncio
import base64

from sqlalchemy import select

from app.config import Settings
from app.models import Chunk, Document
from app.rag.importer import import_sources
from app.rag.summarizer import DESCRIBE_PROMPT
from tests.fakes import FakeEmbedder

#: A real 1×1 transparent PNG (the phase-122 fixture — the importer is
#: content-agnostic, the bytes just need to be plausible image bytes).
PNG_1X1 = base64.b64decode(
    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGP4DwQACfsD/fteaysAAAAASUVORK5CYII="
)

DESCRIPTION = (
    "A network diagram of the homelab VLANs: the core switch, the "
    "router, and three labeled subnets."
)

#: The exact context prefix the task-03 extension appends to the locked
#: prompt (``app.rag.summarizer.describe_image``).
CONTEXT_PREFIX = "\n\nThis image is referenced in documentation with the following context:\n"


class _MockVisionLLM(FakeEmbedder):
    """The phase-122 mock VISION client: the CHAT model answers the
    multimodal describe call with the fixed :data:`DESCRIPTION` (the
    text part is recorded — ``text_parts``); text (``lite``) calls keep
    the ``FakeEmbedder`` behaviour. The REAL
    ``import_sources`` → ``_describe_or_skip`` → ``describe_image``
    chain runs against it (no seam patch)."""

    DESCRIPTION = DESCRIPTION

    def __init__(self) -> None:
        super().__init__()
        self.chat_models: list[str | None] = []
        self.text_parts: list[str] = []

    async def chat(self, messages, model=None):
        self.chat_calls.append(list(messages))
        self.chat_models.append(model)
        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
        if isinstance(user, list):
            self.text_parts.append(user[0]["text"])
            return self.DESCRIPTION
        first = user.split()
        return "Summary of " + (first[0] if first else "<empty>")


def _image_llm[
    LLM: FakeEmbedder
](tmp_path, llm_cls: type[LLM] = _MockVisionLLM, **kwargs) -> LLM:
    """A fake LLM with the phase-122 image knobs (toggle ON; the image
    dir defaults under *tmp_path*). *llm_cls* may be the mock vision
    client (the default — ``text_parts``/``chat_models`` state)."""
    kwargs.setdefault("_env_file", None)
    kwargs.setdefault("images", True)
    kwargs.setdefault("image_dir", str(tmp_path / "images"))
    llm = llm_cls()
    llm.settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return llm


def _cleanup_source(db, source: str) -> None:
    for doc in db.scalars(select(Document).where(Document.source == source)).all():
        db.delete(doc)
    db.commit()


#: A markdown file that references ``pic.png`` on line 3 — one context
#: paragraph before and one after (the task-02 window).
GUIDE_MD = (
    "The restore order diagram explains the backup procedure.\n"
    "\n"
    "![Restore order](pic.png)\n"
    "\n"
    "Only run the restore script after the checks pass.\n"
)


def _expected_block(source: str) -> str:
    """The task-03-shape combined string *GUIDE_MD* yields for
    ``<source>/guide.md`` (the disk scan AND the task-04 pass build
    exactly this)."""
    return (
        f"Referenced in {source}/guide.md:\n"
        '- Alt text: "Restore order"\n'
        "  Context: The restore order diagram explains the backup procedure.\n"
        "\n"
        "Only run the restore script after the checks pass."
    )


#: The ``IMAGE_CONTEXT:`` record the markdown-side bookkeeping (phase
#: 125, task 04, work 1) stores for *GUIDE_MD* — the reference on line
#: 3, 2 context paragraphs.
RECORD = (
    "IMAGE_CONTEXT:\n"
    '- [pic.png] (line 3): "Restore order" — 2 paragraphs of context'
)


def test_markdown_referenced_image_gets_context_and_enriched_chunks(
    db, tmp_path
) -> None:
    """A markdown file referencing an image in the same source → sync
    with ``images=True`` (the REAL seam — no patch) → the vision call
    carries the context after the locked prompt, the image doc's
    ``image_context`` is set, its ``content`` includes the context
    (context first, then the description), and its chunks are embedded
    from the enriched content (the ``is_summary`` chunk mirrors it)."""
    source_root = tmp_path / "anchoredkb"
    source_root.mkdir()
    (source_root / "guide.md").write_text(
        "The backup procedure diagram below shows the restore order.\n"
        "\n"
        "![Backup procedure diagram](diagram.png)\n"
        "\n"
        "Run the restore script only after the diagram's checks pass.\n",
        encoding="utf-8",
    )
    (source_root / "diagram.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path)
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)

        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "diagram.png"
            )
        )
        assert doc is not None and doc.is_image is True and doc.image_path is not None

        expected_context = (
            f"Referenced in {source_root.name}/guide.md:\n"
            '- Alt text: "Backup procedure diagram"\n'
            "  Context: The backup procedure diagram below shows the restore order.\n"
            "\n"
            "Run the restore script only after the diagram's checks pass."
        )
        # The vision model received the context — appended AFTER the
        # locked prompt (the ``IMAGE_DESCRIPTION_MODE`` marker is
        # untouched). The describe call (the ONLY multimodal one) went
        # to the CHAT model (``chat_models`` also carries the markdown
        # doc's ``lite`` summary call — the phase-30 path).
        assert llm.chat_models[0] == llm.settings.llm_chat_model
        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_context}"]

        # ``image_context`` stores the combined context (task 05's
        # retrieval display reads from here).
        assert doc.image_context == expected_context
        # The enriched content — context FIRST, then the description —
        # IS the summary (the phase-122 convention) and the chunks are
        # embedded from it (768 dims, the summary chunk mirrors it). The
        # plain-text packer joins the enriched paragraphs with single
        # newlines (the chunker's normalization, PLAN §5), so the content
        # chunk carries the packed form and the summary chunk the raw
        # text.
        expected_content = f"{expected_context}\n\n{DESCRIPTION}"
        assert doc.content == expected_content
        assert doc.summary == expected_content
        packed_content = "\n".join(expected_content.split("\n\n"))
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
        content_chunks = [c for c in chunks if not c.is_summary]
        assert [c.content for c in content_chunks] == [packed_content]
        summary_chunks = [c for c in chunks if c.is_summary]
        assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
        assert summary_chunks[0].content == expected_content
        # Both enriched forms reached the embedder (the markdown doc's
        # two texts complete the four embedded strings).
        embedded = [t for batch in llm.calls for t in batch]
        assert len(embedded) == 4
        assert packed_content in embedded and expected_content in embedded
        for c in chunks:
            assert c.embedding is not None and len(c.embedding) == 768

        # The referencing markdown stays a plain text doc (the
        # ``image_context`` column is an image-doc field — NULL).
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.is_image is False and md.image_context is None
    finally:
        _cleanup_source(db, source_root.name)


def test_standalone_image_keeps_null_context_and_bare_description(db, tmp_path) -> None:
    """A standalone image (no markdown reference in the source — a
    sibling markdown exists but references nothing) → ``image_context``
    NULL, ``content`` the bare description, the vision message the bare
    locked prompt — byte-identical to pre-phase-125 (the chunk set and
    the embedded text are exactly the phase-122 shape)."""
    source_root = tmp_path / "standalonekb"
    source_root.mkdir()
    (source_root / "notes.md").write_text("# Notes\n\nBody text.\n", encoding="utf-8")
    (source_root / "diagram.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path)
    try:
        summary = asyncio.run(import_sources([source_root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)

        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "diagram.png"
            )
        )
        assert doc is not None and doc.is_image is True
        assert doc.image_context is None, "standalone: the context stays NULL"
        assert doc.content == DESCRIPTION, "standalone: the bare description"
        assert doc.summary == DESCRIPTION
        assert llm.text_parts == [DESCRIBE_PROMPT], "standalone: the bare prompt"
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
        assert all(c.content == DESCRIPTION for c in chunks)
        for c in chunks:
            assert c.embedding is not None and len(c.embedding) == 768
    finally:
        _cleanup_source(db, source_root.name)


# ---------------------------------------------------------------------------
# Phase 125, task 04 — the markdown-FIRST case: the ``IMAGE_CONTEXT:``
# record on the markdown doc + the post-index enrichment pass.
# ---------------------------------------------------------------------------


def test_markdown_indexed_before_image_stores_record_and_image_picks_it_up(
    db, tmp_path
) -> None:
    """Task 04's primary scenario — a markdown is indexed while the
    image it references does not yet exist: the reference is stored on
    the markdown doc's ``image_context`` (the ``IMAGE_CONTEXT:``
    record). The image file then appears and is synced: the image doc
    is enriched with the markdown's context (the task-03 disk scan —
    the markdown is still on disk), and the markdown's record is NOT
    cleared (ASSUMPTION 4)."""
    source_root = tmp_path / "beforekb"
    source_root.mkdir()
    (source_root / "guide.md").write_text(GUIDE_MD, encoding="utf-8")
    llm = _image_llm(tmp_path)
    try:
        # Run 1 — the markdown alone: the record is stored, no image
        # doc exists.
        s1 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s1.files, s1.added) == (1, 1)
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.image_context == RECORD
        assert (
            db.scalar(
                select(Document).where(
                    Document.source == source_root.name, Document.path == "pic.png"
                )
            )
            is None
        )

        # Run 2 — the image appears: enriched at index time (the
        # task-03 disk scan sees the on-disk markdown), the record
        # survives.
        (source_root / "pic.png").write_bytes(PNG_1X1)
        s2 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s2.added, s2.unchanged) == (1, 1)
        block = _expected_block(source_root.name)
        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pic.png"
            )
        )
        assert doc is not None and doc.is_image is True
        assert doc.image_context == block
        expected_content = f"{block}\n\n{DESCRIPTION}"
        assert doc.content == expected_content
        assert doc.summary == expected_content
        # The describe call carried the context after the locked prompt.
        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{block}"]
        md2 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md2 is not None and md2.image_context == RECORD, "the record is NOT cleared"
    finally:
        _cleanup_source(db, source_root.name)


def test_enrichment_pass_enriches_when_markdown_file_is_gone_from_disk(
    db, tmp_path
) -> None:
    """The post-index enrichment pass (task 04, work 3) — the case the
    task-03 disk scan CANNOT cover: the markdown is indexed first (the
    record lands on its doc row), its file is then removed from disk
    (``prune=False`` keeps the row), and only then does the image
    appear. The image is indexed BARE (the disk scan finds no
    reference), and the pass enriches it from the markdown doc's
    STORED rows: the record identifies the referencing markdown, the
    reference is re-extracted from the stored content, and the
    re-describe goes out WITH the context (a second vision call — the
    first, at index time, was bare)."""
    source_root = tmp_path / "passkb"
    source_root.mkdir()
    (source_root / "guide.md").write_text(GUIDE_MD, encoding="utf-8")
    llm = _image_llm(tmp_path)
    try:
        s1 = asyncio.run(import_sources([source_root], llm, session=db))
        assert s1.added == 1
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.image_context == RECORD

        # The markdown file disappears from disk (``prune=False`` keeps
        # the row); the image appears.
        (source_root / "guide.md").unlink()
        (source_root / "pic.png").write_bytes(PNG_1X1)
        s2 = asyncio.run(import_sources([source_root], llm, session=db))
        # The image is added (bare) and then enriched by the pass
        # (updated) — both in this run.
        assert (s2.files, s2.added, s2.updated) == (1, 1, 1)
        block = _expected_block(source_root.name)
        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pic.png"
            )
        )
        assert doc is not None and doc.is_image is True
        assert doc.image_context == block
        expected_content = f"{block}\n\n{DESCRIPTION}"
        assert doc.content == expected_content
        assert doc.summary == expected_content
        # The re-embedded pipeline (the pass replaced the bare chunks).
        packed = "\n".join(expected_content.split("\n\n"))
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
        assert [c.content for c in chunks if not c.is_summary] == [packed]
        assert [c.content for c in chunks if c.is_summary] == [expected_content]
        # Two vision calls: the bare index-time describe (the disk scan
        # saw no reference) and the pass's context-carrying re-describe.
        assert llm.text_parts == [DESCRIBE_PROMPT, f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{block}"]
        # The markdown doc's record survives (ASSUMPTION 4).
        md2 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md2 is not None and md2.image_context == RECORD
    finally:
        _cleanup_source(db, source_root.name)


def test_markdown_referencing_existing_image_keeps_null_record(db, tmp_path) -> None:
    """An EXISTING image doc (indexed before the markdown ever
    existed): the markdown's record stays NULL when it is indexed (the
    task-03 path owns that case — enrichment at IMAGE-index time), the
    unchanged image is not re-scanned in that run, and the image picks
    up the context on its NEXT re-index (bytes changed) via the
    task-03 disk scan — the markdown's record is NULL throughout."""
    source_root = tmp_path / "existingkb"
    source_root.mkdir()
    (source_root / "pic.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path)
    try:
        s1 = asyncio.run(import_sources([source_root], llm, session=db))
        assert s1.added == 1

        # Run 2 — the markdown appears (the image is unchanged).
        (source_root / "guide.md").write_text(GUIDE_MD, encoding="utf-8")
        s2 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s2.added, s2.unchanged) == (1, 1)
        md = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md is not None and md.image_context is None
        doc = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pic.png"
            )
        )
        assert doc is not None and doc.image_context is None and doc.content == DESCRIPTION

        # Run 3 — the image changes: the re-index re-scans the on-disk
        # markdown (task 03) and enriches; the record is still NULL.
        (source_root / "pic.png").write_bytes(PNG_1X1 + b"changed")
        s3 = asyncio.run(import_sources([source_root], llm, session=db))
        assert (s3.added, s3.updated, s3.unchanged) == (0, 1, 1)
        block = _expected_block(source_root.name)
        doc3 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "pic.png"
            )
        )
        assert doc3 is not None and doc3.image_context == block
        assert doc3.content == f"{block}\n\n{DESCRIPTION}"
        md3 = db.scalar(
            select(Document).where(
                Document.source == source_root.name, Document.path == "guide.md"
            )
        )
        assert md3 is not None and md3.image_context is None
    finally:
        _cleanup_source(db, source_root.name)