brain-of-reese/tests/unit/test_image_anchor_context.py

"""Unit: phase 125, task 03 — enriching image descriptions with markdown
context (LOCKED A1/A2/A3).

Task 03 is the PRIMARY enrichment path (LOCKED A2 — enrichment happens
at image-index time): when an image file is (re)indexed, the same
source's importable markdown files are scanned for ``![alt](path)``
references that name it (the task-02
:func:`app.rag.md_images.extract_image_refs`); when found, the combined
context string (``Referenced in <source>/<md>:`` + alt text +
surrounding paragraphs) is (a) appended AFTER the locked
``DESCRIBE_PROMPT`` in the vision call (the model learns what the
image is for — the description remains the primary content, LOCKED
A3) and (b) stored on the doc's ``image_context`` (task 05's retrieval
display), with the doc's ``content`` (and so its ``summary`` and
embedded chunks) becoming ``<context>\\n\\n<description>``. A
standalone image (no markdown reference) stays byte-identical to
pre-phase: ``image_context`` NULL, bare description, bare prompt.

Pinned here: the ``describe_image`` context extension (``None`` →
byte-identical message; provided → exact prefix after the locked
prompt; the fail-soft contract unchanged); the seam's context
forwarding; the context scan's combined-string shape (single/multiple
references, multiple files, non-markdown + hidden + fenced skips,
filename matching); and the full pipeline mechanics through
``import_sources`` (enriched content + stored context, the
standalone no-op, and the changed-image re-scan that re-scans — and
can drop — the context).

Phase 125, task 04 (the markdown-FIRST case, LOCKED A1) — a markdown
file (re)indexed while a reference's image has no document yet in the
source records those references on ITS OWN ``image_context`` (the
``IMAGE_CONTEXT:`` record: resolved path + line + alt text +
context-paragraph count), so the post-index enrichment pass can pick
them up when the image is later indexed. Pinned here: the record
formatter (exact shape, the empty-list → ``""`` contract) and its
parser (round-trip, the fail-soft on malformed input); the reference
resolution rule (root-absolute, nested, escaping); the bookkeeping
(no refs / all-exist → ``None``; missing → the record; re-indexed
markdown with the image now existing → back to ``None``); and the
enrichment pass itself (stored-rows context rebuild, the idempotency
+ selectivity, the fail-soft on a missing copy / failed re-description,
the counters). The task-06 story suite pins the E2E surface; the
task-02 extraction unit pins live in
``tests/unit/test_md_images.py``.
"""
from __future__ import annotations

import asyncio
import base64
from collections.abc import Callable
from pathlib import Path
from typing import Any

import pytest
from sqlalchemy import select

from app.config import Settings
from app.models import Chunk, Document
from app.rag.importer import (
    IMAGE_CONTEXT_HEADER,
    ImportSummary,
    _describe_or_skip,
    _enrich_pending_images,
    _format_missing_image_refs,
    _markdown_image_context,
    _missing_image_context,
    _parse_missing_image_refs,
    _pending_image_context,
    _resolve_ref_path,
    _source_root_for,
    import_sources,
)
from app.rag.llm import EmbeddingError, LLMError
from app.rag.md_images import ImageRef
from app.rag.summarizer import DESCRIBE_PROMPT, describe_image
from tests.fakes import FakeEmbedder

#: A real 1×1 transparent PNG (the phase-122 fixture — the importer is
#: content-agnostic, the bytes just need to be plausible image bytes).
PNG_1X1 = base64.b64decode(
    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGP4DwQACfsD/fteaysAAAAASUVORK5CYII="
)

DESCRIPTION = "A red square on a white background."

#: The exact context prefix the task-03 extension appends to the
#: locked prompt (pinned here AND in the summarizer's code). Phase 137
#: (task 02, ASSUMPTION A5) generalized the wording — it now covers
#: BOTH block kinds (the provenance's ``Extracted from …`` document
#: context and the phase-125 ``Referenced in …`` markdown references).
CONTEXT_PREFIX = (
    "\n\nThis image is part of a document. The following context "
    "describes where it comes from and what it is used for:\n"
)


def _settings(**kwargs: Any) -> Settings:
    """Build Settings without reading a .env file (deterministic tests)."""
    kwargs.setdefault("_env_file", None)
    return Settings(**kwargs)  # pyright: ignore[reportCallIssue]  (kwarg exists at runtime)


def _image_settings(tmp_path: Path, **kwargs: Any) -> Settings:
    """Phase-122 image settings (toggle ON; image dir under *tmp_path*)."""
    kwargs.setdefault("images", True)
    kwargs.setdefault("image_dir", str(tmp_path / "images"))
    return _settings(**kwargs)


def _image_llm[
    LLM: FakeEmbedder
](tmp_path: Path, llm_cls: type[LLM] = FakeEmbedder, **kwargs: Any) -> LLM:
    """A fake LLM whose settings carry the phase-122 image knobs."""
    llm = llm_cls()
    llm.settings = _image_settings(tmp_path, **kwargs)
    return llm


def _cleanup_source(db, source: str) -> None:
    for doc in db.scalars(select(Document).where(Document.source == source)).all():
        db.delete(doc)
    db.commit()


class _VisionMock(FakeEmbedder):
    """Chat-model mock for the ``describe_image`` / seam pins: the
    multimodal message is recorded (the text part is the assertion
    surface for the context prefix) and a fixed reply is returned (or
    the exception raised). Subclasses ``FakeEmbedder`` so it satisfies
    BOTH the summarizer's ``SummaryLLM`` protocol and the importer's
    ``Embedder`` protocol (the ``embed`` surface is inherited, unused
    here)."""

    def __init__(self, reply: str | Exception = DESCRIPTION) -> None:
        super().__init__()
        self.reply = reply
        self.messages: list[Any] = []
        self.text_parts: list[str] = []
        self.model: str | None = None

    async def chat(
        self, messages: list[dict[str, Any]], model: str | None = None
    ) -> str:
        self.messages = list(messages)
        self.model = model
        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
        if isinstance(user, list):
            self.text_parts = [part["text"] for part in user if part["type"] == "text"]
        if isinstance(self.reply, Exception):
            raise self.reply
        return self.reply


class _VisionEmbedder(FakeEmbedder):
    """Fake embedder + vision client for the ``import_sources`` pipeline
    pins: the multimodal describe call gets the fixed
    :data:`DESCRIPTION` (the text part is recorded — ``text_parts``),
    text (``lite``) calls keep the ``FakeEmbedder`` behaviour."""

    DESCRIPTION = DESCRIPTION

    def __init__(self) -> None:
        super().__init__()
        self.text_parts: list[str] = []

    async def chat(
        self, messages: list[dict[str, Any]], model: str | None = None
    ) -> str:
        self.chat_calls.append(list(messages))
        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
        if isinstance(user, list):
            self.text_parts.append(user[0]["text"])
            return self.DESCRIPTION
        first = user.split()
        return "Summary of " + (first[0] if first else "<empty>")


# ---------------------------------------------------------------------------
# The ``describe_image`` extension (summarizer) — the locked prompt stays
# byte-identical; the context is appended AFTER it.
# ---------------------------------------------------------------------------


def test_describe_image_without_context_is_byte_identical() -> None:
    """``context=None`` (the default — a standalone image) sends the BARE
    multimodal message: the text part is exactly ``DESCRIBE_PROMPT`` and
    the image part is unchanged — byte-identical to pre-phase-125."""
    mock = _VisionMock()
    assert (
        asyncio.run(describe_image(mock, data=PNG_1X1, mime="image/png")) == DESCRIPTION
    )
    assert mock.text_parts == [DESCRIBE_PROMPT]
    content = mock.messages[0]["content"]
    assert content[0] == {"type": "text", "text": DESCRIBE_PROMPT}
    b64 = base64.b64encode(PNG_1X1).decode("ascii")
    assert content[1] == {
        "type": "image_url",
        "image_url": {"url": f"data:image/png;base64,{b64}"},
    }
    assert mock.model == mock.settings.llm_chat_model


def test_describe_image_with_context_appends_after_the_locked_prompt() -> None:
    """``context`` is appended to the text part AFTER the locked
    ``DESCRIBE_PROMPT`` (marker + instruction stay byte-identical — the
    ``IMAGE_DESCRIPTION_MODE`` mock key is untouched): the exact task-03
    prefix, then the context. The rest of the message (the image part,
    the CHAT model) is unchanged."""
    context = 'Referenced in S/guide.md:\n- Alt text: "pic"\n  Context: why not'
    mock = _VisionMock()
    assert (
        asyncio.run(
            describe_image(mock, data=PNG_1X1, mime="image/png", context=context)
        )
        == DESCRIPTION
    )
    (text,) = mock.text_parts
    assert text == f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{context}"
    assert text.startswith(DESCRIBE_PROMPT), "the locked prompt is untouched"
    content = mock.messages[0]["content"]
    assert content[0] == {"type": "text", "text": text}
    assert content[1]["type"] == "image_url"
    assert mock.model == mock.settings.llm_chat_model


def test_describe_image_context_keeps_the_fail_soft_contract() -> None:
    """The context changes ONLY the text part: the reply is still
    stripped + capped at ``summary_max_chars``, and a client error still
    fails soft (``None``) — the context neither raises nor leaks."""
    long_reply = "x" * 500
    mock = _VisionMock(long_reply)
    capped = asyncio.run(
        describe_image(
            mock,
            data=PNG_1X1,
            mime="image/png",
            context="ctx",
            settings=_settings(summary_max_chars=100),
        )
    )
    assert capped == "x" * 100
    failing = _VisionMock(LLMError("simulated vision-model failure (test sentinel)"))
    assert (
        asyncio.run(
            describe_image(failing, data=PNG_1X1, mime="image/png", context="ctx")
        )
        is None
    )
    blank = _VisionMock("   \n")
    assert (
        asyncio.run(describe_image(blank, data=PNG_1X1, mime="image/png", context="ctx"))
        is None
    )


# ---------------------------------------------------------------------------
# The seam — ``_describe_or_skip`` forwards the context verbatim.
# ---------------------------------------------------------------------------


def test_describe_or_skip_forwards_the_context() -> None:
    """The seam passes ``context`` through to ``describe_image`` (the
    vision message's text part carries the prefix); without it the
    message stays the bare locked prompt."""
    ctx = "Referenced in S/guide.md:\n- Alt text: \"pic\""
    mock = _VisionMock()
    assert (
        asyncio.run(
            _describe_or_skip(
                mock,
                data=PNG_1X1,
                source="S",
                rel="pic.png",
                full_path=Path("/src/pic.png"),
                context=ctx,
            )
        )
        == DESCRIPTION
    )
    assert mock.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{ctx}"]

    mock_bare = _VisionMock()
    asyncio.run(
        _describe_or_skip(
            mock_bare,
            data=PNG_1X1,
            source="S",
            rel="pic.png",
            full_path=Path("/src/pic.png"),
        )
    )
    assert mock_bare.text_parts == [DESCRIBE_PROMPT]


# ---------------------------------------------------------------------------
# The context scan — ``_source_root_for`` + ``_markdown_image_context``.
# ---------------------------------------------------------------------------


def test_source_root_for_top_level_and_nested() -> None:
    """The walk's ``full_path == root / rel`` invariant: the root is
    *full_path* up by the number of DIRECTORY components of *rel*."""
    assert (
        _source_root_for(Path("/src/pic.png"), "pic.png") == Path("/src")
    )
    assert (
        _source_root_for(Path("/src/sub/dir/guide.md"), "sub/dir/guide.md")
        == Path("/src")
    )


def test_markdown_context_none_when_no_markdown_or_no_reference(
    tmp_path: Path,
) -> None:
    """No markdown at all / markdown with no references / markdown that
    references a DIFFERENT image / a fenced reference (literal text, not
    an image) → ``None`` (the standalone no-op)."""
    root = tmp_path / "src"
    root.mkdir()

    llm = _image_llm(tmp_path)
    assert _markdown_image_context(root, llm, source="src", image_rel="pic.png") is None

    (root / "plain.md").write_text("# Plain\n\nNo references here.\n", encoding="utf-8")
    assert _markdown_image_context(root, llm, source="src", image_rel="pic.png") is None

    (root / "other.md").write_text("See ![the other one](other.png) here.\n", encoding="utf-8")
    assert _markdown_image_context(root, llm, source="src", image_rel="pic.png") is None

    (root / "fenced.md").write_text(
        "A fence:\n\n```\n![literal](pic.png)\n```\n", encoding="utf-8"
    )
    assert _markdown_image_context(root, llm, source="src", image_rel="pic.png") is None


def test_markdown_context_ignores_non_markdown_and_hidden_files(
    tmp_path: Path,
) -> None:
    """Only the source's importable ``.md``/``.markdown`` files are
    scanned: a ``.txt`` reference and a hidden-dir markdown reference
    never contribute context (the A9 scope rules apply exactly as to
    the walk)."""
    root = tmp_path / "src"
    (root / ".hidden").mkdir(parents=True)
    (root / "notes.txt").write_text("![alt](pic.png)\n", encoding="utf-8")
    (root / ".hidden" / "guide.md").write_text("![alt](pic.png)\n", encoding="utf-8")
    llm = _image_llm(tmp_path)
    assert _markdown_image_context(root, llm, source="src", image_rel="pic.png") is None


def test_markdown_context_single_reference_exact_shape(tmp_path: Path) -> None:
    """ONE reference → the exact task-03 combined string: the
    ``Referenced in <source>/<md>:`` header, the ``- Alt text:`` bullet,
    the ``  Context:`` line (the task-02 context window: the paragraphs
    before + after the reference paragraph — here the heading counts as
    a paragraph)."""
    root = tmp_path / "src"
    root.mkdir()
    (root / "guide.md").write_text(
        "# Guide\n"
        "\n"
        "First paragraph explaining the diagram below.\n"
        "\n"
        "![The network diagram](pic.png)\n"
        "\n"
        "Second paragraph after the diagram.\n",
        encoding="utf-8",
    )
    llm = _image_llm(tmp_path)
    context = _markdown_image_context(root, llm, source="src", image_rel="pic.png")
    assert context == (
        "Referenced in src/guide.md:\n"
        '- Alt text: "The network diagram"\n'
        "  Context: # Guide\n"
        "\n"
        "First paragraph explaining the diagram below.\n"
        "\n"
        "Second paragraph after the diagram."
    )


def test_markdown_context_multiple_references_and_files_sorted(tmp_path: Path) -> None:
    """Multiple references to the SAME image in one markdown → one
    bullet pair per reference (in line order); references in MULTIPLE
    markdown files → one ``Referenced in`` block per file, in the
    walk's sorted order, joined with a blank line. A reference with no
    surrounding paragraphs gets NO ``Context:`` line; an empty alt text
    pins ``\"\"``."""
    root = tmp_path / "src"
    root.mkdir()
    (root / "a.md").write_text("![First](pic.png)\n", encoding="utf-8")  # no context
    (root / "b.md").write_text(
        "Intro paragraph.\n"
        "\n"
        "![Alt one](pic.png)\n"
        "\n"
        "Middle paragraph.\n"
        "\n"
        "![](pic.png)\n"
        "\n"
        "Outro paragraph.\n",
        encoding="utf-8",
    )
    llm = _image_llm(tmp_path)
    context = _markdown_image_context(root, llm, source="src", image_rel="pic.png")
    assert context == (
        "Referenced in src/a.md:\n"
        '- Alt text: "First"\n'
        "\n"
        "Referenced in src/b.md:\n"
        '- Alt text: "Alt one"\n'
        "  Context: Intro paragraph.\n"
        "\n"
        "Middle paragraph.\n"
        "\n"
        "![](pic.png)\n"
        '- Alt text: ""\n'
        "  Context: ![Alt one](pic.png)\n"
        "\n"
        "Middle paragraph.\n"
        "\n"
        "Outro paragraph."
    )


def test_markdown_context_resolves_reference_paths_to_the_image(tmp_path: Path) -> None:
    """Path-resolution matching: a reference is resolved against the
    markdown file's directory (a leading ``/`` = source root) and must
    EQUAL the image's source-relative path - a bare reference next to
    the markdown (``docs/guide.md`` -> ``pic.png``) and a folder-carrying
    one from the root (``other.md`` -> ``docs/pic.png``) both name the
    nested image ``docs/pic.png``; neither names a top-level ``pic.png``
    (a reference to a different file that shares the filename is NOT a
    match - LOCKED A1)."""
    root = tmp_path / "src"
    (root / "docs").mkdir(parents=True)
    (root / "docs" / "guide.md").write_text(
        "Bare reference next to the image: ![alt](pic.png)\n", encoding="utf-8"
    )
    (root / "other.md").write_text(
        "Folder-carrying reference from the root: ![alt2](docs/pic.png)\n",
        encoding="utf-8",
    )
    llm = _image_llm(tmp_path)
    context = _markdown_image_context(root, llm, source="src", image_rel="docs/pic.png")
    assert context == (
        "Referenced in src/docs/guide.md:\n"
        '- Alt text: "alt"\n'
        "\n"
        "Referenced in src/other.md:\n"
        '- Alt text: "alt2"'
    )
    # The top-level image (a different file with the same name) is never
    # matched by references that resolve to ``docs/pic.png``.
    assert _markdown_image_context(root, llm, source="src", image_rel="pic.png") is None


def test_markdown_context_root_absolute_reference(tmp_path: Path) -> None:
    """A reference with a leading ``/`` is source-ROOT-relative (a
    root-absolute doc-style reference): ``![alt](/pic.png`` names the
    top-level ``pic.png`` from any markdown depth."""
    root = tmp_path / "src"
    (root / "sub").mkdir(parents=True)
    (root / "sub" / "guide.md").write_text(
        "Root-absolute reference: ![alt](/pic.png)\n", encoding="utf-8"
    )
    llm = _image_llm(tmp_path)
    context = _markdown_image_context(root, llm, source="src", image_rel="pic.png")
    assert context == 'Referenced in src/sub/guide.md:\n- Alt text: "alt"'
    # And it does NOT name a nested image of the same name.
    assert _markdown_image_context(root, llm, source="src", image_rel="sub/pic.png") is None


def test_markdown_context_skips_unreadable_markdown(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    """An unreadable markdown file (OSError) never contributes context
    (fail-soft: the scan continues with the remaining files — the sync
    must not break on a flaky source file)."""
    root = tmp_path / "src"
    root.mkdir()
    (root / "bad.md").write_text("![alt](pic.png)\n", encoding="utf-8")
    (root / "good.md").write_text("Good ref ![alt2](pic.png)\n", encoding="utf-8")
    original = Path.read_text

    def _raising(self: Path, *args: Any, **kwargs: Any) -> str:
        if self.name == "bad.md":
            raise OSError("simulated unreadable file (test sentinel)")
        return original(self, *args, **kwargs)

    monkeypatch.setattr(Path, "read_text", _raising)
    llm = _image_llm(tmp_path)
    context = _markdown_image_context(root, llm, source="src", image_rel="pic.png")
    assert context is not None
    assert "good.md" in context and "bad.md" not in context


# ---------------------------------------------------------------------------
# The pipeline — ``import_sources`` + the real seam (no patch): the
# enriched content, the stored context, and the standalone no-op.
# ---------------------------------------------------------------------------


def test_index_image_referenced_by_markdown_enriches_and_stores_context(
    db, tmp_path: Path
) -> None:
    """LOCKED A2: a markdown file in the same source references the
    image → the vision call receives the context (appended after the
    locked prompt), the doc's ``content`` (and ``summary``) is
    ``<context>\\n\\n<description>``, ``image_context`` stores the
    context, and the ONLY embedded text is the enriched content (the
    content chunk + the ``is_summary`` chunk)."""
    root = tmp_path / "anchored"
    root.mkdir()
    (root / "guide.md").write_text(
        "Why this diagram exists.\n\n![The red square](pic.png)\n\nWhat follows it.\n",
        encoding="utf-8",
    )
    (root / "pic.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path, _VisionEmbedder)
    try:
        summary = asyncio.run(import_sources([root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)

        doc = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "pic.png")
        )
        assert doc is not None and doc.is_image is True
        expected_context = (
            f"Referenced in {root.name}/guide.md:\n"
            '- Alt text: "The red square"\n'
            "  Context: Why this diagram exists.\n"
            "\n"
            "What follows it."
        )
        # (a) the vision model learned what the image is for — the text
        # part carries the locked prompt + the exact context prefix.
        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{expected_context}"]
        # (b) the context is stored on the doc (task 05's display source).
        assert doc.image_context == expected_context
        # (c) the enriched content — context first, then the description —
        # IS the summary (the phase-122 convention) and the only embedded
        # text (the plain-text packer joins the enriched paragraphs with
        # single newlines — the chunker's normalization, PLAN §5 — so the
        # content chunk carries the packed form, the summary chunk the
        # raw text).
        expected_content = f"{expected_context}\n\n{DESCRIPTION}"
        assert doc.content == expected_content
        assert doc.summary == expected_content
        packed_content = "\n".join(expected_content.split("\n\n"))
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
        content_chunks = [c for c in chunks if not c.is_summary]
        assert [c.content for c in content_chunks] == [packed_content]
        summary_chunks = [c for c in chunks if c.is_summary]
        assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
        assert summary_chunks[0].content == expected_content  # stored verbatim
        # The image doc's embedded texts: the packed content chunk + the
        # raw enriched summary chunk (the markdown doc's two texts
        # complete the four).
        embedded = [t for batch in llm.calls for t in batch]
        assert len(embedded) == 4
        assert packed_content in embedded and expected_content in embedded
        for c in chunks:
            assert c.embedding is not None and len(c.embedding) == 768

        # Task 04 (the markdown-FIRST bookkeeping): ``guide.md`` is
        # sorted BEFORE ``pic.png``, so at the markdown's indexing the
        # image doc does not exist yet — the reference is recorded on
        # the markdown doc's OWN ``image_context`` (the ``IMAGE_CONTEXT:``
        # record: resolved path + line + alt text + context-paragraph
        # count). The image was then enriched in the same run through
        # the task-03 disk scan, and the record is NOT cleared after
        # enrichment (ASSUMPTION 4 — it stays the retrieval display's
        # record, task 05).
        md = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "guide.md")
        )
        assert md is not None and md.is_image is False
        assert md.image_context == (
            IMAGE_CONTEXT_HEADER + "\n"
            '- [pic.png] (line 3): "The red square" — 2 paragraphs of context'
        )
    finally:
        _cleanup_source(db, root.name)


def test_standalone_image_keeps_null_context_and_bare_description(
    db, tmp_path: Path
) -> None:
    """The pre-phase byte-identity: an image with NO markdown reference
    (a sibling markdown exists but references nothing) → ``image_context``
    NULL, ``content`` the bare description, the vision message the bare
    locked prompt."""
    root = tmp_path / "standalone"
    root.mkdir()
    (root / "notes.md").write_text("# Notes\n\nBody text.\n", encoding="utf-8")
    (root / "pic.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path, _VisionEmbedder)
    try:
        summary = asyncio.run(import_sources([root], llm, session=db))
        assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)

        doc = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "pic.png")
        )
        assert doc is not None and doc.is_image is True
        assert doc.image_context is None, "standalone: the context stays NULL"
        assert doc.content == DESCRIPTION, "standalone: the bare description"
        assert doc.summary == DESCRIPTION
        assert llm.text_parts == [DESCRIBE_PROMPT], "standalone: the bare prompt"
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all()
        assert all(c.content == DESCRIPTION for c in chunks)
    finally:
        _cleanup_source(db, root.name)


def test_changed_image_re_scans_and_can_drop_the_context(
    db, tmp_path: Path
) -> None:
    """The UPDATE path re-scans (a content change is a new version):
    adding a referencing markdown + changing the bytes enriches the doc;
    deleting the markdown + changing the bytes drops ``image_context``
    back to NULL and the content back to the bare description."""
    root = tmp_path / "rescan"
    root.mkdir()
    (root / "pic.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path, _VisionEmbedder)
    try:
        # 1 — standalone first: NULL context, bare content.
        s1 = asyncio.run(import_sources([root], llm, session=db))
        assert s1.added == 1
        doc1 = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "pic.png")
        )
        assert doc1 is not None
        assert doc1.image_context is None and doc1.content == DESCRIPTION

        # 2 — a markdown appears and the image changes: the update re-
        # scans, the context is set, the content is enriched.
        (root / "guide.md").write_text(
            "The why.\n\n![The red square](pic.png)\n\nThe after.\n", encoding="utf-8"
        )
        (root / "pic.png").write_bytes(PNG_1X1 + b"changed")
        s2 = asyncio.run(import_sources([root], llm, session=db))
        assert (s2.added, s2.updated, s2.images_failed) == (1, 1, 0)
        expected_context = (
            f"Referenced in {root.name}/guide.md:\n"
            '- Alt text: "The red square"\n'
            "  Context: The why.\n"
            "\n"
            "The after."
        )
        doc2 = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "pic.png")
        )
        assert doc2 is not None
        assert doc2.image_context == expected_context
        assert doc2.content == f"{expected_context}\n\n{DESCRIPTION}"
        assert doc2.summary == doc2.content

        # 3 — the markdown is gone and the image changes again: the
        # context drops back to NULL (the scan is the current source
        # state, not a ratchet).
        (root / "guide.md").unlink()
        (root / "pic.png").write_bytes(PNG_1X1 + b"changed-again")
        s3 = asyncio.run(import_sources([root], llm, session=db))
        assert (s3.added, s3.updated, s3.images_failed) == (0, 1, 0)
        doc3 = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "pic.png")
        )
        assert doc3 is not None
        assert doc3.image_context is None
        assert doc3.content == DESCRIPTION
    finally:
        _cleanup_source(db, root.name)


# ---------------------------------------------------------------------------
# Task 04 — the markdown-FIRST bookkeeping: the ``IMAGE_CONTEXT:`` record
# (work 1 + 2), the reference-resolution rule, and the post-index
# enrichment pass (work 3).
# ---------------------------------------------------------------------------

#: A markdown file that references ``pic.png`` on line 3 — one context
#: paragraph before and one after (the task-02 window).
MD_CONTENT = (
    "The restore order diagram explains the backup procedure.\n"
    "\n"
    "![Restore order](pic.png)\n"
    "\n"
    "Only run the restore script after the checks pass.\n"
)


def _expected_block(source: str) -> str:
    """The task-03-shape combined string *MD_CONTENT* yields for
    ``<source>/guide.md`` (the pass rebuilds exactly this from the
    stored rows)."""
    return (
        f"Referenced in {source}/guide.md:\n"
        '- Alt text: "Restore order"\n'
        "  Context: The restore order diagram explains the backup procedure.\n"
        "\n"
        "Only run the restore script after the checks pass."
    )


#: The record work 1 stores on the markdown doc's ``image_context`` for
#: *MD_CONTENT* (the reference on line 3, 2 context paragraphs).
RECORD = (
    "IMAGE_CONTEXT:\n"
    '- [pic.png] (line 3): "Restore order" — 2 paragraphs of context'
)

BLOCK = _expected_block("src")


class _FailingVision(FakeEmbedder):
    """A vision client whose multimodal call ALWAYS fails (the pass's
    fail-soft pin — ``describe_image`` turns the ``LLMError`` into
    ``None``)."""

    async def chat(self, messages, model=None):
        self.chat_calls.append(list(messages))
        raise LLMError("simulated vision-model failure (test sentinel)")


class _EmbedFailingVision(FakeEmbedder):
    """Vision OK (the fixed :data:`DESCRIPTION`), the EMBED endpoint
    fails (the pass's rollback pin)."""

    async def chat(self, messages, model=None):
        self.chat_calls.append(list(messages))
        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
        if isinstance(user, list):
            return DESCRIPTION
        first = user.split()
        return "Summary of " + (first[0] if first else "<empty>")

    async def embed(
        self,
        texts: list[str],
        *,
        on_progress: Callable[[int, int], None] | None = None,
    ) -> list[list[float]]:
        raise EmbeddingError("simulated embed endpoint failure (test sentinel)")


def _seed_doc(db, **kwargs) -> Document:
    kwargs.setdefault("full_path", "/tmp/seed")
    doc = Document(**kwargs)
    db.add(doc)
    db.commit()
    return doc


def _seed_image_doc(db, source: str, path: str, **kwargs) -> Document:
    return _seed_doc(
        db,
        source=source,
        path=path,
        title=path,
        content=DESCRIPTION,
        content_hash="ab" * 32,
        is_image=True,
        **kwargs,
    )


def _seed_md_doc(db, source: str, path: str, content: str, image_context: str | None) -> Document:
    return _seed_doc(
        db,
        source=source,
        path=path,
        title=path,
        content=content,
        content_hash="cd" * 32,
        image_context=image_context,
    )


# ---------------------------------------------------------------------------
# The record — formatter (work 2) + parser (the "task 04b" shape).
# ---------------------------------------------------------------------------


def test_format_missing_image_refs_exact_shape() -> None:
    """The exact task format: header + one bullet per reference in
    reference order — the RESOLVED path in the brackets, the 1-based
    line, the alt between the quotes (``""`` for an empty alt), and the
    context-paragraph count (singular/plural, 0 included). Empty refs →
    ``""`` (the caller stores ``None`` — the pass's ``IS NOT NULL`` gate
    must see "no record" as NULL, not ``""``)."""
    refs = [
        ImageRef(path="pic.png", alt_text="The red square", context="a\n\nb", line_number=42),
        ImageRef(path="diagram.jpg", alt_text="", context="one", line_number=100),
        ImageRef(path="x.gif", alt_text="", context="", line_number=5),
    ]
    assert _format_missing_image_refs(refs, "src") == (
        "IMAGE_CONTEXT:\n"
        '- [pic.png] (line 42): "The red square" — 2 paragraphs of context\n'
        '- [diagram.jpg] (line 100): "" — 1 paragraph of context\n'
        '- [x.gif] (line 5): "" — 0 paragraphs of context'
    )
    assert _format_missing_image_refs([], "src") == ""


def test_parse_missing_image_refs_round_trip_and_fail_soft() -> None:
    """The parser is the formatter's exact inverse on well-formed
    records (path, line, alt — including an alt that itself contains
    ``"``) and yields NO references on anything else (the header, an
    empty value, owner-edited text — fail-soft, never a raise)."""
    refs = [
        ImageRef(path="pic.png", alt_text="The red square", context="a\n\nb", line_number=42),
        ImageRef(path="docs/diagram.jpg", alt_text='He said "hi"', context="one", line_number=7),
    ]
    assert _parse_missing_image_refs(_format_missing_image_refs(refs, "src")) == [
        ("pic.png", 42, "The red square"),
        ("docs/diagram.jpg", 7, 'He said "hi"'),
    ]
    assert _parse_missing_image_refs("IMAGE_CONTEXT:") == []
    assert _parse_missing_image_refs("") == []
    assert _parse_missing_image_refs("owner notes\n- [x.png] (line 1): \"a\" — one paragraph") == []


# ---------------------------------------------------------------------------
# The resolution rule — shared by the scan, the bookkeeping, the pass.
# ---------------------------------------------------------------------------


def test_resolve_ref_path_rules() -> None:
    """Bare reference → the markdown's directory (the source root for a
    top-level markdown); a leading ``/`` → the source root; a
    normalized path that still escapes (``..``) or names no file (``.``)
    → ``None`` (no in-source image doc can ever match it)."""
    assert _resolve_ref_path("pic.png", "guide.md") == "pic.png"
    assert _resolve_ref_path("pic.png", "docs/guide.md") == "docs/pic.png"
    assert _resolve_ref_path("../pic.png", "sub/docs/guide.md") == "sub/pic.png"
    assert _resolve_ref_path("folder/pic.png", "guide.md") == "folder/pic.png"
    assert _resolve_ref_path("/pic.png", "sub/guide.md") == "pic.png"
    assert _resolve_ref_path("../../other/x.png", "sub/guide.md") is None
    assert _resolve_ref_path("../x.png", "guide.md") is None
    assert _resolve_ref_path(".", "guide.md") is None


# ---------------------------------------------------------------------------
# The markdown bookkeeping (work 1) — ``_missing_image_context``.
# ---------------------------------------------------------------------------


def test_missing_image_context_none_without_refs_or_when_all_exist(db) -> None:
    """No references → ``None``; a reference that escapes the source
    records nothing (it names no in-source image doc, ever); a reference
    that DOES resolve to an existing image doc records nothing (the
    task-03 path owns that case — the field stays NULL)."""
    source = "mdbookkeeping"
    try:
        assert (
            _missing_image_context(
                db, source=source, md_rel="guide.md", content="# Plain\n\nNo refs.\n"
            )
            is None
        )
        assert (
            _missing_image_context(
                db, source=source, md_rel="guide.md", content="![a](../../x.png)\n"
            )
            is None
        )
        _seed_image_doc(db, source, "pic.png")
        assert (
            _missing_image_context(db, source=source, md_rel="guide.md", content=MD_CONTENT)
            is None
        ), "the image doc exists: the task-03 path owns it, the record stays NULL"
    finally:
        _cleanup_source(db, source)


def test_missing_image_context_records_missing_refs_resolved(db) -> None:
    """Every in-source reference without an image doc is recorded — the
    RESOLVED source-relative path (a nested markdown resolves against
    its own directory, so the record names the nested image path the
    pass can match against), the line, the alt, the paragraph count."""
    source = "mdbookkeeping2"
    try:
        assert (
            _missing_image_context(db, source=source, md_rel="guide.md", content=MD_CONTENT)
            == RECORD
        )
        record = _missing_image_context(
            db,
            source=source,
            md_rel="docs/guide.md",
            content="![alt](pic.png)\n\n![other](other.png)\n",
        )
        assert record == (
            "IMAGE_CONTEXT:\n"
            '- [docs/pic.png] (line 1): "alt" — 1 paragraph of context\n'
            '- [docs/other.png] (line 3): "other" — 1 paragraph of context'
        )
    finally:
        _cleanup_source(db, source)


# ---------------------------------------------------------------------------
# The pass's context rebuild — ``_pending_image_context`` (stored rows).
# ---------------------------------------------------------------------------


def test_pending_image_context_from_rows(db) -> None:
    """The record is the GATE (the markdown must list the image's
    source-relative path), the stored content is the SOURCE (the
    reference is re-extracted from it — the task-03-shape block)."""
    source = "passcontext"
    try:
        md = _seed_md_doc(db, source, "guide.md", MD_CONTENT, RECORD)
        assert _pending_image_context(source, "pic.png", [md]) == _expected_block(source)
        assert _pending_image_context(source, "other.png", [md]) is None
        assert _pending_image_context(source, "pic.png", []) is None
    finally:
        _cleanup_source(db, source)


# ---------------------------------------------------------------------------
# The post-index enrichment pass (work 3) — ``_enrich_pending_images``.
# ---------------------------------------------------------------------------


def test_enrich_pending_images_enriches_from_stored_rows(db, tmp_path: Path) -> None:
    """The markdown-FIRST case closed: the markdown's STORED record
    names the image doc (``image_context IS NULL``), whose persistent
    copy exists → the pass re-describes WITH the context (appended after
    the locked prompt), sets ``image_context``, makes ``content`` (and
    so the ``summary``) ``<context>\\n\\n<description>``, re-chunks +
    re-embeds, counts ``updated`` + ``chunks`` + ``summaries`` — and
    does NOT clear the markdown's record (ASSUMPTION 4)."""
    source = "src"
    copy = tmp_path / "copy.png"
    copy.write_bytes(PNG_1X1)
    try:
        _seed_md_doc(db, source, "guide.md", MD_CONTENT, RECORD)
        image_doc = _seed_image_doc(db, source, "pic.png", image_path=str(copy))
        llm = _image_llm(tmp_path, _VisionEmbedder)
        summary = ImportSummary()
        asyncio.run(_enrich_pending_images(db, source=source, llm=llm, summary=summary))

        assert image_doc.image_context == BLOCK
        expected_content = f"{BLOCK}\n\n{DESCRIPTION}"
        assert image_doc.content == expected_content
        assert image_doc.summary == expected_content
        assert llm.text_parts == [f"{DESCRIBE_PROMPT}{CONTEXT_PREFIX}{BLOCK}"]
        packed = "\n".join(expected_content.split("\n\n"))
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == image_doc.id)).all()
        assert [c.content for c in chunks if not c.is_summary] == [packed]
        assert [c.content for c in chunks if c.is_summary] == [expected_content]
        assert all(c.embedding is not None and len(c.embedding) == 768 for c in chunks)
        md = db.scalar(
            select(Document).where(Document.source == source, Document.path == "guide.md")
        )
        assert md is not None and md.image_context == RECORD, "the record is NOT cleared"
        assert (summary.updated, summary.chunks, summary.summaries) == (1, 1, 1)
        assert (summary.images_failed, summary.errors) == (0, 0)
    finally:
        _cleanup_source(db, source)


def test_enrich_pending_images_is_idempotent_and_selective(db, tmp_path: Path) -> None:
    """ASSUMPTION 4's selectivity: an image doc with ``image_context``
    SET is never touched (already enriched — the task-03 path or an
    earlier pass); no markdown record at all → no-op; a record that
    lists a DIFFERENT image → no-op for this one; a record that lists
    the image but whose STORED CONTENT no longer references it (the
    record is the index, the content is the source) → no-op. None of
    these makes a vision call."""
    copy = tmp_path / "copy.png"
    copy.write_bytes(PNG_1X1)

    source_a = "passidempotent"
    try:
        _seed_md_doc(db, source_a, "guide.md", MD_CONTENT, RECORD)
        _seed_image_doc(db, source_a, "pic.png", image_path=str(copy), image_context="set")
        llm = _image_llm(tmp_path, _VisionEmbedder)
        asyncio.run(_enrich_pending_images(db, source=source_a, llm=llm, summary=ImportSummary()))
        assert llm.text_parts == [], "already enriched: no re-describe"
    finally:
        _cleanup_source(db, source_a)

    source_b = "passnoms"
    try:
        _seed_image_doc(db, source_b, "pic.png", image_path=str(copy))
        llm = _image_llm(tmp_path, _VisionEmbedder)
        asyncio.run(_enrich_pending_images(db, source=source_b, llm=llm, summary=ImportSummary()))
        assert llm.text_parts == [], "no markdown record: no-op"
    finally:
        _cleanup_source(db, source_b)

    source_c = "passotherref"
    try:
        other = "IMAGE_CONTEXT:\n- [other.png] (line 3): \"other\" — 1 paragraph of context"
        _seed_md_doc(db, source_c, "guide.md", MD_CONTENT, other)
        _seed_image_doc(db, source_c, "pic.png", image_path=str(copy))
        llm = _image_llm(tmp_path, _VisionEmbedder)
        asyncio.run(_enrich_pending_images(db, source=source_c, llm=llm, summary=ImportSummary()))
        assert llm.text_parts == [], "the record does not list this image"
    finally:
        _cleanup_source(db, source_c)

    source_d = "passstale"
    try:
        _seed_md_doc(db, source_d, "guide.md", "# Plain\n\nNo refs.\n", RECORD)
        _seed_image_doc(db, source_d, "pic.png", image_path=str(copy))
        llm = _image_llm(tmp_path, _VisionEmbedder)
        asyncio.run(_enrich_pending_images(db, source=source_d, llm=llm, summary=ImportSummary()))
        assert llm.text_parts == [], "the stored content no longer references it"
    finally:
        _cleanup_source(db, source_d)


def test_enrich_pending_images_fail_soft(db, tmp_path: Path) -> None:
    """A failed re-description, a missing persistent copy (both the
    ``image_path`` itself and a file gone from disk), or an embed error
    keeps the image doc AS-IS (the next sync retries it) — warned /
    counted, never raising, never blocking the sync."""
    source_a = "passfail"
    copy = tmp_path / "copy.png"
    copy.write_bytes(PNG_1X1)
    try:
        _seed_md_doc(db, source_a, "guide.md", MD_CONTENT, RECORD)
        image_doc = _seed_image_doc(db, source_a, "pic.png", image_path=str(copy))
        llm = _image_llm(tmp_path, _FailingVision)
        summary = ImportSummary()
        asyncio.run(_enrich_pending_images(db, source=source_a, llm=llm, summary=summary))
        assert image_doc.content == DESCRIPTION and image_doc.image_context is None
        assert summary.images_failed == 1 and summary.updated == 0
    finally:
        _cleanup_source(db, source_a)

    source_b = "passnobody"
    try:
        _seed_md_doc(db, source_b, "guide.md", MD_CONTENT, RECORD)
        image_doc = _seed_image_doc(
            db, source_b, "pic.png", image_path=str(tmp_path / "gone.png")
        )
        llm = _image_llm(tmp_path, _VisionEmbedder)
        summary = ImportSummary()
        asyncio.run(_enrich_pending_images(db, source=source_b, llm=llm, summary=summary))
        assert llm.text_parts == [], "no bytes: no describe call"
        assert image_doc.content == DESCRIPTION and image_doc.image_context is None
        assert summary.images_failed == 1
    finally:
        _cleanup_source(db, source_b)

    source_c = "passnullpath"
    try:
        _seed_md_doc(db, source_c, "guide.md", MD_CONTENT, RECORD)
        image_doc = _seed_image_doc(db, source_c, "pic.png")  # image_path None
        llm = _image_llm(tmp_path, _VisionEmbedder)
        summary = ImportSummary()
        asyncio.run(_enrich_pending_images(db, source=source_c, llm=llm, summary=summary))
        assert llm.text_parts == []
        assert image_doc.content == DESCRIPTION and image_doc.image_context is None
        assert summary.images_failed == 1
    finally:
        _cleanup_source(db, source_c)

    source_d = "passembedfail"
    copy2 = tmp_path / "copy2.png"
    copy2.write_bytes(PNG_1X1)
    try:
        _seed_md_doc(db, source_d, "guide.md", MD_CONTENT, RECORD)
        image_doc = _seed_image_doc(db, source_d, "pic.png", image_path=str(copy2))
        pre_chunk = Chunk(document_id=image_doc.id, position=0, content=DESCRIPTION)
        db.add(pre_chunk)
        db.commit()
        llm = _image_llm(tmp_path, _EmbedFailingVision)
        summary = ImportSummary()
        asyncio.run(_enrich_pending_images(db, source=source_d, llm=llm, summary=summary))
        assert image_doc.content == DESCRIPTION and image_doc.image_context is None
        assert summary.errors == 1 and summary.updated == 0
        # The rollback left the pre-pass chunk set in place (the pass's
        # replaced rows are gone again).
        chunks = db.scalars(select(Chunk).where(Chunk.document_id == image_doc.id)).all()
        assert [c for c in chunks] == [pre_chunk]
        assert chunks[0].content == DESCRIPTION
    finally:
        _cleanup_source(db, source_d)


# ---------------------------------------------------------------------------
# The full pipeline — markdown vs. image ordering through ``import_sources``.
# ---------------------------------------------------------------------------


def test_markdown_referencing_existing_image_keeps_null_record(db, tmp_path: Path) -> None:
    """The task-03 path owns EXISTING images: with the image doc already
    present when the markdown is indexed, the markdown's record stays
    NULL (the image is unchanged in that run — no re-scan); the image
    then picks up the context on its NEXT (re)index, when the on-disk
    markdown is visible to the task-03 scan — and the markdown's record
    is NULL throughout (it never recorded anything)."""
    root = tmp_path / "existingkb"
    root.mkdir()
    (root / "pic.png").write_bytes(PNG_1X1)
    llm = _image_llm(tmp_path, _VisionEmbedder)
    try:
        s1 = asyncio.run(import_sources([root], llm, session=db))
        assert s1.added == 1
        doc1 = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "pic.png")
        )
        assert doc1 is not None and doc1.image_context is None

        # Run 2 — the markdown appears (the image is unchanged).
        (root / "guide.md").write_text(MD_CONTENT, encoding="utf-8")
        s2 = asyncio.run(import_sources([root], llm, session=db))
        assert (s2.added, s2.unchanged) == (1, 1)
        md = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "guide.md")
        )
        assert md is not None and md.image_context is None, "existing image: the record stays NULL"
        doc2 = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "pic.png")
        )
        assert doc2 is not None
        assert doc2.image_context is None and doc2.content == DESCRIPTION

        # Run 3 — the image changes: the re-index re-scans the on-disk
        # markdown (task 03) and enriches; the markdown record is still
        # NULL (unchanged in this run).
        (root / "pic.png").write_bytes(PNG_1X1 + b"changed")
        s3 = asyncio.run(import_sources([root], llm, session=db))
        assert (s3.added, s3.updated, s3.unchanged) == (0, 1, 1)
        doc3 = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "pic.png")
        )
        assert doc3 is not None and doc3.image_context == _expected_block(root.name)
        assert doc3.content == f"{_expected_block(root.name)}\n\n{DESCRIPTION}"
        md3 = db.scalar(
            select(Document).where(Document.source == root.name, Document.path == "guide.md")
        )
        assert md3 is not None and md3.image_context is None
    finally:
        _cleanup_source(db, root.name)