brain-of-reese/tests/e2e/test_pdf_image_extraction.py

"""Phase 151 E2E (Playwright): a PDF sync produces no blank, no duplicate
image documents — the owner's complaint, proven fixed in the browser
(TODO L4: "it extracted a series of blank white images … there don't
appear to be any blank images in the document, is this a bug?").

Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_pdf_image_extraction.py -v --no-cov

MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported — the vision call
counts are read from the mock's request capture (the
``/v1/e2e/captured`` pair), which only the deterministic mock exposes.

Two module apps (the ``test_pdf_support.py`` per-suite-app pattern —
the conftest env with the leak guards, ``BOR_IMAGES=true`` + the
``pdf`` import token pinned EXPLICITLY on both, the mock LLM only):

* **the story app** (``BOR_PDF_IMAGE_SKIP_BLANK=true`` — the default,
  pinned so an operator's local gitignored ``.env`` cannot leak the
  premise off): the fixture PDF (below) arrives through the REAL admin
  upload flow + the RAG page's **Sync** button (the same user path the
  owner used). Through the product path:
  - the catalog carries EXACTLY ONE image document for the PDF — the
    distinct real image at its FIRST placement (``…#page=1#img=1.png``
    — the dedupe's stability contract: the same-page duplicate at
    ``#img=2`` and the cross-page duplicate on page 2 — the podman
    case — never produced their own documents);
  - NO image document's stored bytes are a blank white image — the
    stored copy is checked with the APP'S OWN predicate
    (``app.rag.pdf.is_blank_image`` imported, not re-derived, so the
    test cannot drift from the product);
  - the mock's vision call count for image description is 1 (the
    duplicate placements and the blank underlay never reached the
    model — the cost the owner paid in "several minutes");
  - a follow-up re-sync (nothing changed) settles on
    ``0 added · 1 unchanged`` and creates nothing new — idempotence
    after the fix.
* **the knob-off app** (``BOR_PDF_IMAGE_SKIP_BLANK=false`` — the
  negative control): the SAME PDF through the SAME user path now
  yields the blank underlay document too (the knob is proven live in
  the product, not just in unit tests): two image documents, the
  underlay's stored bytes blank per the app's own predicate — the
  exact class the story app skipped — and two vision calls (the
  pre-phase cost). The knob only gates the blank gate: the duplicate
  placements still produce no documents (dedupe is a correctness
  fix, not knobed).

The fixture PDF (built with PyMuPDF in the fixture — the house
convention, no binary committed): two text pages; page 1 carries the
near-white 200×200 "decorative underlay" (image index 0 — the blank
gate's target), then the real image — genuinely distinct (colorful
noise) from the underlay — inserted TWICE (indices 1 and 2, the
same-page duplicate placement); page 2 carries one more placement of
the real image (index 0 — the cross-page duplicate, the podman case,
where each image was placed 2–4 times). Four placements, two distinct
images, so the extractor must index exactly ONE image document and
make exactly ONE vision call (the task's "exactly one image document
for the PDF" and "vision call count is 1" pins fix the distinct
non-blank total at one — the colorful real image IS the genuinely
distinct image the task's fixture list names).

Test-order note: both apps share the E2E Postgres — the story seed
runs first, the knob-off seed truncates and re-seeds (the phase-126
isolation pattern), so every story test runs before the knob-off
test, which truncates at the end.
"""
from __future__ import annotations

import json
import os
import re
import subprocess
import sys
import zipfile
from collections.abc import Iterator
from pathlib import Path

import httpx
import pymupdf as fitz  # PyMuPDF — the fixture-PDF builder (house convention)
import pytest
from playwright.sync_api import Browser, Page, expect
from sqlalchemy import select, text

from app.config import Settings
from app.db import SessionLocal
from app.models import Document
from app.rag.pdf import is_blank_image
from e2e.auth_helpers import login
from e2e.conftest import ADMIN_PASSWORD, SESSION_SECRET, USE_REAL_LLM, _wait_http
from e2e.mock_llm import IMAGE_DESCRIPTION_CONTEXT_ANSWER

REPO = Path(__file__).resolve().parents[2]
SOURCE_ON = "e2e-pdf151"  # the story app's archive stem (archive_source_name)
SOURCE_OFF = "e2e-pdf151-off"  # the knob-off app's archive stem
DOC_PATH = "guide.pdf"

#: The synthetic embedded-image paths (the importer's
#: ``<pdf path>#page=<N>#img=<index>.<ext>`` rule). The underlay sits
#: at page 1 index 0; the real image's FIRST placement at page 1
#: index 1; the duplicates at page 1 index 2 and page 2 index 0 (they
#: must never appear as documents).
BLANK_REL = f"{DOC_PATH}#page=1#img=0.png"  # the underlay (skipped by default)
REAL_REL = f"{DOC_PATH}#page=1#img=1.png"  # the real image, first placement
DUP_RELS = (
    f"{DOC_PATH}#page=1#img=2.png",  # the same-page duplicate
    f"{DOC_PATH}#page=2#img=0.png",  # the cross-page duplicate (podman case)
)

PDF_TITLE = "Podman Systemd Guide"  # the metadata title

#: The fixture PDF's page texts (the image's page — page 1 — carries
#: the document context the phase-137 block stores on the image doc).
PAGE_ONE = "The podman systemd guide covers container unit files and their restart policies."
PAGE_TWO = "The guide's appendix lists the journal log locations and the default timeouts."

#: The kept image's stored document block (no markdown reference exists
#: in the source — the doc block alone: provenance origin line + the
#: page's text).
DOC_BLOCK = f"Extracted from {SOURCE_ON}/{DOC_PATH} (page 1):\n{PAGE_ONE}"

SYNCED_LABEL = re.compile(r"^Synced \d{1,2}:\d{2}$")
UPLOAD_TIMEOUT_MS = 30_000
SYNC_TIMEOUT_MS = 60_000


# ---------------------------------------------------------------------------
# Fixture PDF: the images (stdlib-free, PyMuPDF-built — deterministic)
# ---------------------------------------------------------------------------


def _solid_png(width: int, height: int, value: int) -> bytes:
    """A uniform RGB PNG (one ``value`` per channel). At 250 the
    underlay's samples are mean 250 (≥ the 235 near-white floor) and
    100% within any tolerance band — blank by
    :func:`app.rag.pdf.is_blank_image` with room to spare."""
    pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height))
    pix.clear_with(value)
    return pix.tobytes("png")


def _noise_png(width: int, height: int) -> bytes:
    """A deterministic colorful noise PNG (channel means ~127 — the
    "real photo" stand-in: it clears the mean gate with room to spare,
    and its scattered samples clear the uniformity gate). Genuinely
    distinct from the underlay in both channels AND structure."""
    pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height))
    for y in range(height):
        for x in range(width):
            pix.set_pixel(x, y, ((x * 7) % 256, (y * 13) % 256, ((x + y) * 5) % 256))
    return pix.tobytes("png")


#: The two fixture images: the near-white 200×200 decorative underlay
#: (the TODO L4 "blank white" class) and the genuinely distinct
#: colorful real image.
BLANK_PNG = _solid_png(200, 200, 250)
REAL_PNG = _noise_png(96, 64)


def _build_fixture_pdf(path: Path) -> None:
    """The phase-151 fixture: two text pages; page 1 carries the
    underlay (index 0), the real image (index 1), and a SECOND
    placement of the same real image (index 2 — the same-page
    duplicate); page 2 carries one more placement of the real image
    (index 0 — the cross-page duplicate, the podman case). PyMuPDF
    stores the repeated insertions under one xref (the
    repeated-placement path — the digest set is the backstop for
    byte-identical copies under distinct xrefs)."""
    doc = fitz.open()
    page1 = doc.new_page()
    page1.insert_text((72, 72), PAGE_ONE)
    y = 150
    for data in (BLANK_PNG, REAL_PNG, REAL_PNG):
        pix = fitz.Pixmap(data)
        page1.insert_image(
            fitz.Rect(72, y, 72 + pix.width, y + pix.height), stream=data
        )
        y += pix.height + 10
    page2 = doc.new_page()
    page2.insert_text((72, 72), PAGE_TWO)
    pix = fitz.Pixmap(REAL_PNG)
    page2.insert_image(
        fitz.Rect(72, 150, 72 + pix.width, 150 + pix.height), stream=REAL_PNG
    )
    doc.set_metadata({"title": PDF_TITLE})
    doc.save(path)
    doc.close()


# ---------------------------------------------------------------------------
# Module apps: the story (knob ON — the default) and the knob-off
# negative control. Each owns its port + its scratch upload/source/
# image dirs. Both pin BOR_IMPORT_EXTENSIONS WITH the ``pdf`` token
# and BOR_IMAGES=true explicitly; the knob under test is pinned per
# app (a process-wide lru_cached setting — the two apps are the only
# way to prove it live in the product).
# ---------------------------------------------------------------------------


def _app_env(
    mock_llm: int, app_port: int, *, skip_blank: bool, scratch: Path
) -> dict[str, str]:
    """The conftest app env (leak guards included) with the phase-151
    knobs: ``BOR_IMPORT_EXTENSIONS`` pinned to a set carrying the
    ``pdf`` token, ``BOR_IMAGES`` pinned true (the embedded-image pass
    must run on BOTH apps), ``BOR_PDF_IMAGE_SKIP_BLANK`` pinned per
    app (true for the story app, false for the negative control —
    the contract under test cannot leak in either direction), and the
    image/upload homes in the suite's scratch dir (the app under test
    must not write copies into the owner's real ``~/bor-sources``).
    ``BOR_PDF_PICTORIAL`` rides at the conftest pin (``0``) through
    ``dict(os.environ)`` — the fixture's sparse pages stay pre-phase-
    140 content, byte-stable across the re-sync."""
    env = dict(os.environ)
    env.pop("DEBUGPY", None)
    env["BOR_ENVIRONMENT"] = "e2e"
    env["BOR_STATIC_DIR"] = str(REPO / "frontend")
    env["BOR_LLM_BASE_URL"] = (
        "https://aipi.reeseapps.com/v1"
        if USE_REAL_LLM
        else f"http://127.0.0.1:{mock_llm}/v1"
    )
    # Mock-calibrated gate (conftest pattern) — the suite drives no
    # chat turn, but the app boots with the same values every other
    # module app gets (the startup validator's premise).
    env["BOR_RELEVANCE_THRESHOLD"] = "0.30"
    env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15"
    env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15"
    # Phase 67: instant retry waits + the code-default budget.
    env["BOR_LLM_RETRY_DELAY"] = "0"
    env["BOR_LLM_RETRIES"] = str(Settings.model_fields["llm_retries"].default)
    env.setdefault(
        "BOR_DATABASE_URL",
        "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese",
    )
    env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD
    env["BOR_SESSION_SECRET"] = SESSION_SECRET
    # Leak guards (conftest pattern).
    env["BOR_GIT_SOURCES"] = ""
    env["BOR_DOCS_REPO"] = ""
    env["BOR_SUGGESTIONS"] = json.dumps(
        Settings.model_fields["suggestions"].default
    )
    env["BOR_INPUT_PLACEHOLDER"] = Settings.model_fields["input_placeholder"].default
    env["BOR_FOOTER_TEXT"] = Settings.model_fields["footer_text"].default
    # Phase 151: the import scope WITH the ``pdf`` token, the images
    # toggle ON (the embedded-image pass runs on both apps), the knob
    # under test pinned per app, and the suite-private homes.
    env["BOR_IMPORT_EXTENSIONS"] = "md,markdown,txt,pdf"
    env["BOR_IMAGES"] = "true"
    env["BOR_PDF_IMAGE_SKIP_BLANK"] = "true" if skip_blank else "false"
    env["BOR_UPLOAD_DIR"] = str(scratch / "uploads")
    env["BOR_SOURCES_DIR"] = str(scratch / "checkouts")
    env["BOR_IMAGE_DIR"] = str(scratch / "images")
    return env


@pytest.fixture(scope="module")
def app_server(mock_llm: int, tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]:
    """The story app — ``BOR_PDF_IMAGE_SKIP_BLANK=true`` (the default
    ON gate; the module env override — the conftest session app is
    never started in this isolated run, so no port clash)."""
    scratch = tmp_path_factory.mktemp("bor_pdf151_on")
    port = int(os.environ.get("E2E_APP_PORT_PDF151", "8168"))
    proc = subprocess.Popen(
        [sys.executable, "-m", "uvicorn", "app.main:app",
         "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
        cwd=REPO,
        env=_app_env(mock_llm, port, skip_blank=True, scratch=scratch),
    )
    try:
        _wait_http(f"http://127.0.0.1:{port}/api/health")
        yield f"http://127.0.0.1:{port}"
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            proc.kill()


@pytest.fixture(scope="module")
def app_url(app_server: str) -> str:
    return app_server


@pytest.fixture(scope="module")
def off_app_server(
    mock_llm: int, tmp_path_factory: pytest.TempPathFactory
) -> Iterator[str]:
    """The negative-control app — ``BOR_PDF_IMAGE_SKIP_BLANK=false``
    (the kill switch OFF: every extracted image is indexed again —
    the pre-phase-151 behavior; only the knob-off test starts it)."""
    scratch = tmp_path_factory.mktemp("bor_pdf151_off")
    port = int(os.environ.get("E2E_APP_PORT_PDF151_OFF", "8169"))
    proc = subprocess.Popen(
        [sys.executable, "-m", "uvicorn", "app.main:app",
         "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"],
        cwd=REPO,
        env=_app_env(mock_llm, port, skip_blank=False, scratch=scratch),
    )
    try:
        _wait_http(f"http://127.0.0.1:{port}/api/health")
        yield f"http://127.0.0.1:{port}"
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            proc.kill()


@pytest.fixture(scope="module")
def off_app_url(off_app_server: str) -> str:
    return off_app_server


# ---------------------------------------------------------------------------
# The fixture archives: ONE PDF, two archive stems (two source names —
# a leftover row from either seed is then visibly foreign in the other
# app's catalog)
# ---------------------------------------------------------------------------


@pytest.fixture(scope="module")
def fixture_pdf(tmp_path_factory: pytest.TempPathFactory) -> Path:
    """The fixture PDF (two text pages; the underlay + the real image
    twice on page 1, the real image once more on page 2 — built with
    PyMuPDF, the house convention)."""
    root = tmp_path_factory.mktemp("bor_pdf151_pdf")
    pdf_path = root / DOC_PATH
    _build_fixture_pdf(pdf_path)
    return pdf_path


def _zip(fixture_pdf: Path, tmp_path_factory: pytest.TempPathFactory, stem: str) -> Path:
    root = tmp_path_factory.mktemp(f"bor_pdf151_zip_{stem}")
    archive = root / f"{stem}.zip"
    with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf:
        zf.write(fixture_pdf, arcname=DOC_PATH)
    return archive


@pytest.fixture(scope="module")
def zip_on(fixture_pdf: Path, tmp_path_factory: pytest.TempPathFactory) -> Path:
    """Source ``e2e-pdf151`` — the story app's archive (ONE PDF at the
    source root, document path ``guide.pdf``)."""
    return _zip(fixture_pdf, tmp_path_factory, SOURCE_ON)


@pytest.fixture(scope="module")
def zip_off(fixture_pdf: Path, tmp_path_factory: pytest.TempPathFactory) -> Path:
    """Source ``e2e-pdf151-off`` — the SAME PDF for the knob-off app."""
    return _zip(fixture_pdf, tmp_path_factory, SOURCE_OFF)


# ---------------------------------------------------------------------------
# DB + UI + mock-capture helpers
# ---------------------------------------------------------------------------


def _truncate_all() -> None:
    """Fresh KB + registry per test group (the E2E isolation pattern):
    the suites share one Postgres, and a leftover row would corrupt the
    counts. ``sources_meta`` (the KB generation counter) resets with
    the KB — the sync re-creates the row."""
    with SessionLocal() as db:
        db.execute(text(
            "TRUNCATE chunks, documents, query_log, kb_overview, "
            "git_sources, sources_meta, folder_summaries"
        ))
        db.commit()


def _health_db(app_url: str) -> bool:
    try:
        return httpx.get(f"{app_url}/api/health", timeout=5).json()["db"] == "up"
    except Exception:  # noqa: BLE001 — unreachable is the skip case
        return False


def _upload_zip(page: Page, app_url: str, archive: Path, source_name: str) -> None:
    """The real admin upload flow: form login → the git-sources page →
    pick the archive → submit → the terminal result line (phase 90:
    "Uploaded <source> — press Sync sources to import it.")."""
    login(page, app_url, next="/git-sources.html")
    expect(page).to_have_url(app_url + "/git-sources.html", timeout=30_000)
    expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000)
    page.set_input_files("#archive-upload-file", str(archive))
    page.click("#archive-upload-btn")
    result = page.locator("#archive-upload-result")
    expect(result).to_be_visible(timeout=UPLOAD_TIMEOUT_MS)
    expect(result).to_have_text(
        f"Uploaded {source_name} — press Sync sources to import it.",
        timeout=UPLOAD_TIMEOUT_MS,
    )


def _sync_via_ui(page: Page, app_url: str, expected_result: str) -> None:
    """The RAG page's "Sync sources" button (the scan — the same user
    path the owner used): click → "Syncing…" → the terminal
    "Synced HH:MM" label + the counts line settled in
    ``#sync-progress-text`` (the phase-150 single sync text surface —
    the wait barrier, no sleeps)."""
    page.goto(app_url + "/sources.html")
    btn = page.locator("#sync-btn")
    expect(btn).to_be_visible(timeout=30_000)
    expect(page.locator("#sync-label")).to_have_text(
        re.compile(r"^Sync sources$|^Synced \d{1,2}:\d{2}$")
    )
    btn.click()
    expect(btn).to_be_disabled()
    expect(page.locator("#sync-label")).to_have_text("Syncing…")
    expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS)
    expect(btn).to_be_enabled()
    expect(page.locator("#sync-progress-text")).to_have_text(expected_result)


def _drill_to_source(page: Page, name: str) -> None:
    """Top level → the source (the file table then holds its direct
    files — ours are at the source root)."""
    page.locator("#folders-tbody .folder-link").first.wait_for(state="visible")
    page.click(f'#folders-tbody a.folder-link:text-is("{name}")')
    expect(page.locator("#docs-tbody tr").first).to_be_visible(timeout=30_000)


def _reset_capture(mock_port: int) -> None:
    """Clear the mock's request capture (the phase-123 pair) so the
    next sync's window holds ONLY that sync's LLM traffic."""
    httpx.post(f"http://127.0.0.1:{mock_port}/v1/e2e/captured/reset", timeout=5)


def _image_description_calls(mock_port: int) -> int:
    """The vision describe calls in the mock's current capture — the
    ``IMAGE_DESCRIPTION_MODE`` marker is the app's fixed
    ``DESCRIBE_PROMPT`` head (``app.rag.summarizer``), a marker a real
    question is never expected to type, so it identifies the image
    describe calls exactly (no other sync traffic carries it)."""
    bodies = httpx.get(f"http://127.0.0.1:{mock_port}/v1/e2e/captured", timeout=5).json()
    return sum(1 for body in bodies if "IMAGE_DESCRIPTION_MODE" in json.dumps(body))


def _stored_image_is_blank(image_path: str) -> bool:
    """The APP'S OWN blank predicate (``app.rag.pdf.is_blank_image`` —
    imported, not re-derived, so the test cannot drift from the
    product) over the stored copy's samples: the exact gate input the
    extractor measures — alpha dropped first when present, so the mean
    runs over COLOR samples only."""
    pm = fitz.Pixmap(Path(image_path).read_bytes())
    if pm.alpha:
        pm = fitz.Pixmap(pm, 0)
    return is_blank_image(pm.samples)


# ---------------------------------------------------------------------------
# The module seeds: upload + sync through the real UI, once per app
# (the story E2E truncate/re-import-per-module pattern)
# ---------------------------------------------------------------------------


@pytest.fixture(scope="module")
def seeded(
    app_url: str,
    browser: Browser,
    zip_on: Path,
    mock_llm: int,
) -> Iterator[dict[str, object]]:
    """Upload the fixture zip through the admin UI on the story app
    (knob ON), then scan it with the RAG page's Sync button (the
    mock's vision model describes the embedded image in-process).
    Yields the kept image doc's identity + the first sync's vision
    call count. The module app's ``/api/config`` flag is asserted
    first — the env override must have reached the app under test, or
    the whole premise of the suite is void."""
    if not _health_db(app_url):
        pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
    cfg = httpx.get(f"{app_url}/api/config", timeout=5).json()
    assert cfg["images"] is True, "the story app must run with BOR_IMAGES=true"

    _truncate_all()
    page = browser.new_page(viewport={"width": 1280, "height": 800})
    try:
        _upload_zip(page, app_url, zip_on, SOURCE_ON)
        # The capture window: ONLY the first sync's LLM traffic.
        _reset_capture(mock_llm)
        # The PDF + its ONE embedded image (the underlay and both
        # duplicate placements are dropped by the parser — the gate
        # and the dedupe are pre-importer): both count added.
        _sync_via_ui(page, app_url, expected_result="2 added")
        first_sync_vision = _image_description_calls(mock_llm)
        _reset_capture(mock_llm)  # a clean window for the re-sync test
    finally:
        page.close()

    with SessionLocal() as db:
        image_docs = db.scalars(
            select(Document).where(
                Document.source == SOURCE_ON, Document.is_image.is_(True)
            )
        ).all()
        # The seed's sanity pin (the rest — first placement, the
        # no-blank bytes, the vision count — is asserted in the tests
        # below against this state).
        assert [doc.path for doc in image_docs] == [REAL_REL]
        img = image_docs[0]
        assert img.is_image is True and img.image_path is not None
        pdf = db.scalar(
            select(Document).where(
                Document.source == SOURCE_ON, Document.path == DOC_PATH
            )
        )
        assert pdf is not None and pdf.is_image is False
        assert pdf.title == PDF_TITLE
        assert "---PAGE-1---" in pdf.content and PAGE_ONE in pdf.content
    yield {
        "image_url": f"/api/documents/{img.id}/image",
        "image_doc_id": img.id,
        "image_doc_hash": img.content_hash,
        "first_sync_vision_calls": first_sync_vision,
    }


@pytest.fixture(scope="module")
def seeded_off(
    off_app_url: str,
    browser: Browser,
    zip_off: Path,
    mock_llm: int,
) -> Iterator[dict[str, object]]:
    """The SAME fixture PDF through the knob-off app (the negative
    control): truncate + re-seed (the shared-Postgres pattern — this
    runs LAST, after every story test has consumed the story seed),
    upload + sync via the same user path. Yields the sync's vision
    call count (2 — the blank underlay IS described now, the
    pre-phase cost)."""
    if not _health_db(off_app_url):
        pytest.skip("Postgres not reachable — run `podman compose up -d db` first")
    cfg = httpx.get(f"{off_app_url}/api/config", timeout=5).json()
    assert cfg["images"] is True, "the knob-off app must run with BOR_IMAGES=true"

    _truncate_all()
    page = browser.new_page(viewport={"width": 1280, "height": 800})
    try:
        _upload_zip(page, off_app_url, zip_off, SOURCE_OFF)
        _reset_capture(mock_llm)
        # The PDF + BOTH embedded images (the blank gate is OFF — the
        # underlay is indexed again; the duplicate placements are
        # STILL dropped — dedupe is a correctness fix, not knobed):
        # three count added.
        _sync_via_ui(page, off_app_url, expected_result="3 added")
        sync_vision = _image_description_calls(mock_llm)
    finally:
        page.close()

    with SessionLocal() as db:
        image_docs = db.scalars(
            select(Document).where(
                Document.source == SOURCE_OFF, Document.is_image.is_(True)
            )
        ).all()
        assert sorted(doc.path for doc in image_docs) == sorted([BLANK_REL, REAL_REL])
    yield {"vision_calls": sync_vision}


# ---------------------------------------------------------------------------
# 1. Sources page (story app): exactly one image doc, no blank row
# ---------------------------------------------------------------------------


def test_sources_page_lists_exactly_one_image_doc(
    page: Page, app_url: str, seeded: dict[str, object]
) -> None:
    """The owner's complaint, the browser lens: the KB total is 2 docs
    / 5 chunks (the PDF: 2 page-anchored content chunks + the summary
    chunk; the one image: its description chunk + summary chunk — the
    underlay contributes NOTHING, no row, no chunk) and the
    drill-down table lists the PDF (a plain text row, no thumbnail) +
    the ONE image document (the thumbnail served from the persistent
    copy) — and NO row for the underlay's synthetic path
    (``#page=1#img=0``)."""
    login(page, app_url)  # lands on /sources.html
    expect(page.locator("#stat-docs")).to_have_text("2")
    expect(page.locator("#stat-chunks")).to_have_text("5")
    expect(page.locator("#sources-empty")).to_be_hidden()

    _drill_to_source(page, SOURCE_ON)
    expect(page.locator("#docs-tbody tr")).to_have_count(2)

    # The PDF — a plain text row (no thumbnail box at all). Pinned by
    # the EXACT doc-link text (the image row's path carries
    # ``guide.pdf`` as a prefix — a substring match would hit both).
    pdf_row = page.locator(
        "#docs-tbody tr",
        has=page.locator(f'a.doc-link:text-is("{DOC_PATH}")'),
    )
    expect(pdf_row).to_have_count(1)
    expect(pdf_row.locator(".kb-doc-thumb")).to_have_count(0)

    # The ONE image document — the real image's thumbnail row (the
    # bytes route's lazy <img>, alt = the description; visible = the
    # served copy resolved).
    img_row = page.locator(
        "#docs-tbody tr",
        has=page.locator(f'a.doc-link:text-is("{REAL_REL}")'),
    )
    expect(img_row).to_have_count(1)
    img = img_row.locator(".kb-doc-thumb-img")
    expect(img).to_have_count(1)
    expect(img).to_be_visible(timeout=15_000)  # the fetch resolves — no glyph
    expect(img).to_have_attribute("src", str(seeded["image_url"]))

    # The blank underlay's row — the TODO L4 artifact — is ABSENT.
    blank_rows = page.locator(
        "#docs-tbody tr",
        has=page.locator(f'a.doc-link:text-is("{BLANK_REL}")'),
    )
    expect(blank_rows).to_have_count(0)


# ---------------------------------------------------------------------------
# 2. The kept doc is the FIRST placement's; exactly ONE vision call
# ---------------------------------------------------------------------------


def test_kept_doc_is_the_first_placement_with_one_vision_call(
    app_url: str, seeded: dict[str, object]
) -> None:
    """The dedupe's stability contract + the cost pin: the kept image
    doc's synthetic path is the FIRST placement's
    (``…#page=1#img=1.png`` — the real image's first insertion; the
    underlay consumed index 0 and was skipped), the duplicate
    placements' paths (``#page=1#img=2`` and ``#page=2#img=0``) exist
    for NO document, and the first sync's capture held EXACTLY ONE
    vision describe call — the duplicate placements and the blank
    underlay never reached the model (the cost the owner paid in
    "several minutes" — pre-phase, every placement was described)."""
    with SessionLocal() as db:
        img = db.scalar(
            select(Document).where(
                Document.source == SOURCE_ON, Document.path == REAL_REL
            )
        )
        assert img is not None, "the real image's first placement must be a document"
        # Phase 137 regression pin (no behavior change): the document
        # context (provenance origin line + page 1's text) reaches the
        # vision model (the mock's context-aware reply is the stored
        # content), summary === content (the image convention).
        assert img.title == "Embedded image from guide page 1"
        assert img.image_context == DOC_BLOCK
        assert img.content == f"{DOC_BLOCK}\n\n{IMAGE_DESCRIPTION_CONTEXT_ANSWER}"
        assert img.summary == img.content
        for dup_rel in DUP_RELS:
            assert (
                db.scalar(
                    select(Document.id).where(
                        Document.source == SOURCE_ON, Document.path == dup_rel
                    )
                )
                is None
            ), f"the duplicate placement {dup_rel} must not be a document"
    # ONE vision call in the first sync's window (the PDF's text
    # summary and the KB overview are different markers — the
    # IMAGE_DESCRIPTION_MODE count is the describe calls exactly).
    assert seeded["first_sync_vision_calls"] == 1


# ---------------------------------------------------------------------------
# 3. No image document's stored bytes are a blank white image
# ---------------------------------------------------------------------------


def test_no_stored_image_doc_is_blank(app_url: str, seeded: dict[str, object]) -> None:
    """The bytes lens on "no blank white documents": every image
    document's PERSISTENT copy (the row's ``image_path`` — the same
    bytes the bytes route serves) is checked with the app's own
    ``is_blank_image`` predicate (imported — the test cannot drift
    from the product): NOT blank. The single doc's bytes are the
    colorful real image — the underlay's near-white bytes never made
    it into the KB (no row, no copy)."""
    with SessionLocal() as db:
        image_docs = db.scalars(
            select(Document).where(
                Document.source == SOURCE_ON, Document.is_image.is_(True)
            )
        ).all()
    assert len(image_docs) == 1  # the catalog lens (test 1) says the same
    for doc in image_docs:
        assert doc.image_path is not None and doc.image_path != ""
        assert _stored_image_is_blank(doc.image_path) is False, (
            f"image doc {doc.path}'s stored copy is blank by the app's own "
            "predicate — the TODO L4 artifact is back"
        )


# ---------------------------------------------------------------------------
# 4. Re-sync (nothing changed): `unchanged`, creates nothing new
# ---------------------------------------------------------------------------


def test_resync_is_idempotent(
    page: Page, app_url: str, seeded: dict[str, object], mock_llm: int
) -> None:
    """Idempotence after the fix: a second Sync (the PDF is
    byte-identical — the fast path, no image pass, no vision) settles
    on ``0 added · 1 unchanged`` (the PDF doc only — the image doc is
    a synthetic path the walk never sees), burns ZERO vision calls in
    its window, and the KB is exactly what it was before — the same
    two documents, the same image doc row (id + content hash + path
    unchanged), the same counts."""
    login(page, app_url)  # the test's fresh page is a fresh context (no session)
    _reset_capture(mock_llm)
    _sync_via_ui(page, app_url, expected_result="0 added · 1 unchanged")
    # The re-sync's window: no describe call (the image pass never ran
    # — the PDF's digest matched), no vision at all.
    assert _image_description_calls(mock_llm) == 0

    with SessionLocal() as db:
        docs = db.scalars(select(Document).where(Document.source == SOURCE_ON)).all()
        assert len(docs) == 2  # nothing new, nothing pruned
        img = db.scalar(
            select(Document).where(
                Document.source == SOURCE_ON, Document.path == REAL_REL
            )
        )
        assert img is not None
        assert img.id == seeded["image_doc_id"]
        assert img.content_hash == seeded["image_doc_hash"]
    # The counts surface is unchanged.
    expect(page.locator("#stat-docs")).to_have_text("2")
    expect(page.locator("#stat-chunks")).to_have_text("5")


# ---------------------------------------------------------------------------
# 5. Negative control (knob OFF): the blank underlay is indexed again
# ---------------------------------------------------------------------------


def test_knob_off_indexes_the_blank_underlay(
    page: Page, off_app_url: str, seeded_off: dict[str, object]
) -> None:
    """The kill switch, proven live in the product (not just in unit
    tests): the SAME PDF through the SAME user path on
    ``BOR_PDF_IMAGE_SKIP_BLANK=false`` yields the blank underlay
    document TOO — 3 docs / 7 chunks, the underlay's row present in
    the drill-down table (its near-white thumbnail served), TWO vision
    calls (the pre-phase cost — the blank IS described now), and the
    underlay's stored bytes blank per the app's own predicate — the
    exact class the story app's gate dropped. The knob gates ONLY the
    blank gate: the duplicate placements still produce no documents
    (dedupe is a correctness fix), so the catalog is the real image +
    the underlay — not the four placements."""
    try:
        login(page, off_app_url)  # lands on /sources.html
        expect(page.locator("#stat-docs")).to_have_text("3")
        expect(page.locator("#stat-chunks")).to_have_text("7")

        _drill_to_source(page, SOURCE_OFF)
        expect(page.locator("#docs-tbody tr")).to_have_count(3)

        # The underlay's row — absent on the story app (test 1) — is
        # BACK: the knob off restored its indexing (the pre-phase-151
        # behavior the owner's bug report describes).
        blank_row = page.locator(
            "#docs-tbody tr",
            has=page.locator(f'a.doc-link:text-is("{BLANK_REL}")'),
        )
        expect(blank_row).to_have_count(1)
        blank_img = blank_row.locator(".kb-doc-thumb-img")
        expect(blank_img).to_have_count(1)
        expect(blank_img).to_be_visible(timeout=15_000)  # the near-white copy serves
        # The duplicate placements: still no documents (not knobed).
        for dup_rel in DUP_RELS:
            expect(
                page.locator(
                    "#docs-tbody tr",
                    has=page.locator(f'a.doc-link:text-is("{dup_rel}")'),
                )
            ).to_have_count(0)

        # The bytes lens: the underlay's stored copy is blank by the
        # app's own predicate — the exact class the story app skipped
        # (test 3 proved its absence there); the real image's copy is
        # not blank (unchanged by the knob).
        with SessionLocal() as db:
            blank = db.scalar(
                select(Document).where(
                    Document.source == SOURCE_OFF, Document.path == BLANK_REL
                )
            )
            real = db.scalar(
                select(Document).where(
                    Document.source == SOURCE_OFF, Document.path == REAL_REL
                )
            )
        assert blank is not None and blank.image_path is not None
        assert _stored_image_is_blank(blank.image_path) is True
        assert real is not None and real.image_path is not None
        assert _stored_image_is_blank(real.image_path) is False
        # The pre-phase cost: BOTH images described (the blank too).
        assert seeded_off["vision_calls"] == 2
    finally:
        _truncate_all()