brain-of-reese/app/rag/pdf.py

"""PDF parsing utilities (phase 126 — PDF support, locked decision A1).

Pure functions (I/O = reading the PDF file only) for extracting text
and embedded images from PDF files using PyMuPDF (the ``pymupdf``
package, imported as ``fitz`` — its stable API name).

* :func:`extract_pdf_text` — the text of every page that has text,
  each prefixed with :data:`PAGE_SEPARATOR` (``---PAGE-N---``, N is the
  1-indexed page number). The PDF chunker (phase 126 task 03) splits on
  these markers so every chunk keeps its page anchor. Pages without
  text (image-only pages) contribute nothing, so a PDF with no
  extractable text returns ``""``.
* :func:`extract_pdf_page_texts` — one entry per page that has text:
  1-indexed page number → the page's stripped text (the same page read
  as :func:`extract_pdf_text`, so the two agree page-for-page; the
  value for page N is exactly that page's separator-block body). The
  phase-137 image-context block is built from this map (page N's entry
  is the document context for an image embedded on page N).
* :func:`extract_pdf_images` — one :class:`PdfImage` per DISTINCT
  embedded image (page order, first placement wins) — repeated
  placements and byte-identical copies are deduplicated (phase 151
  task 01). ``data`` is the image's original encoded bytes as stored in
  the PDF (no re-encoding), ``mime`` is derived via
  :func:`_mime_from_image_info` — EXCEPT an image stored with a soft
  mask (``/SMask``): ``extract_image`` returns the BASE sample stream
  only (the mask carries the visible shape), so masked images are
  composited to the visible alpha PNG (``mime="image/png"``) — the
  bytes a PDF viewer would show (phase 151 task 02); an image whose
  raw format is unsupported (the ``application/octet-stream``
  fallback, e.g. JPEG2000 ``jpx``) is left untouched (raw bytes, raw
  mime) so the importer's extension gate still decides. THEN the
  blank/decorative gate (phase 151 task 03, ``skip_blank=True`` by
  default — the ``skip_blank=False`` keyword is the function-level
  kill switch): a distinct image whose base samples are near-white AND
  nearly uniform (:func:`is_blank_image`) is SKIPPED entirely — no
  row, no copy, no vision call (ASSUMPTION A1).
* :func:`extract_pdf_images_with_stats` — the stats-carrying variant
  the importer calls (phase 151 task 03): the same extraction plus a
  :class:`PdfImageStats` (``duplicates`` — occurrences dropped by the
  dedupe, ``blank_skipped`` — distinct images dropped by the gate,
  ``total_found`` — occurrences found) so the run's
  ``ImportSummary`` can record them log-only (``pdf_images_deduped`` /
  ``pdf_images_blank_skipped`` — the phase-133 stats convention, the
  sync ``detail`` API shape untouched).
* :func:`is_blank_image` — the phase-151 (task 03) blank/decorative
  predicate: pure, deterministic, no PDF, no I/O (the
  :func:`is_usable_pdf_text` / :func:`pictorial_pages` house style)
  — near-white (mean ≥ 235) AND nearly uniform (≥98% of the samples
  within ±17 of the mean). A dark uniform block is NOT blank (the gate
  targets near-white page underlays); empty samples are not blank
  (never blank a thing we cannot measure).
* :func:`_mime_from_image_info` — maps a PyMuPDF image-info dict to a
  MIME type: the embedded file extension (``ext``) first, then the
  image type code (``type``: 2=JPEG, 3=PNG, 4=GIF, 8=TIFF, 9=WebP),
  else the ``application/octet-stream`` fallback.
* :func:`is_usable_pdf_text` — the phase-133 garbage gate: a pure,
  deterministic predicate over the extracted text layer (no PDF, no
  I/O) — REAL content vs. garbage (the pre-126 XML/binary soup, the
  empty text layer). It is the single trigger for the page-screenshot
  vision fallback (phase 133 task 03).
* :func:`pictorial_pages` — the phase-140 sparse-page gate: the 1-based
  page numbers of a :func:`extract_pdf_page_texts` map whose stripped
  text is STRICTLY under *min_chars* chars (ascending) — pure,
  deterministic, NO LLM call (LOCKED A1): the "way to determine when
  that's necessary" (TODO L5) for the pictorial pass.
* :func:`render_pdf_pages` — the fallback's visual input (phase 133
  task 03): one PNG per page (150 DPI, page order); the phase-140
  optional 1-based *pages* selector renders ONLY the named pages
  (ascending — a 500-page PDF with 30 sparse pages renders 30 PNGs,
  not 500), ``None`` = every page (the phase-133 caller's result is
  byte-identical); an unreadable (or out-of-range named) page yields
  ``b""`` at its position (the caller's placeholder — the result
  stays aligned with the requested pages), an unopenable file yields
  ``[]`` (the caller's existing empty-content path).

Every failure mode is fail-soft: an unreadable file returns an empty
result, an unreadable page or image is logged and skipped — this module
never raises for malformed input (the importer owns the loud failures).
"""
from __future__ import annotations

import hashlib
import logging
import re
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any

import pymupdf as fitz  # PyMuPDF (locked A1); `fitz` is the stable API name

logger = logging.getLogger("app.rag.pdf")

#: Page separator used in extracted text (the chunker splits on this).
#: ``{page}`` is the 1-indexed page number.
PAGE_SEPARATOR = "---PAGE-{page}---"

#: Mapping from PyMuPDF image type codes to MIME types
#: (2=JPEG, 3=PNG, 4=GIF, 8=TIFF, 9=WebP — phase-126 design).
_TYPE_TO_MIME: dict[int, str] = {
    2: "image/jpeg",
    3: "image/png",
    4: "image/gif",
    8: "image/tiff",
    9: "image/webp",
}

#: Mapping from embedded-image file extensions to MIME types.
_EXT_TO_MIME: dict[str, str] = {
    "png": "image/png",
    "jpg": "image/jpeg",
    "jpeg": "image/jpeg",
    "gif": "image/gif",
    "webp": "image/webp",
    "tif": "image/tiff",
    "tiff": "image/tiff",
}

#: Phase 151 (task 03) — the blank/decorative image gate's thresholds
#: (ASSUMPTION A2, owner-locked): a PDF-embedded image is
#: blank/decorative when its COLOR samples are NEAR-WHITE AND NEARLY
#: UNIFORM — :func:`is_blank_image`'s three numbers.
#: Measured basis (probe: ``fitz.Pixmap(...).samples`` on the image's
#: BASE sample stream, the three real PDFs in ``test_sources/``):
#
#: * ``podman-systemd.pdf``'s 9 table-shading underlays: mean 238,
#:   uniq 8, uniform(±8) 0.95–0.995 → blank (the TODO L4 bug — the
#:   "blank white images" the owner saw in the KB);
#: * ``pdf_horse_test.pdf``'s photo: mean 123, uniq 256, uniform ~0.20
#:   → kept (the mean gate);
#: * ``players_handbook.pdf``'s page art: mean 24–221, uniq ≥188,
#:   uniform 0.10–0.59 → kept (the mean gate).
BLANK_IMAGE_MEAN = 235  #: near-white floor (owner-locked A2)
#: "same as the mean" band (owner-locked A2 — see the CALIBRATED
#: DEVIATION note below: the phase overview's ±12 misses the podman
#: PDF's 238/255-blend underlay, ±17 is the smallest fix).
BLANK_IMAGE_TOLERANCE = 17
#: fraction of samples that must sit in the band (owner-locked A2).
BLANK_IMAGE_FRACTION = 0.98

# CALIBRATED DEVIATION (flagged per AGENTS.md rule 3 — never silent):
# the phase overview pinned the band at ±12, but the podman PDF's
# smallest underlay (page 47, 5 300 samples) is a 238/255 blend whose
# 255 level sits 16.2 above its mean (238.81) — at ±12, 4.5% of its
# samples fall out-of-band (uniform 0.954 < 0.98) and the image
# SURVIVES the gate, violating the phase's completion criterion
# (the extractor must return NO images for that file — the owner's
# file is the phase's reason for existing; a 238/255 strip is the
# very "blank white" document the TODO reported, the table text a
# viewer shows is real PDF text drawn on top). ±17 is the smallest
# integer band that keeps every measured real-file pin: all 9 podman
# underlays blank (99.4–99.9% in-band at ±17), the horse photo kept
# (mean 123 < 235 — the mean gate, tolerance irrelevant), and all 586
# players_handbook images kept (means 17–225 — the mean gate). The
# predicate's formula, the mean floor (235), and the fraction (0.98)
# are the owner-locked values, unchanged.


@dataclass
class PdfImage:
    """One embedded image from a PDF page."""

    page: int  #: 1-indexed page number
    index: int  #: image index on the page (0-based, occurrence order)
    width: int  #: native pixel width
    height: int  #: native pixel height
    data: bytes  #: encoded bytes — the original stored bytes, or the
    #: composited (base + soft mask) alpha PNG for masked images
    #: (phase 151 task 02)
    mime: str  #: MIME type (e.g. ``image/png`` — always for composited
    #: masked images, phase 151 task 02)


@dataclass(frozen=True)
class PdfImageStats:
    """The per-PDF drop counts of one
    :func:`extract_pdf_images_with_stats` call (phase 151, task 03).

    The importer records :attr:`duplicates` / :attr:`blank_skipped`
    on the run's :class:`app.rag.importer.ImportSummary` (the
    phase-133 stats log-only convention — the sync ``detail`` API
    shape is untouched).
    """

    duplicates: int  #: image OCCURRENCES dropped by the dedupe (task
    #: 01) — repeated placements + byte-identical copies under
    #: distinct xrefs; per run: sum across the PDFs
    blank_skipped: int  #: DISTINCT images dropped by the
    #: blank/decorative gate (task 03) — no row, no image_dir copy,
    #: no vision call
    total_found: int  #: image occurrences FOUND in the PDF (one per
    #: placement — before dedupe and gate); hence
    #: ``len(images) == total_found - duplicates - blank_skipped``


def _mime_from_image_info(info: dict[str, Any]) -> str:
    """Map a PyMuPDF image-info dict to a MIME type.

    The embedded file extension (``ext``) wins over the image type code
    (``type``); unknown/missing both fall back to
    ``application/octet-stream``.
    """
    ext = str(info.get("ext") or "").lower()
    if ext in _EXT_TO_MIME:
        return _EXT_TO_MIME[ext]
    type_code = info.get("type")
    if isinstance(type_code, int):
        return _TYPE_TO_MIME.get(type_code, "application/octet-stream")
    return "application/octet-stream"


def _page_image_entries(page: Any) -> list[dict[str, Any]]:
    """One info dict per embedded-image occurrence on *page*.

    Prefers ``get_image_info(xrefs=True)`` (one entry per occurrence,
    with the ``xref`` object number); older PyMuPDF releases without the
    ``xrefs`` keyword are served from ``get_images(full=True)`` (one
    entry per unique image).
    """
    try:
        infos: list[dict[str, Any]] = page.get_image_info(xrefs=True)
    except TypeError:
        infos = [
            {"xref": entry[0], "width": entry[2], "height": entry[3]}
            for entry in page.get_images(full=True)
        ]
    return [dict(info) for info in infos]


def _iter_page_texts(doc: Any, pdf_path: Path) -> Iterator[tuple[int, str]]:
    """Yield ``(page, text)`` for every page of *doc* that has text.

    ``page`` is the 1-indexed page number and ``text`` the page's
    stripped text (the ``get_text("text")`` read — it always returns a
    str; the union return type comes from pymupdf's other flavors,
    dict/words/blocks). A page that cannot be read is logged and
    skipped; a page without text contributes nothing. The shared page
    iteration behind :func:`extract_pdf_text` and
    :func:`extract_pdf_page_texts` — the two callers share the exact
    same read and cannot drift.
    """
    for page_num in range(len(doc)):
        try:
            raw = doc[page_num].get_text("text")
        except Exception as exc:
            logger.warning(
                "pdf: could not read text of page %d of %s — %s",
                page_num + 1, pdf_path, exc,
            )
            continue
        text = raw.strip() if isinstance(raw, str) else ""
        if text:
            yield page_num + 1, text


def extract_pdf_text(pdf_path: Path) -> str:
    """Extract the text of all pages of *pdf_path*.

    Every page that has text is prefixed with :data:`PAGE_SEPARATOR`
    (``---PAGE-N---``, N 1-indexed) and the page blocks are joined with
    a blank line. Pages without text contribute nothing, so a PDF with
    no extractable text (or a zero-page file) returns ``""``.

    Fail-soft: an unreadable file returns ``""``; a page that cannot be
    read is logged and skipped.
    """
    try:
        doc = fitz.open(str(pdf_path))
    except Exception as exc:
        logger.warning("pdf: could not open %s — %s", pdf_path, exc)
        return ""

    with doc:
        parts = [
            f"{PAGE_SEPARATOR.format(page=page)}\n{text}"
            for page, text in _iter_page_texts(doc, pdf_path)
        ]
    return "\n\n".join(parts)


def extract_pdf_page_texts(pdf_path: Path) -> dict[int, str]:
    """One entry per page of *pdf_path* that has text.

    Keys are the 1-indexed page numbers, values the page's stripped
    text — the shared :func:`_iter_page_texts` read, so this map agrees
    with :func:`extract_pdf_text` page-for-page: the value for page N
    is exactly the text after that page's ``---PAGE-N---`` separator.
    Pages without text (image-only pages) contribute nothing — no key.

    The phase-137 document context for an embedded image is built from
    this map: one image on page N gets page N's text as the context
    block's body.

    Fail-soft: an unreadable file returns ``{}`` (one warning — the
    :func:`extract_pdf_text` open contract); a page that cannot be
    read is logged and skipped.
    """
    try:
        doc = fitz.open(str(pdf_path))
    except Exception as exc:
        logger.warning("pdf: could not open %s for page texts — %s", pdf_path, exc)
        return {}

    with doc:
        return dict(_iter_page_texts(doc, pdf_path))


#: A line "looks like XML" when it carries a tag-shaped token — the
#: pre-126 garbage's signature (raw XMP blocks, PDF-structure noise).
_XMLISH_LINE_RE = re.compile(r"</?[A-Za-z_][\w:.-]*[\s/>=]")

#: Garbage-gate thresholds (phase 133 task 02): an extracted text
#: layer is UNUSABLE when its XML-ish line fraction reaches this bar.
#: The FINDING's reproduced pre-126 read of the owner's file is 46%
#: XML-ish; a real document ABOUT XML that reaches the bar is a
#: documented false positive (ASSUMPTION A8 — the fallback still
#: extracts it via vision; the cost is a few vision calls).
XMLISH_LINE_FRACTION = 0.30

#: Garbage-gate thresholds (phase 133 task 02): the non-printable
#: fraction (U+FFFD replacement chars + C0 controls over the whole
#: text) at which the layer is UNUSABLE — the ``errors="replace"``
#: binary-soup signal (the pre-126 read is ~19% non-printable; real
#: extracted text is ~0.05%).
NONPRINT_CHAR_FRACTION = 0.05


def is_usable_pdf_text(text: str) -> bool:
    """Whether an extracted PDF text layer is REAL content (not garbage).

    Unusable (→ the page-screenshot fallback) when ANY of:

    * ``text.strip() == ""`` — the empty text layer (a scanned/image-
      only PDF has nothing to read);
    * ≥ :data:`XMLISH_LINE_FRACTION` (0.30) of the non-empty lines
      match :data:`_XMLISH_LINE_RE` — the raw-XMP / binary-soup
      signature (the FINDING's reproduced pre-126 read is 46% —
      ``read_text(errors="replace")`` on the owner's file left
      ``<illustrator:StartupProfile>Web</illustrator:StartupProfile>``
      lines in the preview);
    * ≥ :data:`NONPRINT_CHAR_FRACTION` (0.05) of all characters are
      non-printable — the ``errors="replace"`` binary-soup signal.
      The non-printable set is ``U+FFFD`` + C0 control chars
      (0x00–0x1F) EXCEPT the structural ``\\t``/``\\n``/``\\r`` the
      extractor legitimately emits (the explicit set is deliberate: a
      ``unicodedata`` Cc-category check would catch those too, and
      counting them would flag every prose document).

    Pure + deterministic (no PDF, no I/O) — the unit tests pin the
    thresholds against the REAL reproduced garbage string, not a
    hand-drawn one. A real document ABOUT XML whose tag-like lines
    reach 30% is a documented false positive (ASSUMPTION A8 — the
    fallback still extracts it correctly via vision; the cost is a
    few vision calls).
    """
    if not text.strip():
        return False
    non_empty_lines = [line for line in text.splitlines() if line.strip()]
    xml_ish = sum(1 for line in non_empty_lines if _XMLISH_LINE_RE.search(line))
    if xml_ish / len(non_empty_lines) >= XMLISH_LINE_FRACTION:
        return False
    non_printable = sum(
        1
        for ch in text
        if ch == "\ufffd" or (ord(ch) < 32 and ch not in "\t\n\r")
    )
    return non_printable / len(text) < NONPRINT_CHAR_FRACTION


def pictorial_pages(page_texts: dict[int, str], *, min_chars: int) -> list[int]:
    """The 1-based page numbers of *page_texts* that are PICTORIAL
    candidates: ``len(text.strip()) < min_chars`` — STRICTLY less than
    (a page with exactly *min_chars* stripped chars is NOT sparse; the
    boundary is pinned in the unit tests), ascending.

    The deterministic sparse-page gate (phase 140, task 01, LOCKED A1)
    — the "way to determine when that's necessary" for the pictorial PDF
    pass (TODO L5): NO LLM call, pure, deterministic (no PDF, no I/O —
    the input is a :func:`extract_pdf_page_texts` map, so the gate never
    re-opens the file). A page that carries no text at all has NO key in
    the map and is therefore not a candidate here (the importer's pass
    owns the page-total handling); an empty map yields ``[]`` — a
    clean-text PDF's path never burns a vision call.

    The IKEA example (the phase-140 fixture, ``min_chars=100``):
    ``{1: "BESTÅ", 2: <7 KB of warnings>, 6: "Illustration 6"}`` →
    ``[1, 6, …]`` — page 1 (the title page: the logo + product name) IS
    a candidate: describing it is useful (the logo + product name) and
    never harmful; page 2's 7 KB of multilingual warnings is not.

    ``min_chars=0`` is the documented PLAN A21 special case — **every
    page in the map is a candidate** (the "describe everything" mode).
    The strict-``<`` test cannot express this (no length is below 0), so
    0 is not "a stricter bar": it is the A21 no-cap endpoint, and a
    negative value — unreachable through ``Settings`` (the validator
    fails startup loudly) — takes the same branch defensively.
    """
    if min_chars <= 0:
        return sorted(page_texts)
    return sorted(
        page for page, text in page_texts.items() if len(text.strip()) < min_chars
    )


def is_blank_image(
    samples: bytes,
    *,
    mean_min: int = BLANK_IMAGE_MEAN,
    tolerance: int = BLANK_IMAGE_TOLERANCE,
    fraction: float = BLANK_IMAGE_FRACTION,
) -> bool:
    """Whether *samples* (raw image sample bytes — one byte per sample,
    COLOR channels only, no alpha) are a BLANK/DECORATIVE near-white
    image (phase 151, task 03).

    In the house style of :func:`is_usable_pdf_text` /
    :func:`pictorial_pages`: pure, deterministic, NO PDF, NO I/O, NO
    LLM — unit-testable with hand-built sample buffers.

    Blank when ALL of:

    * ``samples`` is non-empty — empty samples → ``False`` (never blank
      a thing we cannot measure);
    * ``sum(samples) / len(samples) >= mean_min`` — the near-white
      floor (default :data:`BLANK_IMAGE_MEAN` = 235). A DARK uniform
      block is NOT blank by this rule: the gate targets near-white
      PAGE UNDERLAYS (the table-shading rectangles the TODO L4 bug
      indexed) — a solid dark block is unusual and cheaper to keep
      than to explain away;
    * at least ``fraction`` of the samples lie within ``±tolerance``
      of the mean (defaults :data:`BLANK_IMAGE_FRACTION` = 0.98,
      :data:`BLANK_IMAGE_TOLERANCE` = 17) — the "carries no visible
      content" test: a real photo scatters its samples far outside
      the band, a decorative underlay sits on one (near-white) level.

    Measured basis (the constants' docstring carries the full numbers):
    the podman underlays (mean 238, uniq 8, uniform(±8) 0.95–0.995)
    are blank; the horse photo (mean 123, uniq 256, uniform ~0.20) and
    the handbook page art (mean 24–221, uniq ≥188, uniform 0.10–0.59)
    are kept — the mean gate alone carries both real-file "kept" pins,
    the fraction+tolerance carry the "near-white but textured" class.
    """
    if not samples:
        return False
    n = len(samples)
    mean = sum(samples) / n
    if mean < mean_min:
        return False
    uniform = sum(1 for s in samples if abs(s - mean) <= tolerance) / n
    return uniform >= fraction


def render_pdf_pages(
    pdf_path: Path, dpi: int = 150, pages: Sequence[int] | None = None
) -> list[bytes]:
    """Render the selected pages of *pdf_path* to PNGs.

    *pages* (phase 140, task 02): the optional 1-based selector —
    render ONLY the named pages, in ascending order, one PNG each
    (a 500-page PDF with 30 pictorial candidates renders 30 PNGs, not
    500). ``None`` (the default) = every page in page order — the
    phase-133 caller's result is byte-identical, so list index *i* is
    page *i*+1 (with a selector, list index *i* is the *i*-th smallest
    NAMED page).

    The fallback's visual input (phase 133 task 03, ASSUMPTION A8:
    fixed 150 DPI — the balance of vision legibility vs data-URL
    payload). A page that cannot be rendered — including a named page
    outside the document's page range — yields ``b""`` at ITS position
    (the caller's placeholder — the result's order NEVER shifts, so it
    stays aligned with the requested pages; the failure is logged, the
    :func:`extract_pdf_text` fail-soft style). A file that cannot be
    OPENED yields ``[]`` (the caller's existing empty-content path —
    the doc lands with the no-extractable-text behavior, unchanged).
    """
    try:
        doc = fitz.open(str(pdf_path))
    except Exception as exc:
        logger.warning("pdf: could not open %s for page rendering — %s", pdf_path, exc)
        return []

    # 1-based page numbers: the default walks every page in page order
    # (the phase-133 contract — list index *i* is page *i*+1); a
    # selector renders the named pages, ascending (de-duplicated — a
    # repeated page number would render it twice for no reason).
    numbers = (
        range(1, len(doc) + 1)
        if pages is None
        else sorted(set(pages))
    )
    rendered: list[bytes] = []
    with doc:
        for page_number in numbers:
            try:
                rendered.append(
                    doc[page_number - 1].get_pixmap(dpi=dpi).tobytes("png")
                )
            except Exception as exc:
                logger.warning(
                    "pdf: could not render page %d of %s — %s",
                    page_number, pdf_path, exc,
                )
                rendered.append(b"")
    return rendered


def extract_pdf_images_with_stats(
    pdf_path: Path, *, skip_blank: bool = True
) -> tuple[list[PdfImage], PdfImageStats]:
    """Extract all DISTINCT embedded images from *pdf_path*, with the
    per-PDF drop counts (phase 151, task 03 — the stats-carrying
    variant the importer calls; :func:`extract_pdf_images` is the thin
    list-only wrapper for every other caller).

    Returns ``(images, stats)`` — one :class:`PdfImage` per DISTINCT
    embedded image, in page order then 0-based index on the page, plus
    the :class:`PdfImageStats` of the drops (``len(images) ==
    stats.total_found - stats.duplicates - stats.blank_skipped``). The
    FIRST occurrence wins: the kept entry carries the first
    placement's ``page``/``index`` (the importer's synthetic path
    ``…#page=N#img=i.<ext>`` is therefore stable for the first
    placement). Two kinds of duplicate are dropped (phase 151, task
    01) — PyMuPDF reports one info entry per PLACEMENT, so a PDF that
    places the same image N× yields N entries:

    * the same ``xref`` seen again (a repeated placement) — the
      ``seen_xrefs`` set;
    * byte-identical stored bytes under a DIFFERENT ``xref`` (the same
      sha256 of ``data``) — the ``seen_digests`` set (real documents DO
      store the same image under several xrefs — the digest set is what
      catches those; unique-xref count overstates the distinct-image
      count).

    When anything was dropped, ONE INFO line per PDF after the page
    loop (``pdf: dropped N duplicate image occurrence(s) from <path>
    (M unique)``) plus one DEBUG line per drop (``pdf: duplicate image
    xref=… page=… index=… of <path> — skipped``). Dedupe is PER-PDF —
    the seen-sets are local to this call, never across files (two PDFs
    sharing a logo each keep their own image doc: the synthetic path is
    per-PDF, so cross-PDF dedupe would break the upsert key).

    ``data`` is the original encoded bytes as stored in the PDF;
    ``mime`` is derived from the image's extension/type via
    :func:`_mime_from_image_info` — with one exception (phase 151,
    task 02): an image stored with a soft mask (``/SMask``). PyMuPDF's
    ``extract_image`` returns the BASE sample stream only — the mask
    (which carries the visible shape — e.g. the table-shading underlay
    the owner's podman PDF stores as a near-white base) is dropped — so
    for such images the extractor composites base + mask into the
    VISIBLE image (``fitz.Pixmap(base, mask)`` → ``data =
    visible.tobytes("png")``, ``mime = "image/png"``): the bytes a PDF
    viewer shows, not the blank base. Two rules:

    * the RAW-format gate runs FIRST (owner-locked A4): an image whose
      raw format the importer's ``_MIME_TO_EXT`` gate rejects (the
      ``application/octet-stream`` fallback — e.g. JPEG2000 ``jpx``,
      the 586 images in ``players_handbook.pdf``) is NOT composited —
      raw bytes + raw mime, byte-identical to pre-phase; compositing
      must never newly admit an unsupported format (a ``jpx`` image
      with a mask stays skipped);
    * the compositing is fail-soft: any exception (a colorspace the
      combiner cannot merge, a broken mask xref) falls back to the raw
      bytes + raw mime with a warning — the image is still indexed,
      exactly as pre-phase (a decode problem never loses a real
      image). ``width``/``height`` are unchanged either way (the
      composited pixmap has the base's dimensions).

    THEN the blank/decorative gate (phase 151, task 03, gated by
    *skip_blank* — the kill switch at the function level; task 04's
    ``BOR_PDF_IMAGE_SKIP_BLANK`` setting lands here, dedupe and mask
    compositing are NOT knobed): a distinct image whose samples are
    near-white AND nearly uniform (:func:`is_blank_image`) is SKIPPED
    entirely — not returned (ASSUMPTION A1: never indexed, never
    described — no row, no ``image_dir`` copy, no vision call) — with
    one INFO line per skipped image (``pdf: skipped blank/decorative
    image page=… index=… of <path> — mean=… uniform=…``) and one INFO
    line per PDF when anything was skipped (``pdf: skipped N
    blank/decorative image(s) in <path>``). The gate measures the
    image's BASE sample stream (``fitz.Pixmap(doc, xref)``, the alpha
    channel dropped first when present — the mean is over COLOR
    samples only, an alpha channel would drag the mean and produce
    false blanks) for every image — masked or not: for a masked image
    the base is the correct "visible content" signal, because the
    ``/SMask`` only modulates the base's VISIBILITY (a near-white base
    renders near-white over the white page under ANY mask, so a
    near-white base is decorative no matter what the mask says) — and
    the stored composite's color channels are PyMuPDF's
    PREMULTIPLIED base×mask/255, which darkens the transparent regions
    and would miss the very near-white underlays this gate targets
    (measured on the podman underlays: alpha-stripped composite
    mean ≈ 231–234 / uniform ≈ 0.95 vs base mean ≈ 238 / uniform
    ≈ 0.99 — only the base clears the pinned thresholds on every
    underlay). The gate is fail-soft: a pixmap that cannot be built
    skips the gate (the image is KEPT, DEBUG why) — a decode problem
    must never delete a real image.

    Fail-soft: an unreadable file returns ``([], PdfImageStats(0, 0,
    0))``; an image without an ``xref`` or that cannot be extracted is
    logged and skipped.
    """
    try:
        doc = fitz.open(str(pdf_path))
    except Exception as exc:
        logger.warning("pdf: could not open %s for image extraction — %s", pdf_path, exc)
        return [], PdfImageStats(duplicates=0, blank_skipped=0, total_found=0)

    seen_xrefs: set[int] = set()
    seen_digests: set[str] = set()
    dropped = 0
    blank_skipped = 0
    total_found = 0
    images: list[PdfImage] = []
    with doc:
        for page_num in range(len(doc)):
            try:
                entries = _page_image_entries(doc[page_num])
            except Exception as exc:
                logger.warning(
                    "pdf: could not list images on page %d of %s — %s",
                    page_num + 1, pdf_path, exc,
                )
                continue
            for index, info in enumerate(entries):
                xref = int(info.get("xref") or 0)
                if not xref:
                    logger.warning(
                        "pdf: skipping image page=%d index=%d of %s — no xref",
                        page_num + 1, index, pdf_path,
                    )
                    continue
                try:
                    extracted = doc.extract_image(xref)
                except Exception as exc:
                    logger.warning(
                        "pdf: could not extract image page=%d index=%d of %s — %s",
                        page_num + 1, index, pdf_path, exc,
                    )
                    continue
                data = extracted.get("image") or b""
                if not data:
                    logger.warning(
                        "pdf: skipping image page=%d index=%d of %s — empty data",
                        page_num + 1, index, pdf_path,
                    )
                    continue
                # One image occurrence found (valid xref, extracted,
                # non-empty data) — the input to dedupe + gate (task
                # 03's total_found; len(images) = total_found -
                # duplicates - blank_skipped).
                total_found += 1
                # Phase 151 (task 01): one PdfImage per DISTINCT image.
                # Repeated placements share the xref (the first check);
                # byte-identical bytes stored under a different xref are
                # caught by the digest of the stored bytes (the second).
                if xref in seen_xrefs:
                    dropped += 1
                    logger.debug(
                        "pdf: duplicate image xref=%d page=%d index=%d of %s — skipped",
                        xref, page_num + 1, index, pdf_path,
                    )
                    continue
                digest = hashlib.sha256(data).hexdigest()
                if digest in seen_digests:
                    dropped += 1
                    logger.debug(
                        "pdf: duplicate image xref=%d page=%d index=%d of %s — skipped",
                        xref, page_num + 1, index, pdf_path,
                    )
                    continue
                seen_xrefs.add(xref)
                seen_digests.add(digest)
                # The image info's own `ext` is unreliable in some
                # PyMuPDF versions (None); `extract_image` reports the
                # actual format of the returned bytes, so it wins.
                mime = _mime_from_image_info({**info, "ext": extracted.get("ext") or ""})
                # Phase 151 (task 02): a /SMask carries the visible
                # shape — `extract_image` returns the BASE sample
                # stream only (the mask is dropped), so the raw bytes
                # are not what a PDF viewer shows. Composite base +
                # mask to the visible alpha PNG.
                #
                # Ordering rule (owner-locked A4): the raw-format gate
                # runs FIRST — an image whose raw format the importer's
                # `_MIME_TO_EXT` gate will reject (the
                # `application/octet-stream` fallback — e.g. the 586
                # JPEG2000 `jpx` images in players_handbook.pdf) is
                # left EXACTLY as today (raw bytes, raw mime, skipped
                # downstream): compositing must never newly admit an
                # unsupported format.
                smask = int(extracted.get("smask") or 0)
                # The image's BASE pixmap, kept for the blank gate
                # (task 03) when the compositing built it (a second
                # decode of the same xref for the gate would be wasted
                # work; when the compositing never ran — no mask or an
                # unsupported raw format — the gate builds it itself).
                base_pixmap: Any = None
                if smask and mime != "application/octet-stream":
                    try:
                        base = fitz.Pixmap(doc, xref)
                        base_pixmap = base
                        mask = fitz.Pixmap(doc, smask)
                        visible = fitz.Pixmap(base, mask)  # base + alpha-from-mask
                        data = visible.tobytes("png")
                        mime = "image/png"
                        logger.debug(
                            "pdf: composited smask=%d for image page=%d index=%d of %s",
                            smask, page_num + 1, index, pdf_path,
                        )
                    except Exception as exc:
                        # Fail-soft: `data`/`mime` keep their raw
                        # values — the image is still indexed, exactly
                        # as pre-phase (a decode problem never loses a
                        # real image).
                        logger.warning(
                            "pdf: could not composite smask=%d for image page=%d "
                            "index=%d of %s — %s; using raw bytes",
                            smask, page_num + 1, index, pdf_path, exc,
                        )
                # Phase 151 (task 03): the blank/decorative gate — a
                # near-white, nearly-uniform image (a table-shading
                # underlay) carries no visible content: skip it
                # entirely (ASSUMPTION A1 — no row, no copy, no vision
                # call). Gated by `skip_blank` (the function-level kill
                # switch; task 04's settings knob lands here) — dedupe
                # and mask compositing above are NOT knobed.
                if skip_blank:
                    try:
                        # The BASE sample stream (the docstring's gate-
                        # input rule): the base's color content is what
                        # the image SHOWS (a mask only modulates its
                        # visibility), and PyMuPDF's stored composite
                        # is premultiplied (base×mask/255) — darkened
                        # in the transparent regions, where the gate
                        # would miss the very underlays it targets.
                        gate_pixmap = (
                            base_pixmap
                            if base_pixmap is not None
                            else fitz.Pixmap(doc, xref)
                        )
                        if gate_pixmap.alpha:
                            # Drop the alpha FIRST: the mean is over
                            # COLOR samples only (an alpha channel
                            # would drag the mean and produce false
                            # blanks).
                            gate_pixmap = fitz.Pixmap(gate_pixmap, 0)
                        samples = gate_pixmap.samples
                        if is_blank_image(samples):
                            blank_skipped += 1
                            mean = sum(samples) / len(samples)
                            uniform = (
                                sum(
                                    1
                                    for s in samples
                                    if abs(s - mean) <= BLANK_IMAGE_TOLERANCE
                                )
                                / len(samples)
                            )
                            logger.info(
                                "pdf: skipped blank/decorative image page=%d "
                                "index=%d of %s — mean=%.1f uniform=%.3f",
                                page_num + 1, index, pdf_path, mean, uniform,
                            )
                            continue
                    except Exception as exc:
                        # Fail-soft: a pixmap that cannot be built
                        # skips the gate (the image is KEPT — a decode
                        # problem must never delete a real image).
                        logger.debug(
                            "pdf: blank gate could not build pixmap page=%d "
                            "index=%d of %s — %s; keeping image",
                            page_num + 1, index, pdf_path, exc,
                        )
                images.append(
                    PdfImage(
                        page=page_num + 1,
                        index=index,
                        width=int(extracted.get("width") or info.get("width") or 0),
                        height=int(extracted.get("height") or info.get("height") or 0),
                        data=data,
                        mime=mime,
                    )
                )
    if dropped:
        logger.info(
            "pdf: dropped %d duplicate image occurrence(s) from %s (%d unique)",
            dropped, pdf_path, len(images),
        )
    if blank_skipped:
        logger.info(
            "pdf: skipped %d blank/decorative image(s) in %s",
            blank_skipped, pdf_path,
        )
    return images, PdfImageStats(
        duplicates=dropped, blank_skipped=blank_skipped, total_found=total_found
    )


def extract_pdf_images(
    pdf_path: Path, *, skip_blank: bool = True
) -> list[PdfImage]:
    """Extract all DISTINCT embedded images from *pdf_path*.

    Thin wrapper over :func:`extract_pdf_images_with_stats` (phase 151,
    task 03) returning the image list only — the pre-task-03 contract
    for every other caller. *skip_blank* (default ON) passes straight
    through: the kill switch at the function level —
    ``extract_pdf_images(path, skip_blank=False)`` restores the
    pre-phase result (every distinct image returned, gate off; the
    dedupe and mask compositing stay on — they are correctness fixes,
    not knobed). The importer calls the ``_with_stats`` variant so the
    run's :class:`~app.rag.importer.ImportSummary` can record the drop
    counts (the phase-133 stats log-only convention).
    """
    return extract_pdf_images_with_stats(pdf_path, skip_blank=skip_blank)[0]