"""PDF parsing utilities (phase 126 — PDF support, locked decision A1).
Pure functions (I/O = reading the PDF file only) for extracting text
and embedded images from PDF files using PyMuPDF (the ``pymupdf``
package, imported as ``fitz`` — its stable API name).
* :func:`extract_pdf_text` — the text of every page that has text,
each prefixed with :data:`PAGE_SEPARATOR` (``---PAGE-N---``, N is the
1-indexed page number). The PDF chunker (phase 126 task 03) splits on
these markers so every chunk keeps its page anchor. Pages without
text (image-only pages) contribute nothing, so a PDF with no
extractable text returns ``""``.
* :func:`extract_pdf_page_texts` — one entry per page that has text:
1-indexed page number → the page's stripped text (the same page read
as :func:`extract_pdf_text`, so the two agree page-for-page; the
value for page N is exactly that page's separator-block body). The
phase-137 image-context block is built from this map (page N's entry
is the document context for an image embedded on page N).
* :func:`extract_pdf_images` — one :class:`PdfImage` per DISTINCT
embedded image (page order, first placement wins) — repeated
placements and byte-identical copies are deduplicated (phase 151
task 01). ``data`` is the image's original encoded bytes as stored in
the PDF (no re-encoding), ``mime`` is derived via
:func:`_mime_from_image_info` — EXCEPT an image stored with a soft
mask (``/SMask``): ``extract_image`` returns the BASE sample stream
only (the mask carries the visible shape), so masked images are
composited to the visible alpha PNG (``mime="image/png"``) — the
bytes a PDF viewer would show (phase 151 task 02); an image whose
raw format is unsupported (the ``application/octet-stream``
fallback, e.g. JPEG2000 ``jpx``) is left untouched (raw bytes, raw
mime) so the importer's extension gate still decides. THEN the
blank/decorative gate (phase 151 task 03, ``skip_blank=True`` by
default — the ``skip_blank=False`` keyword is the function-level
kill switch): a distinct image whose base samples are near-white AND
nearly uniform (:func:`is_blank_image`) is SKIPPED entirely — no
row, no copy, no vision call (ASSUMPTION A1).
* :func:`extract_pdf_images_with_stats` — the stats-carrying variant
the importer calls (phase 151 task 03): the same extraction plus a
:class:`PdfImageStats` (``duplicates`` — occurrences dropped by the
dedupe, ``blank_skipped`` — distinct images dropped by the gate,
``total_found`` — occurrences found) so the run's
``ImportSummary`` can record them log-only (``pdf_images_deduped`` /
``pdf_images_blank_skipped`` — the phase-133 stats convention, the
sync ``detail`` API shape untouched).
* :func:`is_blank_image` — the phase-151 (task 03) blank/decorative
predicate: pure, deterministic, no PDF, no I/O (the
:func:`is_usable_pdf_text` / :func:`pictorial_pages` house style)
— near-white (mean ≥ 235) AND nearly uniform (≥98% of the samples
within ±17 of the mean). A dark uniform block is NOT blank (the gate
targets near-white page underlays); empty samples are not blank
(never blank a thing we cannot measure).
* :func:`_mime_from_image_info` — maps a PyMuPDF image-info dict to a
MIME type: the embedded file extension (``ext``) first, then the
image type code (``type``: 2=JPEG, 3=PNG, 4=GIF, 8=TIFF, 9=WebP),
else the ``application/octet-stream`` fallback.
* :func:`is_usable_pdf_text` — the phase-133 garbage gate: a pure,
deterministic predicate over the extracted text layer (no PDF, no
I/O) — REAL content vs. garbage (the pre-126 XML/binary soup, the
empty text layer). It is the single trigger for the page-screenshot
vision fallback (phase 133 task 03).
* :func:`pictorial_pages` — the phase-140 sparse-page gate: the 1-based
page numbers of a :func:`extract_pdf_page_texts` map whose stripped
text is STRICTLY under *min_chars* chars (ascending) — pure,
deterministic, NO LLM call (LOCKED A1): the "way to determine when
that's necessary" (TODO L5) for the pictorial pass.
* :func:`render_pdf_pages` — the fallback's visual input (phase 133
task 03): one PNG per page (150 DPI, page order); the phase-140
optional 1-based *pages* selector renders ONLY the named pages
(ascending — a 500-page PDF with 30 sparse pages renders 30 PNGs,
not 500), ``None`` = every page (the phase-133 caller's result is
byte-identical); an unreadable (or out-of-range named) page yields
``b""`` at its position (the caller's placeholder — the result
stays aligned with the requested pages), an unopenable file yields
``[]`` (the caller's existing empty-content path).
Every failure mode is fail-soft: an unreadable file returns an empty
result, an unreadable page or image is logged and skipped — this module
never raises for malformed input (the importer owns the loud failures).
"""
from __future__ import annotations
import hashlib
import logging
import re
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import pymupdf as fitz # PyMuPDF (locked A1); `fitz` is the stable API name
logger = logging.getLogger("app.rag.pdf")
#: Page separator used in extracted text (the chunker splits on this).
#: ``{page}`` is the 1-indexed page number.
PAGE_SEPARATOR = "---PAGE-{page}---"
#: Mapping from PyMuPDF image type codes to MIME types
#: (2=JPEG, 3=PNG, 4=GIF, 8=TIFF, 9=WebP — phase-126 design).
_TYPE_TO_MIME: dict[int, str] = {
2: "image/jpeg",
3: "image/png",
4: "image/gif",
8: "image/tiff",
9: "image/webp",
}
#: Mapping from embedded-image file extensions to MIME types.
_EXT_TO_MIME: dict[str, str] = {
"png": "image/png",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"gif": "image/gif",
"webp": "image/webp",
"tif": "image/tiff",
"tiff": "image/tiff",
}
#: Phase 151 (task 03) — the blank/decorative image gate's thresholds
#: (ASSUMPTION A2, owner-locked): a PDF-embedded image is
#: blank/decorative when its COLOR samples are NEAR-WHITE AND NEARLY
#: UNIFORM — :func:`is_blank_image`'s three numbers.
#: Measured basis (probe: ``fitz.Pixmap(...).samples`` on the image's
#: BASE sample stream, the three real PDFs in ``test_sources/``):
#
#: * ``podman-systemd.pdf``'s 9 table-shading underlays: mean 238,
#: uniq 8, uniform(±8) 0.95–0.995 → blank (the TODO L4 bug — the
#: "blank white images" the owner saw in the KB);
#: * ``pdf_horse_test.pdf``'s photo: mean 123, uniq 256, uniform ~0.20
#: → kept (the mean gate);
#: * ``players_handbook.pdf``'s page art: mean 24–221, uniq ≥188,
#: uniform 0.10–0.59 → kept (the mean gate).
BLANK_IMAGE_MEAN = 235 #: near-white floor (owner-locked A2)
#: "same as the mean" band (owner-locked A2 — see the CALIBRATED
#: DEVIATION note below: the phase overview's ±12 misses the podman
#: PDF's 238/255-blend underlay, ±17 is the smallest fix).
BLANK_IMAGE_TOLERANCE = 17
#: fraction of samples that must sit in the band (owner-locked A2).
BLANK_IMAGE_FRACTION = 0.98
# CALIBRATED DEVIATION (flagged per AGENTS.md rule 3 — never silent):
# the phase overview pinned the band at ±12, but the podman PDF's
# smallest underlay (page 47, 5 300 samples) is a 238/255 blend whose
# 255 level sits 16.2 above its mean (238.81) — at ±12, 4.5% of its
# samples fall out-of-band (uniform 0.954 < 0.98) and the image
# SURVIVES the gate, violating the phase's completion criterion
# (the extractor must return NO images for that file — the owner's
# file is the phase's reason for existing; a 238/255 strip is the
# very "blank white" document the TODO reported, the table text a
# viewer shows is real PDF text drawn on top). ±17 is the smallest
# integer band that keeps every measured real-file pin: all 9 podman
# underlays blank (99.4–99.9% in-band at ±17), the horse photo kept
# (mean 123 < 235 — the mean gate, tolerance irrelevant), and all 586
# players_handbook images kept (means 17–225 — the mean gate). The
# predicate's formula, the mean floor (235), and the fraction (0.98)
# are the owner-locked values, unchanged.
@dataclass
class PdfImage:
"""One embedded image from a PDF page."""
page: int #: 1-indexed page number
index: int #: image index on the page (0-based, occurrence order)
width: int #: native pixel width
height: int #: native pixel height
data: bytes #: encoded bytes — the original stored bytes, or the
#: composited (base + soft mask) alpha PNG for masked images
#: (phase 151 task 02)
mime: str #: MIME type (e.g. ``image/png`` — always for composited
#: masked images, phase 151 task 02)
@dataclass(frozen=True)
class PdfImageStats:
"""The per-PDF drop counts of one
:func:`extract_pdf_images_with_stats` call (phase 151, task 03).
The importer records :attr:`duplicates` / :attr:`blank_skipped`
on the run's :class:`app.rag.importer.ImportSummary` (the
phase-133 stats log-only convention — the sync ``detail`` API
shape is untouched).
"""
duplicates: int #: image OCCURRENCES dropped by the dedupe (task
#: 01) — repeated placements + byte-identical copies under
#: distinct xrefs; per run: sum across the PDFs
blank_skipped: int #: DISTINCT images dropped by the
#: blank/decorative gate (task 03) — no row, no image_dir copy,
#: no vision call
total_found: int #: image occurrences FOUND in the PDF (one per
#: placement — before dedupe and gate); hence
#: ``len(images) == total_found - duplicates - blank_skipped``
def _mime_from_image_info(info: dict[str, Any]) -> str:
"""Map a PyMuPDF image-info dict to a MIME type.
The embedded file extension (``ext``) wins over the image type code
(``type``); unknown/missing both fall back to
``application/octet-stream``.
"""
ext = str(info.get("ext") or "").lower()
if ext in _EXT_TO_MIME:
return _EXT_TO_MIME[ext]
type_code = info.get("type")
if isinstance(type_code, int):
return _TYPE_TO_MIME.get(type_code, "application/octet-stream")
return "application/octet-stream"
def _page_image_entries(page: Any) -> list[dict[str, Any]]:
"""One info dict per embedded-image occurrence on *page*.
Prefers ``get_image_info(xrefs=True)`` (one entry per occurrence,
with the ``xref`` object number); older PyMuPDF releases without the
``xrefs`` keyword are served from ``get_images(full=True)`` (one
entry per unique image).
"""
try:
infos: list[dict[str, Any]] = page.get_image_info(xrefs=True)
except TypeError:
infos = [
{"xref": entry[0], "width": entry[2], "height": entry[3]}
for entry in page.get_images(full=True)
]
return [dict(info) for info in infos]
def _iter_page_texts(doc: Any, pdf_path: Path) -> Iterator[tuple[int, str]]:
"""Yield ``(page, text)`` for every page of *doc* that has text.
``page`` is the 1-indexed page number and ``text`` the page's
stripped text (the ``get_text("text")`` read — it always returns a
str; the union return type comes from pymupdf's other flavors,
dict/words/blocks). A page that cannot be read is logged and
skipped; a page without text contributes nothing. The shared page
iteration behind :func:`extract_pdf_text` and
:func:`extract_pdf_page_texts` — the two callers share the exact
same read and cannot drift.
"""
for page_num in range(len(doc)):
try:
raw = doc[page_num].get_text("text")
except Exception as exc:
logger.warning(
"pdf: could not read text of page %d of %s — %s",
page_num + 1, pdf_path, exc,
)
continue
text = raw.strip() if isinstance(raw, str) else ""
if text:
yield page_num + 1, text
def extract_pdf_text(pdf_path: Path) -> str:
"""Extract the text of all pages of *pdf_path*.
Every page that has text is prefixed with :data:`PAGE_SEPARATOR`
(``---PAGE-N---``, N 1-indexed) and the page blocks are joined with
a blank line. Pages without text contribute nothing, so a PDF with
no extractable text (or a zero-page file) returns ``""``.
Fail-soft: an unreadable file returns ``""``; a page that cannot be
read is logged and skipped.
"""
try:
doc = fitz.open(str(pdf_path))
except Exception as exc:
logger.warning("pdf: could not open %s — %s", pdf_path, exc)
return ""
with doc:
parts = [
f"{PAGE_SEPARATOR.format(page=page)}\n{text}"
for page, text in _iter_page_texts(doc, pdf_path)
]
return "\n\n".join(parts)
def extract_pdf_page_texts(pdf_path: Path) -> dict[int, str]:
"""One entry per page of *pdf_path* that has text.
Keys are the 1-indexed page numbers, values the page's stripped
text — the shared :func:`_iter_page_texts` read, so this map agrees
with :func:`extract_pdf_text` page-for-page: the value for page N
is exactly the text after that page's ``---PAGE-N---`` separator.
Pages without text (image-only pages) contribute nothing — no key.
The phase-137 document context for an embedded image is built from
this map: one image on page N gets page N's text as the context
block's body.
Fail-soft: an unreadable file returns ``{}`` (one warning — the
:func:`extract_pdf_text` open contract); a page that cannot be
read is logged and skipped.
"""
try:
doc = fitz.open(str(pdf_path))
except Exception as exc:
logger.warning("pdf: could not open %s for page texts — %s", pdf_path, exc)
return {}
with doc:
return dict(_iter_page_texts(doc, pdf_path))
#: A line "looks like XML" when it carries a tag-shaped token — the
#: pre-126 garbage's signature (raw XMP blocks, PDF-structure noise).
_XMLISH_LINE_RE = re.compile(r"=]")
#: Garbage-gate thresholds (phase 133 task 02): an extracted text
#: layer is UNUSABLE when its XML-ish line fraction reaches this bar.
#: The FINDING's reproduced pre-126 read of the owner's file is 46%
#: XML-ish; a real document ABOUT XML that reaches the bar is a
#: documented false positive (ASSUMPTION A8 — the fallback still
#: extracts it via vision; the cost is a few vision calls).
XMLISH_LINE_FRACTION = 0.30
#: Garbage-gate thresholds (phase 133 task 02): the non-printable
#: fraction (U+FFFD replacement chars + C0 controls over the whole
#: text) at which the layer is UNUSABLE — the ``errors="replace"``
#: binary-soup signal (the pre-126 read is ~19% non-printable; real
#: extracted text is ~0.05%).
NONPRINT_CHAR_FRACTION = 0.05
def is_usable_pdf_text(text: str) -> bool:
"""Whether an extracted PDF text layer is REAL content (not garbage).
Unusable (→ the page-screenshot fallback) when ANY of:
* ``text.strip() == ""`` — the empty text layer (a scanned/image-
only PDF has nothing to read);
* ≥ :data:`XMLISH_LINE_FRACTION` (0.30) of the non-empty lines
match :data:`_XMLISH_LINE_RE` — the raw-XMP / binary-soup
signature (the FINDING's reproduced pre-126 read is 46% —
``read_text(errors="replace")`` on the owner's file left
``Web``
lines in the preview);
* ≥ :data:`NONPRINT_CHAR_FRACTION` (0.05) of all characters are
non-printable — the ``errors="replace"`` binary-soup signal.
The non-printable set is ``U+FFFD`` + C0 control chars
(0x00–0x1F) EXCEPT the structural ``\\t``/``\\n``/``\\r`` the
extractor legitimately emits (the explicit set is deliberate: a
``unicodedata`` Cc-category check would catch those too, and
counting them would flag every prose document).
Pure + deterministic (no PDF, no I/O) — the unit tests pin the
thresholds against the REAL reproduced garbage string, not a
hand-drawn one. A real document ABOUT XML whose tag-like lines
reach 30% is a documented false positive (ASSUMPTION A8 — the
fallback still extracts it correctly via vision; the cost is a
few vision calls).
"""
if not text.strip():
return False
non_empty_lines = [line for line in text.splitlines() if line.strip()]
xml_ish = sum(1 for line in non_empty_lines if _XMLISH_LINE_RE.search(line))
if xml_ish / len(non_empty_lines) >= XMLISH_LINE_FRACTION:
return False
non_printable = sum(
1
for ch in text
if ch == "\ufffd" or (ord(ch) < 32 and ch not in "\t\n\r")
)
return non_printable / len(text) < NONPRINT_CHAR_FRACTION
def pictorial_pages(page_texts: dict[int, str], *, min_chars: int) -> list[int]:
"""The 1-based page numbers of *page_texts* that are PICTORIAL
candidates: ``len(text.strip()) < min_chars`` — STRICTLY less than
(a page with exactly *min_chars* stripped chars is NOT sparse; the
boundary is pinned in the unit tests), ascending.
The deterministic sparse-page gate (phase 140, task 01, LOCKED A1)
— the "way to determine when that's necessary" for the pictorial PDF
pass (TODO L5): NO LLM call, pure, deterministic (no PDF, no I/O —
the input is a :func:`extract_pdf_page_texts` map, so the gate never
re-opens the file). A page that carries no text at all has NO key in
the map and is therefore not a candidate here (the importer's pass
owns the page-total handling); an empty map yields ``[]`` — a
clean-text PDF's path never burns a vision call.
The IKEA example (the phase-140 fixture, ``min_chars=100``):
``{1: "BESTÅ", 2: <7 KB of warnings>, 6: "Illustration 6"}`` →
``[1, 6, …]`` — page 1 (the title page: the logo + product name) IS
a candidate: describing it is useful (the logo + product name) and
never harmful; page 2's 7 KB of multilingual warnings is not.
``min_chars=0`` is the documented PLAN A21 special case — **every
page in the map is a candidate** (the "describe everything" mode).
The strict-``<`` test cannot express this (no length is below 0), so
0 is not "a stricter bar": it is the A21 no-cap endpoint, and a
negative value — unreachable through ``Settings`` (the validator
fails startup loudly) — takes the same branch defensively.
"""
if min_chars <= 0:
return sorted(page_texts)
return sorted(
page for page, text in page_texts.items() if len(text.strip()) < min_chars
)
def is_blank_image(
samples: bytes,
*,
mean_min: int = BLANK_IMAGE_MEAN,
tolerance: int = BLANK_IMAGE_TOLERANCE,
fraction: float = BLANK_IMAGE_FRACTION,
) -> bool:
"""Whether *samples* (raw image sample bytes — one byte per sample,
COLOR channels only, no alpha) are a BLANK/DECORATIVE near-white
image (phase 151, task 03).
In the house style of :func:`is_usable_pdf_text` /
:func:`pictorial_pages`: pure, deterministic, NO PDF, NO I/O, NO
LLM — unit-testable with hand-built sample buffers.
Blank when ALL of:
* ``samples`` is non-empty — empty samples → ``False`` (never blank
a thing we cannot measure);
* ``sum(samples) / len(samples) >= mean_min`` — the near-white
floor (default :data:`BLANK_IMAGE_MEAN` = 235). A DARK uniform
block is NOT blank by this rule: the gate targets near-white
PAGE UNDERLAYS (the table-shading rectangles the TODO L4 bug
indexed) — a solid dark block is unusual and cheaper to keep
than to explain away;
* at least ``fraction`` of the samples lie within ``±tolerance``
of the mean (defaults :data:`BLANK_IMAGE_FRACTION` = 0.98,
:data:`BLANK_IMAGE_TOLERANCE` = 17) — the "carries no visible
content" test: a real photo scatters its samples far outside
the band, a decorative underlay sits on one (near-white) level.
Measured basis (the constants' docstring carries the full numbers):
the podman underlays (mean 238, uniq 8, uniform(±8) 0.95–0.995)
are blank; the horse photo (mean 123, uniq 256, uniform ~0.20) and
the handbook page art (mean 24–221, uniq ≥188, uniform 0.10–0.59)
are kept — the mean gate alone carries both real-file "kept" pins,
the fraction+tolerance carry the "near-white but textured" class.
"""
if not samples:
return False
n = len(samples)
mean = sum(samples) / n
if mean < mean_min:
return False
uniform = sum(1 for s in samples if abs(s - mean) <= tolerance) / n
return uniform >= fraction
def render_pdf_pages(
pdf_path: Path, dpi: int = 150, pages: Sequence[int] | None = None
) -> list[bytes]:
"""Render the selected pages of *pdf_path* to PNGs.
*pages* (phase 140, task 02): the optional 1-based selector —
render ONLY the named pages, in ascending order, one PNG each
(a 500-page PDF with 30 pictorial candidates renders 30 PNGs, not
500). ``None`` (the default) = every page in page order — the
phase-133 caller's result is byte-identical, so list index *i* is
page *i*+1 (with a selector, list index *i* is the *i*-th smallest
NAMED page).
The fallback's visual input (phase 133 task 03, ASSUMPTION A8:
fixed 150 DPI — the balance of vision legibility vs data-URL
payload). A page that cannot be rendered — including a named page
outside the document's page range — yields ``b""`` at ITS position
(the caller's placeholder — the result's order NEVER shifts, so it
stays aligned with the requested pages; the failure is logged, the
:func:`extract_pdf_text` fail-soft style). A file that cannot be
OPENED yields ``[]`` (the caller's existing empty-content path —
the doc lands with the no-extractable-text behavior, unchanged).
"""
try:
doc = fitz.open(str(pdf_path))
except Exception as exc:
logger.warning("pdf: could not open %s for page rendering — %s", pdf_path, exc)
return []
# 1-based page numbers: the default walks every page in page order
# (the phase-133 contract — list index *i* is page *i*+1); a
# selector renders the named pages, ascending (de-duplicated — a
# repeated page number would render it twice for no reason).
numbers = (
range(1, len(doc) + 1)
if pages is None
else sorted(set(pages))
)
rendered: list[bytes] = []
with doc:
for page_number in numbers:
try:
rendered.append(
doc[page_number - 1].get_pixmap(dpi=dpi).tobytes("png")
)
except Exception as exc:
logger.warning(
"pdf: could not render page %d of %s — %s",
page_number, pdf_path, exc,
)
rendered.append(b"")
return rendered
def extract_pdf_images_with_stats(
pdf_path: Path, *, skip_blank: bool = True
) -> tuple[list[PdfImage], PdfImageStats]:
"""Extract all DISTINCT embedded images from *pdf_path*, with the
per-PDF drop counts (phase 151, task 03 — the stats-carrying
variant the importer calls; :func:`extract_pdf_images` is the thin
list-only wrapper for every other caller).
Returns ``(images, stats)`` — one :class:`PdfImage` per DISTINCT
embedded image, in page order then 0-based index on the page, plus
the :class:`PdfImageStats` of the drops (``len(images) ==
stats.total_found - stats.duplicates - stats.blank_skipped``). The
FIRST occurrence wins: the kept entry carries the first
placement's ``page``/``index`` (the importer's synthetic path
``…#page=N#img=i.`` is therefore stable for the first
placement). Two kinds of duplicate are dropped (phase 151, task
01) — PyMuPDF reports one info entry per PLACEMENT, so a PDF that
places the same image N× yields N entries:
* the same ``xref`` seen again (a repeated placement) — the
``seen_xrefs`` set;
* byte-identical stored bytes under a DIFFERENT ``xref`` (the same
sha256 of ``data``) — the ``seen_digests`` set (real documents DO
store the same image under several xrefs — the digest set is what
catches those; unique-xref count overstates the distinct-image
count).
When anything was dropped, ONE INFO line per PDF after the page
loop (``pdf: dropped N duplicate image occurrence(s) from
(M unique)``) plus one DEBUG line per drop (``pdf: duplicate image
xref=… page=… index=… of — skipped``). Dedupe is PER-PDF —
the seen-sets are local to this call, never across files (two PDFs
sharing a logo each keep their own image doc: the synthetic path is
per-PDF, so cross-PDF dedupe would break the upsert key).
``data`` is the original encoded bytes as stored in the PDF;
``mime`` is derived from the image's extension/type via
:func:`_mime_from_image_info` — with one exception (phase 151,
task 02): an image stored with a soft mask (``/SMask``). PyMuPDF's
``extract_image`` returns the BASE sample stream only — the mask
(which carries the visible shape — e.g. the table-shading underlay
the owner's podman PDF stores as a near-white base) is dropped — so
for such images the extractor composites base + mask into the
VISIBLE image (``fitz.Pixmap(base, mask)`` → ``data =
visible.tobytes("png")``, ``mime = "image/png"``): the bytes a PDF
viewer shows, not the blank base. Two rules:
* the RAW-format gate runs FIRST (owner-locked A4): an image whose
raw format the importer's ``_MIME_TO_EXT`` gate rejects (the
``application/octet-stream`` fallback — e.g. JPEG2000 ``jpx``,
the 586 images in ``players_handbook.pdf``) is NOT composited —
raw bytes + raw mime, byte-identical to pre-phase; compositing
must never newly admit an unsupported format (a ``jpx`` image
with a mask stays skipped);
* the compositing is fail-soft: any exception (a colorspace the
combiner cannot merge, a broken mask xref) falls back to the raw
bytes + raw mime with a warning — the image is still indexed,
exactly as pre-phase (a decode problem never loses a real
image). ``width``/``height`` are unchanged either way (the
composited pixmap has the base's dimensions).
THEN the blank/decorative gate (phase 151, task 03, gated by
*skip_blank* — the kill switch at the function level; task 04's
``BOR_PDF_IMAGE_SKIP_BLANK`` setting lands here, dedupe and mask
compositing are NOT knobed): a distinct image whose samples are
near-white AND nearly uniform (:func:`is_blank_image`) is SKIPPED
entirely — not returned (ASSUMPTION A1: never indexed, never
described — no row, no ``image_dir`` copy, no vision call) — with
one INFO line per skipped image (``pdf: skipped blank/decorative
image page=… index=… of — mean=… uniform=…``) and one INFO
line per PDF when anything was skipped (``pdf: skipped N
blank/decorative image(s) in ``). The gate measures the
image's BASE sample stream (``fitz.Pixmap(doc, xref)``, the alpha
channel dropped first when present — the mean is over COLOR
samples only, an alpha channel would drag the mean and produce
false blanks) for every image — masked or not: for a masked image
the base is the correct "visible content" signal, because the
``/SMask`` only modulates the base's VISIBILITY (a near-white base
renders near-white over the white page under ANY mask, so a
near-white base is decorative no matter what the mask says) — and
the stored composite's color channels are PyMuPDF's
PREMULTIPLIED base×mask/255, which darkens the transparent regions
and would miss the very near-white underlays this gate targets
(measured on the podman underlays: alpha-stripped composite
mean ≈ 231–234 / uniform ≈ 0.95 vs base mean ≈ 238 / uniform
≈ 0.99 — only the base clears the pinned thresholds on every
underlay). The gate is fail-soft: a pixmap that cannot be built
skips the gate (the image is KEPT, DEBUG why) — a decode problem
must never delete a real image.
Fail-soft: an unreadable file returns ``([], PdfImageStats(0, 0,
0))``; an image without an ``xref`` or that cannot be extracted is
logged and skipped.
"""
try:
doc = fitz.open(str(pdf_path))
except Exception as exc:
logger.warning("pdf: could not open %s for image extraction — %s", pdf_path, exc)
return [], PdfImageStats(duplicates=0, blank_skipped=0, total_found=0)
seen_xrefs: set[int] = set()
seen_digests: set[str] = set()
dropped = 0
blank_skipped = 0
total_found = 0
images: list[PdfImage] = []
with doc:
for page_num in range(len(doc)):
try:
entries = _page_image_entries(doc[page_num])
except Exception as exc:
logger.warning(
"pdf: could not list images on page %d of %s — %s",
page_num + 1, pdf_path, exc,
)
continue
for index, info in enumerate(entries):
xref = int(info.get("xref") or 0)
if not xref:
logger.warning(
"pdf: skipping image page=%d index=%d of %s — no xref",
page_num + 1, index, pdf_path,
)
continue
try:
extracted = doc.extract_image(xref)
except Exception as exc:
logger.warning(
"pdf: could not extract image page=%d index=%d of %s — %s",
page_num + 1, index, pdf_path, exc,
)
continue
data = extracted.get("image") or b""
if not data:
logger.warning(
"pdf: skipping image page=%d index=%d of %s — empty data",
page_num + 1, index, pdf_path,
)
continue
# One image occurrence found (valid xref, extracted,
# non-empty data) — the input to dedupe + gate (task
# 03's total_found; len(images) = total_found -
# duplicates - blank_skipped).
total_found += 1
# Phase 151 (task 01): one PdfImage per DISTINCT image.
# Repeated placements share the xref (the first check);
# byte-identical bytes stored under a different xref are
# caught by the digest of the stored bytes (the second).
if xref in seen_xrefs:
dropped += 1
logger.debug(
"pdf: duplicate image xref=%d page=%d index=%d of %s — skipped",
xref, page_num + 1, index, pdf_path,
)
continue
digest = hashlib.sha256(data).hexdigest()
if digest in seen_digests:
dropped += 1
logger.debug(
"pdf: duplicate image xref=%d page=%d index=%d of %s — skipped",
xref, page_num + 1, index, pdf_path,
)
continue
seen_xrefs.add(xref)
seen_digests.add(digest)
# The image info's own `ext` is unreliable in some
# PyMuPDF versions (None); `extract_image` reports the
# actual format of the returned bytes, so it wins.
mime = _mime_from_image_info({**info, "ext": extracted.get("ext") or ""})
# Phase 151 (task 02): a /SMask carries the visible
# shape — `extract_image` returns the BASE sample
# stream only (the mask is dropped), so the raw bytes
# are not what a PDF viewer shows. Composite base +
# mask to the visible alpha PNG.
#
# Ordering rule (owner-locked A4): the raw-format gate
# runs FIRST — an image whose raw format the importer's
# `_MIME_TO_EXT` gate will reject (the
# `application/octet-stream` fallback — e.g. the 586
# JPEG2000 `jpx` images in players_handbook.pdf) is
# left EXACTLY as today (raw bytes, raw mime, skipped
# downstream): compositing must never newly admit an
# unsupported format.
smask = int(extracted.get("smask") or 0)
# The image's BASE pixmap, kept for the blank gate
# (task 03) when the compositing built it (a second
# decode of the same xref for the gate would be wasted
# work; when the compositing never ran — no mask or an
# unsupported raw format — the gate builds it itself).
base_pixmap: Any = None
if smask and mime != "application/octet-stream":
try:
base = fitz.Pixmap(doc, xref)
base_pixmap = base
mask = fitz.Pixmap(doc, smask)
visible = fitz.Pixmap(base, mask) # base + alpha-from-mask
data = visible.tobytes("png")
mime = "image/png"
logger.debug(
"pdf: composited smask=%d for image page=%d index=%d of %s",
smask, page_num + 1, index, pdf_path,
)
except Exception as exc:
# Fail-soft: `data`/`mime` keep their raw
# values — the image is still indexed, exactly
# as pre-phase (a decode problem never loses a
# real image).
logger.warning(
"pdf: could not composite smask=%d for image page=%d "
"index=%d of %s — %s; using raw bytes",
smask, page_num + 1, index, pdf_path, exc,
)
# Phase 151 (task 03): the blank/decorative gate — a
# near-white, nearly-uniform image (a table-shading
# underlay) carries no visible content: skip it
# entirely (ASSUMPTION A1 — no row, no copy, no vision
# call). Gated by `skip_blank` (the function-level kill
# switch; task 04's settings knob lands here) — dedupe
# and mask compositing above are NOT knobed.
if skip_blank:
try:
# The BASE sample stream (the docstring's gate-
# input rule): the base's color content is what
# the image SHOWS (a mask only modulates its
# visibility), and PyMuPDF's stored composite
# is premultiplied (base×mask/255) — darkened
# in the transparent regions, where the gate
# would miss the very underlays it targets.
gate_pixmap = (
base_pixmap
if base_pixmap is not None
else fitz.Pixmap(doc, xref)
)
if gate_pixmap.alpha:
# Drop the alpha FIRST: the mean is over
# COLOR samples only (an alpha channel
# would drag the mean and produce false
# blanks).
gate_pixmap = fitz.Pixmap(gate_pixmap, 0)
samples = gate_pixmap.samples
if is_blank_image(samples):
blank_skipped += 1
mean = sum(samples) / len(samples)
uniform = (
sum(
1
for s in samples
if abs(s - mean) <= BLANK_IMAGE_TOLERANCE
)
/ len(samples)
)
logger.info(
"pdf: skipped blank/decorative image page=%d "
"index=%d of %s — mean=%.1f uniform=%.3f",
page_num + 1, index, pdf_path, mean, uniform,
)
continue
except Exception as exc:
# Fail-soft: a pixmap that cannot be built
# skips the gate (the image is KEPT — a decode
# problem must never delete a real image).
logger.debug(
"pdf: blank gate could not build pixmap page=%d "
"index=%d of %s — %s; keeping image",
page_num + 1, index, pdf_path, exc,
)
images.append(
PdfImage(
page=page_num + 1,
index=index,
width=int(extracted.get("width") or info.get("width") or 0),
height=int(extracted.get("height") or info.get("height") or 0),
data=data,
mime=mime,
)
)
if dropped:
logger.info(
"pdf: dropped %d duplicate image occurrence(s) from %s (%d unique)",
dropped, pdf_path, len(images),
)
if blank_skipped:
logger.info(
"pdf: skipped %d blank/decorative image(s) in %s",
blank_skipped, pdf_path,
)
return images, PdfImageStats(
duplicates=dropped, blank_skipped=blank_skipped, total_found=total_found
)
def extract_pdf_images(
pdf_path: Path, *, skip_blank: bool = True
) -> list[PdfImage]:
"""Extract all DISTINCT embedded images from *pdf_path*.
Thin wrapper over :func:`extract_pdf_images_with_stats` (phase 151,
task 03) returning the image list only — the pre-task-03 contract
for every other caller. *skip_blank* (default ON) passes straight
through: the kill switch at the function level —
``extract_pdf_images(path, skip_blank=False)`` restores the
pre-phase result (every distinct image returned, gate off; the
dedupe and mask compositing stay on — they are correctness fixes,
not knobed). The importer calls the ``_with_stats`` variant so the
run's :class:`~app.rag.importer.ImportSummary` can record the drop
counts (the phase-133 stats log-only convention).
"""
return extract_pdf_images_with_stats(pdf_path, skip_blank=skip_blank)[0]