phase: 133_pdf_screenshot_fallback
All verification is complete — everything is green. No repo-side changes were needed in this pass (the phase's work in the working tree is correct and complete).
## Phase 133 final verification report
- **Verified (already implemented in working tree):** `is_usable_pdf_text` gate (0.30 XML-ish / 0.05 non-printable / empty), `render_pdf_pages`, `_fallback_page_screenshots` + `pdfs_screenshotted` counter in `app/rag/{pdf,importer}.py`; regression pin, unit/integration/E2E suites.
- **Completion criteria:**
1. Real guide clean through real pipeline — ✅ `test_pdf_hnb_guide_regression` + E2E preview/viewer checks (0 XML-ish lines, first-page text present).
2. Garbage PDF → vision fallback end to end — ✅ unit + E2E (stored content = canned per-page descriptions, garbage not stored, sync `detail` shape unchanged).
3. Clean PDF never burns a fallback vision call — ✅ unit + E2E (byte-faithful content).
4. Gates — ✅ `uv run pytest --cov=app --cov-report=term-missing`: **3129 passed, app/ 99%** (>90%); `uv run pytest tests/e2e/test_garbage_pdf.py -v --no-cov`: **3 passed** in isolation.
5. No behavior change to completed phases — ✅ entire suite green (all prior-phase tests included); `ruff check .` + `pyright` clean.
6. Production stale row — post-deploy re-sync verification (live infra, no repo work; no live changes made — deploy of `main` is the owner's step).
- **Documentation:** README.md reviewed in full — every env default cross-checked against `app/config.py`, page routes against `app/main.py`, scripts/Containerfile/compose flags, `.env.example`; the phase-133 PDF bullet added in the working tree is accurate. AGENTS.md verified consistent (all referenced files/commands exist). **No changes needed.**
- **Next pending phase:** `134_sync_cancel`.
This commit is contained in:
@@ -636,13 +636,18 @@ def test_pdf_without_embedded_images_and_unchanged_rerun(db, tmp_path) -> None:
|
||||
_cleanup_source(db, source_root.name)
|
||||
|
||||
|
||||
def test_pdf_without_text_becomes_an_empty_document(db, tmp_path) -> None:
|
||||
"""A PDF with no extractable text (image-only, ``images`` OFF — the
|
||||
embedded image is invisible) → the document is still created, with
|
||||
EMPTY content (the empty byte string's digest), NO content chunks
|
||||
and NO content embedding (there is no text to embed — the only
|
||||
chunk is the best-effort ``lite`` summary's ``is_summary`` row), and
|
||||
the warning line is the signal."""
|
||||
def test_pdf_without_text_takes_the_screenshot_fallback(db, tmp_path) -> None:
|
||||
"""Phase 133 (task 03): a PDF with no extractable text layer
|
||||
(image-only, ``images`` OFF — the embedded image is invisible to
|
||||
the phase-126 A3 pass, but the FALLBACK is not gated by the
|
||||
toggle) → the gate's empty-layer target fires: the single page is
|
||||
rendered to a PNG and vision-described (the mock's fixed
|
||||
:data:`DESCRIPTION`), and the document lands with the
|
||||
``---PAGE-N---``-anchored description as content (one page-anchored
|
||||
content chunk + the ``lite`` summary chunk, both embedded). The
|
||||
phase-126 empty-content behavior is retired for layers that CAN be
|
||||
screenshotted — it survives only for an unopenable file, where the
|
||||
render fails too (the unit suite pins it)."""
|
||||
source_root = tmp_path / "pdfemptykb"
|
||||
source_root.mkdir()
|
||||
_build_pdf(source_root / "pics.pdf", [("", [_png_bytes(40, 30)])])
|
||||
@@ -650,6 +655,7 @@ def test_pdf_without_text_becomes_an_empty_document(db, tmp_path) -> None:
|
||||
try:
|
||||
summary = asyncio.run(import_sources([source_root], llm, session=db))
|
||||
assert (summary.files, summary.added, summary.images_failed) == (1, 1, 0)
|
||||
assert summary.pdfs_screenshotted == 1
|
||||
|
||||
doc = db.scalar(
|
||||
select(Document).where(
|
||||
@@ -657,18 +663,19 @@ def test_pdf_without_text_becomes_an_empty_document(db, tmp_path) -> None:
|
||||
)
|
||||
)
|
||||
assert doc is not None and doc.is_image is False
|
||||
assert doc.content == ""
|
||||
assert doc.content_hash == hashlib.sha256(b"").hexdigest()
|
||||
assert doc.content == f"---PAGE-1---\n{DESCRIPTION}"
|
||||
assert doc.content_hash == hashlib.sha256(doc.content.encode("utf-8")).hexdigest()
|
||||
chunks = db.scalars(
|
||||
select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
|
||||
).all()
|
||||
assert [c for c in chunks if not c.is_summary] == [], (
|
||||
"no text → no content chunks (and no content embedding)"
|
||||
content_chunks = [c for c in chunks if not c.is_summary]
|
||||
assert [c.content for c in content_chunks] == [f"Page 1:\n{DESCRIPTION}"], (
|
||||
"the page-anchored fallback content is chunked like any PDF"
|
||||
)
|
||||
# The phase-30/118 path still runs best-effort on the empty
|
||||
# content — the summary is the only chunk, and it is embedded.
|
||||
assert len(chunks) == 1 and chunks[0].is_summary
|
||||
assert chunks[0].embedding is not None and len(chunks[0].embedding) == 768
|
||||
# The phase-30/118 path runs on the fallback content — the
|
||||
# summary chunk mirrors the stored summary, both embedded.
|
||||
assert len(chunks) == 2
|
||||
assert all(c.embedding is not None and len(c.embedding) == 768 for c in chunks)
|
||||
assert doc.summary is not None
|
||||
finally:
|
||||
_cleanup_source(db, source_root.name)
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Integration: phase 133 (task 01) — the owner's HNB guide regression pin.
|
||||
|
||||
FINDING (the two-line root cause, full evidence in the phase overview):
|
||||
the garbage the owner saw is the PRE-PHASE-126 import — ``pdf`` was
|
||||
already in the owner's ``BOR_IMPORT_EXTENSIONS`` (local, untracked
|
||||
``.env``) before the PDF parser shipped, so the generic text branch
|
||||
read the RAW PDF bytes (``read_text(errors="replace")``) and the
|
||||
uncompressed Illustrator XMP/structure blocks (222 ×
|
||||
``<illustrator:StartupProfile>`` + 138 × ``<x:xmpmeta>`` blocks) were
|
||||
chunked and embedded as literal text. Phase 126+ extracts the same
|
||||
file correctly; the stale production row self-heals on re-sync
|
||||
(content-hash delta → ``updated`` with clean content).
|
||||
|
||||
This suite pins the owner's EXACT file as a permanent regression: the
|
||||
archive's 48-page Illustrator PDF is imported through the REAL
|
||||
importer — the real walk, the real PDF branch (extraction → digest →
|
||||
page-anchored chunking → 768-dim embeds → ``lite`` summary → the
|
||||
phase-126 embedded-image pass, whose vision seam runs against the
|
||||
multimodal-capable fake below) — and must land CLEAN content: the
|
||||
XML-ish line fraction stays far under the task-02 gate's 0.30
|
||||
(0.0 in the real extraction — asserted < 0.05 to be strict), the
|
||||
non-printable fraction stays < 0.01, and the real first-page text is
|
||||
present. If the PDF is ever routed through the text branch again, the
|
||||
stored content is the raw bytes again (~46% XML-ish) and the pin
|
||||
breaks — the pin has teeth.
|
||||
|
||||
Run:
|
||||
uv run pytest tests/integration/test_pdf_hnb_guide_regression.py -v
|
||||
Requires: podman compose up -d db
|
||||
|
||||
Fixture provenance: ``test_sources/hnb_2026_guide.tar.gz`` — the
|
||||
UNTRACKED owner fixture (2.6 MB compressed; one 6.2 MB, 48-page
|
||||
PDFium/Illustrator export, ``HBI-2026-Guide-Year-Round_260704_131221.pdf``
|
||||
inside). The test extracts it into ``tmp_path`` (the test's local
|
||||
source dir) and never touches the repo — ``git status`` stays clean of
|
||||
binaries. The pinned fixture text (the first-page title) fails LOUDLY
|
||||
if the owner ever re-cuts the fixture: that IS the regression signal.
|
||||
|
||||
Config note (why the settings are built the way they are):
|
||||
* ``_env_file=".env"`` — the REAL extension set: ``pdf`` is in the
|
||||
owner's live ``BOR_IMPORT_EXTENSIONS``. A ``_env_file=None`` fake
|
||||
would use the code defaults (no ``pdf`` token) and SILENTLY skip the
|
||||
PDF — the exact trap that hid this class before (FINDING, point 3).
|
||||
* ``images=True`` — the owner's live toggle (``BOR_IMAGES=1`` in the
|
||||
same ``.env``). The suite forces ``BOR_IMAGES`` to the code default
|
||||
(off) in the process environment (the conftest leak class, phase
|
||||
122), and process env beats the ``.env`` file — so the live value is
|
||||
restored with an init kwarg (init kwargs beat both). The embedded-
|
||||
image pass is the phase-126 A3 seam this pin covers.
|
||||
* ``image_dir`` under ``tmp_path`` — the house hermetic pattern
|
||||
(``test_importer.py``'s ``_image_llm``): the embedded-image copies
|
||||
never touch the owner's persistent image home.
|
||||
|
||||
``summary.added`` accounting: the walked file contributes EXACTLY ONE
|
||||
added document (the guide PDF); the phase-126 A3 convention counts
|
||||
each embedded image doc ``added`` too, so the assertion is
|
||||
``added == 1 + len(image_docs)`` together with ``files == 1`` (the
|
||||
walk admitted the one file) — the strict generalization of the naive
|
||||
``added == 1`` for a PDF whose images join the KB.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import tarfile
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.importer import ImportSummary, import_sources
|
||||
from app.rag.pdf import PAGE_SEPARATOR
|
||||
from app.rag.summarizer import DESCRIBE_PROMPT
|
||||
from tests.fakes import FakeEmbedder
|
||||
|
||||
#: The repo root (this file lives in ``tests/integration/``).
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
#: The owner's untracked fixture (the stable reference — the tarball,
|
||||
#: never the extracted PDF, is what the test starts from).
|
||||
TARBALL = REPO_ROOT / "test_sources" / "hnb_2026_guide.tar.gz"
|
||||
|
||||
#: The inner file keeps its name; the temp source dir is named
|
||||
#: ``test_sources`` (the importer's ``source = root.name`` convention —
|
||||
#: the same identity the handbook pin uses).
|
||||
SOURCE = "test_sources"
|
||||
PDF_NAME = "HBI-2026-Guide-Year-Round_260704_131221.pdf"
|
||||
PDF_REL = f"{SOURCE}/{PDF_NAME}"
|
||||
|
||||
#: The pinned fixture text (the real first page). If the owner ever
|
||||
#: re-cuts the fixture, the failure names the regression — the point.
|
||||
FIRST_PAGE_TEXT = "2026 U.S. Benefits Guide"
|
||||
|
||||
#: The title: the stem fallback — the PDF's metadata carries no title
|
||||
#: (only the PDFium creator/producer), so the non-markdown rule applies.
|
||||
TITLE = "HBI-2026-Guide-Year-Round_260704_131221"
|
||||
|
||||
#: The phase-133 XML-ish line heuristic (the task-02 gate's regex):
|
||||
#: ``<tag>``, ``</tag>``, ``<tag attr=``, and the PDF-structure
|
||||
#: ``<</Key…`` noise all match.
|
||||
XMLISH_RE = re.compile(r"</?[A-Za-z_][\w:.-]*[\s/>=]")
|
||||
|
||||
#: Clean-content bars (stricter than the task-02 gate on purpose — the
|
||||
#: real extraction is 0.0 XML-ish / ~0.0005 non-printable, the
|
||||
#: pre-126 garbage read is ~0.46 XML-ish AND heavy on U+FFFD).
|
||||
XML_FRACTION_BAR = 0.05
|
||||
NONPRINT_FRACTION_BAR = 0.01
|
||||
|
||||
#: The task's wall-time guard: 48 pages × in-process fakes must stay
|
||||
#: fast — a drift past this means the test is too heavy.
|
||||
WALL_TIME_BAR_S = 120.0
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not TARBALL.is_file(),
|
||||
reason=(
|
||||
"test_sources/hnb_2026_guide.tar.gz is missing — the untracked "
|
||||
"owner fixture (2.6 MB, one 48-page Illustrator PDF)"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class GuideVisionLLM(FakeEmbedder):
|
||||
"""The task-03-style vision mock the ``fakes.py`` comment
|
||||
anticipates: the base ``chat`` is string-only — the MULTIMODAL user
|
||||
content ``describe_image`` sends (the ``list``-of-parts shape)
|
||||
would crash it — so ``list`` content gets the deterministic canned
|
||||
reply instead. String (``lite``) calls keep the ``FakeEmbedder``
|
||||
behaviour. Mirrors ``test_importer.py``'s ``_MockVisionLLM``
|
||||
pattern; the REAL ``import_sources`` → ``_describe_or_skip`` →
|
||||
``describe_image`` chain runs against it (no seam patch)."""
|
||||
|
||||
VISION_REPLY = "A rendered page of the guide (test vision description)."
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
#: The text part of every multimodal (vision) call received.
|
||||
self.vision_texts: list[str] = []
|
||||
|
||||
async def chat(
|
||||
self, messages: list[dict[str, Any]], model: str | None = None
|
||||
) -> str:
|
||||
self.chat_calls.append(list(messages))
|
||||
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
|
||||
if isinstance(user, list):
|
||||
self.vision_texts.append(
|
||||
next((p["text"] for p in user if p.get("type") == "text"), "")
|
||||
)
|
||||
return self.VISION_REPLY
|
||||
first = user.split()
|
||||
return "Summary of " + (first[0] if first else "<empty>")
|
||||
|
||||
|
||||
def _settings(tmp_path: Path) -> Settings:
|
||||
"""The owner's live import config (the point of the pin — see the
|
||||
module docstring's Config note) + the hermetic image dir."""
|
||||
kwargs: dict[str, Any] = {
|
||||
"_env_file": ".env",
|
||||
"images": True,
|
||||
"image_dir": str(tmp_path / "images"),
|
||||
}
|
||||
return Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
def _xml_frac(content: str) -> float:
|
||||
"""The phase-133 XML-ish line fraction: non-empty lines matching
|
||||
:data:`XMLISH_RE`, over the non-empty lines."""
|
||||
lines = [line for line in content.splitlines() if line.strip()]
|
||||
if not lines:
|
||||
return 0.0
|
||||
return sum(1 for line in lines if XMLISH_RE.search(line)) / len(lines)
|
||||
|
||||
|
||||
def _nonprint_frac(content: str) -> float:
|
||||
"""The binary-soup signal: U+FFFD replacement chars + C0 control
|
||||
chars over the whole text — the whitespace controls (``\\n``/``\\t``/
|
||||
``\\r``) are prose line breaks, not garbage (counting them would
|
||||
flag every document), so they are excluded."""
|
||||
if not content:
|
||||
return 0.0
|
||||
bad = sum(
|
||||
1
|
||||
for ch in content
|
||||
if ch == "\ufffd" or (ord(ch) < 32 and ch not in "\r\n\t")
|
||||
)
|
||||
return bad / len(content)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def guide_kb(
|
||||
db: Session, tmp_path: Path
|
||||
) -> Iterator[tuple[GuideVisionLLM, ImportSummary, float]]:
|
||||
"""The real guide, imported ONCE per test through the REAL importer
|
||||
(the house KB pattern — ``test_read_line_ranges_handbook.py``):
|
||||
``chunks``/``documents``/``folder_summaries`` are truncated around
|
||||
the test, the tarball is extracted into a temp ``test_sources``
|
||||
source dir, and the deterministic in-process
|
||||
:class:`GuideVisionLLM` stands in for the embed/summary/vision
|
||||
calls (the whole 48-page index build stays fast and offline).
|
||||
Yields ``(llm, summary, elapsed_s)``."""
|
||||
db.execute(text("TRUNCATE chunks, documents, folder_summaries"))
|
||||
db.commit()
|
||||
root = tmp_path / SOURCE
|
||||
root.mkdir()
|
||||
with tarfile.open(TARBALL, "r:gz") as tar:
|
||||
tar.extractall(root, filter="data")
|
||||
llm = GuideVisionLLM()
|
||||
llm.settings = _settings(tmp_path)
|
||||
started = time.monotonic()
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
elapsed = time.monotonic() - started
|
||||
yield llm, summary, elapsed
|
||||
db.execute(text("TRUNCATE chunks, documents, folder_summaries"))
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
|
||||
|
||||
def test_hnb_guide_imports_clean_through_the_real_pipeline(
|
||||
db: Session,
|
||||
guide_kb: tuple[GuideVisionLLM, ImportSummary, float],
|
||||
) -> None:
|
||||
"""The owner's exact file lands CLEAN through the real pipeline:
|
||||
the walk admits the one file (``pdf`` token — the PDF branch, never
|
||||
the text branch), the stored content is clean + carries the real
|
||||
first-page text, the page-anchored chunks are embedded, the ``lite``
|
||||
summary is stored, and every embedded-image doc got the fake vision
|
||||
description (the multimodal ``chat`` was called)."""
|
||||
llm, summary, elapsed = guide_kb
|
||||
|
||||
# The task's wall-time guard — the suite must stay fast.
|
||||
assert elapsed < WALL_TIME_BAR_S, (
|
||||
f"the guide import took {elapsed:.1f}s (> {WALL_TIME_BAR_S:.0f}s) — "
|
||||
"the test is too heavy, the suite must stay fast"
|
||||
)
|
||||
|
||||
# The walk ADMITTED the guide as one file with the ``pdf`` token —
|
||||
# the PDF branch (the text branch is unreachable for ``.pdf`` after
|
||||
# phase 126; if it ever ran on this file again, the content bars
|
||||
# below break — the pin has teeth).
|
||||
assert summary.files == 1
|
||||
assert summary.formats == {"pdf": 1}
|
||||
assert summary.errors == 0
|
||||
assert summary.unchanged == 0
|
||||
assert summary.pruned == 0
|
||||
|
||||
doc = db.scalar(
|
||||
select(Document).where(Document.source == SOURCE, Document.path == PDF_NAME)
|
||||
)
|
||||
assert doc is not None, "the guide did not import"
|
||||
assert doc.is_image is False
|
||||
assert doc.title == TITLE # the stem fallback (no metadata title)
|
||||
|
||||
content = doc.content
|
||||
# The stored content is CLEAN — the pre-126 XML-garbage import can
|
||||
# never come back (the garbage read is ~0.46 XML-ish AND heavy on
|
||||
# U+FFFD; the real extraction is 0.0 / ~0.0005).
|
||||
assert _xml_frac(content) < XML_FRACTION_BAR, (
|
||||
f"the guide's content is {_xml_frac(content):.0%} XML-ish — the "
|
||||
"pre-126 raw-bytes import came back (the PDF must not be routed "
|
||||
"through the text branch)"
|
||||
)
|
||||
assert _nonprint_frac(content) < NONPRINT_FRACTION_BAR, (
|
||||
f"the guide's content is {_nonprint_frac(content):.2%} non-printable — "
|
||||
"binary soup, not extracted text"
|
||||
)
|
||||
# The real first-page text is present (the pinned fixture text) and
|
||||
# the content opens with the page-1 marker (the phase-126 A2 shape).
|
||||
assert FIRST_PAGE_TEXT in content
|
||||
assert content.startswith(PAGE_SEPARATOR.format(page=1))
|
||||
|
||||
# The real pipeline ran to the end: page-anchored content chunks
|
||||
# with 768-dim embeddings + exactly one ``is_summary`` chunk at
|
||||
# position −1 (the ``lite`` summary, stored with the pointer line).
|
||||
chunks = db.scalars(
|
||||
select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
|
||||
).all()
|
||||
content_chunks = [c for c in chunks if not c.is_summary]
|
||||
assert content_chunks, "the guide's content produced no chunks"
|
||||
for c in chunks:
|
||||
assert c.embedding is not None and len(c.embedding) == 768
|
||||
summary_chunks = [c for c in chunks if c.is_summary]
|
||||
assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
|
||||
assert doc.summary is not None
|
||||
assert doc.summary.endswith(f"Source: {PDF_REL}")
|
||||
|
||||
# The embedded-image docs (the phase-126 A3 ``#page=…#img=…``
|
||||
# rows): every one of them got the FAKE vision description — the
|
||||
# multimodal ``chat`` was called (the task's literal assertion),
|
||||
# exactly once per image doc, with the locked bare prompt.
|
||||
image_docs = db.scalars(
|
||||
select(Document).where(
|
||||
Document.source == SOURCE,
|
||||
Document.is_image.is_(True),
|
||||
Document.path.like(f"{PDF_NAME}#page=%"),
|
||||
)
|
||||
).all()
|
||||
assert image_docs, "the guide's embedded images did not index"
|
||||
for img in image_docs:
|
||||
assert img.content == GuideVisionLLM.VISION_REPLY
|
||||
assert img.summary == img.content # the image convention
|
||||
assert img.image_path is not None
|
||||
copy = Path(img.image_path)
|
||||
assert copy.parent == Path(llm.settings.image_dir).expanduser()
|
||||
assert copy.exists(), "the embedded image's copy is missing"
|
||||
multimodal_calls = [
|
||||
messages
|
||||
for messages in llm.chat_calls
|
||||
if any(
|
||||
isinstance(m.get("content"), list) for m in messages if m.get("role") == "user"
|
||||
)
|
||||
]
|
||||
assert len(multimodal_calls) >= 1, "the multimodal chat was never called"
|
||||
assert len(llm.vision_texts) == len(image_docs), (
|
||||
"one vision call per embedded image doc (no failures)"
|
||||
)
|
||||
# The bare locked prompt on every call (no markdown reference
|
||||
# exists in this PDF-only source — the phase-126 A3 shape).
|
||||
assert all(t == DESCRIBE_PROMPT for t in llm.vision_texts)
|
||||
assert summary.images_failed == 0
|
||||
|
||||
# The added accounting (see the module docstring): the walked file
|
||||
# is EXACTLY ONE added document (the guide PDF); every other added
|
||||
# doc is an embedded-image byproduct (the phase-126 A3 convention
|
||||
# counts image docs added/updated) — nothing else touched the KB.
|
||||
assert summary.added == 1 + len(image_docs)
|
||||
assert summary.updated == 0
|
||||
Reference in New Issue
Block a user