phase: 137_pdf_image_context
All verification complete — everything is 100% green. Final report: **Phase 137 final verification pass — all tasks already complete, all criteria verified** - **Full suite:** `uv run pytest --cov=app --cov-report=term-missing` → 3272 passed, exit 0; `app/` coverage **99%** (>90% ✓) - **Phase E2E in isolation:** `uv run pytest tests/e2e/test_pdf_image_context.py -v --no-cov` → **4 passed** (forward context, display line, standalone parity, both-blocks markdown case) - **Pre-existing suites in isolation:** `test_pdf_support.py` + `test_image_anchor_context.py` + `test_image_documents.py` → **14 passed**; ripple-edited `test_garbage_pdf.py` → **3 passed** - **Lint/types:** `uv run ruff check .` → all passed; `uv run pyright` → 0 errors **Completion criteria:** (1) forward-path context in vision msg + stored `image_context`/summary — ✓ (unit builder/seam + integration fake-LLM + E2E); (2) both blocks document-first + standalone describe byte-identical — ✓ (combiner + standalone pins); (3) backfill with context + zero-vision idempotent re-sync + parent-gone origin-only — ✓ (6 integration tests); (4) suggestion block leads with `From <source>/<pdf> (page N)` — ✓ (renderer units + E2E); (5) all gates green — ✓ above; (6) no behavior change in phases 122/125/126 + sync `detail` shape untouched — ✓ (verified in code: new counter is log-only); (7) ruff+pyright — ✓; (8) commit — left to the harness per pipeline rules (working tree untouched by me). **Documentation:** README.md audited in full — the new phase-137 bullet verified against code (6 000-char cap, log-only `image_context_backfilled=N`, `<document>`-body locator placement), all env tables/defaults cross-checked vs `app/config.py` and `.env.example` — consistent, no changes needed. AGENTS.md: every rule/command verified — consistent, no changes needed. **Notable deviations (pre-existing, documented in task-05 report, forced by design):** `test_pdf_support.py`/`test_garbage_pdf.py` got minimal ripple edits (the "UNEDITED" pin was unsatisfiable once describe calls carry context by design); the unit `test_image_anchor_context.py` CONTEXT_PREFIX pin generalized per ASSUMPTION A5; one extra mock line (`_seed_context_line` accepts `From …`). **Next pending phase:** none — `todo/` holds only this phase.
This commit is contained in:
+548
-6
@@ -1,7 +1,9 @@
|
||||
"""Unit tests: PDF parser module (phase 126 task 02, locked A1/A2).
|
||||
|
||||
Covers ``app/rag/pdf.py``: text extraction with ``---PAGE-N---``
|
||||
separators (A2), embedded image extraction (original encoded bytes),
|
||||
separators (A2), the per-page text map (phase 137 task 01 —
|
||||
``extract_pdf_page_texts``, the document-context raw material),
|
||||
embedded image extraction (original encoded bytes),
|
||||
MIME mapping (extension first, then image type code, else the
|
||||
``application/octet-stream`` fallback), and the fail-soft contract —
|
||||
unreadable files and broken pages/images return empty results, never
|
||||
@@ -12,6 +14,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import tarfile
|
||||
from collections.abc import Callable, Sequence
|
||||
from datetime import timedelta
|
||||
@@ -27,7 +30,12 @@ from app.config import Settings
|
||||
from app.models import Chunk, Document
|
||||
from app.rag import importer
|
||||
from app.rag import pdf as pdf_mod
|
||||
from app.rag.importer import import_sources
|
||||
from app.rag.importer import (
|
||||
PDF_IMAGE_CONTEXT_MAX_CHARS,
|
||||
_combine_image_contexts,
|
||||
_pdf_image_context_block,
|
||||
import_sources,
|
||||
)
|
||||
from app.rag.llm import EmbeddingError, LLMError
|
||||
from app.rag.pdf import (
|
||||
_XMLISH_LINE_RE,
|
||||
@@ -38,10 +46,13 @@ from app.rag.pdf import (
|
||||
_mime_from_image_info,
|
||||
_page_image_entries,
|
||||
extract_pdf_images,
|
||||
extract_pdf_page_texts,
|
||||
extract_pdf_text,
|
||||
is_usable_pdf_text,
|
||||
render_pdf_pages,
|
||||
)
|
||||
from app.rag.retriever import TRUNCATION_MARKER
|
||||
from app.rag.summarizer import DESCRIBE_PROMPT
|
||||
from tests.fakes import FakeEmbedder
|
||||
|
||||
PNG_MAGIC = b"\x89PNG"
|
||||
@@ -284,8 +295,9 @@ def test_pdf_embedded_images_become_image_documents(db, tmp_path: Path) -> None:
|
||||
pipeline — the single seam describes it (the CHAT model), the copy
|
||||
lands in ``image_dir`` named by the doc id, and the doc row carries
|
||||
the synthetic ``#page=<N>#img=<index>`` path, ``is_image`` +
|
||||
``image_path``, and the description as content AND summary (the
|
||||
phase-122 convention — the summary chunk mirrors it)."""
|
||||
``image_path``, and the enriched content (phase 137, task 02: the
|
||||
document-context block FIRST, then the description — the summary
|
||||
chunk mirrors it, the phase-122 convention)."""
|
||||
root = tmp_path / "pdfimg"
|
||||
root.mkdir()
|
||||
png = _png_bytes(40, 30)
|
||||
@@ -296,11 +308,21 @@ def test_pdf_embedded_images_become_image_documents(db, tmp_path: Path) -> None:
|
||||
# The PDF + its one embedded image: both count added.
|
||||
assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0)
|
||||
|
||||
# Phase 137 (task 02): the document context — the origin line +
|
||||
# page 2's text — is the stored ``image_context`` and the
|
||||
# ``content``/``summary`` enrichment (context first, then the
|
||||
# description — the text the embedder indexes carries it).
|
||||
expected_context = (
|
||||
f"Extracted from {root.name}/report.pdf (page 2):\nText page two."
|
||||
)
|
||||
expected_content = f"{expected_context}\n\n{DESCRIPTION}"
|
||||
|
||||
image_rel = "report.pdf#page=2#img=0.png"
|
||||
img = _doc(db, "pdfimg", image_rel)
|
||||
assert img.is_image is True and img.image_path is not None
|
||||
assert img.title == "Embedded image from report page 2"
|
||||
assert img.content == DESCRIPTION and img.summary == DESCRIPTION
|
||||
assert img.image_context == expected_context
|
||||
assert img.content == expected_content and img.summary == expected_content
|
||||
copy = Path(img.image_path)
|
||||
assert copy.parent == Path(llm.settings.image_dir).expanduser()
|
||||
assert copy.name == f"{img.id}.png" # row and copy agree on the id
|
||||
@@ -310,7 +332,7 @@ def test_pdf_embedded_images_become_image_documents(db, tmp_path: Path) -> None:
|
||||
).all()
|
||||
assert all(c.embedding is not None and len(c.embedding) == 768 for c in chunks)
|
||||
summary_chunks = [c for c in chunks if c.is_summary]
|
||||
assert len(summary_chunks) == 1 and summary_chunks[0].content == DESCRIPTION
|
||||
assert len(summary_chunks) == 1 and summary_chunks[0].content == expected_content
|
||||
|
||||
# The vision describe call went to the CHAT model (the PDF's own
|
||||
# lite summary is the first chat call — the phase-30 path).
|
||||
@@ -745,6 +767,345 @@ def test_pdf_changed_embedded_image_replaces_the_copy(db, tmp_path: Path) -> Non
|
||||
_cleanup_source(db, "pdfrep")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 137 (task 02) — the embedded image's document context
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: The exact generalized context prefix phase 137 (task 02, ASSUMPTION
|
||||
#: A5) appends to the locked prompt (pinned here AND in the
|
||||
#: summarizer's code) — it covers BOTH block kinds (the provenance's
|
||||
#: ``Extracted from …`` document context and the phase-125
|
||||
#: ``Referenced in …`` markdown references).
|
||||
NEUTRAL_CONTEXT_PREFIX = (
|
||||
"\n\nThis image is part of a document. The following context "
|
||||
"describes where it comes from and what it is used for:\n"
|
||||
)
|
||||
|
||||
|
||||
class _CapturingVisionLLM(FakeEmbedder):
|
||||
"""The task-02 capturing vision client: the CHAT model answers
|
||||
every multimodal describe call with the fixed :data:`DESCRIPTION`
|
||||
and records the message's TEXT PART (``text_parts`` — the context's
|
||||
assertion surface: the locked prompt + the prefix + the context);
|
||||
text (``lite``) calls keep the ``FakeEmbedder`` behaviour."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.text_parts: list[str] = []
|
||||
|
||||
async def chat(self, messages, model=None):
|
||||
self.chat_calls.append(list(messages))
|
||||
user = next((m["content"] for m in messages if m.get("role") == "user"), "")
|
||||
if isinstance(user, list):
|
||||
self.text_parts.extend(p["text"] for p in user if p.get("type") == "text")
|
||||
return DESCRIPTION
|
||||
first = user.split()
|
||||
return "Summary of " + (first[0] if first else "<empty>")
|
||||
|
||||
|
||||
# --- the block builder — ``_pdf_image_context_block`` -------------------
|
||||
|
||||
|
||||
def test_pdf_image_context_block_header_and_page_text() -> None:
|
||||
"""A page WITH text → the exact two-line shape: the origin line
|
||||
(a stable single line — NO page-total), then the page's
|
||||
(already-stripped) text."""
|
||||
assert _pdf_image_context_block("src", "report.pdf", 2, {2: "The page text."}) == (
|
||||
"Extracted from src/report.pdf (page 2):\nThe page text."
|
||||
)
|
||||
# The value is stripped by the builder too (defensive: a caller's
|
||||
# unstripped map entry cannot leak trailing newlines into the
|
||||
# block).
|
||||
assert _pdf_image_context_block("s", "r.pdf", 1, {1: " Padded. "}) == (
|
||||
"Extracted from s/r.pdf (page 1):\nPadded."
|
||||
)
|
||||
|
||||
|
||||
def test_pdf_image_context_block_caps_page_text_at_the_cap() -> None:
|
||||
"""ASSUMPTION A3's cap: a ~7000-char page is cut EXACTLY at
|
||||
:data:`PDF_IMAGE_CONTEXT_MAX_CHARS` (6000), with the shared
|
||||
:data:`TRUNCATION_MARKER` on its own line (the summarizer's
|
||||
existing truncation convention); a page that fits exactly at the
|
||||
cap is NOT truncated."""
|
||||
page_text = "w" * 7000
|
||||
assert _pdf_image_context_block("src", "report.pdf", 1, {1: page_text}) == (
|
||||
"Extracted from src/report.pdf (page 1):\n"
|
||||
+ "w" * PDF_IMAGE_CONTEXT_MAX_CHARS
|
||||
+ "\n"
|
||||
+ TRUNCATION_MARKER
|
||||
)
|
||||
exact = "x" * PDF_IMAGE_CONTEXT_MAX_CHARS
|
||||
assert _pdf_image_context_block("s", "r.pdf", 1, {1: exact}) == (
|
||||
f"Extracted from s/r.pdf (page 1):\n{exact}"
|
||||
)
|
||||
|
||||
|
||||
def test_pdf_image_context_block_textless_page_is_origin_line_only() -> None:
|
||||
"""A text-less page (no key in the map — and a whitespace-only
|
||||
value) → the origin line ALONE: the provenance never depends on
|
||||
the page having text (the owner's core complaint)."""
|
||||
assert _pdf_image_context_block("s", "r.pdf", 3, {}) == "Extracted from s/r.pdf (page 3):"
|
||||
assert _pdf_image_context_block("s", "r.pdf", 3, {3: " \n\t "}) == (
|
||||
"Extracted from s/r.pdf (page 3):"
|
||||
)
|
||||
|
||||
|
||||
def test_pdf_image_context_block_unusable_layer_is_origin_line_only() -> None:
|
||||
"""ASSUMPTION A7: the phase-133 gate failed (``text_usable=False``)
|
||||
→ the "page text" is the pre-126 soup and is dropped — the origin
|
||||
line only, even with text present."""
|
||||
assert _pdf_image_context_block(
|
||||
"s", "r.pdf", 1, {1: "<xmp:garbage>value</xmp:garbage>"}, text_usable=False
|
||||
) == "Extracted from s/r.pdf (page 1):"
|
||||
|
||||
|
||||
# --- the combiner — ``_combine_image_contexts`` -------------------------
|
||||
|
||||
|
||||
def test_combine_image_contexts_doc_only() -> None:
|
||||
"""No markdown reference (``None`` or empty) → the doc block
|
||||
alone (the provenance is never empty)."""
|
||||
block = "Extracted from s/r.pdf (page 2):\nText."
|
||||
assert _combine_image_contexts(block, None) == block
|
||||
assert _combine_image_contexts(block, "") == block
|
||||
|
||||
|
||||
def test_combine_image_contexts_document_first_markdown_second() -> None:
|
||||
"""The exact combined shape (A2): the doc block FIRST, a blank
|
||||
line, then the markdown block(s) — the markdown block is
|
||||
BYTE-IDENTICAL inside the combined value (the phase-125 shape is
|
||||
never lost) and precedes nothing (provenance before purpose)."""
|
||||
block = "Extracted from s/r.pdf (page 2):\nText."
|
||||
md = 'Referenced in s/guide.md:\n- Alt text: "pic"\n Context: why'
|
||||
assert _combine_image_contexts(block, md) == f"{block}\n\n{md}"
|
||||
assert md in _combine_image_contexts(block, md), "the markdown block survives verbatim"
|
||||
|
||||
|
||||
# --- the forward path — the capturing fake LLM through ``import_sources`` ---
|
||||
|
||||
|
||||
def test_pdf_embedded_image_vision_call_carries_the_document_context(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""THE forward path (the task's core proof): a PDF with page text +
|
||||
one embedded image → the vision message's text part is
|
||||
``DESCRIBE_PROMPT`` + the (now-neutral) prefix + the exact
|
||||
document-context block (the prefix bytes asserted explicitly), the
|
||||
stored ``image_context`` is the SAME block, and
|
||||
``content``/``summary`` are ``<block>\\n\\n<description>`` — the
|
||||
text the embedder indexes carries the context. A second (unchanged)
|
||||
sync re-describes NOTHING (the phase-126 digest skip is
|
||||
untouched)."""
|
||||
root = tmp_path / "pdfctx"
|
||||
root.mkdir()
|
||||
png = _png_bytes(40, 30)
|
||||
_build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [png])])
|
||||
llm = _pdf_llm(tmp_path, _CapturingVisionLLM)
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0)
|
||||
|
||||
expected_block = f"Extracted from {root.name}/report.pdf (page 2):\nText page two."
|
||||
# (a) the vision model received the context — the locked prompt
|
||||
# + the exact generalized prefix + the block.
|
||||
assert llm.text_parts == [DESCRIBE_PROMPT + NEUTRAL_CONTEXT_PREFIX + expected_block]
|
||||
assert llm.text_parts[0].startswith(DESCRIBE_PROMPT), "the locked prompt is untouched"
|
||||
# (b) the stored image_context IS the same combined value.
|
||||
img = _doc(db, root.name, "report.pdf#page=2#img=0.png")
|
||||
assert img.image_context == expected_block
|
||||
# (c) content/summary carry it too — context first, then the
|
||||
# description (the embedder indexes the same text).
|
||||
expected_content = f"{expected_block}\n\n{DESCRIPTION}"
|
||||
assert img.content == expected_content
|
||||
assert img.summary == expected_content
|
||||
chunks = db.scalars(select(Chunk).where(Chunk.document_id == img.id)).all()
|
||||
packed = "\n".join(expected_content.split("\n\n"))
|
||||
assert [c.content for c in chunks if not c.is_summary] == [packed]
|
||||
assert [c.content for c in chunks if c.is_summary] == [expected_content]
|
||||
|
||||
# The unchanged-image digest skip is untouched: a re-sync of the
|
||||
# same PDF re-describes nothing (no new chat call at all — the
|
||||
# unchanged path skips the image pass entirely).
|
||||
calls1 = len(llm.chat_calls)
|
||||
s2 = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (s2.added, s2.updated, s2.unchanged) == (0, 0, 1)
|
||||
assert len(llm.chat_calls) == calls1, "unchanged: no re-describe"
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_pdf_embedded_image_with_markdown_reference_carries_both_blocks(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""A2's combining at the real seam: a same-source markdown
|
||||
references the image by its synthetic path → the combined context
|
||||
is the DOCUMENT block FIRST, a blank line, then the phase-125
|
||||
``Referenced in`` block — the vision message carries exactly that
|
||||
(generalized prefix + the combined value), and the stored
|
||||
``image_context``/``content``/``summary`` do too (exact-string
|
||||
pin)."""
|
||||
root = tmp_path / "pdfboth"
|
||||
root.mkdir()
|
||||
(root / "guide.md").write_text(
|
||||
"The VLAN map explains the subnet layout.\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"\n"
|
||||
"Only expand the lab subnet after review.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
png = _png_bytes(40, 30)
|
||||
_build_pdf(root / "report.pdf", [("Text page one.", []), ("Text page two.", [png])])
|
||||
llm = _pdf_llm(tmp_path, _CapturingVisionLLM)
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (summary.files, summary.added, summary.images_failed) == (2, 3, 0)
|
||||
|
||||
doc_block = f"Extracted from {root.name}/report.pdf (page 2):\nText page two."
|
||||
md_block = (
|
||||
f"Referenced in {root.name}/guide.md:\n"
|
||||
'- Alt text: "VLAN map"\n'
|
||||
" Context: The VLAN map explains the subnet layout.\n"
|
||||
"\n"
|
||||
"Only expand the lab subnet after review."
|
||||
)
|
||||
combined = f"{doc_block}\n\n{md_block}"
|
||||
assert llm.text_parts == [DESCRIBE_PROMPT + NEUTRAL_CONTEXT_PREFIX + combined]
|
||||
img = _doc(db, root.name, "report.pdf#page=2#img=0.png")
|
||||
assert img.image_context == combined
|
||||
expected_content = f"{combined}\n\n{DESCRIPTION}"
|
||||
assert img.content == expected_content
|
||||
assert img.summary == expected_content
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_pdf_embedded_image_textless_page_still_carries_the_origin_line(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The owner's core complaint, pinned at the real seam: an image
|
||||
on a TEXT-LESS page (page 1 carries text, so the layer is usable —
|
||||
this is the text-less-PAGE case, not the garbage-LAYER case) → the
|
||||
block is the origin line ONLY, and it is STILL stored + enriched
|
||||
(the provenance never depends on the page having text)."""
|
||||
root = tmp_path / "pdftextless"
|
||||
root.mkdir()
|
||||
png = _png_bytes(40, 30)
|
||||
_build_pdf(root / "report.pdf", [("Text page one.", []), ("", [png])])
|
||||
llm = _pdf_llm(tmp_path, _CapturingVisionLLM)
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (summary.files, summary.added, summary.images_failed) == (1, 2, 0)
|
||||
expected_block = f"Extracted from {root.name}/report.pdf (page 2):"
|
||||
assert llm.text_parts == [DESCRIBE_PROMPT + NEUTRAL_CONTEXT_PREFIX + expected_block]
|
||||
img = _doc(db, root.name, "report.pdf#page=2#img=0.png")
|
||||
assert img.image_context == expected_block
|
||||
assert img.content == f"{expected_block}\n\n{DESCRIPTION}"
|
||||
assert img.summary == img.content
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_pdf_embedded_image_garbage_layer_gets_origin_line_only(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""ASSUMPTION A7 at the real seam: a PDF whose text layer fails the
|
||||
phase-133 gate (the XML-soup class) → the pass's gate run yields
|
||||
``text_usable=False`` → the image's block is the origin line ONLY —
|
||||
the garbage "page text" never reaches the vision model (asserted on
|
||||
the captured text part). The document itself takes the
|
||||
page-screenshot fallback (context-free calls — the bare locked
|
||||
prompt)."""
|
||||
root = tmp_path / "pdfgarbage"
|
||||
root.mkdir()
|
||||
doc = fitz.open()
|
||||
page = doc.new_page()
|
||||
for i in range(10):
|
||||
page.insert_text((72, 72 + i * 14), f"<foo:bar>{i}</foo:bar>")
|
||||
pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 40, 30), 0)
|
||||
pix.clear_with(200)
|
||||
page.insert_image(fitz.Rect(72, 220, 112, 250), stream=pix.tobytes("png"))
|
||||
doc.save(root / "report.pdf")
|
||||
doc.close()
|
||||
llm = _pdf_llm(tmp_path, _CapturingVisionLLM)
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert summary.pdfs_screenshotted == 1 # the gate fired on the doc too
|
||||
assert (summary.added, summary.images_failed) == (2, 0)
|
||||
expected_block = f"Extracted from {root.name}/report.pdf (page 1):"
|
||||
# The fallback's page-screenshot call is context-free (the bare
|
||||
# locked prompt); the IMAGE's call carries the origin-line-only
|
||||
# block (A7 — the soup never reaches the model as "text").
|
||||
assert llm.text_parts == [
|
||||
DESCRIBE_PROMPT,
|
||||
DESCRIBE_PROMPT + NEUTRAL_CONTEXT_PREFIX + expected_block,
|
||||
]
|
||||
img = _doc(db, root.name, "report.pdf#page=1#img=0.png")
|
||||
assert img.image_context == expected_block
|
||||
assert img.content == f"{expected_block}\n\n{DESCRIPTION}"
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
# --- the regression pins — the standalone path stays byte-identical ----
|
||||
|
||||
|
||||
def test_standalone_image_vision_call_stays_byte_identical(db, tmp_path: Path) -> None:
|
||||
"""The phase-122 regression pin: a standalone image (no markdown
|
||||
reference, no PDF) → the describe message's text part is the BARE
|
||||
``DESCRIBE_PROMPT`` — byte-identical to pre-phase-137 (the
|
||||
generalized prefix touches ONLY the context-carrying message),
|
||||
``image_context`` stays NULL, and the content is the bare
|
||||
description."""
|
||||
root = tmp_path / "pdfstandalone"
|
||||
root.mkdir()
|
||||
(root / "notes.md").write_text("# Notes\n\nNo references here.\n", encoding="utf-8")
|
||||
(root / "pic.png").write_bytes(_png_bytes(40, 30))
|
||||
llm = _pdf_llm(tmp_path, _CapturingVisionLLM)
|
||||
try:
|
||||
summary = asyncio.run(import_sources([root], llm, session=db))
|
||||
assert (summary.files, summary.added, summary.images_failed) == (2, 2, 0)
|
||||
assert llm.text_parts == [DESCRIBE_PROMPT], "standalone: the bare prompt"
|
||||
img = _doc(db, root.name, "pic.png")
|
||||
assert img.image_context is None, "standalone: the context stays NULL"
|
||||
assert img.content == DESCRIPTION and img.summary == DESCRIPTION
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def test_standalone_markdown_referenced_image_uses_the_neutral_prefix(
|
||||
db, tmp_path: Path
|
||||
) -> None:
|
||||
"""The phase-125 path under the generalized prefix: a standalone
|
||||
image with a markdown reference → the message carries the markdown
|
||||
block with the (now-neutral) prefix, and the STORED values are
|
||||
byte-identical to the phase-125 shape (only the prompt's prefix
|
||||
wording moved — the stored data did not)."""
|
||||
root = tmp_path / "pdfstandmd"
|
||||
root.mkdir()
|
||||
(root / "guide.md").write_text(
|
||||
"Why this diagram exists.\n\n\n\nWhat follows.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "pic.png").write_bytes(_png_bytes(40, 30))
|
||||
llm = _pdf_llm(tmp_path, _CapturingVisionLLM)
|
||||
try:
|
||||
asyncio.run(import_sources([root], llm, session=db))
|
||||
md_block = (
|
||||
f"Referenced in {root.name}/guide.md:\n"
|
||||
'- Alt text: "The red square"\n'
|
||||
" Context: Why this diagram exists.\n"
|
||||
"\n"
|
||||
"What follows."
|
||||
)
|
||||
assert llm.text_parts == [DESCRIBE_PROMPT + NEUTRAL_CONTEXT_PREFIX + md_block]
|
||||
img = _doc(db, root.name, "pic.png")
|
||||
assert img.image_context == md_block, "the stored value is the unchanged data"
|
||||
assert img.content == f"{md_block}\n\n{DESCRIPTION}"
|
||||
finally:
|
||||
_cleanup_source(db, root.name)
|
||||
|
||||
|
||||
def _png_bytes(width: int, height: int, alpha: bool = False) -> bytes:
|
||||
pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, width, height), 1 if alpha else 0)
|
||||
pix.clear_with(200)
|
||||
@@ -1355,3 +1716,184 @@ def test_render_pdf_pages_broken_page_is_zero_bytes_at_its_index(
|
||||
assert render_pdf_pages(path) == [
|
||||
b"\x89PNG-fake-page-1", b"", b"\x89PNG-fake-page-3",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_pdf_page_texts (phase 137 task 01 — the per-page text map)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: The owner's real multi-page fixture (untracked — ``test_sources/``
|
||||
#: is gitignored; unpacked from ``test_sources/pdf_tests.tar``): the
|
||||
#: D&D player's handbook, 293 pages.
|
||||
HANDBOOK = Path(__file__).resolve().parents[2] / "test_sources" / "players_handbook.pdf"
|
||||
|
||||
#: The owner's single-page test PDF (same provenance).
|
||||
HORSE_PDF = Path(__file__).resolve().parents[2] / "test_sources" / "pdf_horse_test.pdf"
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def handbook_texts() -> tuple[str, dict[int, str]]:
|
||||
"""The handbook's ``extract_pdf_text`` output AND page map, each
|
||||
captured ONCE (module-scoped — a full 293-page extraction is ~0.5 s
|
||||
and two tests need the same pair)."""
|
||||
return extract_pdf_text(HANDBOOK), extract_pdf_page_texts(HANDBOOK)
|
||||
|
||||
|
||||
def _parse_page_blocks(joined: str) -> dict[int, str]:
|
||||
"""Parse :func:`extract_pdf_text`'s output back into
|
||||
``{1-indexed page: block body}`` (the page-for-page comparison's
|
||||
reference: the blocks are ``---PAGE-N---\n<body>`` joined with
|
||||
``"\\n\\n"``, the body is stripped — so the body ends 2 chars
|
||||
before the next separator)."""
|
||||
matches = [m for m in re.finditer(r"---PAGE-(\d+)---\n", joined)]
|
||||
blocks: dict[int, str] = {}
|
||||
for i, m in enumerate(matches):
|
||||
end = matches[i + 1].start() - 2 if i + 1 < len(matches) else len(joined)
|
||||
blocks[int(m.group(1))] = joined[m.end():end]
|
||||
return blocks
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HANDBOOK.is_file(), reason="handbook fixture missing")
|
||||
def test_handbook_page_texts_agree_with_extract_pdf_text_page_for_page(
|
||||
handbook_texts: tuple[str, dict[int, str]],
|
||||
) -> None:
|
||||
"""THE page-for-page agreement (the task's core proof): parsing
|
||||
``extract_pdf_text``'s ``---PAGE-N---`` blocks back out and
|
||||
comparing with ``extract_pdf_page_texts`` — the two functions must
|
||||
agree EXACTLY (same keys, same stripped values)."""
|
||||
text, page_texts = handbook_texts
|
||||
assert page_texts == _parse_page_blocks(text)
|
||||
# 293 or fewer entries, 1-indexed keys: the handbook has 293 pages
|
||||
# and two of them (the cover, page 1, and page 280) are image-only
|
||||
# → 291 entries. The keys are the REAL 1-indexed page numbers, so
|
||||
# the first key is 2 (the cover carries no text — no key).
|
||||
assert 0 < len(page_texts) <= 293
|
||||
assert len(page_texts) == 291
|
||||
assert min(page_texts) == 2
|
||||
assert max(page_texts) == 293
|
||||
assert all(isinstance(k, int) and 1 <= k <= 293 for k in page_texts)
|
||||
# every value is non-empty and stripped
|
||||
for value in page_texts.values():
|
||||
assert isinstance(value, str) and value == value.strip() and value
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HANDBOOK.is_file(), reason="handbook fixture missing")
|
||||
def test_handbook_page_texts_spot_check_known_page_body(
|
||||
handbook_texts: tuple[str, dict[int, str]],
|
||||
) -> None:
|
||||
"""Spot-check against the CURRENT (pre-refactor) output: page 2's
|
||||
body (the first text page — the Contents list) and page 293's body
|
||||
(the index tail) carry the known substrings captured from the
|
||||
phase-126 output."""
|
||||
text, page_texts = handbook_texts
|
||||
assert text.startswith(f"{PAGE_SEPARATOR.format(page=2)}\nC o n t e n t s")
|
||||
assert "P r e f a c e" in page_texts[2] # the TOC renders letter-spaced
|
||||
assert "School of Abjuration (wizard)" in page_texts[293]
|
||||
# the page-2 map value is byte-identical to the parsed block body
|
||||
assert page_texts[2] == _parse_page_blocks(text)[2]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HORSE_PDF.is_file(), reason="horse fixture missing")
|
||||
def test_horse_page_texts_match_extract_pdf_text_whatever_it_is() -> None:
|
||||
"""The one-page fixture: asserted AGAINST ``extract_pdf_text``'s
|
||||
output for the same file, whatever it is — a text page means
|
||||
exactly one entry (key 1), a text-less page means ``{}``."""
|
||||
text = extract_pdf_text(HORSE_PDF)
|
||||
page_texts = extract_pdf_page_texts(HORSE_PDF)
|
||||
parsed = _parse_page_blocks(text)
|
||||
assert page_texts == parsed
|
||||
if not parsed:
|
||||
assert page_texts == {}
|
||||
else:
|
||||
assert list(page_texts) == [1] # exactly one entry, 1-indexed
|
||||
assert "Below is a picture of my horse" in page_texts[1]
|
||||
assert page_texts[1] == page_texts[1].strip()
|
||||
|
||||
|
||||
def test_missing_file_page_texts_returns_empty_dict_no_raise(
|
||||
tmp_path: Path, caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Fail-soft: an unreadable file → ``{}`` (no raise, one warning —
|
||||
the ``extract_pdf_text`` open contract, the ``pdf:`` house prefix)."""
|
||||
with caplog.at_level(logging.WARNING, logger="app.rag.pdf"):
|
||||
assert extract_pdf_page_texts(tmp_path / "nope.pdf") == {}
|
||||
warnings = [
|
||||
r for r in caplog.records if "could not open" in r.getMessage()
|
||||
]
|
||||
assert len(warnings) == 1 and warnings[0].levelno == logging.WARNING
|
||||
|
||||
|
||||
def test_garbage_file_page_texts_returns_empty_dict_no_raise(tmp_path: Path) -> None:
|
||||
bad = tmp_path / "garbage.pdf"
|
||||
bad.write_bytes(b"this is not a pdf at all")
|
||||
assert extract_pdf_page_texts(bad) == {}
|
||||
|
||||
|
||||
def test_page_texts_broken_page_is_skipped(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
|
||||
) -> None:
|
||||
"""The shared helper's per-page failure branch, through the NEW
|
||||
call site: a page whose get_text raises is logged/skipped (no key);
|
||||
the other pages keep their (stripped) text."""
|
||||
path = tmp_path / "fake.pdf"
|
||||
path.write_bytes(b"%PDF-1.4 fake")
|
||||
|
||||
class _Page:
|
||||
def __init__(self, text: str | None) -> None:
|
||||
self._text = text
|
||||
|
||||
def get_text(self, flavor: str) -> str:
|
||||
if self._text is None:
|
||||
raise RuntimeError("boom: page unreadable")
|
||||
return self._text
|
||||
|
||||
monkeypatch.setattr(
|
||||
pdf_mod.fitz, "open",
|
||||
lambda _p: _FakeDoc([_Page(None), _Page(" Fine text. ")]),
|
||||
)
|
||||
assert extract_pdf_page_texts(path) == {2: "Fine text."}
|
||||
|
||||
|
||||
# --- the phase-126 regression pin (the shared-iteration refactor) -------
|
||||
# Captured from the PRE-refactor output NOW (phase 137 task 01): the
|
||||
# pins assert the separator structure + length + sha256 — byte-identity
|
||||
# without hardcoding a 300-page string.
|
||||
|
||||
_HANDBOOK_SHA256 = "a0832fc585d4ecf41579198acb3a45761a0f51b0d60d5a282ea8f4d7a84cbeb6"
|
||||
_HANDBOOK_LEN = 1_268_411
|
||||
_HANDBOOK_BLOCKS = 291 # 293 pages minus the two image-only pages (1, 280)
|
||||
|
||||
_HORSE_SHA256 = "ab998617803b71fbd814000f66d0dde83c2b6eb7f679f82b3da7ca06b4d2ecca"
|
||||
_HORSE_EXACT = (
|
||||
f"{PAGE_SEPARATOR.format(page=1)}\nBelow is a picture of my horse. "
|
||||
"It may look like a cat but it\u2019s my horse. I\u2019ve just always "
|
||||
"called it that."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HANDBOOK.is_file(), reason="handbook fixture missing")
|
||||
def test_extract_pdf_text_regression_pin_handbook() -> None:
|
||||
"""``extract_pdf_text`` on the 293-page handbook is BYTE-IDENTICAL
|
||||
to the pre-refactor (phase-126) output: same separator structure
|
||||
(291 blocks, ``---PAGE-2---`` first — the cover is image-only —
|
||||
``---PAGE-293---`` last, ascending), same length, same sha256."""
|
||||
text = extract_pdf_text(HANDBOOK)
|
||||
assert len(text) == _HANDBOOK_LEN
|
||||
assert hashlib.sha256(text.encode("utf-8")).hexdigest() == _HANDBOOK_SHA256
|
||||
headers = [line for line in text.splitlines() if line.startswith("---PAGE-")]
|
||||
assert len(headers) == _HANDBOOK_BLOCKS
|
||||
assert headers[0] == PAGE_SEPARATOR.format(page=2)
|
||||
assert headers[-1] == PAGE_SEPARATOR.format(page=293)
|
||||
assert [int(h[len("---PAGE-"):-3]) for h in headers] == sorted(
|
||||
int(h[len("---PAGE-"):-3]) for h in headers
|
||||
)
|
||||
# the known substring from the first text page (captured pre-refactor)
|
||||
assert text.startswith(f"{PAGE_SEPARATOR.format(page=2)}\nC o n t e n t s")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HORSE_PDF.is_file(), reason="horse fixture missing")
|
||||
def test_extract_pdf_text_regression_pin_horse() -> None:
|
||||
"""The one-page fixture, exact byte pin (short enough to pin in
|
||||
full): the pre-refactor string, character for character."""
|
||||
text = extract_pdf_text(HORSE_PDF)
|
||||
assert text == _HORSE_EXACT
|
||||
assert hashlib.sha256(text.encode("utf-8")).hexdigest() == _HORSE_SHA256
|
||||
|
||||
Reference in New Issue
Block a user