brain-of-reese/tests/integration/test_pdf_hnb_guide_regression.py

"""Integration: phase 133 (task 01) — the owner's HNB guide regression pin.

FINDING (the two-line root cause, full evidence in the phase overview):
the garbage the owner saw is the PRE-PHASE-126 import — ``pdf`` was
already in the owner's ``BOR_IMPORT_EXTENSIONS`` (local, untracked
``.env``) before the PDF parser shipped, so the generic text branch
read the RAW PDF bytes (``read_text(errors="replace")``) and the
uncompressed Illustrator XMP/structure blocks (222 ×
``<illustrator:StartupProfile>`` + 138 × ``<x:xmpmeta>`` blocks) were
chunked and embedded as literal text. Phase 126+ extracts the same
file correctly; the stale production row self-heals on re-sync
(content-hash delta → ``updated`` with clean content).

This suite pins the owner's EXACT file as a permanent regression: the
archive's 48-page Illustrator PDF is imported through the REAL
importer — the real walk, the real PDF branch (extraction → digest →
page-anchored chunking → 768-dim embeds → ``lite`` summary → the
phase-126 embedded-image pass, whose vision seam runs against the
multimodal-capable fake below) — and must land CLEAN content: the
XML-ish line fraction stays far under the task-02 gate's 0.30
(0.0 in the real extraction — asserted < 0.05 to be strict), the
non-printable fraction stays < 0.01, and the real first-page text is
present. If the PDF is ever routed through the text branch again, the
stored content is the raw bytes again (~46% XML-ish) and the pin
breaks — the pin has teeth.

Run:
    uv run pytest tests/integration/test_pdf_hnb_guide_regression.py -v
Requires:  podman compose up -d db

Fixture provenance: ``test_sources/hnb_2026_guide.tar.gz`` — the
UNTRACKED owner fixture (2.6 MB compressed; one 6.2 MB, 48-page
PDFium/Illustrator export, ``HBI-2026-Guide-Year-Round_260704_131221.pdf``
inside). The test extracts it into ``tmp_path`` (the test's local
source dir) and never touches the repo — ``git status`` stays clean of
binaries. The pinned fixture text (the first-page title) fails LOUDLY
if the owner ever re-cuts the fixture: that IS the regression signal.

Config note (why the settings are built the way they are):
* ``_env_file=".env"`` — the REAL extension set: ``pdf`` is in the
  owner's live ``BOR_IMPORT_EXTENSIONS``. A ``_env_file=None`` fake
  would use the code defaults (no ``pdf`` token) and SILENTLY skip the
  PDF — the exact trap that hid this class before (FINDING, point 3).
* ``images=True`` — the owner's live toggle (``BOR_IMAGES=1`` in the
  same ``.env``). The suite forces ``BOR_IMAGES`` to the code default
  (off) in the process environment (the conftest leak class, phase
  122), and process env beats the ``.env`` file — so the live value is
  restored with an init kwarg (init kwargs beat both). The embedded-
  image pass is the phase-126 A3 seam this pin covers.
* ``image_dir`` under ``tmp_path`` — the house hermetic pattern
  (``test_importer.py``'s ``_image_llm``): the embedded-image copies
  never touch the owner's persistent image home.

``summary.added`` accounting: the walked file contributes EXACTLY ONE
added document (the guide PDF); the phase-126 A3 convention counts
each embedded image doc ``added`` too, so the assertion is
``added == 1 + len(image_docs)`` together with ``files == 1`` (the
walk admitted the one file) — the strict generalization of the naive
``added == 1`` for a PDF whose images join the KB.
"""
from __future__ import annotations

import asyncio
import re
import tarfile
import time
from collections import Counter
from collections.abc import Iterator
from pathlib import Path
from typing import Any

import pytest
from sqlalchemy import select, text
from sqlalchemy.orm import Session

from app.config import Settings
from app.models import Chunk, Document
from app.rag.importer import PDF_IMAGE_CONTEXT_MAX_CHARS, ImportSummary, import_sources
from app.rag.pdf import PAGE_SEPARATOR, extract_pdf_page_texts
from app.rag.retriever import TRUNCATION_MARKER
from app.rag.summarizer import DESCRIBE_PROMPT
from tests.fakes import FakeEmbedder

#: The repo root (this file lives in ``tests/integration/``).
REPO_ROOT = Path(__file__).resolve().parents[2]

#: The owner's untracked fixture (the stable reference — the tarball,
#: never the extracted PDF, is what the test starts from).
TARBALL = REPO_ROOT / "test_sources" / "hnb_2026_guide.tar.gz"

#: The inner file keeps its name; the temp source dir is named
#: ``test_sources`` (the importer's ``source = root.name`` convention —
#: the same identity the handbook pin uses).
SOURCE = "test_sources"
PDF_NAME = "HBI-2026-Guide-Year-Round_260704_131221.pdf"
PDF_REL = f"{SOURCE}/{PDF_NAME}"

#: The pinned fixture text (the real first page). If the owner ever
#: re-cuts the fixture, the failure names the regression — the point.
FIRST_PAGE_TEXT = "2026 U.S. Benefits Guide"

#: The title: the stem fallback — the PDF's metadata carries no title
#: (only the PDFium creator/producer), so the non-markdown rule applies.
TITLE = "HBI-2026-Guide-Year-Round_260704_131221"

#: The phase-133 XML-ish line heuristic (the task-02 gate's regex):
#: ``<tag>``, ``</tag>``, ``<tag attr=``, and the PDF-structure
#: ``<</Key…`` noise all match.
XMLISH_RE = re.compile(r"</?[A-Za-z_][\w:.-]*[\s/>=]")

#: Clean-content bars (stricter than the task-02 gate on purpose — the
#: real extraction is 0.0 XML-ish / ~0.0005 non-printable, the
#: pre-126 garbage read is ~0.46 XML-ish AND heavy on U+FFFD).
XML_FRACTION_BAR = 0.05
NONPRINT_FRACTION_BAR = 0.01

#: The task's wall-time guard: 48 pages × in-process fakes must stay
#: fast — a drift past this means the test is too heavy.
WALL_TIME_BAR_S = 120.0

pytestmark = pytest.mark.skipif(
    not TARBALL.is_file(),
    reason=(
        "test_sources/hnb_2026_guide.tar.gz is missing — the untracked "
        "owner fixture (2.6 MB, one 48-page Illustrator PDF)"
    ),
)


#: The phase-137 (task 02, ASSUMPTION A5) generalized context prefix
#: (pinned here AND in the summarizer's code) — it covers BOTH the
#: provenance's document context and the phase-125 markdown reference
#: blocks (this PDF-only source has the former alone).
DESCRIBE_CONTEXT_PREFIX = (
    "\n\nThis image is part of a document. The following context "
    "describes where it comes from and what it is used for:\n"
)


def _expected_context_block(
    pdf_rel: str, page: int, page_texts: dict[int, str]
) -> str:
    """The phase-137 (task 02) document-context block the importer
    builds for an embedded image on *page* of *pdf_rel* (the guide's
    CLEAN text layer: the origin line + the capped page text — the A7
    origin-line-only shape belongs to the garbage layer, never this
    fixture; a text-less page is the origin line alone)."""
    header = f"Extracted from {pdf_rel} (page {page}):"
    text = page_texts.get(page, "").strip()
    if not text:
        return header
    if len(text) > PDF_IMAGE_CONTEXT_MAX_CHARS:
        text = text[:PDF_IMAGE_CONTEXT_MAX_CHARS] + "\n" + TRUNCATION_MARKER
    return f"{header}\n{text}"


class GuideVisionLLM(FakeEmbedder):
    """The task-03-style vision mock the ``fakes.py`` comment
    anticipates: the base ``chat`` is string-only — the MULTIMODAL user
    content ``describe_image`` sends (the ``list``-of-parts shape)
    would crash it — so ``list`` content gets the deterministic canned
    reply instead. String (``lite``) calls keep the ``FakeEmbedder``
    behaviour. Mirrors ``test_importer.py``'s ``_MockVisionLLM``
    pattern; the REAL ``import_sources`` → ``_describe_or_skip`` →
    ``describe_image`` chain runs against it (no seam patch)."""

    VISION_REPLY = "A rendered page of the guide (test vision description)."

    def __init__(self) -> None:
        super().__init__()
        #: The text part of every multimodal (vision) call received.
        self.vision_texts: list[str] = []

    async def chat(
        self, messages: list[dict[str, Any]], model: str | None = None
    ) -> str:
        self.chat_calls.append(list(messages))
        user = next((m["content"] for m in messages if m.get("role") == "user"), "")
        if isinstance(user, list):
            self.vision_texts.append(
                next((p["text"] for p in user if p.get("type") == "text"), "")
            )
            return self.VISION_REPLY
        first = user.split()
        return "Summary of " + (first[0] if first else "<empty>")


def _settings(tmp_path: Path) -> Settings:
    """The owner's live import config (the point of the pin — see the
    module docstring's Config note) + the hermetic image dir."""
    kwargs: dict[str, Any] = {
        "_env_file": ".env",
        "images": True,
        "image_dir": str(tmp_path / "images"),
    }
    return Settings(**kwargs)  # pyright: ignore[reportCallIssue]


def _xml_frac(content: str) -> float:
    """The phase-133 XML-ish line fraction: non-empty lines matching
    :data:`XMLISH_RE`, over the non-empty lines."""
    lines = [line for line in content.splitlines() if line.strip()]
    if not lines:
        return 0.0
    return sum(1 for line in lines if XMLISH_RE.search(line)) / len(lines)


def _nonprint_frac(content: str) -> float:
    """The binary-soup signal: U+FFFD replacement chars + C0 control
    chars over the whole text — the whitespace controls (``\\n``/``\\t``/
    ``\\r``) are prose line breaks, not garbage (counting them would
    flag every document), so they are excluded."""
    if not content:
        return 0.0
    bad = sum(
        1
        for ch in content
        if ch == "\ufffd" or (ord(ch) < 32 and ch not in "\r\n\t")
    )
    return bad / len(content)


@pytest.fixture()
def guide_kb(
    db: Session, tmp_path: Path
) -> Iterator[tuple[GuideVisionLLM, ImportSummary, float, Path]]:
    """The real guide, imported ONCE per test through the REAL importer
    (the house KB pattern — ``test_read_line_ranges_handbook.py``):
    ``chunks``/``documents``/``folder_summaries`` are truncated around
    the test, the tarball is extracted into a temp ``test_sources``
    source dir, and the deterministic in-process
    :class:`GuideVisionLLM` stands in for the embed/summary/vision
    calls (the whole 48-page index build stays fast and offline).
    Yields ``(llm, summary, elapsed_s, pdf_path)``."""
    db.execute(text("TRUNCATE chunks, documents, folder_summaries"))
    db.commit()
    root = tmp_path / SOURCE
    root.mkdir()
    with tarfile.open(TARBALL, "r:gz") as tar:
        tar.extractall(root, filter="data")
    llm = GuideVisionLLM()
    llm.settings = _settings(tmp_path)
    started = time.monotonic()
    summary = asyncio.run(import_sources([root], llm, session=db))
    elapsed = time.monotonic() - started
    yield llm, summary, elapsed, root / PDF_NAME
    db.execute(text("TRUNCATE chunks, documents, folder_summaries"))
    db.commit()
    db.expire_all()


def test_hnb_guide_imports_clean_through_the_real_pipeline(
    db: Session,
    guide_kb: tuple[GuideVisionLLM, ImportSummary, float, Path],
) -> None:
    """The owner's exact file lands CLEAN through the real pipeline:
    the walk admits the one file (``pdf`` token — the PDF branch, never
    the text branch), the stored content is clean + carries the real
    first-page text, the page-anchored chunks are embedded, the ``lite``
    summary is stored, and every embedded-image doc got the fake vision
    description (the multimodal ``chat`` was called) enriched with the
    phase-137 (task 02) document context (the provenance block first —
    the owner's stock photos finally name their document)."""
    llm, summary, elapsed, pdf_path = guide_kb

    # The task's wall-time guard — the suite must stay fast.
    assert elapsed < WALL_TIME_BAR_S, (
        f"the guide import took {elapsed:.1f}s (> {WALL_TIME_BAR_S:.0f}s) — "
        "the test is too heavy, the suite must stay fast"
    )

    # The walk ADMITTED the guide as one file with the ``pdf`` token —
    # the PDF branch (the text branch is unreachable for ``.pdf`` after
    # phase 126; if it ever ran on this file again, the content bars
    # below break — the pin has teeth).
    assert summary.files == 1
    assert summary.formats == {"pdf": 1}
    assert summary.errors == 0
    assert summary.unchanged == 0
    assert summary.pruned == 0

    doc = db.scalar(
        select(Document).where(Document.source == SOURCE, Document.path == PDF_NAME)
    )
    assert doc is not None, "the guide did not import"
    assert doc.is_image is False
    assert doc.title == TITLE  # the stem fallback (no metadata title)

    content = doc.content
    # The stored content is CLEAN — the pre-126 XML-garbage import can
    # never come back (the garbage read is ~0.46 XML-ish AND heavy on
    # U+FFFD; the real extraction is 0.0 / ~0.0005).
    assert _xml_frac(content) < XML_FRACTION_BAR, (
        f"the guide's content is {_xml_frac(content):.0%} XML-ish — the "
        "pre-126 raw-bytes import came back (the PDF must not be routed "
        "through the text branch)"
    )
    assert _nonprint_frac(content) < NONPRINT_FRACTION_BAR, (
        f"the guide's content is {_nonprint_frac(content):.2%} non-printable — "
        "binary soup, not extracted text"
    )
    # The real first-page text is present (the pinned fixture text) and
    # the content opens with the page-1 marker (the phase-126 A2 shape).
    assert FIRST_PAGE_TEXT in content
    assert content.startswith(PAGE_SEPARATOR.format(page=1))

    # The real pipeline ran to the end: page-anchored content chunks
    # with 768-dim embeddings + exactly one ``is_summary`` chunk at
    # position −1 (the ``lite`` summary, stored with the pointer line).
    chunks = db.scalars(
        select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.position)
    ).all()
    content_chunks = [c for c in chunks if not c.is_summary]
    assert content_chunks, "the guide's content produced no chunks"
    for c in chunks:
        assert c.embedding is not None and len(c.embedding) == 768
    summary_chunks = [c for c in chunks if c.is_summary]
    assert len(summary_chunks) == 1 and summary_chunks[0].position == -1
    assert doc.summary is not None
    assert doc.summary.endswith(f"Source: {PDF_REL}")

    # The embedded-image docs (the phase-126 A3 ``#page=…#img=…``
    # rows): every one of them got the FAKE vision description — the
    # multimodal ``chat`` was called (the task's literal assertion),
    # exactly once per image doc — and phase 137 (task 02) enriched it
    # with the document context: the provenance block (the origin line
    # + the page's own text, capped — the guide's layer is CLEAN, so
    # the gate leaves the page texts IN; the A7 origin-line-only shape
    # is the garbage layer's, pinned in the unit suite) FIRST, then
    # the description.
    page_texts = extract_pdf_page_texts(pdf_path)

    def _page_of(path: str) -> int:
        m = re.search(r"#page=(\d+)", path)
        assert m is not None, f"unparseable embedded-image path {path!r}"
        return int(m.group(1))

    image_docs = db.scalars(
        select(Document).where(
            Document.source == SOURCE,
            Document.is_image.is_(True),
            Document.path.like(f"{PDF_NAME}#page=%"),
        )
    ).all()
    assert image_docs, "the guide's embedded images did not index"
    for img in image_docs:
        context = _expected_context_block(PDF_REL, _page_of(img.path), page_texts)
        assert img.image_context == context
        assert img.content == f"{context}\n\n{GuideVisionLLM.VISION_REPLY}"
        assert img.summary == img.content  # the image convention
        assert img.image_path is not None
        copy = Path(img.image_path)
        assert copy.parent == Path(llm.settings.image_dir).expanduser()
        assert copy.exists(), "the embedded image's copy is missing"
    multimodal_calls = [
        messages
        for messages in llm.chat_calls
        if any(
            isinstance(m.get("content"), list) for m in messages if m.get("role") == "user"
        )
    ]
    assert len(multimodal_calls) >= 1, "the multimodal chat was never called"
    # Phase 140 (task 03): the default-ON pictorial pass ALSO runs on
    # this fixture's sparse pages (the fake is seam-agnostic — the
    # same canned reply) — so the pin counts only the EMBEDDED-IMAGE
    # describe calls (the ``DESCRIBE_PROMPT``-headed text parts; the
    # ``PICTORIAL_PAGE_MODE`` parts are the pass's own calls, covered
    # by the phase-140 suites).
    image_vision_texts = [
        t for t in llm.vision_texts if t.startswith(DESCRIBE_PROMPT)
    ]
    assert len(image_vision_texts) == len(image_docs), (
        "one vision call per embedded image doc (no failures)"
    )
    # The locked prompt + the generalized prefix + the document context
    # on EVERY call (no markdown reference exists in this PDF-only
    # source — the phase-137 A2 shape). The multiset is the
    # order-independent exact pin (the rows come back unsorted).
    expected_calls = Counter(
        DESCRIBE_PROMPT + DESCRIBE_CONTEXT_PREFIX
        + _expected_context_block(PDF_REL, _page_of(img.path), page_texts)
        for img in image_docs
    )
    assert Counter(image_vision_texts) == expected_calls
    assert summary.images_failed == 0

    # The added accounting (see the module docstring): the walked file
    # is EXACTLY ONE added document (the guide PDF); every other added
    # doc is an embedded-image byproduct (the phase-126 A3 convention
    # counts image docs added/updated) — nothing else touched the KB.
    assert summary.added == 1 + len(image_docs)
    assert summary.updated == 0