"""Phase 137 E2E (Playwright): PDF-embedded images carry the document context in their summaries — the user path (TODO L3, end to end). Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_pdf_image_context.py -v --no-cov TODO L3: "Images extracted from PDFs still don't have any relevant context in their summaries … Image summaries should always include the document context if the images come from a document (this includes markdown files) so the embedder can more effectively retrieve image sources." The suite's single module app runs with ``BOR_IMAGES=true`` + the ``pdf`` import token (the ``test_pdf_support.py`` per-suite-app pattern — the conftest env with the leak guards): * **Seed 1** (source ``e2e-pdf-imgctx``): ONE fixture PDF at the source root (two text pages + one embedded noise PNG on page 2 — the phase-126 PyMuPDF recipe, built in the fixture) arrives through the REAL admin upload flow + the RAG page's Sync button. The forward path (phase 137 task 02) is the story: the embedded image's describe call carries the document-context block (``Extracted from e2e-pdf-imgctx/report.pdf (page 2):`` + the page's text), so the mock's vision model answers the CONTEXT-AWARE canned reply (the mock's ``Extracted from `` trigger — the deterministic witness the context REACHED the vision model), and the stored ``image_context``/``content``/``summary`` carry it; * **the chat display line** (task 04): a grounded question carrying the mock's ``CONTEXT_LINE_TRIGGER`` marker makes the mock QUOTE THE EXACT LINE of the image doc's ```` block — which now LEADS with the one-line ``From /report.pdf (page 2)`` rendering — so the answer proves the display line was in the prompt; * **Seed 2** (source ``e2e-pdf-imgctx-md`` — a FRESH KB: the phase re-imports the same PDF plus a ``notes.md`` referencing the embedded image by its synthetic path, the phase-125 reference shape): the stored context carries BOTH block kinds with the document block FIRST (ASSUMPTION A2 — provenance before purpose), and the display line carries both clauses in stored order. Test-order note: the standalone-parity witness asserts against SEED 1's KB (exactly the PDF + its one image — the suite seeds no standalone image anywhere), so it runs BEFORE seed 2 truncates and re-imports; the mock's BARE describe branch is pinned by ``test_pdf_support.py`` running green unedited (the regression run — not duplicated here). """ from __future__ import annotations import json import os import re import subprocess import sys import zipfile from collections.abc import Iterator from pathlib import Path from uuid import UUID import httpx import pymupdf as fitz # PyMuPDF — the fixture-PDF builder (the phase-126 recipe) import pytest from playwright.sync_api import Browser, Page, expect from sqlalchemy import select, text from app.config import Settings from app.db import SessionLocal from app.models import Document from e2e.auth_helpers import login from e2e.conftest import ADMIN_PASSWORD, SESSION_SECRET, USE_REAL_LLM, _wait_http from e2e.mock_llm import IMAGE_DESCRIPTION_ANSWER, IMAGE_DESCRIPTION_CONTEXT_ANSWER from e2e.sync_helpers import counts_line REPO = Path(__file__).resolve().parents[2] # --------------------------------------------------------------------------- # Fixtures + markers (defined LOCALLY — the suites must stay # standalone; the shapes mirror test_pdf_support's, the values are # this suite's own). # --------------------------------------------------------------------------- SOURCE = "e2e-pdf-imgctx" # archive 1's stem (archive_source_name) SOURCE_MD = "e2e-pdf-imgctx-md" # archive 2's stem DOC_PATH = "report.pdf" IMAGE_REL = "report.pdf#page=2#img=0.png" # the synthetic embedded-image path MD_PATH = "notes.md" PDF_TITLE = "Homelab Report" # the metadata title #: The fixture PDF's page texts. ``PAGE_TWO`` (the embedded image's #: page) carries the document text that must reach the image's #: ``image_context``; the anchor question's tokens overlap these, so #: the mock's token-overlap gate grounds on the image doc (and FTS #: corroborates). PAGE_ONE = "The quarterly homelab report covers the core switch firmware upgrade." PAGE_TWO = "The homelab backup schedule runs nightly at 02:00 and keeps seven rotations." def _embedded_png() -> bytes: """A small deterministic colorful noise PNG (channel means ~97 — the "real image" stand-in that CLEARS the phase-151 blank/ decorative gate's near-white floor with room to spare). The old 1×1 transparent fixture decomposed in PyMuPDF to a near-white 1-sample base — blank by the owner-locked gate — and is now (correctly) skipped as a decorative underlay, so the suite's pipeline pins ride on a real non-blank image instead (the pipeline is content-agnostic; the well-formed bytes keep the extraction + copy path honest).""" pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 24, 18)) for y in range(18): for x in range(24): pix.set_pixel(x, y, ((x * 7) % 256, (y * 13) % 256, ((x + y) * 5) % 256)) return pix.tobytes("png") #: The embedded fixture image (24×18 colorful noise — non-blank by the #: phase-151 gate; see :func:`_embedded_png`). EMBED_PNG = _embedded_png() # Seed 1's stored document block (NO markdown reference exists in that # source — the doc block alone: the provenance's origin line + the # page's text). DOC_BLOCK = f"Extracted from {SOURCE}/{DOC_PATH} (page 2):\n{PAGE_TWO}" #: The task-04 ONE-LINE rendering of :data:`DOC_BLOCK` — the display #: line that leads the image doc's ```` block (test 2). FROM_LINE = f"From {SOURCE}/{DOC_PATH} (page 2)" FROM_LINE_MD = f"From {SOURCE_MD}/{DOC_PATH} (page 2)" #: Archive 2's markdown — the phase-125 reference shape: the embedded #: image referenced by its SYNTHETIC path (the doc's ``path``), alt #: "network diagram", one paragraph before + one after (the default #: two-paragraph window). NOTES_MD = ( "# Homelab notes\n" "\n" "The network diagram in the report shows the server room layout.\n" "\n" f"![network diagram]({IMAGE_REL})\n" "\n" "Keep the diagram in sync with the VLAN plan.\n" ) MD_BLOCK_HEADER = f"Referenced in {SOURCE_MD}/{MD_PATH}:" #: The grounded question — the mock's ``CONTEXT_LINE_TRIGGER`` marker #: (the zero-read context-line echo — the mock quotes the image doc's #: block's exact first line) + the phase-122/125 grounding phrase #: (the diagram wording is in :data:`IMAGE_DESCRIPTION_CONTEXT_ANSWER`, #: so the token-overlap cosine clears the mock-calibrated gate 0.30; #: FTS corroborates). ANCHOR_QUESTION = ( "quote the image reference — what is shown in the homelab network diagram?" ) #: The honest-deflection opening (the gate must NOT fire on the marker #: question — a deflected prompt carries no ```` block, so #: the mock would never see the image doc). DEFLECT_OPENING = "I haven't done anything like that" UPLOAD_TIMEOUT_MS = 30_000 SYNC_TIMEOUT_MS = 60_000 SYNCED_LABEL = re.compile(r"^Synced \d{1,2}:\d{2}$") # --------------------------------------------------------------------------- # Module app: images ON — the conftest env pattern with the phase-126 # knobs + leak guards (the conftest session app is never started in # this isolated run, so no port clash). # --------------------------------------------------------------------------- def _app_env(mock_llm: int, app_port: int, scratch: Path) -> dict[str, str]: """The conftest app env (leak guards included) with the phase-126 knobs: ``BOR_IMPORT_EXTENSIONS`` pinned to a set carrying the ``pdf`` token (the feature is gated by the token — an operator's local gitignored ``.env`` cannot leak the scope in either direction), ``BOR_IMAGES`` pinned EXPLICITLY true (the story), and the image/upload homes in the suite's scratch dir (the app under test must not write copies into the owner's real ``~/bor-sources``).""" env = dict(os.environ) env.pop("DEBUGPY", None) env["BOR_ENVIRONMENT"] = "e2e" env["BOR_STATIC_DIR"] = str(REPO / "frontend") env["BOR_LLM_BASE_URL"] = ( "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else f"http://127.0.0.1:{mock_llm}/v1" ) # Mock-calibrated gate (the test_image_anchor_context values) — # the anchor question grounds on these (see ANCHOR_QUESTION). env["BOR_RELEVANCE_THRESHOLD"] = "0.30" env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15" env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15" # Phase 67: instant retry waits + the code-default budget. env["BOR_LLM_RETRY_DELAY"] = "0" env["BOR_LLM_RETRIES"] = str(Settings.model_fields["llm_retries"].default) env.setdefault( "BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", ) env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD env["BOR_SESSION_SECRET"] = SESSION_SECRET # Leak guards (conftest pattern). env["BOR_GIT_SOURCES"] = "" env["BOR_DOCS_REPO"] = "" env["BOR_SUGGESTIONS"] = json.dumps( Settings.model_fields["suggestions"].default ) env["BOR_INPUT_PLACEHOLDER"] = Settings.model_fields["input_placeholder"].default env["BOR_FOOTER_TEXT"] = Settings.model_fields["footer_text"].default # The import scope WITH the ``pdf`` token + the toggle under test # + the suite-private homes. env["BOR_IMPORT_EXTENSIONS"] = "md,markdown,txt,pdf" env["BOR_IMAGES"] = "true" env["BOR_UPLOAD_DIR"] = str(scratch / "uploads") env["BOR_SOURCES_DIR"] = str(scratch / "checkouts") env["BOR_IMAGE_DIR"] = str(scratch / "images") return env @pytest.fixture(scope="module") def app_server(mock_llm: int, tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: """The module app — ``BOR_IMAGES=true`` + the ``pdf`` token (the module env override).""" scratch = tmp_path_factory.mktemp("bor_pdf_imgctx") port = int(os.environ.get("E2E_APP_PORT_PDF_IMG_CTX", "8154")) proc = subprocess.Popen( [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], cwd=REPO, env=_app_env(mock_llm, port, scratch), ) try: _wait_http(f"http://127.0.0.1:{port}/api/health") yield f"http://127.0.0.1:{port}" finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() @pytest.fixture(scope="module") def app_url(app_server: str) -> str: return app_server # --------------------------------------------------------------------------- # Fixture archives (built in tmp dirs — ``test_sources/`` is never # touched, the house rule) # --------------------------------------------------------------------------- def _build_pdf(root: Path) -> Path: """The fixture PDF (the phase-126 recipe, built LOCALLY): two text pages + one embedded noise PNG on page 2 + the metadata title.""" pdf_path = root / DOC_PATH doc = fitz.open() doc.new_page().insert_text((72, 72), PAGE_ONE) page2 = doc.new_page() page2.insert_text((72, 72), PAGE_TWO) page2.insert_image(fitz.Rect(72, 150, 96, 168), stream=EMBED_PNG) doc.set_metadata({"title": PDF_TITLE}) doc.save(pdf_path) doc.close() return pdf_path @pytest.fixture(scope="module") def zip_path(tmp_path_factory: pytest.TempPathFactory) -> Path: """Seed 1's archive: ONE PDF at the root — source ``e2e-pdf-imgctx``, document path ``report.pdf``.""" root = tmp_path_factory.mktemp("bor_pdfimgctx_zip") pdf_path = _build_pdf(root) archive = root / f"{SOURCE}.zip" with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf: zf.write(pdf_path, arcname=DOC_PATH) return archive @pytest.fixture(scope="module") def zip_md_path(tmp_path_factory: pytest.TempPathFactory) -> Path: """Seed 2's archive: the SAME PDF + ``notes.md`` at the root (the markdown references the embedded image by its synthetic path, with a paragraph of context before and after) — source ``e2e-pdf-imgctx-md``.""" root = tmp_path_factory.mktemp("bor_pdfimgctx_md_zip") pdf_path = _build_pdf(root) (root / MD_PATH).write_text(NOTES_MD, encoding="utf-8") archive = root / f"{SOURCE_MD}.zip" with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf: zf.write(pdf_path, arcname=DOC_PATH) zf.write(root / MD_PATH, arcname=MD_PATH) return archive # --------------------------------------------------------------------------- # DB + UI helpers (the test_pdf_support pattern) # --------------------------------------------------------------------------- def _truncate_all() -> None: """Fresh KB + registry per seed (the E2E isolation pattern): the suites share one Postgres, and a leftover row would corrupt the counts. ``sources_meta`` (the KB generation counter) resets with the KB — the sync re-creates the row.""" with SessionLocal() as db: db.execute(text( "TRUNCATE chunks, documents, query_log, kb_overview, " "git_sources, sources_meta, folder_summaries" )) db.commit() def _health_db(app_url: str) -> bool: try: return httpx.get(f"{app_url}/api/health", timeout=5).json()["db"] == "up" except Exception: # noqa: BLE001 — unreachable is the skip case return False def _ready_for_sync_line(source: str) -> re.Pattern[str]: """Phase 154 (task 02): the ready-for-sync line carrying the completion stamp — "Uploaded () — press Sync sources to import it." (the fresh 202 settle). In E2E the stamp is ALWAYS the same-day ``HH:MM`` form (the run settles within seconds; the date-qualified forms are unit-pinned on a fixed clock — E2E never depends on the wall clock).""" return re.compile( rf"^Uploaded {re.escape(source)} \(\d{{1,2}}:\d{{2}}\)" r" — press Sync sources to import it\.$" ) def _upload_zip(page: Page, app_url: str, archive: Path, source: str) -> None: """The real admin upload flow: form login → the git-sources page → pick the archive → submit → the terminal result line (phase 90 A3, phase 154 task 02: "Uploaded () — press Sync sources to import it." — the completion stamp).""" login(page, app_url, next="/git-sources.html") expect(page).to_have_url(app_url + "/git-sources.html", timeout=30_000) expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000) page.set_input_files("#archive-upload-file", str(archive)) page.click("#archive-upload-btn") result = page.locator("#archive-upload-result") expect(result).to_be_visible(timeout=UPLOAD_TIMEOUT_MS) expect(result).to_have_text( _ready_for_sync_line(source), timeout=UPLOAD_TIMEOUT_MS, ) def _sync_via_ui(page: Page, app_url: str, expected_result: str) -> None: """The RAG page's "Sync sources" button (the scan): click → "Syncing…" → the terminal "Synced HH:MM" label + the fresh counts in #sync-progress (the never-stale contract).""" page.goto(app_url + "/sources.html") btn = page.locator("#sync-btn") expect(btn).to_be_visible(timeout=30_000) expect(page.locator("#sync-label")).to_have_text( re.compile(r"^Sync sources$|^Synced \d{1,2}:\d{2}$") ) btn.click() expect(btn).to_be_disabled() expect(page.locator("#sync-label")).to_have_text("Syncing…") expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) expect(btn).to_be_enabled() expect(page.locator("#sync-progress-text")).to_have_text(counts_line(expected_result)) def _ask(page: Page, app_url: str, question: str) -> None: """The chat user path: login → the question → Send → the final brain bubble (the Send button re-enabled = the turn is over).""" login(page, app_url, next="/") expect(page.locator("#kb-banner")).to_be_hidden() page.fill("#message-input", question) page.click("#send-btn") bubble = page.locator(".msg.brain .bubble").last expect(page.locator("#send-btn")).to_be_enabled(timeout=60_000) expect(page.locator("#send-label")).to_have_text("Send") expect(bubble).to_contain_text("Deterministic mock answer", timeout=30_000) # --------------------------------------------------------------------------- # The module seeds: upload + sync the fixture archives through the # real UI, once each (the story E2E truncate/re-import-per-module # pattern). # --------------------------------------------------------------------------- @pytest.fixture(scope="module") def seeded( app_url: str, browser: Browser, zip_path: Path, ) -> Iterator[dict[str, UUID]]: """Seed 1: upload the single-PDF zip through the admin UI, then scan it with the RAG page's Sync button (the mock's vision model describes the embedded image in-process, WITH the document context). The module app's ``/api/config`` flag is asserted first — the env override must have reached the app under test, or the whole premise of the suite is void.""" if not _health_db(app_url): pytest.skip("Postgres not reachable — run `podman compose up -d db` first") cfg = httpx.get(f"{app_url}/api/config", timeout=5).json() assert cfg["images"] is True, "the module app must run with BOR_IMAGES=true" _truncate_all() page = browser.new_page(viewport={"width": 1280, "height": 800}) try: _upload_zip(page, app_url, zip_path, SOURCE) # The PDF + its one embedded image (phase 126 A3): both added. _sync_via_ui(page, app_url, expected_result="2 added") with SessionLocal() as db: img = db.scalar( select(Document).where( Document.source == SOURCE, Document.path == IMAGE_REL ) ) assert img is not None, "the embedded image must become an image doc" finally: page.close() yield {"image_id": img.id} @pytest.fixture(scope="module") def seeded_md( app_url: str, browser: Browser, zip_md_path: Path, ) -> Iterator[dict[str, UUID]]: """Seed 2: a FRESH KB (seed 1's rows are truncated — the phase re-imports through the SAME UI) seeded with archive 2: the same PDF plus the markdown that references its embedded image by the synthetic path (the combined-context case).""" _truncate_all() page = browser.new_page(viewport={"width": 1280, "height": 800}) try: _upload_zip(page, app_url, zip_md_path, SOURCE_MD) # The PDF + its one embedded image + the markdown doc: three added. _sync_via_ui(page, app_url, expected_result="3 added") with SessionLocal() as db: img = db.scalar( select(Document).where( Document.source == SOURCE_MD, Document.path == IMAGE_REL ) ) assert img is not None, "the embedded image must become an image doc" finally: page.close() yield {"image_id": img.id} _truncate_all() # --------------------------------------------------------------------------- # 1. The forward path (the TODO's core complaint): the embedded image's # stored context + summary carry the document context # --------------------------------------------------------------------------- def test_embedded_image_doc_stores_the_document_context( app_url: str, seeded: dict[str, UUID] ) -> None: """The owner's complaint, gone, on the ROW: the embedded image's ``image_context`` leads with the provenance's origin line (the document it came from — source + PDF path + page) and carries the page's own text; its ``content`` is ``\\n\\n`` where the description is the mock's CONTEXT-AWARE canned reply (the ``Extracted from `` trigger — the deterministic witness the describe call carried the context; the bare pre-137 reply must NOT appear), and ``summary`` === ``content`` (the image convention).""" with SessionLocal() as db: img = db.scalar( select(Document).where( Document.source == SOURCE, Document.path == IMAGE_REL ) ) assert img is not None, "the embedded image must be a document" assert img.is_image is True and img.image_path is not None assert img.image_context is not None # The origin line is ALWAYS present (even for a text-less page) # — here with the page's text on the following line(s). assert img.image_context.startswith( f"Extracted from {SOURCE}/{DOC_PATH} (page 2):\n" ) assert PAGE_TWO in img.image_context # No markdown references this image in seed 1 — the doc block # alone (the exact value). assert img.image_context == DOC_BLOCK # The context-aware reply is the witness the context reached # the vision model (the diagram wording is kept, so retrieval # still grounds — but it NAMES the document + page). assert img.content == f"{DOC_BLOCK}\n\n{IMAGE_DESCRIPTION_CONTEXT_ANSWER}" assert img.summary == img.content assert IMAGE_DESCRIPTION_ANSWER not in img.content # --------------------------------------------------------------------------- # 2. Chat: the suggestion block for the image leads with the one-line # From … (page N) display line # --------------------------------------------------------------------------- def test_chat_suggestion_leads_with_the_from_line( page: Page, app_url: str, seeded: dict[str, UUID] ) -> None: """The mock's ``CONTEXT_LINE_TRIGGER`` (the phase-125 marker — no new display mock branch): the ZERO-READ echo QUOTES THE EXACT LINE of the image doc's ```` block — which now LEADS with the one-line ``From /report.pdf (page 2)`` rendering (phase 137 task 04) BEFORE the description. The quoted line in the answer proves the display line was in the prompt; the standalone witness ("stands alone") is absent — the block is NOT the bare description.""" _ask(page, app_url, ANCHOR_QUESTION) bubble = page.locator(".msg.brain .bubble").last expect(bubble).to_contain_text(FROM_LINE, timeout=30_000) expect(bubble).not_to_contain_text("stands alone") # Grounded (not deflected) — a deflected prompt would carry no # block at all (the mock would never see the image # doc, and would answer with the deflection opening). expect(bubble).not_to_contain_text(DEFLECT_OPENING) # --------------------------------------------------------------------------- # 4. Standalone parity witness (against seed 1's KB — it must run # BEFORE seed 2 truncates and re-imports) # --------------------------------------------------------------------------- def test_kb_is_exactly_the_pdf_and_its_image( app_url: str, seeded: dict[str, UUID] ) -> None: """Seed 1's KB is EXACTLY the PDF + its one embedded image — the suite seeds no standalone image anywhere, so the context-aware reply fired for the EMBEDDED image's describe call alone (no standalone image in this KB to witness the bare branch here). The bare branch's byte-identity is pinned by ``test_pdf_support.py`` running green unedited (the regression run) — not duplicated.""" with SessionLocal() as db: docs = db.scalars(select(Document)).all() assert len(docs) == 2, "seed 1's KB is exactly the PDF + its image" image_docs = [d for d in docs if d.is_image] assert len(image_docs) == 1 assert image_docs[0].source == SOURCE assert image_docs[0].path == IMAGE_REL # --------------------------------------------------------------------------- # 3. Seed 2: a markdown reference COMBINES with the document context — # the document block first (ASSUMPTION A2), both clauses on the # display line # --------------------------------------------------------------------------- def test_markdown_reference_combines_the_document_context_first( page: Page, app_url: str, seeded_md: dict[str, UUID] ) -> None: """The same-source markdown references the embedded image by its synthetic path (the phase-125 reference shape) → the stored context carries BOTH block kinds: the document block (the provenance's origin line + the page text) FIRST, then the ``Referenced in e2e-pdf-imgctx-md/notes.md:`` block (provenance before purpose — ASSUMPTION A2), and the display line carries both clauses in stored order (the mock quotes the block's exact first line — the task-04 combined rendering).""" with SessionLocal() as db: img = db.scalar( select(Document).where( Document.source == SOURCE_MD, Document.path == IMAGE_REL ) ) assert img is not None, "the embedded image must be a document" assert img.is_image is True ctx = img.image_context assert ctx is not None # STARTS-WITH the document block (origin line + the page text), # the Referenced-in block present, and the document block's # index BEFORE the markdown block's (document first — A2). assert ctx.startswith( f"Extracted from {SOURCE_MD}/{DOC_PATH} (page 2):\n{PAGE_TWO}" ) assert MD_BLOCK_HEADER in ctx assert ctx.index("Extracted from ") < ctx.index(MD_BLOCK_HEADER) # The same describe-call witness as seed 1 (the combined # context carries the provenance header): the context-aware # reply, content = context + description, summary === content. assert img.content == f"{ctx}\n\n{IMAGE_DESCRIPTION_CONTEXT_ANSWER}" assert img.summary == img.content # The display line: the task-04 combined rendering — the document # clause first, the reference clause after. _ask(page, app_url, ANCHOR_QUESTION) bubble = page.locator(".msg.brain .bubble").last expect(bubble).to_contain_text(FROM_LINE_MD, timeout=30_000) expect(bubble).to_contain_text(MD_BLOCK_HEADER) expect(bubble).not_to_contain_text(DEFLECT_OPENING)