"""Phase 125 E2E (Playwright): image-anchor context — the user path. Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_image_anchor_context.py -v --no-cov The suite's app instance runs with ``BOR_IMAGES=true`` and ``BOR_ANSWER_IMAGES=false`` (module env override — the conftest per-suite-app pattern, leak guards included): the fixture archive carries ``docs/guide.md`` — a markdown file whose ``![alt](path)`` reference NAMES ``docs/diagram.png`` in the SAME source — so the real upload → sync flow drives the whole phase-125 pipeline (LOCKED A1/A2): the task-03 disk scan finds the reference at image-index time, the vision model receives the context after the locked describe prompt, and the image doc's ``image_context`` (and so its ``content``, ``summary`` and embedded chunks) becomes ``\\n\n``. * the sync leaves the image doc ENRICHED (the ``Referenced in`` block on ``image_context``, the enriched content + chunks, the markdown doc's record NULL — the image was indexed first, the task-03 path owns that case); * a grounded chat turn whose suggested tier carries the enriched image doc gets the task-05 context line: the mock (the ``CONTEXT_LINE_TRIGGER`` branch — the prompt parser extended for this phase) QUOTES THE EXACT LINE of the image doc's ```` block, so the rendered answer AND the captured wire prompt prove the block LEADS with the one-line ``Referenced in /: ""`` rendering before the summary (the stored block's shape carries the alt on the next line as a bullet — the quote pins the task-05 rendering, not the stored block); * a STANDALONE image (a second phase of the same app — the KB is re-seeded with a single PNG no markdown references) stays byte-identical to pre-phase: ``image_context`` NULL, the bare description everywhere, the ```` block the description alone, and the mock's "no reference line" answer (the deterministic witness that no context line was in the prompt). """ from __future__ import annotations import base64 import json import os import re import subprocess import sys import zipfile from collections.abc import Iterator from pathlib import Path import httpx import pytest from playwright.sync_api import Browser, Page, expect from sqlalchemy import select, text from app.config import Settings from app.db import SessionLocal from app.models import Chunk, Document from e2e.auth_helpers import login from e2e.conftest import ADMIN_PASSWORD, SESSION_SECRET, USE_REAL_LLM, _wait_http from e2e.mock_llm import IMAGE_DESCRIPTION_ANSWER REPO = Path(__file__).resolve().parents[2] GIT_SOURCES_URL = "/git-sources.html" #: Phase A — the anchored source: the markdown references the image #: from the SAME folder (``diagram.png`` next to ``guide.md`` → the #: resolved in-source path ``docs/diagram.png``). SOURCE_NAME = "e2e-anchor" # the archive stem (archive_source_name) MD_PATH = "docs/guide.md" DOC_PATH = "docs/diagram.png" #: Phase B — the standalone source: a single PNG no markdown names. STANDALONE_SOURCE_NAME = "e2e-anchor-standalone" LOGO_PATH = "logo.png" #: The fixture markdown — the reference on line 5, two context #: paragraphs (the task-02 default window). GUIDE_MD = ( "# Backup runbook\n" "\n" "The restore order diagram explains the backup procedure.\n" "\n" "![Restore order](diagram.png)\n" "\n" "Only run the restore script after the checks pass.\n" ) #: The task-03-shape combined context the same-source disk scan builds #: for ``docs/diagram.png`` (one referring markdown, one reference, #: the task-02 window as the ``Context:`` body — the ``# Backup #: runbook`` heading is the first of the two paragraphs before the #: reference; the N=2 window keeps it). BLOCK = ( "Referenced in e2e-anchor/docs/guide.md:\n" '- Alt text: "Restore order"\n' " Context: # Backup runbook\n" "\n" "The restore order diagram explains the backup procedure.\n" "\n" "Only run the restore script after the checks pass." ) #: The enriched image doc's stored ``content`` (AND ``summary`` — the #: phase-122 ``summary = content`` convention): the context FIRST #: (for retrieval), then the vision description. ENRICHED = f"{BLOCK}\n\n{IMAGE_DESCRIPTION_ANSWER}" #: The content chunk's stored text — the plain-text packer joins the #: enriched paragraphs with single newlines (the chunker's #: normalization, PLAN §5); the ``is_summary`` chunk keeps the raw #: enriched text. PACKED = "\n".join(ENRICHED.split("\n\n")) #: The task-05 ONE-LINE rendering of :data:`BLOCK` (the context line #: that leads the enriched image doc's ```` block body): #: the referring markdown's identity + the reference's alt, the #: ``Context:`` body never rendered. CONTEXT_LINE = 'Referenced in e2e-anchor/docs/guide.md: "Restore order"' #: The grounded question — the mock's ``CONTEXT_LINE_TRIGGER`` marker #: (the zero-read context-line echo) + the phase-122 grounding phrase #: (token-overlap cosine ≈0.47 against the enriched diagram doc and #: ≈0.37 against the standalone logo's bare description — both clear #: the mock-calibrated gate (0.30), the FTS leg corroborates). ANCHOR_QUESTION = ( "quote the image reference — what is shown in the homelab network diagram?" ) #: The mock's two deterministic answers for the marker (the exact #: branch texts — the phase-125 pins key off them). ANSWER_WITH_LINE = ( f"The notes reference the image thus: {CONTEXT_LINE} " "(Deterministic mock answer for E2E.)" ) ANSWER_STANDALONE = ( "The image stands alone — the notes carry no reference line for it. " "(Deterministic mock answer for E2E.)" ) #: The honest-deflection opening (the gate must NOT fire on the #: marker question — a deflected prompt carries no ```` #: block, so the mock would never see the image doc). DEFLECT_OPENING = "I haven't done anything like that" UPLOAD_TIMEOUT_MS = 30_000 SYNC_TIMEOUT_MS = 60_000 SYNCED_LABEL = re.compile(r"^Synced \d{1,2}:\d{2}$") #: A real 1×1 transparent PNG (the phase-122 fixture — the pipeline #: is content-agnostic, the bytes just need to be plausible image #: bytes). PNG_1X1 = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGP4DwQACfsD/fteaysAAAAASUVORK5CYII=" ) # --------------------------------------------------------------------------- # Module app: images ON (the story) — the conftest env pattern with the # phase-122/125 knobs + leak guards. # --------------------------------------------------------------------------- def _app_env(mock_llm: int, app_port: int, scratch: Path) -> dict[str, str]: """The conftest app env (leak guards included) with the phase-122 knobs: ``BOR_IMAGES`` pinned EXPLICITLY true (the story) and ``BOR_ANSWER_IMAGES`` pinned EXPLICITLY false (the default — the auto-cite/figure behavior is phase 124's story, not this phase's, and an operator's local gitignored ``.env`` cannot leak it in: both toggles are the contract under test's neighbors), the image/upload homes in the suite's scratch dir (the app under test must not write copies into the owner's real ``~/bor-sources``).""" env = dict(os.environ) env.pop("DEBUGPY", None) env["BOR_ENVIRONMENT"] = "e2e" env["BOR_STATIC_DIR"] = str(REPO / "frontend") env["BOR_LLM_BASE_URL"] = ( "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else f"http://127.0.0.1:{mock_llm}/v1" ) # Mock-calibrated gate (conftest pattern) — the marker question # grounds on these values (see ANCHOR_QUESTION). env["BOR_RELEVANCE_THRESHOLD"] = "0.30" env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15" env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15" # Phase 67: instant retry waits + the code-default budget. env["BOR_LLM_RETRY_DELAY"] = "0" env["BOR_LLM_RETRIES"] = str(Settings.model_fields["llm_retries"].default) env.setdefault( "BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", ) env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD env["BOR_SESSION_SECRET"] = SESSION_SECRET # Leak guards (conftest pattern). env["BOR_GIT_SOURCES"] = "" env["BOR_DOCS_REPO"] = "" env["BOR_SUGGESTIONS"] = json.dumps(Settings.model_fields["suggestions"].default) env["BOR_INPUT_PLACEHOLDER"] = Settings.model_fields["input_placeholder"].default env["BOR_FOOTER_TEXT"] = Settings.model_fields["footer_text"].default # Phase 122/124 toggles + the suite-private homes. env["BOR_IMAGES"] = "true" env["BOR_ANSWER_IMAGES"] = "false" env["BOR_UPLOAD_DIR"] = str(scratch / "uploads") env["BOR_SOURCES_DIR"] = str(scratch / "checkouts") env["BOR_IMAGE_DIR"] = str(scratch / "images") return env @pytest.fixture(scope="module") def app_server(mock_llm: int, tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: """The story app — ``BOR_IMAGES=true`` (the module env override; the conftest session app is never started in this isolated run, so no port clash).""" scratch = tmp_path_factory.mktemp("bor_anchor") port = int(os.environ.get("E2E_APP_PORT_ANCHOR", "8160")) proc = subprocess.Popen( [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], cwd=REPO, env=_app_env(mock_llm, port, scratch=scratch), ) try: _wait_http(f"http://127.0.0.1:{port}/api/health") yield f"http://127.0.0.1:{port}" finally: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() @pytest.fixture(scope="module") def app_url(app_server: str) -> str: return app_server # --------------------------------------------------------------------------- # Fixture archives # --------------------------------------------------------------------------- @pytest.fixture(scope="module") def anchor_zip(tmp_path_factory: pytest.TempPathFactory) -> Path: """Phase A's archive: the markdown that references the image in the SAME source (``docs/guide.md`` → ``docs/diagram.png``).""" root = tmp_path_factory.mktemp("bor_anchor_zip") docs = root / "docs" docs.mkdir() (docs / "guide.md").write_text(GUIDE_MD, encoding="utf-8") (docs / "diagram.png").write_bytes(PNG_1X1) archive = root / f"{SOURCE_NAME}.zip" with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf: zf.write(docs / "guide.md", arcname="docs/guide.md") zf.write(docs / "diagram.png", arcname="docs/diagram.png") return archive @pytest.fixture(scope="module") def standalone_zip(tmp_path_factory: pytest.TempPathFactory) -> Path: """Phase B's archive: ONE standalone PNG at the source root — no markdown in the source references it (the byte-identical pre-phase case).""" root = tmp_path_factory.mktemp("bor_anchor_standalone_zip") (root / LOGO_PATH).write_bytes(PNG_1X1) archive = root / f"{STANDALONE_SOURCE_NAME}.zip" with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf: zf.write(root / LOGO_PATH, arcname=LOGO_PATH) return archive # --------------------------------------------------------------------------- # DB + UI helpers # --------------------------------------------------------------------------- def _truncate_all() -> None: """Fresh KB + registry per phase (the E2E isolation pattern): the suites share one Postgres, and a leftover row would corrupt the counts. ``sources_meta`` (the KB generation counter) resets with the KB — the sync re-creates the row.""" with SessionLocal() as db: db.execute(text( "TRUNCATE chunks, documents, query_log, kb_overview, " "git_sources, sources_meta, folder_summaries" )) db.commit() def _health_db(app_url: str) -> bool: try: return httpx.get(f"{app_url}/api/health", timeout=5).json()["db"] == "up" except Exception: # noqa: BLE001 — unreachable is the skip case return False def _upload_zip(page: Page, app_url: str, archive: Path, source: str) -> None: """The real admin upload flow: form login → the git-sources page → pick the archive → submit → the terminal result line (phase 90: "Uploaded — press Sync sources to import it.").""" login(page, app_url, next=GIT_SOURCES_URL) expect(page).to_have_url(app_url + GIT_SOURCES_URL, timeout=30_000) expect(page.locator("#sign-out-btn")).to_be_visible(timeout=15_000) page.set_input_files("#archive-upload-file", str(archive)) page.click("#archive-upload-btn") result = page.locator("#archive-upload-result") expect(result).to_be_visible(timeout=UPLOAD_TIMEOUT_MS) expect(result).to_have_text( f"Uploaded {source} — press Sync sources to import it.", timeout=UPLOAD_TIMEOUT_MS, ) def _sync_via_ui(page: Page, app_url: str, expected_result: str) -> None: """The RAG page's "Sync sources" button (the scan — phase 90 A3): click → "Syncing…" → the terminal "Synced HH:MM" label + the fresh counts in #sync-result (the never-stale contract).""" page.goto(app_url + "/sources.html") btn = page.locator("#sync-btn") expect(btn).to_be_visible(timeout=30_000) expect(page.locator("#sync-label")).to_have_text( re.compile(r"^Sync sources$|^Synced \d{1,2}:\d{2}$") ) btn.click() expect(btn).to_be_disabled() expect(page.locator("#sync-label")).to_have_text("Syncing…") expect(page.locator("#sync-label")).to_have_text(SYNCED_LABEL, timeout=SYNC_TIMEOUT_MS) expect(btn).to_be_enabled() expect(page.locator("#sync-result")).to_have_text(expected_result) def _captured_systems(mock_llm: int) -> list[str]: """The system prompts of the captured chat requests (the mock's ring buffer — the wire-level lens on the EXACT prompt the app built; only ``/v1/chat/completions`` bodies are captured, so after a reset + one grounded zero-read turn there is exactly one).""" bodies = httpx.get(f"http://127.0.0.1:{mock_llm}/v1/e2e/captured", timeout=10).json() systems: list[str] = [] for body in bodies: for msg in body.get("messages", []): if msg.get("role") == "system" and isinstance(msg.get("content"), str): systems.append(msg["content"]) return systems def _reset_capture(mock_llm: int) -> None: httpx.post(f"http://127.0.0.1:{mock_llm}/v1/e2e/captured/reset", timeout=10) def _ask(page: Page, app_url: str, question: str) -> None: """The chat user path: login → the question → Send → the final brain bubble (the Send button re-enabled = the turn is over).""" login(page, app_url, next="/") expect(page.locator("#kb-banner")).to_be_hidden() page.fill("#message-input", question) page.click("#send-btn") bubble = page.locator(".msg.brain .bubble").last expect(page.locator("#send-btn")).to_be_enabled(timeout=60_000) expect(page.locator("#send-label")).to_have_text("Send") expect(bubble).to_contain_text("Deterministic mock answer", timeout=30_000) # --------------------------------------------------------------------------- # The module seeds: phase A (the anchored archive) and phase B (the # standalone archive) — each owns its truncate/re-import window. # --------------------------------------------------------------------------- @pytest.fixture(scope="module") def seeded( app_url: str, browser: Browser, anchor_zip: Path, ) -> Iterator[dict[str, str]]: """Phase A: upload the anchored archive through the admin UI, then scan it with the RAG page's Sync button (the mock's vision model describes the image in-process, with the markdown context). The module app's ``/api/config`` flags are asserted first — the env overrides must have reached the app under test, or the whole premise of the suite is void.""" if not _health_db(app_url): pytest.skip("Postgres not reachable — run `podman compose up -d db` first") cfg = httpx.get(f"{app_url}/api/config", timeout=5).json() assert cfg["images"] is True, "the module app must run with BOR_IMAGES=true" _truncate_all() page = browser.new_page(viewport={"width": 1280, "height": 800}) try: _upload_zip(page, app_url, anchor_zip, SOURCE_NAME) _sync_via_ui(page, app_url, expected_result="2 added") with SessionLocal() as db: doc = db.scalar( select(Document).where( Document.source == SOURCE_NAME, Document.path == DOC_PATH ) ) assert doc is not None, "the uploaded image must become a document" assert doc.is_image is True and doc.image_path is not None finally: page.close() yield {"image_url": f"/api/documents/{doc.id}/image"} _truncate_all() @pytest.fixture(scope="module") def standalone( app_url: str, browser: Browser, standalone_zip: Path, ) -> Iterator[dict[str, str]]: """Phase B: a FRESH KB (the phase-A rows are truncated) seeded with the standalone archive — a single PNG no markdown in the source references (the byte-identical pre-phase case).""" _truncate_all() page = browser.new_page(viewport={"width": 1280, "height": 800}) try: _upload_zip(page, app_url, standalone_zip, STANDALONE_SOURCE_NAME) _sync_via_ui(page, app_url, expected_result="1 added") with SessionLocal() as db: doc = db.scalar( select(Document).where( Document.source == STANDALONE_SOURCE_NAME, Document.path == LOGO_PATH ) ) assert doc is not None, "the uploaded image must become a document" assert doc.is_image is True and doc.image_path is not None finally: page.close() yield {"image_url": f"/api/documents/{doc.id}/image"} _truncate_all() # --------------------------------------------------------------------------- # 1. Sync: the image doc is enriched with the markdown context # --------------------------------------------------------------------------- def test_sync_enriches_the_image_doc_with_markdown_context( page: Page, app_url: str, seeded: dict[str, str] ) -> None: """LOCKED A1/A2 end-to-end: the markdown in the SAME source references the image → after the sync the image doc's ``image_context`` is the ``Referenced in`` block, its ``content`` (AND ``summary`` — the phase-122 convention) is ``\\n\n`` (context first — for retrieval — then the vision description), the content chunk is embedded from the enriched text (the packer's single-newline form) and the ``is_summary`` chunk mirrors the raw enriched text. The referring markdown's OWN ``image_context`` stays NULL: the image was indexed BEFORE the markdown (``docs/diagram.png`` sorts first), so its reference names an existing image doc — the task-03 path owns that case, no pending record is written. The Sources page counts the KB: 2 docs, 4 chunks (one content + one summary chunk per doc).""" with SessionLocal() as db: doc = db.scalar( select(Document).where( Document.source == SOURCE_NAME, Document.path == DOC_PATH ) ) assert doc is not None # The stored context (task 05's display source). assert doc.image_context == BLOCK # The enriched content — context FIRST, then the description — # IS the summary (the phase-122 image convention). assert doc.content == ENRICHED assert doc.summary == ENRICHED chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all() content_chunks = [c for c in chunks if not c.is_summary] assert [c.content for c in content_chunks] == [PACKED], ( "the content chunk is embedded from the ENRICHED text" ) summary_chunks = [c for c in chunks if c.is_summary] assert len(summary_chunks) == 1 and summary_chunks[0].position == -1 assert summary_chunks[0].content == ENRICHED for c in chunks: assert c.embedding is not None and len(c.embedding) == 768 # The referring markdown — the image doc existed at its # indexing (sort order), so NO pending record was written. md = db.scalar( select(Document).where( Document.source == SOURCE_NAME, Document.path == MD_PATH ) ) assert md is not None and md.is_image is False assert md.image_context is None, "existing image: the record stays NULL" # The KB on the Sources page: 2 docs (the markdown + the image), # 4 chunks (one content + one is_summary per doc). login(page, app_url) expect(page.locator("#stat-docs")).to_have_text("2") expect(page.locator("#stat-chunks")).to_have_text("4") # --------------------------------------------------------------------------- # 2. Chat: the enriched image doc's block leads with the # context line — the answer AND the wire prompt prove it # --------------------------------------------------------------------------- def test_chat_answer_quotes_the_context_line( page: Page, app_url: str, mock_llm: int, seeded: dict[str, str] ) -> None: """LOCKED A4 end-to-end: a grounded turn whose suggested tier carries the ENRICHED image doc → the ```` block LEADS with the ONE-LINE context line (the task-05 rendering of the stored block: the referring markdown's identity + the reference's alt, BEFORE the summary — for an enriched image doc the summary is the enriched content, so the stored block rides in the body too, but the header is the one-line rendering, never the stored multi-line header). The mock's marker branch QUOTES THE EXACT LINE of the block — so the rendered answer proves the one-line rendering was in the prompt (the quoted alt sits on the header line — the stored block's shape carries it on the next line as a bullet), and the captured wire prompt pins the exact block shape: line, blank line, enriched summary.""" _reset_capture(mock_llm) _ask(page, app_url, ANCHOR_QUESTION) bubble = page.locator(".msg.brain .bubble").last # The mock's enriched answer — the EXACT context line, quoted. expect(bubble).to_contain_text(ANSWER_WITH_LINE, timeout=30_000) # Grounded (not deflected) — a deflected prompt would carry no # block at all. expect(bubble).not_to_contain_text(DEFLECT_OPENING) # The wire prompt: exactly one chat request (grounded, zero tool # calls), and the image doc's block leads with the # ONE-LINE context line — then a blank line, then the summary # (for an enriched image doc the summary IS the enriched content — # the phase-122 ``summary = content`` convention — so the stored # block rides along in the body, but the block HEADS with the # one-line rendering, never the stored header line). systems = _captured_systems(mock_llm) assert len(systems) == 1, "one grounded zero-read turn → one chat request" system = systems[0] start = system.index(f'\n", start) + 2 body_end = system.index("\n", body_start) block_body = system[body_start:body_end] # The exact block shape: the one-line context line (the alt QUOTED # on the same line — the stored block's shape carries the alt on # the NEXT line as a bullet, so this pins the task-05 rendering), # a blank line, then the enriched summary. assert block_body == f"{CONTEXT_LINE}\n\n{ENRICHED}" # The line is inside , before the summary's block. assert system.index("") < system.index(CONTEXT_LINE) assert system.index(CONTEXT_LINE) < system.index("\n" + BLOCK) # --------------------------------------------------------------------------- # 3. Standalone image: the byte-identical pre-phase row # --------------------------------------------------------------------------- def test_standalone_image_keeps_null_context_and_bare_description( page: Page, app_url: str, standalone: dict[str, str] ) -> None: """LOCKED A4 (the byte-identity criterion) on the ROW: a standalone image (no markdown in the source references it) → ``image_context`` NULL, ``content`` (AND ``summary``) the BARE vision description, every chunk the bare description — exactly the phase-122 shape. The Sources page counts 1 doc / 2 chunks (one content + one summary).""" with SessionLocal() as db: doc = db.scalar( select(Document).where( Document.source == STANDALONE_SOURCE_NAME, Document.path == LOGO_PATH ) ) assert doc is not None assert doc.image_context is None, "standalone: the context stays NULL" assert doc.content == IMAGE_DESCRIPTION_ANSWER, "standalone: the bare description" assert doc.summary == IMAGE_DESCRIPTION_ANSWER chunks = db.scalars(select(Chunk).where(Chunk.document_id == doc.id)).all() assert len(chunks) == 2 assert all(c.content == IMAGE_DESCRIPTION_ANSWER for c in chunks) for c in chunks: assert c.embedding is not None and len(c.embedding) == 768 login(page, app_url) expect(page.locator("#stat-docs")).to_have_text("1") expect(page.locator("#stat-chunks")).to_have_text("2") # --------------------------------------------------------------------------- # 4. Standalone image: the block is byte-identical in the # prompt — no context line, the bare description alone # --------------------------------------------------------------------------- def test_standalone_answer_carries_no_context_line( page: Page, app_url: str, mock_llm: int, standalone: dict[str, str] ) -> None: """LOCKED A4 (the byte-identity criterion) in the PROMPT: the SAME marker question against the standalone KB → the image doc's ```` block body is EXACTLY the bare description (no context line, no stray blank — the pre-phase block), and the mock — which quotes the block's first line when it is a ``Referenced in`` line — answers that the image carries no reference line: the deterministic witness that the line was NOT in the prompt. ``Referenced in`` does not appear ANYWHERE in the system prompt.""" _reset_capture(mock_llm) _ask(page, app_url, ANCHOR_QUESTION) bubble = page.locator(".msg.brain .bubble").last expect(bubble).to_contain_text(ANSWER_STANDALONE, timeout=30_000) expect(bubble).not_to_contain_text(DEFLECT_OPENING) systems = _captured_systems(mock_llm) assert len(systems) == 1, "one grounded zero-read turn → one chat request" system = systems[0] start = system.index( f'\n", start) + 2 body_end = system.index("\n", body_start) block_body = system[body_start:body_end] # The pre-phase block body: the description alone, verbatim. assert block_body == IMAGE_DESCRIPTION_ANSWER assert "Referenced in" not in system