"""Phase 24 E2E (Playwright): a matched document reaches the LLM whole. Story: ``.agent/user_stories/whole-document-context.md`` Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_whole_document_context.py -v --no-cov The mock LLM's tail-echo trigger (``END_OF_NOTES_TRIGGER``, see ``tests/e2e/mock_llm.py``) makes the model quote the last 160 chars of the document context. A sentinel placed on the *last line* of a document therefore appears in the rendered answer **iff the entire document was in the prompt** — which is what makes the no-truncation contract (A7 revised, owner permission 2026-08-24: matched parent documents are never cut) provable end-to-end. The oversized documents are seeded directly via SQLAlchemy (a ``documents`` row + 2–3 ``chunks`` rows whose embeddings are the mock's own deterministic bag-of-words vectors, so the question's live mock embedding genuinely overlaps — no fixture files added: ``tests/fixtures/docs/`` stays at its 13 files (phase 47), other suites pin ``summary.added == 13``). """ from __future__ import annotations import asyncio import hashlib from collections.abc import Callable, Sequence from datetime import UTC, datetime from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import select, text from sqlalchemy.orm import Session from app.config import Settings from app.db import SessionLocal from app.models import Chunk, Document, QueryLog from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.retriever import TRUNCATION_MARKER from tests.e2e.mock_llm import embed_text REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" #: The pre-phase-24 ``BOR_MAX_CONTEXT_CHARS`` default — the budget this #: suite proves is gone from the document path. OLD_CONTEXT_CAP = 24_000 QUESTION = "Show the end of your notes about the gitlab install playbook, please." SMALL_QUESTION = "How is my Kubernetes cluster set up?" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" # --- Content builders (deterministic, token-controlled) ------------------- def _repeated(line: str, min_chars: int) -> str: """"line" (newline-terminated) repeated until at least min_chars chars.""" unit = line + "\n" return unit * max(1, -(-min_chars // len(unit))) def _doc_slices(content: str, n: int) -> list[str]: """Even slices of *content* (the last slice keeps the final line).""" step = len(content) // n return [content[i * step : (i + 1) * step] for i in range(n - 1)] + [ content[(n - 1) * step :] ] def _gitlab_30k_doc(sentinel: str) -> str: """A ~30 000-char document (past the old 24k cap): a body of repeated "gitlab install playbook" lines — the same tokens the question carries, so hybrid retrieval genuinely hits — whose LAST line is a unique sentinel only a tail echo can surface.""" body = _repeated( "gitlab install playbook: run the gitlab install playbook on the homelab host.", OLD_CONTEXT_CAP + 6_000, ) return body + sentinel + "\n" def _pair_doc(strong_line: str, filler_line: str, sentinel: str) -> tuple[str, list[str]]: """A ~16 000-char document: a ~2 000-char first chunk carrying the question's key tokens, a ~14 000-char low-overlap remainder, and a unique sentinel as the last line. Returns (content, chunk_texts).""" chunk0 = _repeated(strong_line, 2_000) chunk1 = _repeated(filler_line, 14_000) return chunk0 + chunk1 + sentinel + "\n", [chunk0, chunk1] # --- DB seeding (TRUNCATE-then-seed, cf. test_chat_rag.py) ----------------- def _seed_doc( db: Session, source: str, path: str, title: str, content: str, chunk_texts: Sequence[str], ) -> None: """One ``documents`` row + one ``chunks`` row per chunk text. Each chunk's embedding is the mock's own ``embed_text`` vector, so the app's live mock embedding of the question genuinely overlaps. """ doc = Document( source=source, path=path, full_path=f"/tmp/{path}", title=title, content=content, content_hash=hashlib.sha256(content.encode("utf-8")).hexdigest(), indexed_at=datetime.now(UTC), ) db.add(doc) db.flush() db.add_all( Chunk(document_id=doc.id, position=i, content=chunk, embedding=embed_text(chunk)) for i, chunk in enumerate(chunk_texts) ) def _reset_db(seed: Callable[[Session], None] | None = None) -> None: """Truncate the KB (and query log), then optionally run *seed*.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() if seed is not None: seed(db) db.commit() # --- Importer + thread helpers (test_chat_rag.py pattern) ------------------ async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = { "_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1", } settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _ask(page: Page, app_url: str, question: str) -> Any: """Submit *question* and wait for the streamed brain bubble.""" page.set_default_timeout(30_000) page.goto(app_url) page.fill("#message-input", question) page.click("#send-btn") bubble = page.locator(".msg.brain .bubble") bubble.first.wait_for(state="visible", timeout=30_000) return bubble.first def _last_query_log() -> QueryLog: with SessionLocal() as db: rows = db.scalars(select(QueryLog)).all() assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}" return rows[0] # --- Story tests ------------------------------------------------------------- def test_whole_document_over_old_cap_reaches_llm( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """A 30k document (past the old 24k cap) reaches the LLM whole: its tail sentinel — the last 160 chars of the context — is echoed back.""" sentinel = "WHOLE-DOC-TAIL-GITLAB-30K" content = _gitlab_30k_doc(sentinel) assert len(content) > OLD_CONTEXT_CAP # this is the point of the test def seed(db: Session) -> None: _seed_doc( db, "Homelab", "gitlab-30k.md", "GitLab Install Playbook (30k)", content, _doc_slices(content, 3), ) _reset_db(seed) bubble = _ask(page, app_url, QUESTION) # The tail sentinel exists only on the document's last line — its # presence proves the entire 30k document was in the LLM prompt. expect(bubble).to_contain_text(sentinel, timeout=30_000) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER) expect(bubble).not_to_contain_text(TRUNCATION_MARKER) # Grounded: the document's source chip renders under the bubble. chip = page.locator(".msg.brain .source-chip", has_text="gitlab-30k.md") expect(chip).to_have_count(1) expect(chip.first).to_contain_text("Homelab/gitlab-30k.md") # Button recovers (never stale) and the turn was grounded. expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") row = _last_query_log() assert row.question == QUESTION assert row.deflected is False assert "Homelab/gitlab-30k.md" in row.sources def test_second_document_of_over_cap_pair_reaches_llm( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """Two ~16k documents (32k combined — under the old budget the lower-ranked one was truncated in place): the SECOND document, the last block inside , reaches the LLM whole — its sentinel is the one the tail echo surfaces.""" sentinel_a = "WHOLE-DOC-TAIL-PAIR-A" sentinel_b = "WHOLE-DOC-TAIL-PAIR-B" # Doc A's first chunk carries the question's key tokens → it ranks # first in both candidate lists → it comes first in . content_a, chunks_a = _pair_doc( "gitlab install playbook: run the gitlab install playbook end to end.", "the server room keeps a steady temperature and the racks are labelled.", sentinel_a, ) # Doc B's first chunk has only a weaker overlap ("playbook", "notes") # → it ranks second → it is the LAST block inside . content_b, chunks_b = _pair_doc( "playbook notes: the playbook notes track what changed and where.", "the rack elevation drawing shows cable trays and pdu positions.", sentinel_b, ) assert len(content_a) + len(content_b) > OLD_CONTEXT_CAP # 32k > 24k def seed(db: Session) -> None: _seed_doc( db, "Homelab", "gitlab-install-playbook.md", "GitLab Install Playbook", content_a, chunks_a, ) _seed_doc( db, "Homelab", "playbook-notes.md", "Playbook Notes", content_b, chunks_b, ) _reset_db(seed) bubble = _ask(page, app_url, QUESTION) # The tail echo quotes doc B's sentinel (the last block's tail) — doc B # was in the prompt whole, past the old budget. Doc A's sentinel sits # mid-prompt, so it must NOT be in the quoted tail: that is what pins # the rank order (A first, B last inside ). expect(bubble).to_contain_text(sentinel_b, timeout=30_000) expect(bubble).not_to_contain_text(sentinel_a) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER) expect(bubble).not_to_contain_text(TRUNCATION_MARKER) # Both documents are cited (top-2), in rank order. chips = page.locator(".msg.brain .source-chip") expect(chips).to_have_count(2) expect(page.locator(".msg.brain .source-chip", has_text="gitlab-install-playbook.md")).to_have_count(1) expect(page.locator(".msg.brain .source-chip", has_text="playbook-notes.md")).to_have_count(1) row = _last_query_log() assert row.question == QUESTION assert row.deflected is False assert "Homelab/gitlab-install-playbook.md" in row.sources assert "Homelab/playbook-notes.md" in row.sources def test_small_document_path_unchanged( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """Regression: the standard (small) fixtures still take the grounded path, byte-identical to before — no marker, kubernetes.md cited.""" _reset_db(None) summary = _run_in_thread(_import_fixtures(mock_llm)) assert summary.added == 13 # A9 formats (phase 47 added quadlet+j2) bubble = _ask(page, app_url, SMALL_QUESTION) expect(bubble).to_contain_text(SMALL_QUESTION, timeout=30_000) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER) expect(bubble).not_to_contain_text(TRUNCATION_MARKER) chip = page.locator(".msg.brain .source-chip", has_text="kubernetes.md") expect(chip).to_have_count(1) row = _last_query_log() assert row.deflected is False assert "docs/homelab/kubernetes.md" in row.sources