"""Phase 30 E2E (Playwright): a summary hit delivers the full source doc. Story: ``.agent/user_stories/document-summaries.md`` Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_document_summaries.py -v --no-cov The fixture KB is a story-dedicated directory (``tests/fixtures/summary_kb/`` — the shared ``tests/fixtures/docs/`` stays at its 9 pinned files) with two documents: * ``quadlet/qwen-llamacpp.yaml`` — a non-markdown A9 doc. At import the mock ``lite`` model (``SUMMARY_MODE`` marker, ``tests/e2e/mock_llm.py``) reduces it to a deterministic 24-token digest, stored on ``documents.summary`` and indexed as one ``is_summary`` chunk. The raw yaml body is deliberately token-diluted, so the document's best fused chunk is its summary chunk. The sentinel ``RESE-SUMMARY-SENTINEL-7f3a`` sits on the document's LAST line — outside the 24-token digest, unreachable from the summary. * ``notes/qwen-llamacpp-notes.md`` — a markdown control doc (never summarized) that ranks first, which puts the yaml document LAST inside ````. The mock LLM's tail-echo trigger (``END_OF_NOTES_TRIGGER``) makes the answer quote the last 160 chars of the document context — the tail of the LAST selected document. The sentinel therefore appears in the rendered answer **iff the entire yaml source document (not the summary digest) reached the LLM prompt** — the summary→parent-document resolution through the unchanged chunk→document mapping (A7 revised: never truncated), which is what this story is about. """ from __future__ import annotations import asyncio from collections.abc import Callable, Sequence from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import select, text from sqlalchemy.orm import Session from app.config import Settings from app.db import SessionLocal from app.models import Document, QueryLog from app.rag.chunker import chunk_document from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.retriever import RetrievedChunk, retrieve from tests.e2e.mock_llm import TOKEN_RE, embed_text REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "summary_kb" QUESTION = ( "What are the optimal parameters for qwen 3.8 on llama.cpp? " "show the end of your notes" ) SENTINEL = "RESE-SUMMARY-SENTINEL-7f3a" SOURCE = "summary_kb" YAML_PATH = "quadlet/qwen-llamacpp.yaml" MD_PATH = "notes/qwen-llamacpp-notes.md" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" #: The importer's chunk policy (Settings defaults; the mock never trips #: the endpoint token-cap retry, so the target is never halved). CHUNK_TARGET = 2_000 CHUNK_OVERLAP = 200 # --- Importer + thread helpers (test_whole_document_context.py pattern) --- async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = { "_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1", } settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(seed: Callable[[Session], None] | None = None) -> None: """Truncate the KB (and query log + steering), then optionally seed.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes")) db.commit() if seed is not None: seed(db) db.commit() def _ask(page: Page, app_url: str, question: str) -> Any: """Submit *question* and wait for the streamed brain bubble.""" page.set_default_timeout(30_000) page.goto(app_url) page.fill("#message-input", question) page.click("#send-btn") bubble = page.locator(".msg.brain .bubble") bubble.first.wait_for(state="visible", timeout=30_000) return bubble.first def _last_query_log() -> QueryLog: with SessionLocal() as db: rows = db.scalars(select(QueryLog)).all() assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}" return rows[0] def _doc(db: Session, path: str) -> Document: doc = db.scalar(select(Document).where(Document.path == path)) assert doc is not None, f"fixture doc {path!r} was not imported" return doc def _expected_summary(content: str, source: str, path: str) -> str: """The mock lite model's byte-stable digest + the code pointer line. Mirrors ``mock_llm.compose_answer``'s ``SUMMARY_MODE`` branch (first 24 tokens of the document content) plus the summarizer's deterministic ``Source:`` line — no model output is ever trusted. """ digest = " ".join(TOKEN_RE.findall(content.lower())[:24]) return f"This document covers {digest}.\nSource: {source}/{path}" def _chunks_by_path(chunks: Sequence[RetrievedChunk], path: str) -> list[RetrievedChunk]: return [c for c in chunks if c.document.path == path] # --- Story tests ------------------------------------------------------------- def test_summary_hit_retrieves_full_source_document( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """A question whose best yaml match is the summary chunk yields an answer grounded in the FULL yaml source document: its tail sentinel — which the summary digest cannot contain — is echoed back, and the source chip cites the yaml path (deflected: false).""" _reset_db() summary = _run_in_thread(_import_fixtures(mock_llm)) assert summary.added == 2 # yaml + md control assert summary.summaries == 1 and summary.summary_errors == 0 assert summary.errors == 0 # Import state: exactly one embedded ``is_summary`` chunk (position # −1) whose text is the byte-stable mock digest + the deterministic # pointer line. yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8") assert SENTINEL in yaml_content.splitlines()[-1] # last line, by design with SessionLocal() as db: yaml_doc = _doc(db, YAML_PATH) schunks = [c for c in yaml_doc.chunks if c.is_summary] assert len(schunks) == 1 assert schunks[0].position == -1 assert schunks[0].embedding is not None assert yaml_doc.summary == _expected_summary(yaml_content, SOURCE, YAML_PATH) # Retrieval state: the summary chunk is the yaml document's best fused # chunk — the document enters the context through its summary, not # through the diluted raw yaml chunks. with SessionLocal() as db: chunks = retrieve(db, QUESTION, embed_text(QUESTION)) yaml_chunks = _chunks_by_path(chunks, YAML_PATH) best_yaml = max(yaml_chunks, key=lambda c: c.score) assert best_yaml.is_summary assert len(yaml_chunks) >= 2 # summary + at least one raw candidate bubble = _ask(page, app_url, QUESTION) # The tail sentinel exists only on the document's last line and # cannot be in the summary digest — its presence proves the entire # source document was in the LLM prompt (summary→parent resolution). expect(bubble).to_contain_text(SENTINEL, timeout=30_000) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER) # Grounded: the yaml source chip renders (the md doc ranks first, so # both fixtures are cited). chip = page.locator(".msg.brain .source-chip", has_text=YAML_PATH) expect(chip).to_have_count(1) expect(chip.first).to_contain_text(f"{SOURCE}/{YAML_PATH}") expect(page.locator(".msg.brain .source-chip", has_text=MD_PATH)).to_have_count(1) # Button recovers (never stale) and the turn was grounded, not # deflected. expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") row = _last_query_log() assert row.question == QUESTION assert row.deflected is False assert f"{SOURCE}/{YAML_PATH}" in row.sources assert f"{SOURCE}/{MD_PATH}" in row.sources def test_markdown_control_doc_gets_no_summary_chunk( mock_llm: int, db_ready: None ) -> None: """Control: in the same KB the markdown doc gets no summary at all — its chunk count is exactly the raw chunks; the yaml doc has exactly one ``is_summary`` row and its raw chunk count is untouched by the summary.""" _reset_db() summary = _run_in_thread(_import_fixtures(mock_llm)) assert summary.added == 2 md_content = (FIXTURES / MD_PATH).read_text(encoding="utf-8") yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8") with SessionLocal() as db: md_doc = _doc(db, MD_PATH) yaml_doc = _doc(db, YAML_PATH) md_chunks = [c for c in md_doc.chunks if not c.is_summary] yaml_raw = [c for c in yaml_doc.chunks if not c.is_summary] yaml_summary = [c for c in yaml_doc.chunks if c.is_summary] # Markdown: never summarized (phase 30 scope — A9 non-markdown only). assert md_doc.summary is None assert len(md_chunks) == len( chunk_document(md_content, MD_PATH, CHUNK_TARGET, CHUNK_OVERLAP) ) # YAML: raw chunks exactly as chunked by the importer policy, plus # exactly one summary chunk (position −1, embedded, on the doc row). assert len(yaml_raw) == len( chunk_document(yaml_content, YAML_PATH, CHUNK_TARGET, CHUNK_OVERLAP) ) assert sorted(c.position for c in yaml_raw) == list(range(len(yaml_raw))) assert len(yaml_summary) == 1 assert yaml_summary[0].position == -1 assert yaml_summary[0].embedding is not None assert yaml_doc.summary is not None assert yaml_doc.summary == yaml_summary[0].content assert yaml_doc.summary.endswith(f"\nSource: {SOURCE}/{YAML_PATH}")