"""Phase 30 E2E (Playwright) — phase 118 re-targeted (A2/A6): a summary hit seeds the SUMMARY into the prompt — never the full source doc. Story: ``.agents/user_stories/document-summaries.md`` Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_document_summaries.py -v --no-cov The fixture KB is a story-dedicated directory (``tests/fixtures/summary_kb/`` — the shared ``tests/fixtures/docs/`` stays at its pinned files) with two documents: * ``quadlet/qwen-llamacpp.yaml`` — at import the mock ``lite`` model (``SUMMARY_MODE`` marker, ``tests/e2e/mock_llm.py``) reduces it to a deterministic 24-token digest, stored on ``documents.summary`` and indexed as one ``is_summary`` chunk. The raw yaml body is deliberately token-diluted, so the document's top fused chunks are the summary and the one lexical-hitting raw chunk. The sentinel ``RESE-SUMMARY-SENTINEL-7f3a`` sits on the document's LAST line — outside the 24-token digest, unreachable from the summary. * ``notes/qwen-llamacpp-notes.md`` — a markdown doc (phase 118 A2: it is summarized TOO — the phase-30 non-markdown-only scope is retired) that ranks first, which puts the yaml document LAST inside ```` (a two-doc KB → both docs are suggested, the related tier is empty). The mock LLM's tail-echo trigger (``END_OF_NOTES_TRIGGER``) makes the answer quote the last 160 chars of the seeded ```` block — under the phase-118 summary-seed contract (A6) that block carries the suggested docs' SUMMARIES, never their full texts, so the echoed tail is the LAST suggested doc's summary (the yaml doc's byte-stable digest tail + ``Source:`` pointer line). The sentinel therefore appears in the rendered answer **only if the entire yaml source document (not the summary) reached the LLM prompt** — under the locked contract it must be ABSENT (full text enters the context only through the capped ``read`` tool), which is the inverse of the retired phase-30/24 full-text pin and what this story is about now. """ from __future__ import annotations import asyncio from collections.abc import Callable, Sequence from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import select, text from sqlalchemy.orm import Session from app.config import Settings from app.db import SessionLocal from app.models import Document, QueryLog from app.rag.chunker import chunk_document from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.retriever import RetrievedChunk, retrieve, select_suggested from e2e.auth_helpers import login from tests.e2e.mock_llm import TOKEN_RE, embed_text REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "summary_kb" QUESTION = ( "What are the optimal parameters for qwen 3.8 on llama.cpp? " "show the end of your notes" ) SENTINEL = "RESE-SUMMARY-SENTINEL-7f3a" SOURCE = "summary_kb" YAML_PATH = "quadlet/qwen-llamacpp.yaml" MD_PATH = "notes/qwen-llamacpp-notes.md" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" #: The importer's chunk policy (Settings defaults; the mock never trips #: the endpoint token-cap retry, so the target is never halved). CHUNK_TARGET = 2_000 CHUNK_OVERLAP = 200 # --- Importer + thread helpers (test_whole_document_context.py pattern) --- async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = { "_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1", } settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(seed: Callable[[Session], None] | None = None) -> None: """Truncate the KB (and query log + steering), then optionally seed.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes")) db.commit() if seed is not None: seed(db) db.commit() def _ask(page: Page, app_url: str, question: str) -> Any: """Submit *question* and wait for the streamed brain bubble.""" page.set_default_timeout(30_000) login(page, app_url, next="/") page.fill("#message-input", question) page.click("#send-btn") bubble = page.locator(".msg.brain .bubble") bubble.first.wait_for(state="visible", timeout=30_000) return bubble.first def _last_query_log() -> QueryLog: with SessionLocal() as db: rows = db.scalars(select(QueryLog)).all() assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}" return rows[0] def _doc(db: Session, path: str) -> Document: doc = db.scalar(select(Document).where(Document.path == path)) assert doc is not None, f"fixture doc {path!r} was not imported" return doc def _expected_summary(content: str, source: str, path: str) -> str: """The mock lite model's byte-stable digest + the code pointer line. Mirrors ``mock_llm.compose_answer``'s ``SUMMARY_MODE`` branch (first 24 tokens of the document content) plus the summarizer's deterministic ``Source:`` line — no model output is ever trusted. """ digest = " ".join(TOKEN_RE.findall(content.lower())[:24]) return f"This document covers {digest}.\nSource: {source}/{path}" def _chunks_by_path(chunks: Sequence[RetrievedChunk], path: str) -> list[RetrievedChunk]: return [c for c in chunks if c.document.path == path] # --- Story tests ------------------------------------------------------------- def test_summary_hit_seeds_the_summary_not_the_full_text( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """A question whose best yaml match is its summary chunk yields a grounded answer seeded from the SUMMARY — the mock's tail echo quotes the last suggested doc's summary tail (digest + pointer), and the sentinel (the yaml doc's last line, outside the digest) is ABSENT: the full source doc never reached the prompt (A6). Both fixture docs are suggested (two-doc KB, no floor) and both chips render (deflected: false).""" _reset_db() summary = _run_in_thread(_import_fixtures(mock_llm)) assert summary.added == 2 # yaml + md control assert summary.summaries == 2 and summary.summary_errors == 0 # A2: md too assert summary.errors == 0 # Import state: exactly one embedded ``is_summary`` chunk (position # −1) whose text is the byte-stable mock digest + the deterministic # pointer line. yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8") assert SENTINEL in yaml_content.splitlines()[-1] # last line, by design with SessionLocal() as db: yaml_doc = _doc(db, YAML_PATH) schunks = [c for c in yaml_doc.chunks if c.is_summary] assert len(schunks) == 1 assert schunks[0].position == -1 assert schunks[0].embedding is not None assert yaml_doc.summary == _expected_summary(yaml_content, SOURCE, YAML_PATH) # Retrieval state (phase 118, A2/A6): the EMBEDDED summary chunk is # a retrieval candidate (the seeded text ranks on its own), and the # suggested tier is the two docs in rank order — md first, yaml # LAST (so the yaml's summary is the tail of the block, # the mock echo's target). The document enters the prompt through # its SUMMARY block, not through the diluted raw yaml chunks. with SessionLocal() as db: chunks = retrieve(db, QUESTION, embed_text(QUESTION)) yaml_chunks = _chunks_by_path(chunks, YAML_PATH) assert any(c.is_summary for c in yaml_chunks) # the embedded summary ranks assert len(yaml_chunks) >= 2 # summary + at least one raw candidate assert [d.path for d in select_suggested(chunks)] == [MD_PATH, YAML_PATH] bubble = _ask(page, app_url, QUESTION) # Phase 118 (A6): the seed is the SUMMARY, not the full text — the # mock's tail echo quotes the last 160 chars of the # block, which end in the LAST suggested doc's summary: the yaml # doc's byte-stable digest tail + pointer line. The sentinel # (document's last line, outside the digest) is therefore ABSENT — # the full source doc never reached the prompt (full text enters # only through the capped read tool; the inverse of the retired # phase-30/24 full-text pin). The bubble renders the answer as # markdown, which collapses the summary's newline — so pin each # LINE separately (the digest line's tail sits inside the echoed # 160 chars; the pointer line is single-line too). expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) yaml_summary = _expected_summary(yaml_content, SOURCE, YAML_PATH) yaml_digest_line = yaml_summary.split("\n", 1)[0] expect(bubble).to_contain_text(f"Source: {SOURCE}/{YAML_PATH}") expect(bubble).to_contain_text(yaml_digest_line[-80:]) expect(bubble).not_to_contain_text(SENTINEL) # Grounded: both fixture docs are suggested (two-doc KB — no floor) # and both chips render, in rank order (md first, yaml last). chip = page.locator(".msg.brain .source-chip", has_text=YAML_PATH) expect(chip).to_have_count(1) expect(chip.first).to_contain_text(f"{SOURCE}/{YAML_PATH}") expect(page.locator(".msg.brain .source-chip", has_text=MD_PATH)).to_have_count(1) expect(page.locator(".msg.brain .source-chip")).to_have_count(2) # No rank-6+ doc in a two-doc KB → the related row is absent. expect(page.locator(".msg.brain .related-docs")).to_have_count(0) # Button recovers (never stale) and the turn was grounded, not # deflected. expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") row = _last_query_log() assert row.question == QUESTION assert row.deflected is False # The durable record: suggested + related + read (deduped) — here # exactly the two suggested docs, in rank order (LOCKED A3). assert row.sources == ( f"{SOURCE}/{MD_PATH}, {SOURCE}/{YAML_PATH}" ), row.sources def test_markdown_control_doc_gets_a_summary_chunk( mock_llm: int, db_ready: None ) -> None: """Phase 118 (A2): in the same KB the markdown doc gets a summary TOO — the phase-30 non-markdown-only scope is retired (markdown docs backfill + summarize like every other doc): the same byte-stable digest + pointer line, exactly one ``is_summary`` chunk, and the raw chunk count untouched. The yaml doc keeps its exactly-one summary row with its raw chunks untouched.""" _reset_db() summary = _run_in_thread(_import_fixtures(mock_llm)) assert summary.added == 2 assert summary.summaries == 2 # A2: the markdown doc is summarized too md_content = (FIXTURES / MD_PATH).read_text(encoding="utf-8") yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8") with SessionLocal() as db: md_doc = _doc(db, MD_PATH) yaml_doc = _doc(db, YAML_PATH) md_chunks = [c for c in md_doc.chunks if not c.is_summary] md_summary = [c for c in md_doc.chunks if c.is_summary] yaml_raw = [c for c in yaml_doc.chunks if not c.is_summary] yaml_summary = [c for c in yaml_doc.chunks if c.is_summary] # Markdown: NOW summarized (phase 118 A2) — the same byte-stable # digest + deterministic pointer line, exactly one is_summary chunk # (position −1, embedded), raw chunks untouched. expected_md_summary = _expected_summary(md_content, SOURCE, MD_PATH) assert md_doc.summary == expected_md_summary assert len(md_summary) == 1 assert md_summary[0].position == -1 assert md_summary[0].embedding is not None assert md_summary[0].content == expected_md_summary assert expected_md_summary.endswith(f"\nSource: {SOURCE}/{MD_PATH}") assert len(md_chunks) == len( chunk_document(md_content, MD_PATH, CHUNK_TARGET, CHUNK_OVERLAP) ) # YAML: raw chunks exactly as chunked by the importer policy, plus # exactly one summary chunk (position −1, embedded, on the doc row). assert len(yaml_raw) == len( chunk_document(yaml_content, YAML_PATH, CHUNK_TARGET, CHUNK_OVERLAP) ) assert sorted(c.position for c in yaml_raw) == list(range(len(yaml_raw))) assert len(yaml_summary) == 1 assert yaml_summary[0].position == -1 assert yaml_summary[0].embedding is not None assert yaml_doc.summary is not None assert yaml_doc.summary == yaml_summary[0].content assert yaml_doc.summary.endswith(f"\nSource: {SOURCE}/{YAML_PATH}")