feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc

tests/e2e/mock_llm.py

@@ -11,6 +11,10 @@ Implements just enough of the aipi surface:
   off markers in the system prompt:
   - user message containing ``write a long answer`` -> a ~900-word
     deterministic numbered answer (long-answers story, phase 11)
+  - ``SUMMARY_MODE``  -> the deterministic summary digest: the first 24
+    tokens of the user message (the summarizer puts the capped document
+    content there) — byte-stable for a given fixture (document summaries,
+    phase 30)
   - ``DEFLECT_MODE``  -> honest "I haven't done anything like that" answer
   - otherwise         -> upbeat answer quoting the provided document context
   - user message containing ``pretend to think slowly`` -> 3s warm-up delay
@@ -156,6 +160,18 @@ def compose_answer(body: dict[str, Any]) -> str:
     user = _user(body)
     if LONG_ANSWER_TRIGGER in user.lower():
         answer = long_answer()
+    elif "SUMMARY_MODE" in system:
+        # Document summaries (phase 30): the ``lite`` stand-in returns a
+        # deterministic digest — the first 24 tokens of the user message
+        # (the summarizer puts the capped document content there). Byte-
+        # stable for a given fixture, so the summary chunk's retrieval
+        # rank is a pure function of the fixture text. Checked BEFORE the
+        # DEFLECT_MODE branch (task 06) so a deflection prompt that ever
+        # carries the marker cannot shadow the summary call.
+        answer = (
+            f"This document covers "
+            f"{' '.join(TOKEN_RE.findall(user.lower())[:24])}."
+        )
     elif "DEFLECT_MODE" in system:
         answer = (
             "Ah — I haven't done anything like that, so I don't want to make stuff up! "

tests/e2e/test_document_summaries.py

@@ -0,0 +1,257 @@
+"""Phase 30 E2E (Playwright): a summary hit delivers the full source doc.
+
+Story: ``.agent/user_stories/document-summaries.md``
+Run in isolation (DB must be up: ``podman compose up -d db``):
+
+    uv run pytest tests/e2e/test_document_summaries.py -v --no-cov
+
+The fixture KB is a story-dedicated directory
+(``tests/fixtures/summary_kb/`` — the shared ``tests/fixtures/docs/``
+stays at its 8 pinned files) with two documents:
+
+* ``quadlet/qwen-llamacpp.yaml`` — a non-markdown A9 doc. At import the
+  mock ``lite`` model (``SUMMARY_MODE`` marker, ``tests/e2e/mock_llm.py``)
+  reduces it to a deterministic 24-token digest, stored on
+  ``documents.summary`` and indexed as one ``is_summary`` chunk. The raw
+  yaml body is deliberately token-diluted, so the document's best fused
+  chunk is its summary chunk. The sentinel ``RESE-SUMMARY-SENTINEL-7f3a``
+  sits on the document's LAST line — outside the 24-token digest,
+  unreachable from the summary.
+* ``notes/qwen-llamacpp-notes.md`` — a markdown control doc (never
+  summarized) that ranks first, which puts the yaml document LAST inside
+  ``<documents>``.
+
+The mock LLM's tail-echo trigger (``END_OF_NOTES_TRIGGER``) makes the
+answer quote the last 160 chars of the document context — the tail of
+the LAST selected document. The sentinel therefore appears in the
+rendered answer **iff the entire yaml source document (not the summary
+digest) reached the LLM prompt** — the summary→parent-document resolution
+through the unchanged chunk→document mapping (A7 revised: never
+truncated), which is what this story is about.
+"""
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable, Sequence
+from pathlib import Path
+from threading import Thread
+from typing import Any
+
+from playwright.sync_api import Page, expect
+from sqlalchemy import select, text
+from sqlalchemy.orm import Session
+
+from app.config import Settings
+from app.db import SessionLocal
+from app.models import Document, QueryLog
+from app.rag.chunker import chunk_document
+from app.rag.importer import ImportSummary, import_sources
+from app.rag.llm import LLMClient
+from app.rag.retriever import RetrievedChunk, retrieve
+from tests.e2e.mock_llm import TOKEN_RE, embed_text
+
+REPO = Path(__file__).resolve().parents[2]
+FIXTURES = REPO / "tests" / "fixtures" / "summary_kb"
+
+QUESTION = (
+    "What are the optimal parameters for qwen 3.8 on llama.cpp? "
+    "show the end of your notes"
+)
+SENTINEL = "RESE-SUMMARY-SENTINEL-7f3a"
+SOURCE = "summary_kb"
+YAML_PATH = "quadlet/qwen-llamacpp.yaml"
+MD_PATH = "notes/qwen-llamacpp-notes.md"
+MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
+
+#: The importer's chunk policy (Settings defaults; the mock never trips
+#: the endpoint token-cap retry, so the target is never halved).
+CHUNK_TARGET = 2_000
+CHUNK_OVERLAP = 200
+
+
+# --- Importer + thread helpers (test_whole_document_context.py pattern) ---
+
+
+async def _import_fixtures(mock_port: int) -> ImportSummary:
+    kwargs: dict[str, Any] = {
+        "_env_file": None,
+        "llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
+    }
+    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
+    return await import_sources([FIXTURES], LLMClient(settings))
+
+
+def _run_in_thread(coro: Any) -> Any:
+    """Run a coroutine on a worker thread.
+
+    Playwright's sync API keeps an asyncio loop running on the test thread,
+    so ``asyncio.run`` cannot be called directly from a test body.
+    """
+    box: dict[str, Any] = {}
+
+    def runner() -> None:
+        try:
+            box["value"] = asyncio.run(coro)
+        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
+            box["error"] = e
+
+    t = Thread(target=runner)
+    t.start()
+    t.join()
+    if "error" in box:
+        raise box["error"]
+    return box["value"]
+
+
+def _reset_db(seed: Callable[[Session], None] | None = None) -> None:
+    """Truncate the KB (and query log + steering), then optionally seed."""
+    with SessionLocal() as db:
+        db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
+        db.commit()
+        if seed is not None:
+            seed(db)
+            db.commit()
+
+
+def _ask(page: Page, app_url: str, question: str) -> Any:
+    """Submit *question* and wait for the streamed brain bubble."""
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+    page.fill("#message-input", question)
+    page.click("#send-btn")
+    bubble = page.locator(".msg.brain .bubble")
+    bubble.first.wait_for(state="visible", timeout=30_000)
+    return bubble.first
+
+
+def _last_query_log() -> QueryLog:
+    with SessionLocal() as db:
+        rows = db.scalars(select(QueryLog)).all()
+    assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
+    return rows[0]
+
+
+def _doc(db: Session, path: str) -> Document:
+    doc = db.scalar(select(Document).where(Document.path == path))
+    assert doc is not None, f"fixture doc {path!r} was not imported"
+    return doc
+
+
+def _expected_summary(content: str, source: str, path: str) -> str:
+    """The mock lite model's byte-stable digest + the code pointer line.
+
+    Mirrors ``mock_llm.compose_answer``'s ``SUMMARY_MODE`` branch (first
+    24 tokens of the document content) plus the summarizer's deterministic
+    ``Source:`` line — no model output is ever trusted.
+    """
+    digest = " ".join(TOKEN_RE.findall(content.lower())[:24])
+    return f"This document covers {digest}.\nSource: {source}/{path}"
+
+
+def _chunks_by_path(chunks: Sequence[RetrievedChunk], path: str) -> list[RetrievedChunk]:
+    return [c for c in chunks if c.document.path == path]
+
+
+# --- Story tests -------------------------------------------------------------
+
+
+def test_summary_hit_retrieves_full_source_document(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """A question whose best yaml match is the summary chunk yields an
+    answer grounded in the FULL yaml source document: its tail sentinel —
+    which the summary digest cannot contain — is echoed back, and the
+    source chip cites the yaml path (deflected: false)."""
+    _reset_db()
+    summary = _run_in_thread(_import_fixtures(mock_llm))
+    assert summary.added == 2  # yaml + md control
+    assert summary.summaries == 1 and summary.summary_errors == 0
+    assert summary.errors == 0
+
+    # Import state: exactly one embedded ``is_summary`` chunk (position
+    # −1) whose text is the byte-stable mock digest + the deterministic
+    # pointer line.
+    yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8")
+    assert SENTINEL in yaml_content.splitlines()[-1]  # last line, by design
+    with SessionLocal() as db:
+        yaml_doc = _doc(db, YAML_PATH)
+        schunks = [c for c in yaml_doc.chunks if c.is_summary]
+        assert len(schunks) == 1
+        assert schunks[0].position == -1
+        assert schunks[0].embedding is not None
+        assert yaml_doc.summary == _expected_summary(yaml_content, SOURCE, YAML_PATH)
+
+    # Retrieval state: the summary chunk is the yaml document's best fused
+    # chunk — the document enters the context through its summary, not
+    # through the diluted raw yaml chunks.
+    with SessionLocal() as db:
+        chunks = retrieve(db, QUESTION, embed_text(QUESTION))
+    yaml_chunks = _chunks_by_path(chunks, YAML_PATH)
+    best_yaml = max(yaml_chunks, key=lambda c: c.score)
+    assert best_yaml.is_summary
+    assert len(yaml_chunks) >= 2  # summary + at least one raw candidate
+
+    bubble = _ask(page, app_url, QUESTION)
+
+    # The tail sentinel exists only on the document's last line and
+    # cannot be in the summary digest — its presence proves the entire
+    # source document was in the LLM prompt (summary→parent resolution).
+    expect(bubble).to_contain_text(SENTINEL, timeout=30_000)
+    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
+
+    # Grounded: the yaml source chip renders (the md doc ranks first, so
+    # both fixtures are cited).
+    chip = page.locator(".msg.brain .source-chip", has_text=YAML_PATH)
+    expect(chip).to_have_count(1)
+    expect(chip.first).to_contain_text(f"{SOURCE}/{YAML_PATH}")
+    expect(page.locator(".msg.brain .source-chip", has_text=MD_PATH)).to_have_count(1)
+
+    # Button recovers (never stale) and the turn was grounded, not
+    # deflected.
+    expect(page.locator("#send-btn")).to_be_enabled()
+    expect(page.locator("#send-label")).to_have_text("Send")
+    row = _last_query_log()
+    assert row.question == QUESTION
+    assert row.deflected is False
+    assert f"{SOURCE}/{YAML_PATH}" in row.sources
+    assert f"{SOURCE}/{MD_PATH}" in row.sources
+
+
+def test_markdown_control_doc_gets_no_summary_chunk(
+    mock_llm: int, db_ready: None
+) -> None:
+    """Control: in the same KB the markdown doc gets no summary at all —
+    its chunk count is exactly the raw chunks; the yaml doc has exactly
+    one ``is_summary`` row and its raw chunk count is untouched by the
+    summary."""
+    _reset_db()
+    summary = _run_in_thread(_import_fixtures(mock_llm))
+    assert summary.added == 2
+
+    md_content = (FIXTURES / MD_PATH).read_text(encoding="utf-8")
+    yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8")
+    with SessionLocal() as db:
+        md_doc = _doc(db, MD_PATH)
+        yaml_doc = _doc(db, YAML_PATH)
+        md_chunks = [c for c in md_doc.chunks if not c.is_summary]
+        yaml_raw = [c for c in yaml_doc.chunks if not c.is_summary]
+        yaml_summary = [c for c in yaml_doc.chunks if c.is_summary]
+
+    # Markdown: never summarized (phase 30 scope — A9 non-markdown only).
+    assert md_doc.summary is None
+    assert len(md_chunks) == len(
+        chunk_document(md_content, MD_PATH, CHUNK_TARGET, CHUNK_OVERLAP)
+    )
+
+    # YAML: raw chunks exactly as chunked by the importer policy, plus
+    # exactly one summary chunk (position −1, embedded, on the doc row).
+    assert len(yaml_raw) == len(
+        chunk_document(yaml_content, YAML_PATH, CHUNK_TARGET, CHUNK_OVERLAP)
+    )
+    assert sorted(c.position for c in yaml_raw) == list(range(len(yaml_raw)))
+    assert len(yaml_summary) == 1
+    assert yaml_summary[0].position == -1
+    assert yaml_summary[0].embedding is not None
+    assert yaml_doc.summary is not None
+    assert yaml_doc.summary == yaml_summary[0].content
+    assert yaml_doc.summary.endswith(f"\nSource: {SOURCE}/{YAML_PATH}")

tests/e2e/test_import_documents.py

@@ -92,7 +92,11 @@ def test_sources_page_lists_indexed_docs(
 
     login(page, app_url)  # phase 16: the catalog is admin-only
     expect(page.locator("#stat-docs")).to_have_text("8")
-    expect(page.locator("#stat-chunks")).to_have_text(str(summary.chunks))
+    # Phase 30: non-markdown fixtures each gained one ``is_summary`` chunk,
+    # so the Sources total is content chunks + summary chunks.
+    expect(page.locator("#stat-chunks")).to_have_text(
+        str(summary.chunks + summary.summaries)
+    )
     expect(page.locator("#stat-last")).not_to_have_text("–")
     expect(page.locator("#sources-empty")).to_be_hidden()