brain-of-reese/tests/e2e/test_document_summaries.py

"""Phase 30 E2E (Playwright) — phase 118 re-targeted (A2/A6): a summary
hit seeds the SUMMARY into the prompt — never the full source doc.

Story: ``.agents/user_stories/document-summaries.md``
Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_document_summaries.py -v --no-cov

The fixture KB is a story-dedicated directory
(``tests/fixtures/summary_kb/`` — the shared ``tests/fixtures/docs/``
stays at its pinned files) with two documents:

* ``quadlet/qwen-llamacpp.yaml`` — at import the mock ``lite`` model
  (``SUMMARY_MODE`` marker, ``tests/e2e/mock_llm.py``) reduces it to a
  deterministic 24-token digest, stored on ``documents.summary`` and
  indexed as one ``is_summary`` chunk. The raw yaml body is deliberately
  token-diluted, so the document's top fused chunks are the summary and
  the one lexical-hitting raw chunk. The sentinel
  ``RESE-SUMMARY-SENTINEL-7f3a`` sits on the document's LAST line —
  outside the 24-token digest, unreachable from the summary.
* ``notes/qwen-llamacpp-notes.md`` — a markdown doc (phase 118 A2: it is
  summarized TOO — the phase-30 non-markdown-only scope is retired) that
  ranks first, which puts the yaml document LAST inside
  ``<documents>`` (a two-doc KB → both docs are suggested, the related
  tier is empty).

The mock LLM's tail-echo trigger (``END_OF_NOTES_TRIGGER``) makes the
answer quote the last 160 chars of the seeded ``<documents>`` block —
under the phase-118 summary-seed contract (A6) that block carries the
suggested docs' SUMMARIES, never their full texts, so the echoed tail is
the LAST suggested doc's summary (the yaml doc's byte-stable digest tail
+ ``Source:`` pointer line). The sentinel therefore appears in the
rendered answer **only if the entire yaml source document (not the
summary) reached the LLM prompt** — under the locked contract it must be
ABSENT (full text enters the context only through the capped ``read``
tool), which is the inverse of the retired phase-30/24 full-text pin
and what this story is about now.
"""
from __future__ import annotations

import asyncio
from collections.abc import Callable, Sequence
from pathlib import Path
from threading import Thread
from typing import Any

from playwright.sync_api import Page, expect
from sqlalchemy import select, text
from sqlalchemy.orm import Session

from app.config import Settings
from app.db import SessionLocal
from app.models import Document, QueryLog
from app.rag.chunker import chunk_document
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from app.rag.retriever import RetrievedChunk, retrieve, select_suggested
from e2e.auth_helpers import login
from tests.e2e.mock_llm import TOKEN_RE, embed_text

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "summary_kb"

QUESTION = (
    "What are the optimal parameters for qwen 3.8 on llama.cpp? "
    "show the end of your notes"
)
SENTINEL = "RESE-SUMMARY-SENTINEL-7f3a"
SOURCE = "summary_kb"
YAML_PATH = "quadlet/qwen-llamacpp.yaml"
MD_PATH = "notes/qwen-llamacpp-notes.md"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"

#: The importer's chunk policy (Settings defaults; the mock never trips
#: the endpoint token-cap retry, so the target is never halved).
CHUNK_TARGET = 2_000
CHUNK_OVERLAP = 200


# --- Importer + thread helpers (test_whole_document_context.py pattern) ---


async def _import_fixtures(mock_port: int) -> ImportSummary:
    kwargs: dict[str, Any] = {
        "_env_file": None,
        "llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
    }
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return await import_sources([FIXTURES], LLMClient(settings))


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread.

    Playwright's sync API keeps an asyncio loop running on the test thread,
    so ``asyncio.run`` cannot be called directly from a test body.
    """
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(seed: Callable[[Session], None] | None = None) -> None:
    """Truncate the KB (and query log + steering), then optionally seed."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
        db.commit()
        if seed is not None:
            seed(db)
            db.commit()


def _ask(page: Page, app_url: str, question: str) -> Any:
    """Submit *question* and wait for the streamed brain bubble."""
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")
    page.fill("#message-input", question)
    page.click("#send-btn")
    bubble = page.locator(".msg.brain .bubble")
    bubble.first.wait_for(state="visible", timeout=30_000)
    return bubble.first


def _last_query_log() -> QueryLog:
    with SessionLocal() as db:
        rows = db.scalars(select(QueryLog)).all()
    assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
    return rows[0]


def _doc(db: Session, path: str) -> Document:
    doc = db.scalar(select(Document).where(Document.path == path))
    assert doc is not None, f"fixture doc {path!r} was not imported"
    return doc


def _expected_summary(content: str, source: str, path: str) -> str:
    """The mock lite model's byte-stable digest + the code pointer line.

    Mirrors ``mock_llm.compose_answer``'s ``SUMMARY_MODE`` branch (first
    24 tokens of the document content) plus the summarizer's deterministic
    ``Source:`` line — no model output is ever trusted.
    """
    digest = " ".join(TOKEN_RE.findall(content.lower())[:24])
    return f"This document covers {digest}.\nSource: {source}/{path}"


def _chunks_by_path(chunks: Sequence[RetrievedChunk], path: str) -> list[RetrievedChunk]:
    return [c for c in chunks if c.document.path == path]


# --- Story tests -------------------------------------------------------------


def test_summary_hit_seeds_the_summary_not_the_full_text(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """A question whose best yaml match is its summary chunk yields a
    grounded answer seeded from the SUMMARY — the mock's tail echo
    quotes the last suggested doc's summary tail (digest + pointer), and
    the sentinel (the yaml doc's last line, outside the digest) is
    ABSENT: the full source doc never reached the prompt (A6). Both
    fixture docs are suggested (two-doc KB, no floor) but chip
    NOTHING — the turn read nothing, so (phase 119, LOCKED A1) the
    citation surface is empty (deflected: false)."""
    _reset_db()
    summary = _run_in_thread(_import_fixtures(mock_llm))
    assert summary.added == 2  # yaml + md control
    assert summary.summaries == 2 and summary.summary_errors == 0  # A2: md too
    assert summary.errors == 0

    # Import state: exactly one embedded ``is_summary`` chunk (position
    # −1) whose text is the byte-stable mock digest + the deterministic
    # pointer line.
    yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8")
    assert SENTINEL in yaml_content.splitlines()[-1]  # last line, by design
    with SessionLocal() as db:
        yaml_doc = _doc(db, YAML_PATH)
        schunks = [c for c in yaml_doc.chunks if c.is_summary]
        assert len(schunks) == 1
        assert schunks[0].position == -1
        assert schunks[0].embedding is not None
        assert yaml_doc.summary == _expected_summary(yaml_content, SOURCE, YAML_PATH)

    # Retrieval state (phase 118, A2/A6): the EMBEDDED summary chunk is
    # a retrieval candidate (the seeded text ranks on its own), and the
    # suggested tier is the two docs in rank order — md first, yaml
    # LAST (so the yaml's summary is the tail of the <documents> block,
    # the mock echo's target). The document enters the prompt through
    # its SUMMARY block, not through the diluted raw yaml chunks.
    with SessionLocal() as db:
        chunks = retrieve(db, QUESTION, embed_text(QUESTION))
    yaml_chunks = _chunks_by_path(chunks, YAML_PATH)
    assert any(c.is_summary for c in yaml_chunks)  # the embedded summary ranks
    assert len(yaml_chunks) >= 2  # summary + at least one raw candidate
    assert [d.path for d in select_suggested(chunks)] == [MD_PATH, YAML_PATH]

    bubble = _ask(page, app_url, QUESTION)

    # Phase 118 (A6): the seed is the SUMMARY, not the full text — the
    # mock's tail echo quotes the last 160 chars of the <documents>
    # block, which end in the LAST suggested doc's summary: the yaml
    # doc's byte-stable digest tail + pointer line. The sentinel
    # (document's last line, outside the digest) is therefore ABSENT —
    # the full source doc never reached the prompt (full text enters
    # only through the capped read tool; the inverse of the retired
    # phase-30/24 full-text pin). The bubble renders the answer as
    # markdown, which collapses the summary's newline — so pin each
    # LINE separately (the digest line's tail sits inside the echoed
    # 160 chars; the pointer line is single-line too).
    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000)
    yaml_summary = _expected_summary(yaml_content, SOURCE, YAML_PATH)
    yaml_digest_line = yaml_summary.split("\n", 1)[0]
    expect(bubble).to_contain_text(f"Source: {SOURCE}/{YAML_PATH}")
    expect(bubble).to_contain_text(yaml_digest_line[-80:])
    expect(bubble).not_to_contain_text(SENTINEL)

    # Phase 119 (LOCKED A1): the turn read nothing, so ZERO citation
    # chips — both fixture docs are suggested (two-doc KB — no floor,
    # md first, yaml last) but suggested docs are seed context, not
    # citations (the retired phase-118 A4 suggested-chip union is gone);
    # their rank order is pinned by the durable record below (LOCKED
    # A3, untouched).
    expect(page.locator(".msg.brain .source-chip")).to_have_count(0)
    # No rank-6+ doc in a two-doc KB → the related row is absent.
    expect(page.locator(".msg.brain .related-docs")).to_have_count(0)

    # Button recovers (never stale) and the turn was grounded, not
    # deflected.
    expect(page.locator("#send-btn")).to_be_enabled()
    expect(page.locator("#send-label")).to_have_text("Send")
    row = _last_query_log()
    assert row.question == QUESTION
    assert row.deflected is False
    # The durable record: suggested + related + read (deduped) — here
    # exactly the two suggested docs, in rank order (LOCKED A3).
    assert row.sources == (
        f"{SOURCE}/{MD_PATH}, {SOURCE}/{YAML_PATH}"
    ), row.sources


def test_markdown_control_doc_gets_a_summary_chunk(
    mock_llm: int, db_ready: None
) -> None:
    """Phase 118 (A2): in the same KB the markdown doc gets a summary TOO
    — the phase-30 non-markdown-only scope is retired (markdown docs
    backfill + summarize like every other doc): the same byte-stable
    digest + pointer line, exactly one ``is_summary`` chunk, and the raw
    chunk count untouched. The yaml doc keeps its exactly-one summary
    row with its raw chunks untouched."""
    _reset_db()
    summary = _run_in_thread(_import_fixtures(mock_llm))
    assert summary.added == 2
    assert summary.summaries == 2  # A2: the markdown doc is summarized too

    md_content = (FIXTURES / MD_PATH).read_text(encoding="utf-8")
    yaml_content = (FIXTURES / YAML_PATH).read_text(encoding="utf-8")
    with SessionLocal() as db:
        md_doc = _doc(db, MD_PATH)
        yaml_doc = _doc(db, YAML_PATH)
        md_chunks = [c for c in md_doc.chunks if not c.is_summary]
        md_summary = [c for c in md_doc.chunks if c.is_summary]
        yaml_raw = [c for c in yaml_doc.chunks if not c.is_summary]
        yaml_summary = [c for c in yaml_doc.chunks if c.is_summary]

    # Markdown: NOW summarized (phase 118 A2) — the same byte-stable
    # digest + deterministic pointer line, exactly one is_summary chunk
    # (position −1, embedded), raw chunks untouched.
    expected_md_summary = _expected_summary(md_content, SOURCE, MD_PATH)
    assert md_doc.summary == expected_md_summary
    assert len(md_summary) == 1
    assert md_summary[0].position == -1
    assert md_summary[0].embedding is not None
    assert md_summary[0].content == expected_md_summary
    assert expected_md_summary.endswith(f"\nSource: {SOURCE}/{MD_PATH}")
    assert len(md_chunks) == len(
        chunk_document(md_content, MD_PATH, CHUNK_TARGET, CHUNK_OVERLAP)
    )

    # YAML: raw chunks exactly as chunked by the importer policy, plus
    # exactly one summary chunk (position −1, embedded, on the doc row).
    assert len(yaml_raw) == len(
        chunk_document(yaml_content, YAML_PATH, CHUNK_TARGET, CHUNK_OVERLAP)
    )
    assert sorted(c.position for c in yaml_raw) == list(range(len(yaml_raw)))
    assert len(yaml_summary) == 1
    assert yaml_summary[0].position == -1
    assert yaml_summary[0].embedding is not None
    assert yaml_doc.summary is not None
    assert yaml_doc.summary == yaml_summary[0].content
    assert yaml_doc.summary.endswith(f"\nSource: {SOURCE}/{YAML_PATH}")