"""Phase 31 E2E (Playwright): the stored KB overview reaches every turn. Story: ``.agents/user_stories/kb-overview-prompt.md`` Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_kb_overview.py -v --no-cov The story: a single-row ``kb_overview`` table holds a lite-generated plain-text outline of the KB's basic categories; every chat turn — HIGH (grounded) and LOW (deflected) alike — injects it into the system prompt as the ```` section, so the agent knows roughly what the KB contains before retrieval returns. The mock LLM echoes the section's first bullet into its answer (`` (kb: )`` — the ``(tuning: …)`` steering-echo precedent), which makes the prompt change observable in the rendered UI deterministically. The row is seeded **directly in the DB** so the INJECTION path is what is under test (the import-time trigger is integration-tested in ``tests/integration/test_import_docs_overview.py``). The outline is multi-line so the echo must pick the FIRST bullet — not the intro line, not a later line. The last test additionally runs the real ``regenerate_overview`` against the mock to cover the mock's ``KB_OVERVIEW_MODE`` generation branch end-to-end (stored outline is the byte-stable 8-token digest of the generator's document list). Test → story mapping (Playwright Mapping Rule): 1. ``test_on_topic_answer_echoes_kb_overview`` 2. ``test_deflected_answer_still_echoes_kb_overview`` 3. ``test_absent_row_yields_no_kb_echo`` 4. ``test_mock_generated_outline_is_stored_and_echoed`` """ from __future__ import annotations import asyncio import re from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import select, text from app.config import Settings from app.db import SessionLocal from app.models import Document, KbOverview, QueryLog from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.overview import build_overview_prompt, regenerate_overview from tests.e2e.mock_llm import TOKEN_RE REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" QUESTION = "How is my Kubernetes cluster set up?" OFF_TOPIC = "How do I bake sourdough bread?" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" #: The stored outline: multi-line ``-`` bullets (the generator's locked #: format). The echo must pick the FIRST bullet — the parsing (skip the #: intro line, strip the dash) is part of the story. OVERVIEW = ( "- Kubernetes cluster and node maintenance notes\n" "- Backup schedules and restore runbooks\n" "- Networking: static DNS and kafkabridge" ) FIRST_BULLET = "Kubernetes cluster and node maintenance notes" KB_ECHO = f"(kb: {FIRST_BULLET})" # --- Importer + thread helpers (test_document_summaries.py pattern) --- async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = { "_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1", } settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db( mock_port: int, seed: bool, overview: str | None = None ) -> ImportSummary | None: """Truncate the KB (+ query log, steering, kb_overview), re-import the fixtures, and optionally seed the single ``kb_overview`` row.""" with SessionLocal() as db: db.execute( text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview") ) db.commit() if overview is not None: db.add(KbOverview(id=1, content=overview)) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) def _ask(page: Page, app_url: str, question: str, done_text: str) -> Any: """Submit *question* and wait until the streamed answer has fully landed. ``done_text`` is the deterministic last thing the answer contains for its path: the ``(kb: …)`` echo (grounded AND deflected turns with a stored row — it is appended after everything else) or the mock marker (the no-row control, where the echo is absent by design). """ page.set_default_timeout(30_000) page.goto(app_url) page.fill("#message-input", question) page.click("#send-btn") bubble = page.locator(".msg.brain .bubble") bubble.first.wait_for(state="visible", timeout=30_000) expect(bubble.first).to_contain_text(done_text, timeout=30_000) expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") return bubble.first def _last_query_log() -> QueryLog: with SessionLocal() as db: rows = db.scalars(select(QueryLog)).all() assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}" return rows[0] def _overview_row() -> KbOverview | None: with SessionLocal() as db: return db.get(KbOverview, 1) def _doc_rows() -> list[tuple[str, str, str, str | None]]: with SessionLocal() as db: rows = db.execute( select(Document.source, Document.path, Document.title, Document.summary) .order_by(Document.source, Document.path) ).all() return [(source, path, title, summary) for source, path, title, summary in rows] # --- 1. Injection: grounded turn ------------------------------------------ def test_on_topic_answer_echoes_kb_overview( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """On-topic question with a stored row: the rendered answer ENDS with the mock's echo of the ```` section's first bullet — only possible if the section reached the LLM prompt.""" summary = _reset_db(mock_llm, seed=True, overview=OVERVIEW) assert summary is not None and summary.added == 13 # phase 47 added quadlet+j2 assert _overview_row() is not None # the row the turn must inject bubble = _ask(page, app_url, QUESTION, KB_ECHO) # The echo is the LAST thing in the bubble (the section was in the # HIGH prompt) and names the FIRST outline bullet (not the intro, # not a later line). expect(bubble).to_have_text( re.compile(rf"{re.escape(KB_ECHO)}\s*$"), timeout=30_000 ) # Grounded, not deflected — the overview does not change the gate. expect(page.locator(".msg.brain.is-deflected")).to_have_count(0) row = _last_query_log() assert row.question == QUESTION assert row.deflected is False # --- 2. Injection: deflected turn ------------------------------------------ def test_deflected_answer_still_echoes_kb_overview( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """Off-topic question (LOW path): the deflected answer STILL carries the echo — the ```` section is in the LOW prompt too (that is what lets the model offer real alternatives).""" _reset_db(mock_llm, seed=True, overview=OVERVIEW) # The deflection answer carries no mock marker — the kb echo is its # deterministic end, so it doubles as the completion signal. bubble = _ask(page, app_url, OFF_TOPIC, KB_ECHO) expect(page.locator(".msg.brain.is-deflected")).to_have_count(1) expect(bubble).to_have_text(re.compile(r"haven't done anything like that")) expect(bubble).to_have_text( re.compile(rf"{re.escape(KB_ECHO)}\s*$"), timeout=30_000 ) row = _last_query_log() assert row.question == OFF_TOPIC assert row.deflected is True # --- 3. Absence: no row → no echo ------------------------------------------ def test_absent_row_yields_no_kb_echo( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """Delete the row: a fresh question's answer carries NO ``(kb: …)`` suffix — the section is absent and the prompt behaves exactly like the pre-phase text (byte-identity is unit-asserted; this proves it end-to-end through the rendered answer).""" summary = _reset_db(mock_llm, seed=True, overview=OVERVIEW) assert summary is not None # Now delete the row (the absence control). with SessionLocal() as db: db.execute(text("TRUNCATE kb_overview")) db.commit() assert _overview_row() is None bubble = _ask(page, app_url, QUESTION, MOCK_ANSWER_MARKER) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER) expect(bubble).not_to_contain_text("(kb:") row = _last_query_log() assert row.deflected is False # grounded either way — only the echo is gone # --- 4. Generation: KB_OVERVIEW_MODE branch through the real regenerator ---- def test_mock_generated_outline_is_stored_and_echoed( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """The mock's ``KB_OVERVIEW_MODE`` branch is exercised end-to-end: the real ``regenerate_overview`` (``LLMClient`` pointed at the mock) stores the byte-stable 8-token digest of the generator's document list, and a chat turn echoes its first bullet.""" summary = _reset_db(mock_llm, seed=True, overview=None) assert summary is not None and summary.added == 13 # phase 47 added quadlet+j2 assert _overview_row() is None # direct import never regenerates kwargs: dict[str, Any] = { "_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_llm}/v1", } settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] ok = _run_in_thread(regenerate_overview(LLMClient(settings))) assert ok is True row = _overview_row() assert row is not None and row.id == 1 # Byte-stable expectation: the mock digests the generator's user # message (the same document list, the same cap) to its first 8 tokens. _, user = build_overview_prompt(_doc_rows()) expected = "Knowledge base outline:\n- " + " ".join( TOKEN_RE.findall(user.lower())[:8] ) assert row.content == expected first_bullet = row.content.splitlines()[1].removeprefix("- ").strip() assert first_bullet # the digest line is non-empty # A grounded chat turn echoes the STORED outline's first bullet. bubble = _ask(page, app_url, QUESTION, f"(kb: {first_bullet})") expect(bubble).to_have_text( re.compile(rf"\(kb: {re.escape(first_bullet)}\)\s*$"), timeout=30_000 ) expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)