brain-of-reese/tests/e2e/test_kb_overview.py

"""Phase 31 E2E (Playwright): the stored KB overview reaches every turn.

Story: ``.agents/user_stories/kb-overview-prompt.md``
Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_kb_overview.py -v --no-cov

The story: a single-row ``kb_overview`` table holds a lite-generated
plain-text outline of the KB's basic categories; every chat turn — HIGH
(grounded) and LOW (deflected) alike — injects it into the system prompt
as the ``<knowledge_base>`` section, so the agent knows roughly what the
KB contains before retrieval returns. The mock LLM echoes the section's
first bullet into its answer (`` (kb: <first bullet>)`` — the
``(tuning: …)`` steering-echo precedent), which makes the prompt change
observable in the rendered UI deterministically.

The row is seeded **directly in the DB** so the INJECTION path is what
is under test (the import-time trigger is integration-tested in
``tests/integration/test_import_docs_overview.py``). The outline is
multi-line so the echo must pick the FIRST bullet — not the intro line,
not a later line. The last test additionally runs the real
``regenerate_overview`` against the mock to cover the mock's
``KB_OVERVIEW_MODE`` generation branch end-to-end (stored outline is the
byte-stable 8-token digest of the generator's document list — re-pinned
in phase 142, task 02: the generator's list is now RECENT-FIRST, so the
digest is of the NEWEST document's line, not the lexicographic-first).

Test → story mapping (Playwright Mapping Rule):
1. ``test_on_topic_answer_echoes_kb_overview``
2. ``test_deflected_answer_still_echoes_kb_overview``
3. ``test_absent_row_yields_no_kb_echo``
4. ``test_mock_generated_outline_is_stored_and_echoed``
"""
from __future__ import annotations

import asyncio
import re
from datetime import UTC, datetime
from pathlib import Path
from threading import Thread
from typing import Any

from playwright.sync_api import Page, expect
from sqlalchemy import select, text

from app.config import Settings
from app.db import SessionLocal
from app.models import Document, KbOverview, QueryLog
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from app.rag.overview import build_overview_prompt, regenerate_overview
from e2e.auth_helpers import login

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
QUESTION = "How is my Kubernetes cluster set up?"
OFF_TOPIC = "How do I bake sourdough bread?"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"

#: The stored outline: multi-line ``-`` bullets (the generator's locked
#: format). The echo must pick the FIRST bullet — the parsing (skip the
#: intro line, strip the dash) is part of the story.
OVERVIEW = (
    "- Kubernetes cluster and node maintenance notes\n"
    "- Backup schedules and restore runbooks\n"
    "- Networking: static DNS and kafkabridge"
)
FIRST_BULLET = "Kubernetes cluster and node maintenance notes"
KB_ECHO = f"(kb: {FIRST_BULLET})"


# --- Importer + thread helpers (test_document_summaries.py pattern) ---


async def _import_fixtures(mock_port: int) -> ImportSummary:
    kwargs: dict[str, Any] = {
        "_env_file": None,
        "llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
    }
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return await import_sources([FIXTURES], LLMClient(settings))


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread.

    Playwright's sync API keeps an asyncio loop running on the test thread,
    so ``asyncio.run`` cannot be called directly from a test body.
    """
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(
    mock_port: int, seed: bool, overview: str | None = None
) -> ImportSummary | None:
    """Truncate the KB (+ query log, steering, kb_overview), re-import the
    fixtures, and optionally seed the single ``kb_overview`` row."""
    with SessionLocal() as db:
        db.execute(
            text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
        )
        db.commit()
        if overview is not None:
            db.add(KbOverview(id=1, content=overview))
            db.commit()
    if not seed:
        return None
    return _run_in_thread(_import_fixtures(mock_port))


def _ask(page: Page, app_url: str, question: str, done_text: str) -> Any:
    """Submit *question* and wait until the streamed answer has fully
    landed.

    ``done_text`` is the deterministic last thing the answer contains for
    its path: the ``(kb: …)`` echo (grounded AND deflected turns with a
    stored row — it is appended after everything else) or the mock
    marker (the no-row control, where the echo is absent by design).
    """
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")
    page.fill("#message-input", question)
    page.click("#send-btn")
    bubble = page.locator(".msg.brain .bubble")
    bubble.first.wait_for(state="visible", timeout=30_000)
    expect(bubble.first).to_contain_text(done_text, timeout=30_000)
    expect(page.locator("#send-btn")).to_be_enabled()
    expect(page.locator("#send-label")).to_have_text("Send")
    return bubble.first


def _last_query_log() -> QueryLog:
    with SessionLocal() as db:
        rows = db.scalars(select(QueryLog)).all()
    assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
    return rows[0]


def _overview_row() -> KbOverview | None:
    with SessionLocal() as db:
        return db.get(KbOverview, 1)


def _doc_rows() -> list[tuple[str, str, str, str | None]]:
    """The catalogue the way the generator sees it (phase 142, task 02):
    RECENT-FIRST by the document's date, ``(source, path)`` tie-break —
    the same order ``regenerate_overview`` SELECTs."""
    with SessionLocal() as db:
        rows = db.execute(
            select(Document.source, Document.path, Document.title, Document.summary)
            .order_by(Document.created_at.desc(), Document.source, Document.path)
        ).all()
    return [(source, path, title, summary) for source, path, title, summary in rows]


# --- 1. Injection: grounded turn ------------------------------------------


def test_on_topic_answer_echoes_kb_overview(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """On-topic question with a stored row: the rendered answer ENDS with
    the mock's echo of the ``<knowledge_base>`` section's first bullet —
    only possible if the section reached the LLM prompt."""
    summary = _reset_db(mock_llm, seed=True, overview=OVERVIEW)
    assert summary is not None and summary.added == 13  # phase 47 added quadlet+j2
    assert _overview_row() is not None  # the row the turn must inject

    bubble = _ask(page, app_url, QUESTION, KB_ECHO)

    # The echo is the LAST thing in the bubble (the section was in the
    # HIGH prompt) and names the FIRST outline bullet (not the intro,
    # not a later line).
    expect(bubble).to_have_text(
        re.compile(rf"{re.escape(KB_ECHO)}\s*$"), timeout=30_000
    )
    # Grounded, not deflected — the overview does not change the gate.
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)
    row = _last_query_log()
    assert row.question == QUESTION
    assert row.deflected is False


# --- 2. Injection: deflected turn ------------------------------------------


def test_deflected_answer_still_echoes_kb_overview(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """Off-topic question (LOW path): the deflected answer STILL carries
    the echo — the ``<knowledge_base>`` section is in the LOW prompt too
    (that is what lets the model offer real alternatives)."""
    _reset_db(mock_llm, seed=True, overview=OVERVIEW)

    # The deflection answer carries no mock marker — the kb echo is its
    # deterministic end, so it doubles as the completion signal.
    bubble = _ask(page, app_url, OFF_TOPIC, KB_ECHO)

    expect(page.locator(".msg.brain.is-deflected")).to_have_count(1)
    expect(bubble).to_have_text(re.compile(r"haven't done anything like that"))
    expect(bubble).to_have_text(
        re.compile(rf"{re.escape(KB_ECHO)}\s*$"), timeout=30_000
    )
    row = _last_query_log()
    assert row.question == OFF_TOPIC
    assert row.deflected is True


# --- 3. Absence: no row → no echo ------------------------------------------


def test_absent_row_yields_no_kb_echo(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """Delete the row: a fresh question's answer carries NO ``(kb: …)``
    suffix — the section is absent and the prompt behaves exactly like
    the pre-phase text (byte-identity is unit-asserted; this proves it
    end-to-end through the rendered answer)."""
    summary = _reset_db(mock_llm, seed=True, overview=OVERVIEW)
    assert summary is not None
    # Now delete the row (the absence control).
    with SessionLocal() as db:
        db.execute(text("TRUNCATE kb_overview"))
        db.commit()
    assert _overview_row() is None

    bubble = _ask(page, app_url, QUESTION, MOCK_ANSWER_MARKER)

    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
    expect(bubble).not_to_contain_text("(kb:")
    row = _last_query_log()
    assert row.deflected is False  # grounded either way — only the echo is gone


# --- 4. Generation: KB_OVERVIEW_MODE branch through the real regenerator ----


def test_mock_generated_outline_is_stored_and_echoed(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The mock's ``KB_OVERVIEW_MODE`` branch is exercised end-to-end:
    the real ``regenerate_overview`` (``LLMClient`` pointed at the mock)
    stores the byte-stable 8-token digest of the generator's document
    list, and a chat turn echoes its first bullet.

    Phase 142, task 02 (recency-first re-pin): the generator's list is
    now ``created_at`` desc, ``(source, path)`` tie-break. The fixture
    files' on-disk mtimes are checkout-dependent (git preserves none),
    so the documents' dates are pinned explicitly BEFORE the
    regeneration: every doc shares one date EXCEPT
    ``homelab/tables.md`` — the lexicographic LAST path, the owner's
    phase-dir shape where the newest sorts last — which gets the newer
    one. The stored digest is therefore of THAT doc's line, not of the
    pre-fix lexicographic-first one (``deployments/new-service.md``).
    """
    summary = _reset_db(mock_llm, seed=True, overview=None)
    assert summary is not None and summary.added == 13  # phase 47 added quadlet+j2
    assert _overview_row() is None  # direct import never regenerates

    # Pin the document dates (raw SQL — ``normalize_doc_date`` is an
    # importer concern, the ordering reads the stored column): one
    # shared date for all but the newest doc, a strictly newer date for
    # it. Without this the order — and the digest — would depend on the
    # fixture files' checkout mtimes.
    with SessionLocal() as db:
        db.execute(
            text("UPDATE documents SET created_at = :d WHERE path != :newest"),
            {"d": datetime(2026, 1, 1, tzinfo=UTC), "newest": "homelab/tables.md"},
        )
        db.execute(
            text("UPDATE documents SET created_at = :d WHERE path = :newest"),
            {"d": datetime(2026, 9, 1, tzinfo=UTC), "newest": "homelab/tables.md"},
        )
        db.commit()

    kwargs: dict[str, Any] = {
        "_env_file": None,
        "llm_base_url": f"http://127.0.0.1:{mock_llm}/v1",
    }
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    ok = _run_in_thread(regenerate_overview(LLMClient(settings)))
    assert ok is True

    row = _overview_row()
    assert row is not None and row.id == 1
    # Byte-stable expectation, re-pinned for recency-first: the mock
    # digests the generator's user message (the same document list, the
    # same cap) to its first 8 tokens — and the list now LEADS with the
    # newest doc, so the digest is of ITS line:
    #   "docs — homelab/tables.md — Service Port Table — This document
    #    covers …" → "docs homelab tables md service port table this"
    # (pre-fix the lexicographic-first doc was deployments/new-service.md).
    _, user = build_overview_prompt(_doc_rows())
    assert user.splitlines()[0].startswith(
        "docs — homelab/tables.md — Service Port Table"
    ), "the generator's list must lead with the NEWEST doc (recency-first)"
    expected = (
        "Knowledge base outline:\n- docs homelab tables md service port table this"
    )
    assert row.content == expected
    first_bullet = row.content.splitlines()[1].removeprefix("- ").strip()
    assert first_bullet  # the digest line is non-empty

    # A grounded chat turn echoes the STORED outline's first bullet.
    bubble = _ask(page, app_url, QUESTION, f"(kb: {first_bullet})")
    expect(bubble).to_have_text(
        re.compile(rf"\(kb: {re.escape(first_bullet)}\)\s*$"), timeout=30_000
    )
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)