brain-of-reese/tests/e2e/test_kb_overview_freshness.py

"""Phase 142 E2E (Playwright): the forced-cap KB outline leads with the
NEWEST document, and the honest ``<knowledge_base>`` intro is served.

Source: ``TODO.md`` L4 (the "Summarize the last 10 phases" failure — the
stored outline described a KB that "ends at phase 113" because the input
cap cut the lexicographic prefix, so the newest documents never reached
the ``lite`` model).
Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_kb_overview_freshness.py -v --no-cov

The whole phase, end to end through the real prompt path:

* the fixture KB is shaped like the owner's — one source, an OLDER batch
  whose paths sort lexicographically FIRST (``old/a..c.md``, past file
  mtimes) and a NEWER batch whose paths sort LAST (``zz_new/y..z.md``,
  fresh mtimes) — phase 106 sources ``created_at`` from the local file's
  mtime, so the pre-phase-142 lexicographic order and the recency order
  DISAGREE about which document is first;
* ``BOR_OVERVIEW_INPUT_MAX_CHARS`` is forced small (the E2E settings
  override: process env + ``get_settings`` cache clear around the
  regeneration — the generator's ``build_overview_prompt`` reads the cap
  from ``get_settings``) so the document list is CUT — the cap must bite
  or the ordering is unobservable;
* test 1: the real ``regenerate_overview`` against the mock's
  ``KB_OVERVIEW_MODE`` digest — the stored outline's 8 tokens come from
  the NEWEST document's line (recency-first, end to end);
* test 2: the ``<knowledge_base>`` section builder, called here directly
  against the app (the phase-31 file's own convention) — the section's
  lead line IS the new honest intro verbatim (no "generated at import
  time"), tag + cap behavior intact;
* test 3: the seeded row's section reaches a RENDERED turn — the
  phase-31 first-bullet echo, re-asserted here against the seeded row so
  this file stands alone on the injection surface too.

The pre-existing phase-31 file (``tests/e2e/test_kb_overview.py``) is
NOT modified here (its re-pins landed in task 02).

Test → phase mapping:
1. ``test_regenerated_outline_leads_with_newest_doc`` (the objective)
2. ``test_kb_section_carries_honest_intro``
3. ``test_kb_section_reaches_a_rendered_turn``
"""
from __future__ import annotations

import asyncio
import os
import re
from datetime import UTC, datetime, timedelta
from pathlib import Path
from threading import Thread
from typing import Any

import pytest
from playwright.sync_api import Page, expect
from sqlalchemy import select, text

from app.config import Settings, get_settings
from app.db import SessionLocal
from app.models import Document, KbOverview, QueryLog
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from app.rag.overview import build_overview_prompt, regenerate_overview
from app.rag.prompts import _KB_INTRO, build_kb_section
from app.rag.retriever import TRUNCATION_MARKER
from e2e.auth_helpers import login
from tests.e2e.mock_llm import TOKEN_RE

# --- the owner-shaped fixture --------------------------------------------

#: The fixture dir's basename — the ``source`` the importer stores (the
#: phase-31 fixture is "docs" for ``tests/fixtures/docs``).
SOURCE_NAME = "reese"

#: The OLDER batch: paths that sort lexicographically FIRST, past
#: mtimes (phase 106's mtime fallback makes the documents' dates old).
OLD_PATHS = ("old/a.md", "old/b.md", "old/c.md")
#: The NEWER batch: paths that sort lexicographically LAST, fresh
#: mtimes. ``zz_new/z.md`` is the NEWEST document of the whole KB.
NEW_PATHS = ("zz_new/y.md", "zz_new/z.md")
NEWEST_PATH = "zz_new/z.md"
NEWEST_TITLE = "Zz New Z"

#: The generator line the NEWEST doc must occupy FIRST —
#: ``source — path — title`` (the summary tail is the mock's).
NEWEST_LINE_PREFIX = f"{SOURCE_NAME} — {NEWEST_PATH} — {NEWEST_TITLE}"

#: The forced input cap: the 5-doc list (~650 chars) exceeds it, so the
#: cap BITES (``[…truncated…]`` lands in the model's input), while the
#: ~130-char newest line survives intact (the digest is of ITS tokens).
OVERVIEW_INPUT_CAP = 300

#: The stored outline the mock's ``KB_OVERVIEW_MODE`` branch writes when
#: the newest doc leads: header + ONE bullet of the user message's first
#: 8 tokens — all 8 of them from ``NEWEST_LINE_PREFIX`` (source + path +
#: title supply exactly 8), computed from the fixture, not hardcoded.
EXPECTED_DIGEST = (
    "Knowledge base outline:\n- "
    + " ".join(TOKEN_RE.findall(NEWEST_LINE_PREFIX.lower())[:8])
)

# --- test 2/3 constants ----------------------------------------------------

#: The phase-142 honest intro, pinned verbatim here (the same text the
#: unit suite pins — WITHOUT the trailing newline; the app's
#: ``_KB_INTRO`` carries it). It says what the outline actually is (a
#: compact picture that favors recent documents, a map not an inventory)
#: and teaches the verify-first behavior for "the last N …" questions.
E2E_KB_INTRO = (
    'A compact picture of the knowledge base\'s categories and topics — '
    'it favors recent documents and may omit older or lesser ones: a map '
    'of what exists, not a complete inventory. For questions about the '
    'latest state or the complete set ("the last N …", "everything '
    'about …", "all the …"), verify with `ls` (drill into the folders) '
    'or `grep` before answering from the outline:'
)

#: The seeded stored outline (test 3): multi-line ``-`` bullets so the
#: echo must pick the FIRST bullet — the phase-31 story's parse contract.
SEED_OVERVIEW = (
    "- Newest zeta gateway routing and edge proxy notes\n"
    "- Older ypsilon wireguard tunnel peers and keepalive tuning\n"
    "- Old kubernetes cluster and node maintenance"
)
FIRST_BULLET = "Newest zeta gateway routing and edge proxy notes"
KB_ECHO = f"(kb: {FIRST_BULLET})"

QUESTION = "What do my newest notes cover?"


# --- the owner-shaped fixture ----------------------------------------------


def _write_doc(path: Path, title: str, body: str, age_days: float) -> None:
    """Write one markdown doc with a pinned mtime *age_days* in the past.

    Phase 106 sources ``created_at`` from the local file's mtime
    (``file_mtime_datetime`` → ``normalize_doc_date``), so the mtime IS
    the document's date. The ages are RELATIVE to the run (the
    integration fixture's pattern): they stay in the past forever, while
    the staggered order — and thus the digest — is a pure function of
    the fixture.
    """
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(f"# {title}\n\n{body}\n", encoding="utf-8")
    ts = (datetime.now(UTC) - timedelta(days=age_days)).timestamp()
    os.utime(path, (ts, ts))


@pytest.fixture()
def owner_kb(tmp_path: Path) -> Path:
    """The owner's KB shape at fixture scale: one source whose OLDER
    documents sort lexicographically FIRST (``old/*``, ≈91–95 days old)
    and whose NEWER documents sort LAST (``zz_new/*``, 1–3 days old).
    Recency strictly OPPOSES the lexicographic path order — pre-fix, the
    capped list led with ``old/a.md``; post-fix it leads with
    ``zz_new/z.md`` (the newest)."""
    root = tmp_path / SOURCE_NAME
    _write_doc(
        root / OLD_PATHS[0], "Old A",
        "Old alpha notes: kubernetes cluster nodes and etcd quorum settings.",
        95,
    )
    _write_doc(
        root / OLD_PATHS[1], "Old B",
        "Old beta notes: borg backup schedules and restore runbook steps.",
        93,
    )
    _write_doc(
        root / OLD_PATHS[2], "Old C",
        "Old gamma notes: kafkabridge routing and static DNS records.",
        91,
    )
    _write_doc(
        root / NEW_PATHS[0], "Zz New Y",
        "Newer ypsilon notes: wireguard tunnel peers and keepalive tuning.",
        3,
    )
    _write_doc(
        root / NEW_PATHS[1], NEWEST_TITLE,
        "Newest zeta notes: edge proxy routing and gateway header policy.",
        1,
    )
    return root


# --- Importer + thread helpers (test_kb_overview.py pattern) ----------------


def _settings_for(mock_port: int) -> Settings:
    kwargs: dict[str, Any] = {
        "_env_file": None,
        "llm_base_url": f"http://127.0.0.1:{mock_port}/v1",
    }
    return Settings(**kwargs)  # pyright: ignore[reportCallIssue]


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread.

    Playwright's sync API keeps an asyncio loop running on the test thread,
    so ``asyncio.run`` cannot be called directly from a test body.
    """
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(
    mock_port: int, root: Path, overview: str | None = None
) -> ImportSummary:
    """Truncate the KB (+ query log, steering, kb_overview), re-import the
    owner-shaped fixture, and optionally seed the single ``kb_overview``
    row (directly, so the injection path is what a test exercises)."""
    with SessionLocal() as db:
        db.execute(
            text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
        )
        db.commit()
        if overview is not None:
            db.add(KbOverview(id=1, content=overview))
            db.commit()
    return _run_in_thread(import_sources([root], LLMClient(_settings_for(mock_port))))


def _ask(page: Page, app_url: str, question: str, done_text: str) -> Any:
    """Submit *question* and wait until the streamed answer has fully
    landed (the phase-31 file's helper): ``done_text`` is the
    deterministic last thing the answer contains — the ``(kb: …)`` echo,
    which the mock appends after EVERYTHING else whenever the system
    prompt carries ``<knowledge_base>`` (HIGH and LOW alike)."""
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")
    page.fill("#message-input", question)
    page.click("#send-btn")
    bubble = page.locator(".msg.brain .bubble")
    bubble.first.wait_for(state="visible", timeout=30_000)
    expect(bubble.first).to_contain_text(done_text, timeout=30_000)
    expect(page.locator("#send-btn")).to_be_enabled()
    expect(page.locator("#send-label")).to_have_text("Send")
    return bubble.first


def _last_query_log() -> QueryLog:
    with SessionLocal() as db:
        rows = db.scalars(select(QueryLog)).all()
    assert len(rows) == 1, f"expected exactly one query_log row, got {len(rows)}"
    return rows[0]


def _overview_row() -> KbOverview | None:
    with SessionLocal() as db:
        return db.get(KbOverview, 1)


def _doc_rows() -> list[tuple[str, str, str, str | None]]:
    """The catalogue the way the generator sees it (phase 142, task 02):
    RECENT-FIRST by the document's date, ``(source, path)`` tie-break —
    the same order ``regenerate_overview`` SELECTs."""
    with SessionLocal() as db:
        rows = db.execute(
            select(Document.source, Document.path, Document.title, Document.summary)
            .order_by(Document.created_at.desc(), Document.source, Document.path)
        ).all()
    return [(source, path, title, summary) for source, path, title, summary in rows]


# --- 1. The objective: with the cap biting, the outline leads with the newest ---


def test_regenerated_outline_leads_with_newest_doc(
    page: Page,
    app_url: str,
    mock_llm: int,
    db_ready: None,
    owner_kb: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The owner's "ends at phase 113" failure, end to end: with
    ``BOR_OVERVIEW_INPUT_MAX_CHARS`` forced to bite, the real
    ``regenerate_overview`` (``LLMClient`` pointed at the mock's
    ``KB_OVERVIEW_MODE`` digest) stores an outline whose 8 tokens come
    from the NEWEST document's line — ``zz_new/z.md`` sorts
    lexicographically LAST, so only the task-02 recency ordering can put
    it first. Pre-fix the digest would have been of ``old/a.md``'s line
    (the lexicographic prefix the cap kept) — the shape this pin
    forbids."""
    summary = _reset_db(mock_llm, owner_kb)
    assert summary.added == 5  # the whole owner-shaped fixture
    assert _overview_row() is None  # direct import never regenerates

    # The E2E settings override: the generator's ``build_overview_prompt``
    # reads the cap from ``get_settings`` (the process cache), so the
    # forced cap rides in via the env + a cache clear around the call.
    monkeypatch.setenv("BOR_OVERVIEW_INPUT_MAX_CHARS", str(OVERVIEW_INPUT_CAP))
    get_settings.cache_clear()
    try:
        ok = _run_in_thread(regenerate_overview(LLMClient(_settings_for(mock_llm))))
    finally:
        get_settings.cache_clear()  # restore the default-cap settings after
    assert ok is True

    # The generator's list leads with the NEWEST doc (the ordering that
    # makes the cap honest) — and its line renders uncut:
    rows = _doc_rows()
    assert rows[0][1] == NEWEST_PATH
    _, first_line = build_overview_prompt([rows[0]])
    assert first_line.startswith(f"{NEWEST_LINE_PREFIX} — This document covers")

    # The cap BIT on the full list (the repro's precondition) while the
    # leading line survived intact — so the mock digested THIS line:
    _, user = build_overview_prompt(rows)
    assert TRUNCATION_MARKER in user
    assert len(user) <= OVERVIEW_INPUT_CAP + 1 + len(TRUNCATION_MARKER)
    assert user.splitlines()[0] == first_line

    row = _overview_row()
    assert row is not None and row.id == 1
    # The stored outline is the mock's 8-token digest of the capped list
    # — all 8 tokens from the NEWEST doc's source/path/title (computed
    # from the fixture above, not hardcoded wording) — and phase 162
    # (LOCKED A2) appends the deterministic source-inventory line after
    # it: every fixture source named with its exact count.
    assert row.content == EXPECTED_DIGEST + "\nSources: reese (5 docs)"
    assert "old" not in row.content  # no oldest-batch token survives the cut


# --- 2. The section builder serves the new honest intro ----------------------


def test_kb_section_carries_honest_intro(db_ready: None) -> None:
    """The intro is a prompt constant — the deterministic surface is the
    section builder itself, called here directly against the app (the
    phase-31 file's own convention for ``build_overview_prompt`` /
    ``regenerate_overview``). The section's lead line IS the phase-142
    honest intro verbatim — "(generated at import time)" is GONE — and
    the tag + cap behavior around it is intact (byte-stable contract)."""
    # The app's constant, pinned verbatim (the trailing newline included):
    assert _KB_INTRO == E2E_KB_INTRO + "\n"

    section = build_kb_section(SEED_OVERVIEW)

    # The lead line IS the new intro — verbatim, colon-terminated,
    # between the open tag and the outline:
    assert section.startswith(f"<knowledge_base>\n{E2E_KB_INTRO}\n")
    assert section.splitlines()[1] == E2E_KB_INTRO
    assert section.index(E2E_KB_INTRO) < section.index(SEED_OVERVIEW.splitlines()[0])
    # The false claim is gone from the served section (and thus the
    # prompt), and the three contracts it must carry are present:
    assert "(generated at import time)" not in section
    assert "generated at import time" not in section
    assert "favors recent documents" in section
    assert "not a complete inventory" in section
    assert "verify with `ls` (drill into the folders) or `grep`" in section
    # Tag + body intact: the exact section is open tag, the new intro,
    # the seed's bullets in order, close tag (the byte-stable shape):
    assert section == f"<knowledge_base>\n{E2E_KB_INTRO}\n{SEED_OVERVIEW}\n</knowledge_base>"
    # Cap behavior byte-stable: exact fit keeps the marker out …
    exact = build_kb_section(SEED_OVERVIEW, max_chars=len(section))
    assert exact == section and TRUNCATION_MARKER not in exact
    # … a tighter budget cuts with the shared marker, within the cap,
    # the new intro still leading:
    tight = build_kb_section(SEED_OVERVIEW, max_chars=len(section) - 10)
    assert len(tight) <= len(section) - 10
    assert TRUNCATION_MARKER in tight
    assert tight.startswith(f"<knowledge_base>\n{E2E_KB_INTRO}\n-")
    # Empty overview still omits the whole section (byte-identity):
    assert build_kb_section("") == ""


# --- 3. The seeded section reaches a rendered turn ---------------------------


def test_kb_section_reaches_a_rendered_turn(
    page: Page, app_url: str, mock_llm: int, db_ready: None, owner_kb: Path
) -> None:
    """The phase-31 injection path, re-asserted here so this file stands
    alone on it: the SEEDED row's outline reaches a rendered turn — the
    mock echoes the ``<knowledge_base>`` section's first bullet into its
    answer (`` (kb: <first bullet>)``), which only happens when the
    section (lead line and all) was in the LLM prompt."""
    summary = _reset_db(mock_llm, owner_kb, overview=SEED_OVERVIEW)
    assert summary.added == 5
    assert _overview_row() is not None  # the row the turn must inject

    # The echo is the deterministic end of BOTH the grounded and the
    # deflected mock answer — the completion signal and the proof:
    bubble = _ask(page, app_url, QUESTION, KB_ECHO)
    expect(bubble).to_have_text(
        re.compile(rf"{re.escape(KB_ECHO)}\s*$"), timeout=30_000
    )
    row = _last_query_log()
    assert row.question == QUESTION