brain-of-reese/tests/e2e/test_llm_history.py

"""Phase 74 E2E (Playwright): prior turns + prior thinking reach the LLM.

TODO.md L4 (owner 2026-09-05): "Chat history isn't being passed to the
LLM. When the LLM responds and you ask a follow-up question the
previous question/answer isn't passed to the model. Since my models
support preserve thinking, make sure to pass previous thinking blocks
as well."

Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_llm_history.py -v --no-cov

The mock's ``echo my history`` marker (``HISTORY_TRIGGER``) answers
with a deterministic echo of the history block the model received —
``history: N prior messages; last answer tail: <last 24 chars of the
prior answer>; thinking: yes|no`` — so every assertion below is a
byte-exact pin on the wire contents. The prior answer's tail is
derived from the conversation record the client persisted
(localStorage ``bor.chat.v1``) — the SAME array task 02 maps into the
request body's ``history``, so what the record shows IS what the model
received (``thinking`` travels as ``reasoning_content`` on the
assistant message — A4).

The marker is checked BEFORE the mock's ``DEFLECT_MODE`` branch, so
the echo fires on BOTH turn branches — the branch under test is
discriminated separately (the grounded source chip / the
``is-deflected`` bubble class). The echo answers carry no tool
markup, so no marker tool flow is re-triggered by the now-always-
present (user/assistant-only) history.

The file name deliberately differs from phase 50's
``test_chat_history.py`` (save & view chat history — a different
story).
"""
from __future__ import annotations

import asyncio
import json
from pathlib import Path
from threading import Thread
from typing import Any

from playwright.sync_api import Page, expect
from sqlalchemy import text

from app.config import Settings
from app.db import SessionLocal
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
STORAGE_KEY = "bor.chat.v1"

#: Turn 1 (both follow-up stories): on-topic (HIGH gate -> grounded)
#: and carries the phase-17 thinking trigger, so the brain record
#: streams a deterministic scratchpad into its ``thinking`` key.
T1 = "think out loud — how is my Kubernetes cluster set up?"
#: Turn 2, grounded story: on-topic + the phase-74 history echo marker.
T2_GROUNDED = "echo my history about my kubernetes cluster"
#: Turn 2, deflected story: OFF-topic (LOW gate -> deflected branch) +
#: the marker — ASSUMPTION A3: BOTH branches carry the history, and
#: the marker fires before the DEFLECT_MODE branch, so this is the
#: deflected path under test.
T2_DEFLECTED = "echo my history — how do I bake sourdough bread?"


async def _import_fixtures(mock_port: int) -> ImportSummary:
    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return await import_sources([FIXTURES], LLMClient(settings))


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread.

    Playwright's sync API keeps an asyncio loop running on the test
    thread, so ``asyncio.run`` cannot be called directly from a test
    body.
    """
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
    """Truncate the KB (+ query log + steering notes — deterministic
    mock answers), then optionally re-import fixtures."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
        db.commit()
    if not seed:
        return None
    return _run_in_thread(_import_fixtures(mock_port))


def _ask(page: Page, question: str) -> None:
    """Send one turn and wait until the answer has fully landed (the
    ``done`` event restored the Send button)."""
    page.fill("#message-input", question)
    page.click("#send-btn")
    expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
    expect(page.locator(".msg.brain .bubble").last).to_contain_text(
        MOCK_ANSWER_MARKER, timeout=60_000
    )
    expect(page.locator("#send-label")).to_have_text("Send")


def _record(page: Page) -> dict[str, Any]:
    """The persisted ``bor.chat.v1`` record (the same array task 02
    maps into the request body's ``history``)."""
    raw = page.evaluate(f"localStorage.getItem({STORAGE_KEY!r})")
    return json.loads(raw) if raw else {"messages": []}


def _wait_record(page: Page, n_messages: int) -> dict[str, Any]:
    """Wait until the persisted record carries ``n_messages`` turns
    (the ``done`` event's save point has landed in localStorage)."""
    page.wait_for_function(
        """([key, n]) => {
            const raw = localStorage.getItem(key);
            const rec = raw ? JSON.parse(raw) : null;
            return !!rec && rec.messages.length >= n;
        }""",
        arg=[STORAGE_KEY, n_messages],
        timeout=15_000,
    )
    return _record(page)


def test_followup_receives_history_and_thinking(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    summary = _reset_db(mock_llm, seed=True)
    assert summary is not None and summary.added == 13  # A9 formats (phase 47 added quadlet+j2)
    page.set_default_timeout(30_000)
    # Cold start: no restored conversation — every prior turn the model
    # sees on turn 2 is the one this test just sent.
    page.add_init_script("localStorage.clear()")
    page.goto(app_url)

    # Turn 1 — grounded + the thinking trigger: the brain record must
    # carry the streamed scratchpad in its ``thinking`` key.
    _ask(page, T1)
    brain1 = _wait_record(page, 2)["messages"][1]
    assert brain1["who"] == "brain"
    assert brain1["thinking"], "turn 1 must have streamed thinking into the record"
    assert MOCK_ANSWER_MARKER in brain1["text"]
    answer_tail = brain1["text"][-24:]

    # Turn 2 — grounded + the echo marker: the model receives
    # [system, user(T1), assistant(A1, reasoning_content), user(T2)]
    # and the echo proves it byte-exactly.
    _ask(page, T2_GROUNDED)
    bubble = page.locator(".msg.brain .bubble").last
    expect(bubble).to_contain_text("history: 2 prior messages", timeout=30_000)
    expect(bubble).to_contain_text(f"last answer tail: {answer_tail}")
    expect(bubble).to_contain_text("thinking: yes")
    # Grounded proof — the echo fires in BOTH branches, so the branch
    # is discriminated by the kubernetes.md source chip (the deflected
    # turn carries no cited sources). Scoped to the LAST brain message:
    # turn 1 cited kubernetes.md too.
    expect(
        page.locator(".msg.brain").last.locator(".source-chip", has_text="kubernetes.md")
    ).to_have_count(1)


def test_first_question_has_no_history(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    summary = _reset_db(mock_llm, seed=True)
    assert summary is not None and summary.added == 13  # A9 formats (phase 47 added quadlet+j2)
    page.set_default_timeout(30_000)
    page.add_init_script("localStorage.clear()")
    page.goto(app_url)

    # Cold start: the request body's history is empty — no phantom
    # prior turns, no phantom thinking.
    _ask(page, T2_GROUNDED)
    bubble = page.locator(".msg.brain .bubble").last
    expect(bubble).to_contain_text("history: 0 prior messages", timeout=30_000)
    expect(bubble).to_contain_text("last answer tail: none")
    expect(bubble).to_contain_text("thinking: no")
    # Grounded: the echo question is on-topic (the chip proves the
    # HIGH gate, not a deflection).
    expect(
        page.locator(".msg.brain").last.locator(".source-chip", has_text="kubernetes.md")
    ).to_have_count(1)


def test_deflected_followup_receives_history(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    summary = _reset_db(mock_llm, seed=True)
    assert summary is not None and summary.added == 13  # A9 formats (phase 47 added quadlet+j2)
    page.set_default_timeout(30_000)
    page.add_init_script("localStorage.clear()")
    page.goto(app_url)

    _ask(page, T1)
    brain1 = _wait_record(page, 2)["messages"][1]
    assert brain1["thinking"], "turn 1 must have streamed thinking into the record"
    answer_tail = brain1["text"][-24:]

    # Turn 2 — OFF-topic (LOW gate -> deflected branch) + the marker:
    # the echo still arrives with the SAME history block (A3: both
    # branches carry it — the marker is checked before the
    # DEFLECT_MODE branch, so this test proves the deflected path).
    _ask(page, T2_DEFLECTED)
    bubble = page.locator(".msg.brain.is-deflected .bubble").last
    bubble.wait_for(state="visible", timeout=30_000)
    expect(bubble).to_contain_text("history: 2 prior messages", timeout=30_000)
    expect(bubble).to_contain_text(f"last answer tail: {answer_tail}")
    expect(bubble).to_contain_text("thinking: yes")