"""Phase 74 E2E (Playwright): prior turns + prior thinking reach the LLM. TODO.md L4 (owner 2026-09-05): "Chat history isn't being passed to the LLM. When the LLM responds and you ask a follow-up question the previous question/answer isn't passed to the model. Since my models support preserve thinking, make sure to pass previous thinking blocks as well." Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_llm_history.py -v --no-cov The mock's ``echo my history`` marker (``HISTORY_TRIGGER``) answers with a deterministic echo of the history block the model received — ``history: N prior messages; last answer tail: ; thinking: yes|no`` — so every assertion below is a byte-exact pin on the wire contents. The prior answer's tail is derived from the conversation record the client persisted (localStorage ``bor.chat.v1``) — the SAME array task 02 maps into the request body's ``history``, so what the record shows IS what the model received (``thinking`` travels as ``reasoning_content`` on the assistant message — A4). The marker is checked BEFORE the mock's ``DEFLECT_MODE`` branch, so the echo fires on BOTH turn branches — the branch under test is discriminated separately (the grounded source chip / the ``is-deflected`` bubble class). The echo answers carry no tool markup, so no marker tool flow is re-triggered by the now-always- present (user/assistant-only) history. The file name deliberately differs from phase 50's ``test_chat_history.py`` (save & view chat history — a different story). """ from __future__ import annotations import asyncio import json from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import text from app.config import Settings from app.db import SessionLocal from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" STORAGE_KEY = "bor.chat.v1" #: Turn 1 (both follow-up stories): on-topic (HIGH gate -> grounded) #: and carries the phase-17 thinking trigger, so the brain record #: streams a deterministic scratchpad into its ``thinking`` key. T1 = "think out loud — how is my Kubernetes cluster set up?" #: Turn 2, grounded story: on-topic + the phase-74 history echo marker. T2_GROUNDED = "echo my history about my kubernetes cluster" #: Turn 2, deflected story: OFF-topic (LOW gate -> deflected branch) + #: the marker — ASSUMPTION A3: BOTH branches carry the history, and #: the marker fires before the DEFLECT_MODE branch, so this is the #: deflected path under test. T2_DEFLECTED = "echo my history — how do I bake sourdough bread?" async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None: """Truncate the KB (+ query log + steering notes — deterministic mock answers), then optionally re-import fixtures.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes")) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) def _ask(page: Page, question: str) -> None: """Send one turn and wait until the answer has fully landed (the ``done`` event restored the Send button).""" page.fill("#message-input", question) page.click("#send-btn") expect(page.locator(".msg.user .bubble").last).to_contain_text(question) expect(page.locator(".msg.brain .bubble").last).to_contain_text( MOCK_ANSWER_MARKER, timeout=60_000 ) expect(page.locator("#send-label")).to_have_text("Send") def _record(page: Page) -> dict[str, Any]: """The persisted ``bor.chat.v1`` record (the same array task 02 maps into the request body's ``history``).""" raw = page.evaluate(f"localStorage.getItem({STORAGE_KEY!r})") return json.loads(raw) if raw else {"messages": []} def _wait_record(page: Page, n_messages: int) -> dict[str, Any]: """Wait until the persisted record carries ``n_messages`` turns (the ``done`` event's save point has landed in localStorage).""" page.wait_for_function( """([key, n]) => { const raw = localStorage.getItem(key); const rec = raw ? JSON.parse(raw) : null; return !!rec && rec.messages.length >= n; }""", arg=[STORAGE_KEY, n_messages], timeout=15_000, ) return _record(page) def test_followup_receives_history_and_thinking( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: summary = _reset_db(mock_llm, seed=True) assert summary is not None and summary.added == 13 # A9 formats (phase 47 added quadlet+j2) page.set_default_timeout(30_000) # Cold start: no restored conversation — every prior turn the model # sees on turn 2 is the one this test just sent. page.add_init_script("localStorage.clear()") page.goto(app_url) # Turn 1 — grounded + the thinking trigger: the brain record must # carry the streamed scratchpad in its ``thinking`` key. _ask(page, T1) brain1 = _wait_record(page, 2)["messages"][1] assert brain1["who"] == "brain" assert brain1["thinking"], "turn 1 must have streamed thinking into the record" assert MOCK_ANSWER_MARKER in brain1["text"] answer_tail = brain1["text"][-24:] # Turn 2 — grounded + the echo marker: the model receives # [system, user(T1), assistant(A1, reasoning_content), user(T2)] # and the echo proves it byte-exactly. _ask(page, T2_GROUNDED) bubble = page.locator(".msg.brain .bubble").last expect(bubble).to_contain_text("history: 2 prior messages", timeout=30_000) expect(bubble).to_contain_text(f"last answer tail: {answer_tail}") expect(bubble).to_contain_text("thinking: yes") # Grounded proof — the echo fires in BOTH branches, so the branch # is discriminated by the kubernetes.md source chip (the deflected # turn carries no cited sources). Scoped to the LAST brain message: # turn 1 cited kubernetes.md too. expect( page.locator(".msg.brain").last.locator(".source-chip", has_text="kubernetes.md") ).to_have_count(1) def test_first_question_has_no_history( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: summary = _reset_db(mock_llm, seed=True) assert summary is not None and summary.added == 13 # A9 formats (phase 47 added quadlet+j2) page.set_default_timeout(30_000) page.add_init_script("localStorage.clear()") page.goto(app_url) # Cold start: the request body's history is empty — no phantom # prior turns, no phantom thinking. _ask(page, T2_GROUNDED) bubble = page.locator(".msg.brain .bubble").last expect(bubble).to_contain_text("history: 0 prior messages", timeout=30_000) expect(bubble).to_contain_text("last answer tail: none") expect(bubble).to_contain_text("thinking: no") # Grounded: the echo question is on-topic (the chip proves the # HIGH gate, not a deflection). expect( page.locator(".msg.brain").last.locator(".source-chip", has_text="kubernetes.md") ).to_have_count(1) def test_deflected_followup_receives_history( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: summary = _reset_db(mock_llm, seed=True) assert summary is not None and summary.added == 13 # A9 formats (phase 47 added quadlet+j2) page.set_default_timeout(30_000) page.add_init_script("localStorage.clear()") page.goto(app_url) _ask(page, T1) brain1 = _wait_record(page, 2)["messages"][1] assert brain1["thinking"], "turn 1 must have streamed thinking into the record" answer_tail = brain1["text"][-24:] # Turn 2 — OFF-topic (LOW gate -> deflected branch) + the marker: # the echo still arrives with the SAME history block (A3: both # branches carry it — the marker is checked before the # DEFLECT_MODE branch, so this test proves the deflected path). _ask(page, T2_DEFLECTED) bubble = page.locator(".msg.brain.is-deflected .bubble").last bubble.wait_for(state="visible", timeout=30_000) expect(bubble).to_contain_text("history: 2 prior messages", timeout=30_000) expect(bubble).to_contain_text(f"last answer tail: {answer_tail}") expect(bubble).to_contain_text("thinking: yes")