"""Phase 17 E2E (Playwright, mock-only): the model's "thinking" display. Story: ``.agents/user_stories/thinking-display.md`` Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_thinking_display.py -v --no-cov MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported here. The real ``turbo`` thinks on *every* turn, which would break the no-thinking regression test (scenario 4) — the deterministic mock's ``think out loud`` trigger (mock_llm.py) keeps all five scenarios reproducible. Test → story mapping (Playwright Mapping Rule): 1. ``test_thinking_block_streams_open_then_collapses`` 2. ``test_thinking_toggle_after_done`` 3. ``test_thinking_restored_after_reload`` 4. ``test_no_thinking_block_without_trigger`` 5. ``test_thinking_with_deflection`` Determinism note: the mock paces every SSE frame at 0.02s and the thinking text is ~2 700 chars (≈ 230 frames ≈ 4.5s — lengthened in phase 21 so the scratchpad overflows the 320px window) before the first content frame, so "attach → assert open" runs well inside the open window on headless Chromium; all other assertions are made after the send button re-enables (fully settled state). """ from __future__ import annotations import asyncio import json import re from collections.abc import Iterator from pathlib import Path from threading import Thread from typing import Any import pytest from playwright.sync_api import Page, expect from sqlalchemy import text from app.config import Settings from app.db import SessionLocal from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from e2e.auth_helpers import login REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" THINK_QUESTION = "think out loud — how is my kubernetes cluster set up?" PLAIN_QUESTION = "How is my Kubernetes cluster set up?" THINK_DEFLECT_QUESTION = "think out loud — tell me about quantum wormhole cooling" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" DEFLECT_PHRASE = r"haven't done anything like that" #: Line fragment the mock's deterministic scratchpad must carry — the #: suite keys off it (mock_llm.compose_thinking). THINKING_FRAGMENT = "Step 2: Check my notes" STORAGE_KEY = "bor.chat.v1" async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None: """Truncate the KB (and query log), then optionally re-import fixtures.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) @pytest.fixture() def seeded_kb(mock_llm: int, db_ready: None) -> Iterator[None]: """A fresh KB seeded from ``tests/fixtures/docs`` (13 docs since phase 47, A9 formats), truncated again on teardown. ``db_ready`` (conftest) skips with clear instructions when Postgres is down.""" summary = _reset_db(mock_llm, seed=True) assert summary is not None and summary.added == 13 yield _reset_db(mock_llm, seed=False) def send_and_wait(page: Page, question: str) -> None: """Type into #message-input, submit via #composer, then wait until the last brain message settles (send button re-enabled, label "Send").""" page.fill("#message-input", question) page.evaluate("() => document.querySelector('#composer').requestSubmit()") expect(page.locator(".msg.user .bubble").last).to_contain_text(question) # The mock streams at 0.02s/chunk, so thinking + answer land in a few # seconds — 30s is generous on headless Chromium. Phase 48: the label # assertion carries the settle wait with an explicit timeout — the # in-flight button is the enabled Stop control (never disabled), so # to_be_enabled no longer blocks until the turn settles. expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000) expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000) expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000) # --------------------------------------------------------------------------- # 1. Streaming: the block attaches OPEN at the first thinking event, then # auto-collapses when the first answer token lands # --------------------------------------------------------------------------- def test_thinking_block_streams_open_then_collapses( page: Page, app_url: str, seeded_kb: None ) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") page.fill("#message-input", THINK_QUESTION) page.evaluate("() => document.querySelector('#composer').requestSubmit()") expect(page.locator(".msg.user .bubble").last).to_contain_text(THINK_QUESTION) # The block attaches at the FIRST thinking event — before any answer # token — and is created OPEN. details = page.locator(".msg.brain").last.locator("details.thinking") details.wait_for(state="attached", timeout=10_000) # The ~2 700-char thinking stream (≈4.5s, phase 21) keeps the block # open right after attach — assert while it is still streaming. expect(details).to_have_attribute("open", "") expect(details.locator(".thinking-text")).not_to_have_text("") # First answer token: the block auto-collapses and stays closed. bubble = page.locator(".msg.brain .bubble").last expect(bubble).not_to_have_text("", timeout=30_000) expect(details).not_to_have_attribute("open") # Settled: full scratchpad, grounded mock answer, ZERO citation # chips (phase 119, LOCKED A1 — the zero-read turn chips nothing; # the retired phase-118 A4 union is gone), and the re-enabled send # button. expect(details.locator(".thinking-text")).to_contain_text(THINKING_FRAGMENT) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER) expect(page.locator(".msg.brain .source-chip")).to_have_count(0) expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") # --------------------------------------------------------------------------- # 2. Toggle: after settle the block is closed; the summary re-opens it # (a real keyboard-focusable control) and closes it again # --------------------------------------------------------------------------- def test_thinking_toggle_after_done(page: Page, app_url: str, seeded_kb: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") send_and_wait(page, THINK_QUESTION) last = page.locator(".msg.brain").last details = last.locator("details.thinking") expect(details).to_have_count(1) expect(details).not_to_have_attribute("open") # auto-collapsed at first token # The summary is a real, keyboard-focusable control. details.locator("summary").focus() assert page.evaluate("() => document.activeElement.tagName") == "SUMMARY" # Open: the full scratchpad is visible. details.locator("summary").click() expect(details).to_have_attribute("open", "") text_el = details.locator(".thinking-text") expect(text_el).to_be_visible() expect(text_el).to_contain_text(THINKING_FRAGMENT) expect(text_el).to_contain_text("nothing is invented") # Closed again — user control in both directions. details.locator("summary").click() expect(details).not_to_have_attribute("open") expect(text_el).not_to_be_visible() # --------------------------------------------------------------------------- # 3. Persistence: the thinking block (and its text) survives a reload, # restored COLLAPSED — phase-14 restore path + phase-17 field # --------------------------------------------------------------------------- def test_thinking_restored_after_reload(page: Page, app_url: str, seeded_kb: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") send_and_wait(page, THINK_QUESTION) # The live block is collapsed; capture what it shows and what the # turn persisted (raw text, same as what was rendered). details = page.locator(".msg.brain").last.locator("details.thinking") expect(details).not_to_have_attribute("open") captured = details.locator(".thinking-text").text_content() assert captured # The persisted raw text is the same scratchpad (renderMarkdown turns # the line breaks into
, which textContent drops — compare without # whitespace). raw = json.loads( page.evaluate(f"() => localStorage.getItem('{STORAGE_KEY}')") )["messages"][1]["thinking"] assert re.sub(r"\s+", "", raw) == re.sub(r"\s+", "", captured) page.reload() expect(page.locator("#empty-state")).to_be_hidden() restored = page.locator(".msg.brain").last.locator("details.thinking") expect(restored).to_have_count(1) expect(restored).not_to_have_attribute("open") # restored COLLAPSED expect(restored.locator(".thinking-text")).to_have_text(captured) # Answer bubble is intact after the restore (phase-14 restore path) # — with ZERO citation chips (phase 119, LOCKED A1: the zero-read # turn persisted an empty sources list; the retired phase-118 A4 # union is gone). expect(page.locator(".msg.brain .bubble").last).to_contain_text(MOCK_ANSWER_MARKER) expect(page.locator(".msg.brain .source-chip")).to_have_count(0) # --------------------------------------------------------------------------- # 4. No thinking, no block: a model/turn that emits no reasoning renders # exactly as before (no layout regression) # --------------------------------------------------------------------------- def test_no_thinking_block_without_trigger(page: Page, app_url: str, seeded_kb: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") send_and_wait(page, PLAIN_QUESTION) # No trigger → no thinking events → no block anywhere on the page. expect(page.locator("details.thinking")).to_have_count(0) # The turn itself is complete and grounded, exactly as before phase # 17 — with ZERO citation chips (phase 119, LOCKED A1: the zero-read # turn chips nothing; the retired phase-118 A4 union is gone). expect(page.locator(".msg.brain .source-chip")).to_have_count(0) # --------------------------------------------------------------------------- # 5. Coexistence: the honesty gate (deflection) and the thinking block # on the same turn # --------------------------------------------------------------------------- def test_thinking_with_deflection(page: Page, app_url: str, seeded_kb: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") send_and_wait(page, THINK_DEFLECT_QUESTION) last = page.locator(".msg.brain").last # The honesty gate fired: amber deflected bubble + "Maybe try" chips. expect(last).to_have_class(re.compile(r"is-deflected")) expect(last.locator(".bubble")).to_contain_text( re.compile(DEFLECT_PHRASE, re.IGNORECASE) ) chips = last.locator(".maybe-try .suggestion-chip") expect(chips.first).to_be_visible() assert chips.count() >= 2 # And the thinking block came along, closed, with its scratchpad. details = last.locator("details.thinking") expect(details).to_have_count(1) expect(details).not_to_have_attribute("open") expect(details.locator(".thinking-text")).to_contain_text(THINKING_FRAGMENT)