"""Phase 17 E2E (Playwright, mock-only): the model's "thinking" display. Story: ``.agents/user_stories/thinking-display.md`` Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_thinking_display.py -v --no-cov MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported here. The real ``turbo`` thinks on *every* turn, which would break the no-thinking regression test (scenario 4) — the deterministic mock's ``think out loud`` trigger (mock_llm.py) keeps all five scenarios reproducible. Test → story mapping (Playwright Mapping Rule): 1. ``test_thinking_block_streams_in_the_open_container`` 2. ``test_thinking_toggle_after_done`` 3. ``test_thinking_restored_after_reload`` 4. ``test_no_thinking_block_without_trigger`` 5. ``test_thinking_with_deflection`` Determinism note: the mock paces every SSE frame at 0.02s and the thinking text is ~2 700 chars (≈ 230 frames ≈ 4.5s — lengthened in phase 21 so the scratchpad overflows the 320px window) before the first content frame, so "attach → assert open" runs well inside the open window on headless Chromium; all other assertions are made after the send button re-enables (fully settled state). """ from __future__ import annotations import asyncio import json import re from collections.abc import Iterator from pathlib import Path from threading import Thread from typing import Any import pytest from playwright.sync_api import Page, expect from sqlalchemy import text from app.config import Settings from app.db import SessionLocal from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from e2e.auth_helpers import login REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" THINK_QUESTION = "think out loud — how is my kubernetes cluster set up?" PLAIN_QUESTION = "How is my Kubernetes cluster set up?" THINK_DEFLECT_QUESTION = "think out loud — tell me about quantum wormhole cooling" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" DEFLECT_PHRASE = r"haven't done anything like that" #: Line fragment the mock's deterministic scratchpad must carry — the #: suite keys off it (mock_llm.compose_thinking). THINKING_FRAGMENT = "Step 2: Check my notes" STORAGE_KEY = "bor.chat.v1" async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None: """Truncate the KB (and query log), then optionally re-import fixtures.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) @pytest.fixture() def seeded_kb(mock_llm: int, db_ready: None) -> Iterator[None]: """A fresh KB seeded from ``tests/fixtures/docs`` (13 docs since phase 47, A9 formats), truncated again on teardown. ``db_ready`` (conftest) skips with clear instructions when Postgres is down.""" summary = _reset_db(mock_llm, seed=True) assert summary is not None and summary.added == 13 yield _reset_db(mock_llm, seed=False) def send_and_wait(page: Page, question: str) -> None: """Type into #message-input, submit via #composer, then wait until the last brain message settles (send button re-enabled, label "Send").""" page.fill("#message-input", question) page.evaluate("() => document.querySelector('#composer').requestSubmit()") expect(page.locator(".msg.user .bubble").last).to_contain_text(question) # The mock streams at 0.02s/chunk, so thinking + answer land in a few # seconds — 30s is generous on headless Chromium. Phase 48: the label # assertion carries the settle wait with an explicit timeout — the # in-flight button is the enabled Stop control (never disabled), so # to_be_enabled no longer blocks until the turn settles. expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000) expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000) expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000) # --------------------------------------------------------------------------- # 1. Streaming: the block attaches at the first thinking event (nested # in the open reasoning container — D1); the container STAYS OPEN # through the first answer token (D2) and the block is ALWAYS # visible (the D5 refinement — a plain div, not collapsible); the # container folds at done. # --------------------------------------------------------------------------- def test_thinking_block_streams_in_the_open_container( page: Page, app_url: str, seeded_kb: None ) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") page.fill("#message-input", THINK_QUESTION) page.evaluate("() => document.querySelector('#composer').requestSubmit()") expect(page.locator(".msg.user .bubble").last).to_contain_text(THINK_QUESTION) # The block attaches at the FIRST thinking event — before any # answer token — nested in the container created OPEN (D1). block = page.locator(".msg.brain").last.locator("div.thinking") block.wait_for(state="attached", timeout=10_000) container = page.locator(".msg.brain").last.locator("details.reasoning") expect(container).to_have_attribute("open", "") # The ~2 700-char thinking stream (≈4.5s, phase 21) keeps the # container open right after attach — assert while it is still # streaming (the D5 refinement: the container is the ONE open # disclosure — zero
inside it). expect(container).to_have_attribute("open", "") expect( page.locator(".msg.brain").last.locator("details.reasoning details") ).to_have_count(0) expect(block.locator(".thinking-text")).not_to_have_text("") # First answer token: the container STAYS OPEN (D2 — the mid-round # delta does not fold it; only done / stop / the failed-turn # funnel do). bubble = page.locator(".msg.brain .bubble").last expect(bubble).not_to_have_text("", timeout=30_000) expect(container).to_have_attribute("open", "") # Settled: the container FOLDS (done); the block is not # collapsible (a div — the D5 refinement); the full scratchpad is # intact, the grounded mock answer is present, ZERO citation chips # (phase 119, LOCKED A1), and the re-enabled send button. expect(container).not_to_have_attribute("open") expect(block.locator(".thinking-text")).to_contain_text(THINKING_FRAGMENT) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER) expect(page.locator(".msg.brain .source-chip")).to_have_count(0) expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") # --------------------------------------------------------------------------- # 2. Toggle: after settle the container is folded; its summary # re-opens it (a real keyboard-focusable control) and closes it # again. The D5 refinement (owner 2026-10-06): the inner run is a # plain div — NOT collapsible — so the container's summary is the # ONE toggle (the per-run "summary" click is gone; the block is # always visible once the container opens). # --------------------------------------------------------------------------- def test_thinking_toggle_after_done(page: Page, app_url: str, seeded_kb: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") send_and_wait(page, THINK_QUESTION) last = page.locator(".msg.brain").last block = last.locator("div.thinking") expect(block).to_have_count(1) # The D5 refinement: the inner run is not a
at all (zero #
inside the container — the block is a plain div). expect(last.locator("details.reasoning details")).to_have_count(0) # Phase 161 (D2): the run nests in the reasoning container, which # folds at done (the at-rest focusable surface is the container's # summary — the ONE toggle). container = last.locator("details.reasoning") expect(container).to_have_count(1) expect(container).not_to_have_attribute("open") container.locator("summary.reasoning-summary").focus() assert page.evaluate("() => document.activeElement.tagName") == "SUMMARY" container.locator("summary.reasoning-summary").click() expect(container).to_have_attribute("open", "") # Open: the full scratchpad is visible — ONE toggle (the D5 # refinement — no second, per-run toggle; the block is always # visible once the container opens). text_el = block.locator(".thinking-text") expect(text_el).to_be_visible() expect(text_el).to_contain_text(THINKING_FRAGMENT) expect(text_el).to_contain_text("nothing is invented") # Closed again — user control in both directions (the container's # summary hides the always-visible block with it). container.locator("summary.reasoning-summary").click() expect(container).not_to_have_attribute("open") expect(text_el).not_to_be_visible() # --------------------------------------------------------------------------- # 3. Persistence: the thinking block (and its text) survives a reload, # restored COLLAPSED — phase-14 restore path + phase-17 field # --------------------------------------------------------------------------- def test_thinking_restored_after_reload(page: Page, app_url: str, seeded_kb: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") send_and_wait(page, THINK_QUESTION) # The live block is a plain div (the D5 refinement — always visible # while the container is live-open); capture what it shows and what # the turn persisted (raw text, same as what was rendered). block = page.locator(".msg.brain").last.locator("div.thinking") expect(block).to_have_count(1) captured = block.locator(".thinking-text").text_content() assert captured # The persisted raw text is the same scratchpad (renderMarkdown turns # the line breaks into
, which textContent drops — compare without # whitespace). raw = json.loads( page.evaluate(f"() => localStorage.getItem('{STORAGE_KEY}')") )["messages"][1]["thinking"] assert re.sub(r"\s+", "", raw) == re.sub(r"\s+", "", captured) page.reload() expect(page.locator("#empty-state")).to_be_hidden() restored = page.locator(".msg.brain").last.locator("div.thinking") expect(restored).to_have_count(1) # Restored COLLAPSED — the at-rest fold is the container's (the # D5 refinement — the inner block is not a
at all). restored_wrap = page.locator(".msg.brain").last expect( restored_wrap.locator("details.reasoning") ).not_to_have_attribute("open") expect(restored_wrap.locator("details.reasoning details")).to_have_count(0) expect(restored.locator(".thinking-text")).to_have_text(captured) # Answer bubble is intact after the restore (phase-14 restore path) # — with ZERO citation chips (phase 119, LOCKED A1: the zero-read # turn persisted an empty sources list; the retired phase-118 A4 # union is gone). expect(page.locator(".msg.brain .bubble").last).to_contain_text(MOCK_ANSWER_MARKER) expect(page.locator(".msg.brain .source-chip")).to_have_count(0) # --------------------------------------------------------------------------- # 4. No thinking, no block: a model/turn that emits no reasoning renders # exactly as before (no layout regression) # --------------------------------------------------------------------------- def test_no_thinking_block_without_trigger(page: Page, app_url: str, seeded_kb: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") send_and_wait(page, PLAIN_QUESTION) # No trigger → no thinking events → no block anywhere on the page. expect(page.locator(".thinking")).to_have_count(0) # The turn itself is complete and grounded, exactly as before phase # 17 — with ZERO citation chips (phase 119, LOCKED A1: the zero-read # turn chips nothing; the retired phase-118 A4 union is gone). expect(page.locator(".msg.brain .source-chip")).to_have_count(0) # --------------------------------------------------------------------------- # 5. Coexistence: the honesty gate (deflection) and the thinking block # on the same turn # --------------------------------------------------------------------------- def test_thinking_with_deflection(page: Page, app_url: str, seeded_kb: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/") send_and_wait(page, THINK_DEFLECT_QUESTION) last = page.locator(".msg.brain").last # The honesty gate fired: amber deflected bubble + "Maybe try" chips. expect(last).to_have_class(re.compile(r"is-deflected")) expect(last.locator(".bubble")).to_contain_text( re.compile(DEFLECT_PHRASE, re.IGNORECASE) ) chips = last.locator(".maybe-try .suggestion-chip") expect(chips.first).to_be_visible() assert chips.count() >= 2 # And the thinking block came along — a plain div (the D5 # refinement — not collapsible; the container is the ONE fold) — # with its scratchpad. block = last.locator("div.thinking") expect(block).to_have_count(1) expect(last.locator("details.reasoning details")).to_have_count(0) expect(block.locator(".thinking-text")).to_contain_text(THINKING_FRAGMENT)