282 lines
12 KiB
Python
282 lines
12 KiB
Python
"""Phase 17 E2E (Playwright, mock-only): the model's "thinking" display.
|
||
|
||
Story: ``.agent/user_stories/thinking-display.md``
|
||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||
|
||
uv run pytest tests/e2e/test_thinking_display.py -v --no-cov
|
||
|
||
MOCK-ONLY suite: ``E2E_REAL_LLM=1`` is not supported here. The real
|
||
``turbo`` thinks on *every* turn, which would break the no-thinking
|
||
regression test (scenario 4) — the deterministic mock's ``think out loud``
|
||
trigger (mock_llm.py) keeps all five scenarios reproducible.
|
||
|
||
Test → story mapping (Playwright Mapping Rule):
|
||
1. ``test_thinking_block_streams_open_then_collapses``
|
||
2. ``test_thinking_toggle_after_done``
|
||
3. ``test_thinking_restored_after_reload``
|
||
4. ``test_no_thinking_block_without_trigger``
|
||
5. ``test_thinking_with_deflection``
|
||
|
||
Determinism note: the mock paces every SSE frame at 0.02s and the thinking
|
||
text is ~700–900 chars (≈ 60–75 frames ≈ 1.2–1.5s) before the first
|
||
content frame, so "attach → assert open" runs well inside the open window
|
||
on headless Chromium; all other assertions are made after the send button
|
||
re-enables (fully settled state).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import re
|
||
from collections.abc import Iterator
|
||
from pathlib import Path
|
||
from threading import Thread
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from playwright.sync_api import Page, expect
|
||
from sqlalchemy import text
|
||
|
||
from app.config import Settings
|
||
from app.db import SessionLocal
|
||
from app.rag.importer import ImportSummary, import_sources
|
||
from app.rag.llm import LLMClient
|
||
|
||
REPO = Path(__file__).resolve().parents[2]
|
||
FIXTURES = REPO / "tests" / "fixtures" / "docs"
|
||
THINK_QUESTION = "think out loud — how is my kubernetes cluster set up?"
|
||
PLAIN_QUESTION = "How is my Kubernetes cluster set up?"
|
||
THINK_DEFLECT_QUESTION = "think out loud — tell me about quantum wormhole cooling"
|
||
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
|
||
DEFLECT_PHRASE = r"haven't done anything like that"
|
||
#: Line fragment the mock's deterministic scratchpad must carry — the
|
||
#: suite keys off it (mock_llm.compose_thinking).
|
||
THINKING_FRAGMENT = "Step 2: Check my notes"
|
||
STORAGE_KEY = "bor.chat.v1"
|
||
#: Phase-10 viewer URL + phase-13 back=/ (byte-identical to the chip the
|
||
#: persistence suite pins — grounded-turn sources are unchanged by 17).
|
||
CHIP_HREF = "/document.html?source=docs&path=homelab%2Fkubernetes.md&back=%2F"
|
||
|
||
|
||
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
|
||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||
return await import_sources([FIXTURES], LLMClient(settings))
|
||
|
||
|
||
def _run_in_thread(coro: Any) -> Any:
|
||
"""Run a coroutine on a worker thread.
|
||
|
||
Playwright's sync API keeps an asyncio loop running on the test thread,
|
||
so ``asyncio.run`` cannot be called directly from a test body.
|
||
"""
|
||
box: dict[str, Any] = {}
|
||
|
||
def runner() -> None:
|
||
try:
|
||
box["value"] = asyncio.run(coro)
|
||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||
box["error"] = e
|
||
|
||
t = Thread(target=runner)
|
||
t.start()
|
||
t.join()
|
||
if "error" in box:
|
||
raise box["error"]
|
||
return box["value"]
|
||
|
||
|
||
def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
|
||
"""Truncate the KB (and query log), then optionally re-import fixtures."""
|
||
with SessionLocal() as db:
|
||
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
||
db.commit()
|
||
if not seed:
|
||
return None
|
||
return _run_in_thread(_import_fixtures(mock_port))
|
||
|
||
|
||
@pytest.fixture()
|
||
def seeded_kb(mock_llm: int, db_ready: None) -> Iterator[None]:
|
||
"""A fresh KB seeded from ``tests/fixtures/docs`` (8 docs, A9 formats),
|
||
truncated again on teardown. ``db_ready`` (conftest) skips with clear
|
||
instructions when Postgres is down."""
|
||
summary = _reset_db(mock_llm, seed=True)
|
||
assert summary is not None and summary.added == 8
|
||
yield
|
||
_reset_db(mock_llm, seed=False)
|
||
|
||
|
||
def send_and_wait(page: Page, question: str) -> None:
|
||
"""Type into #message-input, submit via #composer, then wait until the
|
||
last brain message settles (send button re-enabled, label "Send")."""
|
||
page.fill("#message-input", question)
|
||
page.evaluate("() => document.querySelector('#composer').requestSubmit()")
|
||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||
# The mock streams at 0.02s/chunk, so thinking + answer land in a few
|
||
# seconds — 30s is generous on headless Chromium.
|
||
expect(page.locator(".msg.brain .bubble").last).not_to_have_text("", timeout=30_000)
|
||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||
expect(page.locator("#send-label")).to_have_text("Send")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 1. Streaming: the block attaches OPEN at the first thinking event, then
|
||
# auto-collapses when the first answer token lands
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_thinking_block_streams_open_then_collapses(
|
||
page: Page, app_url: str, seeded_kb: None
|
||
) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
page.fill("#message-input", THINK_QUESTION)
|
||
page.evaluate("() => document.querySelector('#composer').requestSubmit()")
|
||
expect(page.locator(".msg.user .bubble").last).to_contain_text(THINK_QUESTION)
|
||
|
||
# The block attaches at the FIRST thinking event — before any answer
|
||
# token — and is created OPEN.
|
||
details = page.locator(".msg.brain").last.locator("details.thinking")
|
||
details.wait_for(state="attached", timeout=10_000)
|
||
# The ~800-char thinking stream (≈1.3s) keeps the block open right
|
||
# after attach — assert while it is still streaming.
|
||
expect(details).to_have_attribute("open", "")
|
||
expect(details.locator(".thinking-text")).not_to_have_text("")
|
||
|
||
# First answer token: the block auto-collapses and stays closed.
|
||
bubble = page.locator(".msg.brain .bubble").last
|
||
expect(bubble).not_to_have_text("", timeout=30_000)
|
||
expect(details).not_to_have_attribute("open")
|
||
|
||
# Settled: full scratchpad, grounded mock answer, source chip(s),
|
||
# and the re-enabled send button.
|
||
expect(details.locator(".thinking-text")).to_contain_text(THINKING_FRAGMENT)
|
||
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
|
||
chip = page.locator(".msg.brain .source-chip", has_text="kubernetes.md")
|
||
expect(chip.first).to_be_visible()
|
||
expect(chip.first).to_have_attribute("href", CHIP_HREF)
|
||
expect(page.locator("#send-btn")).to_be_enabled()
|
||
expect(page.locator("#send-label")).to_have_text("Send")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 2. Toggle: after settle the block is closed; the summary re-opens it
|
||
# (a real keyboard-focusable control) and closes it again
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_thinking_toggle_after_done(page: Page, app_url: str, seeded_kb: None) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
send_and_wait(page, THINK_QUESTION)
|
||
|
||
last = page.locator(".msg.brain").last
|
||
details = last.locator("details.thinking")
|
||
expect(details).to_have_count(1)
|
||
expect(details).not_to_have_attribute("open") # auto-collapsed at first token
|
||
|
||
# The summary is a real, keyboard-focusable control.
|
||
details.locator("summary").focus()
|
||
assert page.evaluate("() => document.activeElement.tagName") == "SUMMARY"
|
||
|
||
# Open: the full scratchpad is visible.
|
||
details.locator("summary").click()
|
||
expect(details).to_have_attribute("open", "")
|
||
text_el = details.locator(".thinking-text")
|
||
expect(text_el).to_be_visible()
|
||
expect(text_el).to_contain_text(THINKING_FRAGMENT)
|
||
expect(text_el).to_contain_text("nothing is invented")
|
||
|
||
# Closed again — user control in both directions.
|
||
details.locator("summary").click()
|
||
expect(details).not_to_have_attribute("open")
|
||
expect(text_el).not_to_be_visible()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3. Persistence: the thinking block (and its text) survives a reload,
|
||
# restored COLLAPSED — phase-14 restore path + phase-17 field
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_thinking_restored_after_reload(page: Page, app_url: str, seeded_kb: None) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
send_and_wait(page, THINK_QUESTION)
|
||
|
||
# The live block is collapsed; capture what it shows and what the
|
||
# turn persisted (raw text, same as what was rendered).
|
||
details = page.locator(".msg.brain").last.locator("details.thinking")
|
||
expect(details).not_to_have_attribute("open")
|
||
captured = details.locator(".thinking-text").text_content()
|
||
assert captured
|
||
# The persisted raw text is the same scratchpad (renderMarkdown turns
|
||
# the line breaks into <br>, which textContent drops — compare without
|
||
# whitespace).
|
||
raw = json.loads(
|
||
page.evaluate(f"() => localStorage.getItem('{STORAGE_KEY}')")
|
||
)["messages"][1]["thinking"]
|
||
assert re.sub(r"\s+", "", raw) == re.sub(r"\s+", "", captured)
|
||
|
||
page.reload()
|
||
expect(page.locator("#empty-state")).to_be_hidden()
|
||
|
||
restored = page.locator(".msg.brain").last.locator("details.thinking")
|
||
expect(restored).to_have_count(1)
|
||
expect(restored).not_to_have_attribute("open") # restored COLLAPSED
|
||
expect(restored.locator(".thinking-text")).to_have_text(captured)
|
||
|
||
# Answer bubble + source chip are intact (phase-14 restore path).
|
||
expect(page.locator(".msg.brain .bubble").last).to_contain_text(MOCK_ANSWER_MARKER)
|
||
chip = page.locator(".msg.brain .source-chip", has_text="kubernetes.md")
|
||
expect(chip.first).to_have_attribute("href", CHIP_HREF)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 4. No thinking, no block: a model/turn that emits no reasoning renders
|
||
# exactly as before (no layout regression)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_no_thinking_block_without_trigger(page: Page, app_url: str, seeded_kb: None) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
send_and_wait(page, PLAIN_QUESTION)
|
||
|
||
# No trigger → no thinking events → no block anywhere on the page.
|
||
expect(page.locator("details.thinking")).to_have_count(0)
|
||
|
||
# The turn itself is complete and grounded, exactly as before phase 17.
|
||
chip = page.locator(".msg.brain .source-chip", has_text="kubernetes.md")
|
||
expect(chip.first).to_be_visible()
|
||
expect(chip.first).to_have_attribute("href", CHIP_HREF)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 5. Coexistence: the honesty gate (deflection) and the thinking block
|
||
# on the same turn
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_thinking_with_deflection(page: Page, app_url: str, seeded_kb: None) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
send_and_wait(page, THINK_DEFLECT_QUESTION)
|
||
|
||
last = page.locator(".msg.brain").last
|
||
# The honesty gate fired: amber deflected bubble + "Maybe try" chips.
|
||
expect(last).to_have_class(re.compile(r"is-deflected"))
|
||
expect(last.locator(".bubble")).to_contain_text(
|
||
re.compile(DEFLECT_PHRASE, re.IGNORECASE)
|
||
)
|
||
chips = last.locator(".maybe-try .suggestion-chip")
|
||
expect(chips.first).to_be_visible()
|
||
assert chips.count() >= 2
|
||
|
||
# And the thinking block came along, closed, with its scratchpad.
|
||
details = last.locator("details.thinking")
|
||
expect(details).to_have_count(1)
|
||
expect(details).not_to_have_attribute("open")
|
||
expect(details.locator(".thinking-text")).to_contain_text(THINKING_FRAGMENT)
|