feat(ui): explicit chat state machine — typing indicator, streaming progress, timeout and error recovery

2026-08-21 18:45:27 -04:00
parent 2364e1ee7d
commit e5810b0bcf
8 changed files with 642 additions and 40 deletions
@@ -0,0 +1,311 @@
+"""Phase 06 E2E (Playwright): loading feedback & progress.
+
+Story: ``.agent/user_stories/loading-feedback.md``
+Run in isolation (DB must be up: ``podman compose up -d db``):
+
+    uv run pytest tests/e2e/test_loading_feedback.py -v --no-cov
+
+Determinism comes from two mock-LLM behaviors (tests/e2e/mock_llm.py):
+
+* ``pretend to think slowly`` in the user message → a 3s warm-up before
+  the first token, wide enough to assert the pre-token UI (typing dots +
+  busy "Thinking…" button) at a known timestamp;
+* the ``POST /__shutdown__`` hook → the ``llm_down`` fixture stops the
+  shared mock to simulate an LLM outage, then restores a fresh instance
+  on the same port so later tests keep working.
+
+The state machine under test lives in frontend/assets/app.js:
+``idle → thinking → streaming → done | error → idle`` (PLAN §7.4).
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+import subprocess
+import sys
+import time
+from collections.abc import Iterator
+from pathlib import Path
+from threading import Thread
+from typing import Any
+
+import httpx
+import pytest
+from playwright.sync_api import Page, expect
+from sqlalchemy import text
+
+from app.config import Settings
+from app.db import SessionLocal
+from app.rag.importer import ImportSummary, import_sources
+from app.rag.llm import LLMClient
+
+REPO = Path(__file__).resolve().parents[2]
+FIXTURES = REPO / "tests" / "fixtures" / "docs"
+MOCK_PORT = int(os.environ.get("E2E_MOCK_PORT", "8901"))
+
+# The mock warms up for 3s when the question contains this marker, so the
+# pre-token window is observable. (Retrieval may answer or deflect — the
+# feedback states are identical either way.)
+SLOW_QUESTION = "pretend to think slowly then tell me about kubernetes"
+ON_TOPIC = "How is my Kubernetes cluster set up?"
+
+TYPING = "#typing-indicator"
+# The typing indicator is itself a .msg.brain — exclude its bubble.
+ANSWER = ".msg.brain .bubble:not(.typing)"
+
+
+# --------------------------------------------------------------------------
+# KB seeding (same pattern as the phase 02/03 story suites)
+# --------------------------------------------------------------------------
+
+
+async def _import_fixtures(mock_port: int) -> ImportSummary:
+    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
+    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
+    return await import_sources([FIXTURES], LLMClient(settings))
+
+
+def _run_in_thread(coro: Any) -> Any:
+    """Run a coroutine on a worker thread.
+
+    Playwright's sync API keeps an asyncio loop running on the test thread,
+    so ``asyncio.run`` cannot be called directly from a test body.
+    """
+    box: dict[str, Any] = {}
+
+    def runner() -> None:
+        try:
+            box["value"] = asyncio.run(coro)
+        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
+            box["error"] = e
+
+    t = Thread(target=runner)
+    t.start()
+    t.join()
+    if "error" in box:
+        raise box["error"]
+    return box["value"]
+
+
+def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
+    """Truncate the KB (and query log), then optionally re-import fixtures."""
+    with SessionLocal() as db:
+        db.execute(text("TRUNCATE chunks, documents, query_log"))
+        db.commit()
+    if not seed:
+        return None
+    return _run_in_thread(_import_fixtures(mock_port))
+
+
+# --------------------------------------------------------------------------
+# LLM-down fixture: stop the shared mock, restore it for later tests
+# --------------------------------------------------------------------------
+
+_replacement_mocks: list[subprocess.Popen] = []
+
+
+@pytest.fixture(scope="session")
+def _cleanup_replacement_mocks() -> Iterator[None]:
+    """Terminate mock instances spawned to replace a stopped one."""
+    yield
+    for proc in _replacement_mocks:
+        proc.terminate()
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+
+def _wait_http(url: str, timeout: float = 40.0) -> None:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        try:
+            httpx.get(url, timeout=2.0)
+            return
+        except Exception:  # noqa: BLE001 — retry until deadline
+            time.sleep(0.5)
+    raise RuntimeError(f"server at {url} did not come up")
+
+
+def _spawn_mock() -> subprocess.Popen:
+    env = dict(os.environ)
+    env.pop("DEBUGPY", None)
+    return subprocess.Popen(
+        [sys.executable, "-m", "uvicorn", "tests.e2e.mock_llm:app",
+         "--host", "127.0.0.1", "--port", str(MOCK_PORT), "--log-level", "warning"],
+        cwd=REPO,
+        env=env,
+    )
+
+
+@pytest.fixture()
+def llm_down(mock_llm: int, _cleanup_replacement_mocks: None) -> Iterator[None]:
+    """Simulate the LLM going down for one test (stop the shared mock via
+    its ``__shutdown__`` hook), then restore a fresh instance on the same
+    port so the rest of the session keeps working."""
+    r = httpx.post(f"http://127.0.0.1:{MOCK_PORT}/__shutdown__", timeout=10)
+    assert r.status_code == 200
+
+    stopped = False
+    deadline = time.monotonic() + 15
+    while time.monotonic() < deadline:
+        try:
+            httpx.get(f"http://127.0.0.1:{MOCK_PORT}/v1/models", timeout=1.0)
+        except Exception:  # noqa: BLE001 — connection refused == stopped
+            stopped = True
+            break
+        time.sleep(0.2)
+    assert stopped, "mock LLM did not stop in time"
+
+    yield
+
+    proc = _spawn_mock()
+    _replacement_mocks.append(proc)
+    _wait_http(f"http://127.0.0.1:{MOCK_PORT}/v1/models")
+
+
+# --------------------------------------------------------------------------
+# Tests (story → test mapping, .agent/user_stories/loading-feedback.md)
+# --------------------------------------------------------------------------
+
+
+def test_typing_indicator_during_slow_think(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """AC1/AC5: the 3s mock warm-up must show the typing indicator for
+    >=2s before any text appears, then it is gone once the answer lands."""
+    summary = _reset_db(mock_llm, seed=True)
+    assert summary is not None and summary.added == 3
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    page.fill("#message-input", SLOW_QUESTION)
+    page.click("#send-btn")
+
+    # Visible within 500ms of submit — the indicator is added synchronously
+    # by setUiState("thinking").
+    typing = page.locator(TYPING)
+    expect(typing).to_be_visible(timeout=500)
+    bubble = typing.locator(".bubble")
+    expect(bubble).to_have_attribute("role", "status")
+    expect(bubble).to_have_attribute("aria-label", "Brain of Reese is thinking")
+
+    # ~2s in: still pre-token (the mock is in its 3s warm-up).
+    page.wait_for_timeout(2000)
+    assert typing.is_visible(), "typing indicator must persist through the pre-token window"
+
+    # First token lands (~3s): dots gone, answer text present.
+    answer = page.locator(ANSWER)
+    answer.wait_for(state="visible", timeout=30_000)
+    expect(typing).to_be_hidden()
+    assert answer.inner_text().strip(), "answer bubble must contain text"
+
+
+def test_button_state_machine(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """AC1/AC3: disabled + spinner + 'Thinking…' while in flight; enabled
+    + 'Send' + focused input after done."""
+    _reset_db(mock_llm, seed=True)
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    page.fill("#message-input", SLOW_QUESTION)
+    page.click("#send-btn")
+
+    btn = page.locator("#send-btn")
+    label = page.locator("#send-label")
+    expect(btn).to_be_disabled(timeout=500)
+    expect(label).to_have_text("Thinking…")
+    expect(btn.locator(".spinner")).to_be_visible()
+    expect(page.locator("#send-status")).to_contain_text("thinking")
+
+    # Done: button recovers and the input is focused back.
+    page.locator(ANSWER).wait_for(state="visible", timeout=30_000)
+    expect(label).to_have_text("Send", timeout=30_000)
+    expect(btn).to_be_enabled()
+    expect(btn.locator(".spinner")).to_be_hidden()
+    expect(page.locator("#message-input")).to_be_focused()
+
+
+def test_streaming_appends_live(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """AC2: text appends live — a sample taken mid-stream must be strictly
+    shorter than a later one (no 'whole answer appears at once')."""
+    _reset_db(mock_llm, seed=True)
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    page.fill("#message-input", ON_TOPIC)
+    page.click("#send-btn")
+
+    answer = page.locator(ANSWER)
+    answer.wait_for(state="visible", timeout=30_000)
+    first = answer.inner_text()
+    assert first.strip(), "bubble should carry text as soon as it appears"
+
+    second = first
+    deadline = time.monotonic() + 15
+    while time.monotonic() < deadline:
+        second = answer.inner_text()
+        if len(second) > len(first):
+            break
+        time.sleep(0.05)
+    assert len(second) > len(first), (
+        "answer text never grew after the first sample — no live streaming visible"
+    )
+
+    # The turn settles: button back to 'Send', final text no shorter.
+    expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
+    final = answer.inner_text()
+    assert len(final) >= len(second)
+
+
+def test_reduced_motion_keeps_feedback(
+    page: Page, app_url: str, mock_llm: int, db_ready: None
+) -> None:
+    """AC7: under prefers-reduced-motion the dots/spinner stay visible
+    (static/slower) — feedback is never removed, only calmed."""
+    _reset_db(mock_llm, seed=True)
+    page.emulate_media(reduced_motion="reduce")
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    page.fill("#message-input", SLOW_QUESTION)
+    page.click("#send-btn")
+
+    expect(page.locator(TYPING)).to_be_visible(timeout=500)
+    expect(page.locator(TYPING).locator(".bubble span")).to_have_count(3)
+    expect(page.locator("#send-btn .spinner")).to_be_visible()
+
+    # And the turn still completes and recovers.
+    expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)
+    expect(page.locator("#send-btn")).to_be_enabled()
+    expect(page.locator(TYPING)).to_be_hidden()
+
+
+def test_error_banner_on_llm_down(
+    page: Page, app_url: str, mock_llm: int, db_ready: None, llm_down: None
+) -> None:
+    """AC4a: with the LLM down the turn ends in the red role=alert banner
+    with the actionable retry hint, and the button recovers (never a
+    zombie)."""
+    page.set_default_timeout(30_000)
+    page.goto(app_url)
+
+    page.fill("#message-input", ON_TOPIC)
+    page.click("#send-btn")
+
+    banner = page.locator("#kb-banner")
+    expect(banner).to_be_visible(timeout=30_000)
+    expect(banner).to_have_attribute("role", "alert")
+    expect(banner).to_have_class(re.compile(r"kb-banner.*is-error"))
+    expect(banner).to_contain_text("Try again")
+    expect(banner).to_contain_text("check the LLM is reachable")
+
+    # Recovery: button enabled, label 'Send', no lingering indicator.
+    expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
+    expect(page.locator("#send-label")).to_have_text("Send")
+    expect(page.locator(TYPING)).to_be_hidden()