357 lines
14 KiB
Python
357 lines
14 KiB
Python
"""Phase 18 E2E (Playwright, mock-only): the chat follows the bottom.
|
||
|
||
Story: ``.agent/user_stories/follow-bottom-scroll.md``
|
||
Run in isolation (DB must be up: ``podman compose up -d db``):
|
||
|
||
uv run pytest tests/e2e/test_follow_bottom_scroll.py -v --no-cov
|
||
|
||
The follow-the-bottom contract (owner choice 2026-08-23, option 1 — no
|
||
"↓ new content" pill): the page auto-scrolls *only while the user is
|
||
already pinned at the bottom*; submitting a question reveals the user's
|
||
own message; once the user scrolls up, nothing auto-scrolls for the rest
|
||
of the turn (thinking or answer); a restored conversation still lands on
|
||
the latest message.
|
||
|
||
MOCK-ONLY suite: the scenarios key off the deterministic mock's
|
||
``write a long answer`` trigger (~900 words ≈ 8s of streaming — a wide,
|
||
reliable window to scroll away in) and, for scenario 4, the phase-17
|
||
``think out loud`` trigger (both fire independently). ``E2E_REAL_LLM=1``
|
||
would make the scroll-away windows unpredictable, so it is not supported
|
||
here.
|
||
|
||
Measurement convention: the scroller is the DOCUMENT — there is no inner
|
||
scroll container (``body`` is ``min-height: 100dvh``; the page scrolls on
|
||
the window). Scroll position is read via ``page.evaluate`` as
|
||
``{ y: window.scrollY, sh: document.documentElement.scrollHeight,
|
||
ch: window.innerHeight }``; "near bottom" = ``sh - y - ch <= 200``
|
||
(mirrors the frontend's ``NEAR_BOTTOM_PX``); scrolling to the top is
|
||
``page.evaluate("() => window.scrollTo(0, 0)")``.
|
||
|
||
Real-user flow: the user submits from the composer — i.e. pinned at the
|
||
bottom (a normal ``fill`` + ``Enter``/click) — and only *after* the
|
||
stream starts do they scroll up to read earlier messages. The no-yank
|
||
scenarios follow exactly that sequence, so no off-screen input
|
||
manipulation is needed (and Playwright's own click/fill auto-scroll
|
||
never fires, because the composer is already in view).
|
||
|
||
Determinism note: the mock paces every SSE frame at 0.02s, so the long
|
||
answer streams for several seconds — "mid-stream" assertions land
|
||
comfortably inside the window on headless Chromium. Every "held still"
|
||
assertion compares against the exact ``scrollTo(0, 0)`` position
|
||
(tolerance 5px for rounding).
|
||
|
||
Test → story mapping (Playwright Mapping Rule):
|
||
1. ``test_submit_reveals_new_message``
|
||
2. ``test_stream_follows_while_pinned_at_bottom``
|
||
3. ``test_no_yank_while_scrolled_up_during_answer_stream``
|
||
4. ``test_no_yank_while_scrolled_up_during_thinking``
|
||
5. ``test_restore_lands_on_latest_message``
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import time
|
||
from collections.abc import Iterator
|
||
from pathlib import Path
|
||
from threading import Thread
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from playwright.sync_api import Page, expect
|
||
from sqlalchemy import text
|
||
|
||
from app.config import Settings
|
||
from app.db import SessionLocal
|
||
from app.rag.importer import ImportSummary, import_sources
|
||
from app.rag.llm import LLMClient
|
||
|
||
REPO = Path(__file__).resolve().parents[2]
|
||
FIXTURES = REPO / "tests" / "fixtures" / "docs"
|
||
|
||
#: Mirror of app.js's exported ``NEAR_BOTTOM_PX`` (the 200px composer-zone
|
||
#: band that counts as "pinned to the bottom").
|
||
NEAR_BOTTOM_PX = 200
|
||
|
||
#: Mock long-answer trigger (~900 words ≈ 8s of streaming at the mock's
|
||
#: 0.02s/frame pace) — the wide, deterministic window to scroll away in.
|
||
LONG_QUESTION = "write a long answer about my kubernetes cluster"
|
||
#: Phase-17 thinking prefix + the long-answer trigger: both mock triggers
|
||
#: fire independently (a ~4.5s reasoning stream — lengthened in phase 21 —
|
||
#: then the long answer).
|
||
THINK_LONG_QUESTION = "think out loud — write a long answer about my kubernetes cluster"
|
||
#: The mock long answer's unique final line (mock_llm.LONG_ANSWER_END) —
|
||
#: proves the whole stream landed even while the viewport was at the top.
|
||
LONG_ANSWER_END = "LONG-ANSWER-END"
|
||
#: Line fragment the mock's deterministic scratchpad carries
|
||
#: (mock_llm.compose_thinking) — same key phase 17's suite uses.
|
||
THINKING_FRAGMENT = "Step 2: Check my notes"
|
||
#: Tolerance for "the viewport held still at the top" (rounding).
|
||
HOLD_TOLERANCE_PX = 5
|
||
|
||
|
||
async def _import_fixtures(mock_port: int) -> ImportSummary:
|
||
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
|
||
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
|
||
return await import_sources([FIXTURES], LLMClient(settings))
|
||
|
||
|
||
def _run_in_thread(coro: Any) -> Any:
|
||
"""Run a coroutine on a worker thread.
|
||
|
||
Playwright's sync API keeps an asyncio loop running on the test thread,
|
||
so ``asyncio.run`` cannot be called directly from a test body.
|
||
"""
|
||
box: dict[str, Any] = {}
|
||
|
||
def runner() -> None:
|
||
try:
|
||
box["value"] = asyncio.run(coro)
|
||
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
|
||
box["error"] = e
|
||
|
||
t = Thread(target=runner)
|
||
t.start()
|
||
t.join()
|
||
if "error" in box:
|
||
raise box["error"]
|
||
return box["value"]
|
||
|
||
|
||
def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
|
||
"""Truncate the KB (and query log), then optionally re-import fixtures."""
|
||
with SessionLocal() as db:
|
||
db.execute(text("TRUNCATE chunks, documents, query_log"))
|
||
db.commit()
|
||
if not seed:
|
||
return None
|
||
return _run_in_thread(_import_fixtures(mock_port))
|
||
|
||
|
||
@pytest.fixture()
|
||
def seeded_kb(mock_llm: int, db_ready: None) -> Iterator[None]:
|
||
"""A fresh KB seeded from ``tests/fixtures/docs`` (8 docs, A9 formats),
|
||
truncated again on teardown. ``db_ready`` (conftest) skips with clear
|
||
instructions when Postgres is down."""
|
||
summary = _reset_db(mock_llm, seed=True)
|
||
assert summary is not None and summary.added == 8
|
||
yield
|
||
_reset_db(mock_llm, seed=False)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Measurement + flow helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def scroll_state(page: Page) -> dict[str, float]:
|
||
"""The document scroller's state (there is no inner scroll container)."""
|
||
return page.evaluate(
|
||
"() => ({ y: window.scrollY, "
|
||
"sh: document.documentElement.scrollHeight, "
|
||
"ch: window.innerHeight })"
|
||
)
|
||
|
||
|
||
def near_bottom(state: dict[str, float]) -> bool:
|
||
"""Mirror of app.js's ``isNearBottom`` — the NEAR_BOTTOM_PX band."""
|
||
return state["sh"] - state["y"] - state["ch"] <= NEAR_BOTTOM_PX
|
||
|
||
|
||
def held_at_top(page: Page) -> bool:
|
||
"""The viewport has not moved from ``window.scrollTo(0, 0)`` (±5px)."""
|
||
return scroll_state(page)["y"] <= HOLD_TOLERANCE_PX
|
||
|
||
|
||
def wait_settled(page: Page) -> None:
|
||
"""The turn is over: the never-stale contract re-enabled the button."""
|
||
expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
|
||
expect(page.locator("#send-label")).to_have_text("Send")
|
||
|
||
|
||
def submit(page: Page, question: str) -> None:
|
||
"""Submit from the composer — the real-user flow (pinned at the
|
||
bottom, so Playwright's click/fill auto-scroll never kicks in)."""
|
||
page.fill("#message-input", question)
|
||
page.click("#send-btn")
|
||
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
|
||
|
||
|
||
def brain_bubble_longer_than(n: int) -> str:
|
||
"""JS predicate: the LAST brain bubble's rendered text is > n chars
|
||
(i.e. that far into the stream)."""
|
||
return (
|
||
"() => { const els = document.querySelectorAll('.msg.brain .bubble');"
|
||
f" const el = els[els.length - 1]; return !!el && el.innerText.length > {n}; }}"
|
||
)
|
||
|
||
|
||
def brain_message_in_view(page: Page) -> bool:
|
||
"""The last brain message intersects the viewport vertically. Partial
|
||
visibility counts: a long answer is taller than the window, and the
|
||
contract is that it is revealed (its lower edge in view), not that it
|
||
fits."""
|
||
box = page.locator(".msg.brain").last.bounding_box()
|
||
if box is None:
|
||
return False
|
||
ch = scroll_state(page)["ch"]
|
||
return box["y"] < ch and box["y"] + box["height"] > 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 1. Submit: the user's message and the answer reveal into view
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_submit_reveals_new_message(page: Page, app_url: str, seeded_kb: None) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
# A fresh chat page starts pinned at the bottom (short conversation —
|
||
# the composer, i.e. the user, sits in the band).
|
||
assert near_bottom(scroll_state(page))
|
||
|
||
submit(page, LONG_QUESTION)
|
||
wait_settled(page)
|
||
|
||
# The last brain message is inside the viewport ...
|
||
assert brain_message_in_view(page), (
|
||
"the answer must be revealed — the last brain message is not in view"
|
||
)
|
||
# ... and the page is still pinned at the bottom.
|
||
assert near_bottom(scroll_state(page))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 2. Follow: while pinned, the page keeps up with the stream
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_stream_follows_while_pinned_at_bottom(page: Page, app_url: str, seeded_kb: None) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
submit(page, LONG_QUESTION)
|
||
|
||
# ~2s into the stream: the answer bubble already carries >200 chars
|
||
# (the mock paces frames at 0.02s).
|
||
page.wait_for_function(brain_bubble_longer_than(200), timeout=30_000)
|
||
# Let the smooth follow scroll settle before measuring.
|
||
time.sleep(0.3)
|
||
# The follow behavior is alive — not accidentally removed.
|
||
assert near_bottom(scroll_state(page)), (
|
||
"the page must follow the stream while the user is pinned at the bottom"
|
||
)
|
||
|
||
wait_settled(page)
|
||
assert near_bottom(scroll_state(page))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3. No yank: scrolled up mid-ANSWER — the viewport holds for the rest
|
||
# of the turn (the answer finishes off-screen below, by design)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_no_yank_while_scrolled_up_during_answer_stream(
|
||
page: Page, app_url: str, seeded_kb: None
|
||
) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
|
||
# Turn 1 (settled) makes the document overflow the 800px viewport.
|
||
submit(page, LONG_QUESTION)
|
||
wait_settled(page)
|
||
state = scroll_state(page)
|
||
assert state["sh"] > state["ch"], "a long answer must make the document scrollable"
|
||
assert near_bottom(state), "follow was active: the settled turn ends pinned"
|
||
|
||
# Turn 2: submit from the composer (pinned — normal flow), then let
|
||
# the new answer stream a bit.
|
||
submit(page, LONG_QUESTION)
|
||
page.wait_for_function(brain_bubble_longer_than(200), timeout=30_000)
|
||
|
||
# The user goes up to read while the stream is running.
|
||
page.evaluate("() => window.scrollTo(0, 0)")
|
||
# The stream kept running at the top ...
|
||
page.wait_for_function(brain_bubble_longer_than(600), timeout=30_000)
|
||
assert held_at_top(page), "the viewport must hold still while scrolled up"
|
||
|
||
# ... and nothing scrolls for the rest of the turn — the answer
|
||
# finishes off-screen below, by design.
|
||
wait_settled(page)
|
||
assert held_at_top(page)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 4. No yank: scrolled up during THINKING — the whole reasoning stream
|
||
# plus the answer's start happen at the top
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_no_yank_while_scrolled_up_during_thinking(
|
||
page: Page, app_url: str, seeded_kb: None
|
||
) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
|
||
# One settled turn first, so the document overflows (scrollable).
|
||
submit(page, LONG_QUESTION)
|
||
wait_settled(page)
|
||
|
||
# The thinking turn: submit pinned (normal flow) ...
|
||
submit(page, THINK_LONG_QUESTION)
|
||
details = page.locator(".msg.brain").last.locator("details.thinking")
|
||
details.wait_for(state="attached", timeout=10_000)
|
||
# ... and, while the reasoning stream is still open (phase-17
|
||
# behavior: created open, ~1.3s before the first answer token) ...
|
||
expect(details).to_have_attribute("open", "")
|
||
# ... the user goes up to read.
|
||
page.evaluate("() => window.scrollTo(0, 0)")
|
||
|
||
# The whole thinking stream plus the answer's start happen at the top.
|
||
bubble = page.locator(".msg.brain").last.locator(".bubble")
|
||
expect(bubble).not_to_have_text("", timeout=30_000)
|
||
assert held_at_top(page), "the viewport must hold still during thinking"
|
||
|
||
# Settled: still at the top, and everything landed (off-screen,
|
||
# which is the point of the story).
|
||
wait_settled(page)
|
||
assert held_at_top(page)
|
||
expect(details.locator(".thinking-text")).to_contain_text(THINKING_FRAGMENT)
|
||
expect(bubble).to_contain_text(LONG_ANSWER_END)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 5. Restore: the one-shot landing still puts the latest message in view
|
||
# (phase 14 behavior preserved — pinned so a future "remove all
|
||
# scrolling" change fails loudly instead of silently)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_restore_lands_on_latest_message(page: Page, app_url: str, seeded_kb: None) -> None:
|
||
page.set_default_timeout(30_000)
|
||
page.goto(app_url)
|
||
|
||
# Two settled turns (user + brain × 2) — the document overflows.
|
||
submit(page, LONG_QUESTION)
|
||
wait_settled(page)
|
||
submit(page, LONG_QUESTION)
|
||
wait_settled(page)
|
||
|
||
page.reload()
|
||
# Restore re-renders from localStorage; wait until the last restored
|
||
# brain answer is fully back.
|
||
expect(page.locator(".msg.brain .bubble").last).to_contain_text(
|
||
LONG_ANSWER_END, timeout=30_000
|
||
)
|
||
wait_settled(page)
|
||
expect(page.locator(".msg.user .bubble")).to_have_count(2)
|
||
state = scroll_state(page)
|
||
assert state["sh"] > state["ch"]
|
||
|
||
# The forced one-shot landing (the only `force`d scrolls) puts the
|
||
# last brain message back in view ...
|
||
assert brain_message_in_view(page), (
|
||
"a restored conversation must land on its latest message"
|
||
)
|
||
# ... and the page sits at the bottom.
|
||
assert near_bottom(state)
|