"""Phase 49 E2E (Playwright): retry the last answer (redo in place). Source: ``TODO.md`` L4 — "Need a retry button to retry the last answer, like a redo button" (no user story file — TODO-derived phase). Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_retry_answer.py -v --no-cov Mock-only, no admin login (chat is public — Retry is NOT admin-gated, unlike Tune). The owner-locked contract (2026-08-29) under test: * the LAST brain bubble (completed, deflected, or a phase-48 stopped partial) carries the Retry button in its meta row; earlier brain bubbles do not (``markLastRetryable`` is last-bubble-only); * clicking Retry redoes the turn IN PLACE — the old answer leaves the DOM and the persisted ``bor.chat.v1`` record (the pop is saved before the rerun, so a crash never resurrects the replaced answer), the preceding question is re-asked WITHOUT being duplicated, and the fresh answer streams into the old bubble's place; the new bubble is the new last and carries the Retry button; * a stopped partial is the prime retry candidate: the redo replaces it with a fresh ``done`` answer (no ``stopped`` marker, the full text — the long answer's unique final line arrives); * Retry is inert while a turn is in flight — no second turn starts, no bubble or record duplication, the in-flight turn completes to its own ``done``. Determinism: the mock quotes the asked question into its grounded answer (ending in the ``Deterministic mock answer for E2E`` marker), the deflection answer is a fixed honest phrase, and the long answer ("write a long answer" trigger) streams ~900 words over ~8 s (12 chars / 0.02 s) with a unique final line (``LONG-ANSWER-END``) absent from any partial. OLD-vs-fresh bubble identity is proved with a test-only ``data-retry-marker`` attribute set on the old wrap before the click — the mock answers are byte-stable, so a redo of the same question is textually indistinguishable from the original and only the DOM element (and the storage record) can prove the replacement. """ from __future__ import annotations import asyncio import json import re import time from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import text from app.config import Settings from app.db import SessionLocal from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" #: Two grounded (on-topic) questions — the phase-03 phrasing and the #: phase-14 follow-up, both retrieve the kubernetes fixture doc. Q1 = "How is my Kubernetes cluster set up?" Q2 = "What about the nodes?" #: Phase-04 phrasing: no token overlap with the fixtures → the honesty #: gate is LOW → the deterministic deflection answer. OFF_TOPIC = "How do I bake sourdough bread?" #: On-topic + the mock's long-answer trigger (~900 words, ~8 s at the #: mock's pacing — the phase-48 stop window). LONG_QUESTION = "How is my Kubernetes cluster set up? write a long answer" #: Phase-06 phrasing: the mock's 3 s warm-up before the first token — #: a comfortable in-flight window to click the stale Retry button. SLOW_QUESTION = "pretend to think slowly then tell me about kubernetes" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" DEFLECT_PHRASE = r"haven't done anything like that" LONG_ANSWER_END = "LONG-ANSWER-END" STORAGE_KEY = "bor.chat.v1" #: The typing indicator is itself a .msg.brain — exclude its bubble. ANSWER = ".msg.brain .bubble:not(.typing)" #: The brain message wraps, in rendered order (the typing indicator is #: absent in every settled state these tests read at). BRAIN_WRAPS = "#messages > .msg.brain" # --------------------------------------------------------------------------- # KB seeding (same pattern as the phase 02/03/04/14/48 story suites) # --------------------------------------------------------------------------- async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int) -> ImportSummary: with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() return _run_in_thread(_import_fixtures(mock_port)) def _stored_parsed(page: Page) -> dict[str, Any]: raw = page.evaluate(f"() => localStorage.getItem('{STORAGE_KEY}')") assert raw is not None, "the conversation key must exist in localStorage" return json.loads(raw) def _assert_no_error_banner(page: Page) -> None: """A retry settles through the normal done path — never the red role=alert error banner (the KB-offline banner is a separate, health-driven state the db_ready fixture keeps away).""" banner = page.locator("#kb-banner") expect(banner).to_be_hidden() expect(banner).not_to_have_attribute("role", "alert") expect(banner).not_to_have_class(re.compile(r"is-error")) def _tag_last_brain_wrap(page: Page, marker: str) -> None: """Tag the rendered wrap of the LAST brain bubble (settled state — no typing indicator present) so the test can prove the OLD element leaves the DOM: the mock answers are byte-stable, so a redo of the same question is textually indistinguishable from the original.""" page.evaluate( """(marker) => { const wraps = document.querySelectorAll("#messages > .msg.brain"); wraps[wraps.length - 1].setAttribute("data-retry-marker", marker); }""", marker, ) # --------------------------------------------------------------------------- # Shared flows # --------------------------------------------------------------------------- def _ask(page: Page, question: str) -> None: """Send one grounded turn and wait until the answer has fully landed (marker = last chunk streamed; "Send" = the done save point settled).""" page.fill("#message-input", question) page.click("#send-btn") expect(page.locator(".msg.user .bubble").last).to_contain_text(question) expect(page.locator(ANSWER).last).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") def _ask_deflected(page: Page, question: str) -> None: """Send an off-topic turn and wait until the deflected answer landed.""" page.fill("#message-input", question) page.click("#send-btn") bubble = page.locator(".msg.brain.is-deflected .bubble").first bubble.wait_for(state="visible", timeout=30_000) expect(bubble).to_contain_text(re.compile(DEFLECT_PHRASE, re.IGNORECASE), timeout=30_000) expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") def _stop_long_answer_mid_stream(page: Page) -> str: """Ask the long on-topic question and Stop it once a few words of answer have streamed (the phase-48 flow). Returns the rendered partial text after the stop settles.""" page.fill("#message-input", LONG_QUESTION) page.click("#send-btn") answer = page.locator(ANSWER) answer.wait_for(state="visible", timeout=30_000) partial = "" deadline = time.monotonic() + 15 while time.monotonic() < deadline: partial = answer.inner_text() if len(partial.split()) >= 8: break time.sleep(0.05) assert len(partial.split()) >= 8, "no answer deltas before the stop" btn = page.locator("#send-btn") expect(btn).to_have_class(re.compile(r"is-stop")) btn.click() expect(page.locator("#send-label")).to_have_text("Send", timeout=5_000) _assert_no_error_banner(page) stopped_text = answer.inner_text() assert "Step 1:" in stopped_text assert LONG_ANSWER_END not in stopped_text return stopped_text # --------------------------------------------------------------------------- # 1. Grounded redo: last-bubble-only button, redo in place, no duplicate # --------------------------------------------------------------------------- def test_retry_redoes_in_place( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: summary = _reset_db(mock_llm) assert summary is not None and summary.added == 13 # A9 formats (phase 47 added quadlet+j2) page.set_default_timeout(30_000) page.goto(app_url) # Two grounded turns: the conversation is [u1, b1, u2, b2]. _ask(page, Q1) _ask(page, Q2) # Last-bubble-only: exactly ONE Retry button, on the LAST brain # bubble (the "Retry" text is the accessible name); the first brain # bubble carries none. expect(page.locator(".retry-btn")).to_have_count(1) expect(page.locator(".retry-btn").first).to_contain_text("Retry") first_wrap = page.locator(BRAIN_WRAPS).first last_wrap = page.locator(BRAIN_WRAPS).last expect(first_wrap.locator(".retry-btn")).to_have_count(0) expect(last_wrap.locator(".retry-btn")).to_have_count(1) # Tag the old last wrap: the redo must remove THIS element (the # fresh answer is textually identical — the mock is byte-stable). _tag_last_brain_wrap(page, "old-b2") page.locator(".retry-btn").click() # The redo is in flight immediately (the button is the Stop control), # then settles to the fresh answer. expect(page.locator("#send-label")).to_have_text("Stop", timeout=5_000) expect(page.locator("#send-btn")).to_have_class(re.compile(r"is-stop")) expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000) _assert_no_error_banner(page) # The old answer left the DOM (the tagged wrap is gone) and the fresh # answer streamed into its place: two brain bubbles, both grounded, # the first one intact (it still quotes Q1). expect(page.locator("[data-retry-marker='old-b2']")).to_have_count(0) expect(page.locator(ANSWER)).to_have_count(2) expect(page.locator(ANSWER).first).to_contain_text(Q1) expect(page.locator(ANSWER).last).to_contain_text(MOCK_ANSWER_MARKER) # The conversation never duplicated the question: exactly one user # bubble for the retried question, two in total. expect(page.locator(".msg.user .bubble")).to_have_count(2) expect(page.locator(".msg.user .bubble", has_text=Q2)).to_have_count(1) # Storage: [u1, b1, u2, b2'] — the retried question occurs once, the # last brain record is the fresh answer. msgs = _stored_parsed(page)["messages"] assert [m["who"] for m in msgs] == ["user", "brain", "user", "brain"] assert msgs[0]["text"] == Q1 assert msgs[2]["text"] == Q2 assert [m["text"] for m in msgs if m["who"] == "user"].count(Q2) == 1 assert MOCK_ANSWER_MARKER in msgs[1]["text"] assert MOCK_ANSWER_MARKER in msgs[3]["text"] # The new answer is the new last — it carries the Retry button, the # first bubble still does not. expect(page.locator(".retry-btn")).to_have_count(1) expect(page.locator(BRAIN_WRAPS).first.locator(".retry-btn")).to_have_count(0) expect(page.locator(BRAIN_WRAPS).last.locator(".retry-btn")).to_have_count(1) # --------------------------------------------------------------------------- # 2. A stopped partial (phase 48) redoes to a fresh full done answer # --------------------------------------------------------------------------- def test_retry_on_stopped_partial( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: _reset_db(mock_llm) page.set_default_timeout(30_000) page.goto(app_url) _stop_long_answer_mid_stream(page) # The stopped partial carries the "Stopped" note AND the Retry # button (the prime retry candidate — owner-locked). expect(page.locator(".msg.brain .stopped-note")).to_have_count(1) expect(page.locator(".retry-btn")).to_have_count(1) msgs = _stored_parsed(page)["messages"] assert [m["who"] for m in msgs] == ["user", "brain"] assert msgs[-1]["stopped"] is True assert LONG_ANSWER_END not in msgs[-1]["text"] _tag_last_brain_wrap(page, "old-partial") page.locator(".retry-btn").click() # The turn re-runs to a fresh `done` answer (the full long answer — # ~8 s of streaming at the mock's pacing). expect(page.locator("#send-label")).to_have_text("Stop", timeout=5_000) expect(page.locator("#send-label")).to_have_text("Send", timeout=60_000) _assert_no_error_banner(page) # The partial is gone — DOM, "Stopped" note, and stored marker — and # the fresh full answer stands in its place. expect(page.locator("[data-retry-marker='old-partial']")).to_have_count(0) expect(page.locator(".msg.brain .stopped-note")).to_have_count(0) bubble = page.locator(ANSWER) expect(bubble).to_have_count(1) text = bubble.inner_text() assert "Step 1:" in text assert LONG_ANSWER_END in text, "the redo ran to a fresh full done answer" # Storage: the stored partial is replaced — the last brain record # has no `stopped` flag and carries the full answer. msgs = _stored_parsed(page)["messages"] assert [m["who"] for m in msgs] == ["user", "brain"] assert msgs[-1].get("stopped") is not True assert LONG_ANSWER_END in msgs[-1]["text"] # The fresh answer is the new last brain bubble — it carries Retry. expect(page.locator(".retry-btn")).to_have_count(1) expect(page.locator(BRAIN_WRAPS).last.locator(".retry-btn")).to_have_count(1) # --------------------------------------------------------------------------- # 3. A deflected answer redoes: still deflected, chips re-rendered # --------------------------------------------------------------------------- def test_retry_deflected( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: _reset_db(mock_llm) page.set_default_timeout(30_000) page.goto(app_url) _ask_deflected(page, OFF_TOPIC) # The deflected bubble (with its Maybe-try chips; weak-hit source # chips ride along per A8) carries the Retry button. chips_before = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip") expect(chips_before.first).to_be_visible() assert chips_before.count() >= 2 expect(page.locator(".retry-btn")).to_have_count(1) _tag_last_brain_wrap(page, "old-deflect") page.locator(".retry-btn").click() expect(page.locator("#send-label")).to_have_text("Stop", timeout=5_000) expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000) _assert_no_error_banner(page) # Redo mechanics: the old bubble is gone, the fresh deflected bubble # stands in its place (still deflected — the mock is deterministic), # the question was not duplicated. expect(page.locator("[data-retry-marker='old-deflect']")).to_have_count(0) expect(page.locator(".msg.user .bubble")).to_have_count(1) expect(page.locator(".msg.user .bubble")).to_contain_text(OFF_TOPIC) fresh = page.locator(".msg.brain.is-deflected .bubble") expect(fresh).to_have_count(1) expect(fresh.first).to_contain_text(re.compile(DEFLECT_PHRASE, re.IGNORECASE)) # The "Maybe try" chips are re-rendered on the fresh bubble. chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip") assert chips.count() >= 2, "the redo must re-render the maybe-try chips" group = page.locator(".msg.brain.is-deflected .maybe-try") expect(group).to_have_count(1) expect(group.first).to_have_attribute("aria-label", "Maybe try") # The fresh bubble is the new last — it carries the Retry button. expect(page.locator(".retry-btn")).to_have_count(1) expect(page.locator(BRAIN_WRAPS).last.locator(".retry-btn")).to_have_count(1) # Storage: one question, one (still deflected) answer — no duplicate. msgs = _stored_parsed(page)["messages"] assert [m["who"] for m in msgs] == ["user", "brain"] assert msgs[0]["text"] == OFF_TOPIC assert msgs[-1]["deflected"] is True assert len(msgs[-1]["suggestions"]) >= 2 # --------------------------------------------------------------------------- # 4. Retry is inert while a turn is in flight # --------------------------------------------------------------------------- def test_retry_inert_while_in_flight( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: _reset_db(mock_llm) page.set_default_timeout(30_000) page.goto(app_url) # One completed turn: the last (only) brain bubble carries Retry. _ask(page, Q1) expect(page.locator(".retry-btn")).to_have_count(1) _tag_last_brain_wrap(page, "first-turn") first_answer = page.locator(ANSWER).inner_text() # Start the slow second turn (3 s pre-token warm-up = the window). page.fill("#message-input", SLOW_QUESTION) page.click("#send-btn") expect(page.locator("#send-label")).to_have_text("Stop", timeout=5_000) # Retry on the previous last bubble while in flight: a no-op — no # second turn, the previous bubble (answer + button) stays put. page.locator(".retry-btn").click() expect(page.locator("[data-retry-marker='first-turn']")).to_have_count(1) expect(page.locator("[data-retry-marker='first-turn'] .bubble")).to_have_text( first_answer ) # Still in flight — the in-flight turn, not a retry, owns the button. expect(page.locator("#send-label")).to_have_text("Stop") _assert_no_error_banner(page) # The in-flight turn completes to its own done answer. expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000) _assert_no_error_banner(page) # No second turn started, no duplication: exactly two user bubbles — # the in-flight question occurs once; the first turn is intact. expect(page.locator(".msg.user .bubble")).to_have_count(2) expect(page.locator(".msg.user .bubble", has_text=SLOW_QUESTION)).to_have_count(1) expect(page.locator(ANSWER)).to_have_count(2) expect(page.locator("[data-retry-marker='first-turn'] .bubble")).to_have_text( first_answer ) msgs = _stored_parsed(page)["messages"] assert [m["text"] for m in msgs if m["who"] == "user"] == [Q1, SLOW_QUESTION] assert [m["who"] for m in msgs] == ["user", "brain", "user", "brain"] # The settled conversation is again last-bubble-only retryable. expect(page.locator(".retry-btn")).to_have_count(1) expect(page.locator(BRAIN_WRAPS).last.locator(".retry-btn")).to_have_count(1)