"""Phase 53 E2E (Playwright): invalidate saved chats on sources sync. TODO.md L4 (owner 2026-08-30): "Make sure the saved chats are invalidated if the docs are synced, that way it generates a new answer with new data". Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_stale_saved_chats.py -v --no-cov The full user-visible invalidation loop under test: * **Auto-save (fresh, phase 55)** — as admin: ask (the mock answers deterministically) — the conversation auto-saves at the save points (the Save pill is gone); ``GET /api/chats`` (admin cookie) reports the row with ``stale: false``; opening the FRESH row at ``/?chat=`` shows NO banner; * **KB change** — the test process (which shares the app's environment) bumps the ``sources_meta`` seed row through ``bump_sources_version`` over a short ``SessionLocal()`` — the exact helper BOTH real sync paths (the Sync button, ``scripts/import_docs``) call change-gated. The bump GATES are pinned by this phase's integration tests (task 02) and the Sync button's end-to-end clone/import path by ``test_sync_button.py``; the E2E proves the user-visible loop, not git plumbing (the phase's recorded ASSUMPTION); * **Stale surfaced** — ``/history.html``: the row now carries the rose Stale pill (``aria-label``d for screen readers); ``GET /api/chats`` and ``GET /api/chats/`` carry ``stale: true`` (computed server-side — the client never computes staleness); * **Regenerate** — opening the row (``/?chat=``, the same URL the History table links) reveals ``#stale-banner`` with the Regenerate button; clicking it re-streams the last answer IN PLACE against the new index (the phase-49 redo-in-place: the old bubble leaves the DOM, the question is not duplicated), and the handler then auto re-saves the linked row — the server re-stamps ``sources_version`` → the banner clears, ``GET /api/chats/`` reports ``stale: false`` with the last brain message being the fresh answer, and the History pill is gone; * **Guard** — a stale row whose conversation has NO brain record (user-only) reveals the banner text WITHOUT the Regenerate button (``retryLastTurn`` has nothing to redo); * **Anonymous** — sharing the (now fresh) chat and opening ``/shared/`` without a session renders the phase-51 snapshot with NO staleness surface — neither in the DOM nor on the public ``SharedChatOut`` wire (the snapshot is frozen by design). DB isolation: the shared e2e Postgres keeps ``saved_chats`` rows across suites, so every test here uses a DISTINCTIVE question text (its auto-title is therefore unique), never asserts on absolute row counts, and deletes the rows it creates in a ``finally`` (admin cookie). The KB tables are truncated + re-seeded the house way (deterministic mock embeddings); ``saved_chats`` and ``sources_meta`` are never touched by the reset — the version is monotonic by design, and rows stamped against an older generation are simply stale (that is the point). Determinism: the mock quotes the asked question into its grounded answer (ending in the ``Deterministic mock answer for E2E`` marker), so the regenerated answer is textually identical to the stale one — OLD-vs-fresh bubble identity is proved the phase-49 way, with a test-only ``data-retry-marker`` attribute set on the old wrap before the click. """ from __future__ import annotations import asyncio import time import uuid from pathlib import Path from threading import Thread from typing import Any import httpx from playwright.sync_api import Browser, BrowserContext, Page, expect from sqlalchemy import text from app.config import Settings from app.db import SessionLocal from app.models import SavedChat from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from app.rag.sources_meta import bump_sources_version, current_sources_version from e2e.auth_helpers import login REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" #: The banner's line (frontend/index.html, task 05). BANNER_TEXT = "The sources have been updated since this chat was saved." #: The live-region outcome of a successful Regenerate (app.js, task 05). REGEN_STATUS = "Regenerated — the answer now reflects the current sources." #: The fresh row's Stale cell (history.js, task 04). FRESH_STALE_CELL = "—" #: The stale row's Stale cell aria-label (history.js, task 04). STALE_CELL_ARIA = "Stale — sources have changed since this chat was saved" #: The typing indicator is itself a .msg.brain — exclude its bubble. ANSWER = ".msg.brain .bubble:not(.typing)" async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread. Playwright's sync API keeps an asyncio loop running on the test thread, so ``asyncio.run`` cannot be called directly from a test body. """ box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None: """Truncate the KB (and query log + steering notes — deterministic mock answers), then optionally re-import fixtures. ``saved_chats`` and ``sources_meta`` are deliberately NOT touched: rows persist across suites (every test cleans up after itself) and the version is monotonic (the invalidation marker).""" with SessionLocal() as db: db.execute( text("TRUNCATE chunks, documents, query_log, steering_notes") ) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) def _bump_sources_version() -> int: """The test-only stand-in for a KB-changing sync (task 02): the SAME ``bump_sources_version`` helper both real sync paths call change-gated, over a short session (bump flushes, the caller commits). The bump GATES — changed/unchanged/`--limit`/failed — are pinned by this phase's integration tests, not by the E2E.""" with SessionLocal() as db: v = bump_sources_version(db) db.commit() return v def _current_version() -> int: with SessionLocal() as db: return current_sources_version(db) def _row_stamp(chat_id: str) -> int: """The saved row's ``sources_version`` stamp (helper read — the same pattern as the phase's integration tests).""" with SessionLocal() as db: row = db.get(SavedChat, uuid.UUID(chat_id)) assert row is not None return row.sources_version def _ask(page: Page, question: str) -> None: """Send one turn and wait until the grounded answer has fully landed (the ``done`` event restored the Send button).""" page.fill("#message-input", question) page.click("#send-btn") expect(page.locator(".msg.user .bubble").last).to_contain_text(question) expect(page.locator(ANSWER).last).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) expect(page.locator("#send-btn")).to_be_enabled() expect(page.locator("#send-label")).to_have_text("Send") def _wait_saved_row( app_url: str, cookies: dict[str, str], title: str, messages: int = 2, ) -> dict[str, Any]: """Wait for the auto-saved row (phase 55: auto-saves are SILENT — A2 — so there is no status line to wait on). The upsert is fire-and-forget from the UI's point of view, so poll the admin list until the row with the conversation's auto-title appears with the expected message count.""" deadline = time.monotonic() + 15 last: dict[str, Any] | None = None while time.monotonic() < deadline: last = _find_row(_chats(app_url, cookies), title) if last is not None and last["message_count"] >= messages: return last time.sleep(0.2) raise AssertionError(f"no auto-saved row for {title!r} (last: {last!r})") def _admin_cookies(page: Page) -> dict[str, str]: """The signed session cookies the browser holds after a form login — used to call the admin API with plain httpx (the test's API side sees exactly what the signed-in browser sees).""" return { c["name"]: c["value"] for c in page.context.cookies() if "name" in c and "value" in c } def _chats(app_url: str, cookies: dict[str, str]) -> list[dict[str, Any]]: r = httpx.get(f"{app_url}/api/chats", timeout=10, cookies=cookies) assert r.status_code == 200 return r.json()["chats"] def _chat(app_url: str, cookies: dict[str, str], chat_id: str) -> dict[str, Any]: r = httpx.get(f"{app_url}/api/chats/{chat_id}", timeout=10, cookies=cookies) assert r.status_code == 200 return r.json() def _auto_title(question: str) -> str: """The phase-50 auto-title convention: the first question, whitespace-collapsed, capped at 120 chars.""" return " ".join(question.split())[:120] def _find_row( rows: list[dict[str, Any]], title: str ) -> dict[str, Any] | None: return next((c for c in rows if c["title"] == title), None) def _delete_chat(app_url: str, cookies: dict[str, str], chat_id: str) -> None: """Best-effort row cleanup (a 404 — already deleted — is fine).""" httpx.delete(f"{app_url}/api/chats/{chat_id}", timeout=10, cookies=cookies) def _history_row(page: Page, chat_id: str) -> Any: """The History table row for this chat (located through the Open link — ``/?chat=`` — the same URL the table links).""" return page.locator( "#history-tbody tr", has=page.locator(f"a[href='/?chat={chat_id}']") ) def _assert_no_error_banner(page: Page) -> None: """Regenerate settles through the normal done path — never the red role=alert error banner (the KB-offline banner is a separate, health-driven state the db_ready fixture keeps away).""" banner = page.locator("#kb-banner") expect(banner).to_be_hidden() expect(banner).not_to_have_class("is-error") def _tag_last_brain_wrap(page: Page, marker: str) -> None: """Tag the rendered wrap of the LAST brain bubble (settled state — no typing indicator present) so the test can prove the OLD element leaves the DOM: the mock answers are byte-stable, so the redo of the same question is textually indistinguishable from the original.""" page.evaluate( """(marker) => { const wraps = document.querySelectorAll("#messages > .msg.brain"); wraps[wraps.length - 1].setAttribute("data-retry-marker", marker); }""", marker, ) def _regenerate_in_place(page: Page, marker: str) -> None: """Click the banner's Regenerate and wait for the FULL phase-49 redo-in-place + auto re-save to settle: the turn goes in-flight (the button IS Stop), the old bubble leaves the DOM before the first fresh token, the fresh answer lands, and the handler's post-turn PUT (the server re-stamps ``sources_version``) clears the banner and lands the outcome on the live region.""" _tag_last_brain_wrap(page, marker) regen = page.locator("#stale-regenerate") expect(regen).to_be_enabled() regen.click() # In flight: the redo owns the Send/Stop control… expect(page.locator("#send-label")).to_have_text("Stop", timeout=5_000) # …and the double-click guard holds (one regenerate at a time). expect(regen).to_be_disabled() # Redo in place: the OLD wrap is already gone (the phase-49 # contract — removal precedes the rerun). expect(page.locator(f"[data-retry-marker='{marker}']")).to_have_count(0) # The fresh answer streams into its place and the turn settles. expect(page.locator(ANSWER).last).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000) _assert_no_error_banner(page) # Post-turn: the linked row was re-saved (the server re-stamped # it), so the banner cleared and the outcome is announced. expect(page.locator("#stale-banner")).to_be_hidden(timeout=15_000) expect(page.locator("#send-status")).to_have_text(REGEN_STATUS) # --------------------------------------------------------------------------- # 1. The full loop: save (fresh) → KB change → stale surfaced → Regenerate # → fresh again (UI + API agree at every step) # --------------------------------------------------------------------------- def test_full_invalidation_loop( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: _reset_db(mock_llm, seed=True) page.set_default_timeout(30_000) login(page, app_url, next="/") expect(page).to_have_url(app_url + "/", timeout=30_000) q = "How is my Kubernetes cluster set up? (stale-loop)" _ask(page, q) # --- Auto-save (fresh, phase 55 — no Save pill): the API agrees, # and a fresh open shows NO banner. cookies = _admin_cookies(page) row = _wait_saved_row(app_url, cookies, _auto_title(q)) assert row["message_count"] == 2 assert row["stale"] is False, "a just-saved chat is fresh, not stale" chat_id: str = row["id"] # The DB agrees: Save stamped the row with the CURRENT generation # (helper read — the integration suite pins the same contract). assert _row_stamp(chat_id) == _current_version(), ( "Save must stamp sources_version with the current generation" ) try: # A fresh row opened at /?chat= shows NO stale banner. page.goto(app_url + f"/?chat={chat_id}") expect(page.locator(".msg.user .bubble")).to_have_count(1) expect(page.locator(".msg.user .bubble")).to_contain_text(q) expect(page.locator(ANSWER)).to_have_count(1) expect(page.locator("#stale-banner")).to_be_hidden() expect(page.locator("#stale-regenerate")).to_be_hidden() # --- KB change: the version bumps (the test-only stand-in for a # KB-changing sync — see the module docstring / task ASSUMPTION). bumped = _bump_sources_version() assert bumped > 0 # The API agrees the row is stale NOW (server-computed — both # the detail and the list shapes carry the flag): detail = _chat(app_url, cookies, chat_id) assert detail["stale"] is True mine = _find_row(_chats(app_url, cookies), _auto_title(q)) assert mine is not None and mine["stale"] is True # --- History: the row carries the rose Stale pill (and its # aria-label, so the marker is conveyed without the visual). page.goto(app_url + "/history.html") tr = _history_row(page, chat_id) expect(tr.locator("a.history-title-link")).to_be_visible(timeout=15_000) pill = tr.locator(".stale-pill") expect(pill).to_have_count(1) expect(pill).to_have_text("Stale") expect(tr.locator(".history-stale-cell")).to_have_attribute( "aria-label", STALE_CELL_ARIA ) # --- Open the row (the SAME URL the History table links): the # banner reveals, with the Regenerate button. tr.locator("a.history-title-link").click() banner = page.locator("#stale-banner") expect(banner).to_be_visible(timeout=15_000) expect(banner).to_have_attribute("role", "status") expect(banner).to_contain_text(BANNER_TEXT) expect(page.locator("#stale-regenerate")).to_have_text("Regenerate") # The conversation restored in full (one question, one answer). expect(page.locator(".msg.user .bubble")).to_have_count(1) expect(page.locator(ANSWER)).to_have_count(1) # --- Regenerate: the fresh answer streams in place against the # new index, and the handler re-saves the linked row. _regenerate_in_place(page, "stale-old") # Redo mechanics: the question was NOT duplicated, the fresh # answer stands alone in the last bubble (it quotes the # question — the mock is deterministic). expect(page.locator(".msg.user .bubble")).to_have_count(1) expect(page.locator(ANSWER)).to_have_count(1) expect(page.locator(ANSWER).last).to_contain_text(q) expect(page.locator("#stale-regenerate")).to_be_enabled() # --- The API agrees the row is fresh again, with the fresh # answer as its last brain message (the re-save re-stamped the # row — the DB agrees). detail = _chat(app_url, cookies, chat_id) assert detail["stale"] is False, "the re-saved row must be fresh" msgs = detail["messages"] assert [m["who"] for m in msgs] == ["user", "brain"], ( "the re-save must not duplicate the question" ) assert msgs[0]["text"] == q assert MOCK_ANSWER_MARKER in msgs[-1]["text"] assert q in msgs[-1]["text"], "the fresh answer quotes the re-asked question" assert _row_stamp(chat_id) == _current_version(), ( "the Regenerate re-save must re-stamp sources_version" ) # --- History: the Stale pill is gone (the em-dash returns). page.goto(app_url + "/history.html") tr = _history_row(page, chat_id) expect(tr.locator("a.history-title-link")).to_be_visible(timeout=15_000) expect(tr.locator(".stale-pill")).to_have_count(0) expect(tr.locator(".history-stale-cell")).to_have_text(FRESH_STALE_CELL) finally: _delete_chat(app_url, cookies, chat_id) # --------------------------------------------------------------------------- # 2. Guard: a stale chat with NO brain record reveals the banner text # WITHOUT the Regenerate button (nothing to regenerate) # --------------------------------------------------------------------------- def test_stale_chat_without_brain_answer_is_text_only( page: Page, app_url: str, db_ready: None ) -> None: page.set_default_timeout(30_000) js_errors: list[str] = [] page.on("pageerror", lambda e: js_errors.append(str(e))) login(page, app_url, next="/") expect(page).to_have_url(app_url + "/", timeout=30_000) cookies = _admin_cookies(page) # A user-only saved conversation — the same wire shape the Save pill # posts (auto-title from the first user message). q = "How is my Kubernetes cluster set up? (stale-nobrain)" r = httpx.post( f"{app_url}/api/chats", timeout=10, cookies=cookies, json={"messages": [{"who": "user", "text": q}]}, ) assert r.status_code == 201, r.text chat_id = r.json()["id"] assert r.json()["stale"] is False # freshly stamped on create try: _bump_sources_version() assert _chat(app_url, cookies, chat_id)["stale"] is True # Opening the stale user-only row: the banner reveals… page.goto(app_url + f"/?chat={chat_id}") banner = page.locator("#stale-banner") expect(banner).to_be_visible(timeout=15_000) expect(banner).to_contain_text(BANNER_TEXT) # …but the Regenerate button is REMOVED — with no brain record # there is nothing to regenerate (retryLastTurn is never called). expect(page.locator("#stale-regenerate")).to_have_count(0) # The conversation restored: the user question, no brain bubble. expect(page.locator(".msg.user .bubble")).to_have_count(1) expect(page.locator(".msg.user .bubble")).to_contain_text(q) expect(page.locator(ANSWER)).to_have_count(0) assert not js_errors, f"the text-only banner must not throw: {js_errors}" finally: _delete_chat(app_url, cookies, chat_id) # --------------------------------------------------------------------------- # 3. Anonymous: the shared (now fresh) chat renders at /shared/ # with NO staleness surface — neither in the DOM nor on the public # SharedChatOut wire (phase 51 unchanged) # --------------------------------------------------------------------------- def test_anonymous_shared_snapshot_has_no_staleness_surface( page: Page, browser: Browser, app_url: str, mock_llm: int, db_ready: None, ) -> None: _reset_db(mock_llm, seed=True) page.set_default_timeout(30_000) login(page, app_url, next="/") expect(page).to_have_url(app_url + "/", timeout=30_000) q = "How is my Kubernetes cluster set up? (stale-share)" _ask(page, q) cookies = _admin_cookies(page) row = _wait_saved_row(app_url, cookies, _auto_title(q)) # auto-saved (phase 55) chat_id: str = row["id"] anon_ctx: BrowserContext | None = None try: # Make the chat stale, then bring it back CURRENT through the # full Regenerate loop — the chat shared below is fresh. _bump_sources_version() page.goto(app_url + f"/?chat={chat_id}") expect(page.locator("#stale-banner")).to_be_visible(timeout=15_000) _regenerate_in_place(page, "stale-share-old") assert _chat(app_url, cookies, chat_id)["stale"] is False # Share from the History row's Share column (Create link → the # cell re-renders to Copy + Unshare). page.goto(app_url + "/history.html") tr = _history_row(page, chat_id) create = tr.locator("button.history-share-create") expect(create).to_be_visible(timeout=15_000) create.click() expect(tr.locator("button.history-share-copy")).to_be_visible(timeout=15_000) share_url = _chat(app_url, cookies, chat_id).get("share_url") assert share_url is not None, "the Create link must have shared the row" token = share_url.removeprefix("/shared/") # --- The FRESH anonymous context (no session): the phase-51 # snapshot renders in full… anon_ctx = browser.new_context() anon = anon_ctx.new_page() anon.set_default_timeout(30_000) anon.goto(app_url + share_url) expect(anon.locator("#shared-title")).to_have_text(_auto_title(q)) expect(anon.locator(".msg.user .bubble")).to_have_count(1) expect(anon.locator(".msg.user .bubble")).to_contain_text(q) expect(anon.locator(".msg.brain .bubble").first).to_contain_text( MOCK_ANSWER_MARKER, timeout=30_000 ) # …with NO staleness surface anywhere: none of the phase-53 # elements exist on the shared page, and the banner's line never # appears (the snapshot is frozen by design — phase 51). (The # check is on the UI strings, not the bare word: the question # text is quoted into the mock's answer and is data, not a # staleness surface.) stale_selectors = ( "#stale-banner, #stale-regenerate, " ".stale-banner, .stale-pill, .stale-regenerate" ) expect(anon.locator(stale_selectors)).to_have_count(0) body_text = anon.locator("body").inner_text().lower() assert "sources have been updated since this chat was saved" not in body_text, ( "the shared snapshot must carry no staleness surface" ) anon_ctx.close() anon_ctx = None # …and the public wire agrees: the anonymous read carries # title + messages ONLY (no stale flag, no staleness surface). r = httpx.get(f"{app_url}/api/shared/{token}", timeout=10) assert r.status_code == 200 body = r.json() assert set(body) == {"title", "messages"}, ( f"the public snapshot must stay minimal: {set(body)}" ) assert "stale" not in body assert body["title"] == _auto_title(q) assert MOCK_ANSWER_MARKER in body["messages"][-1]["text"] finally: if anon_ctx is not None: anon_ctx.close() _delete_chat(app_url, cookies, chat_id)