"""Phase 11 E2E (Playwright): long answers stream to completion. Story: ``.agents/user_stories/long-answers.md`` Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_long_answers.py -v --no-cov The mock LLM honors ``max_tokens`` (token ≈ word) like a real endpoint, and emits a ~900-word deterministic answer for the "write a long answer" trigger. Under the old hard 700-token cap the answer loses its tail (the final line never arrives); with ``BOR_MAX_OUTPUT_TOKENS`` defaulting to 32 768 the full answer streams to completion. Test → story mapping (Playwright Mapping Rule): 1. ``test_long_answer_streams_to_completion`` — trigger question → full ~900-word answer, final line intact, >700 words rendered. 2. ``test_normal_answer_unaffected`` — a regular question still streams a complete short answer. """ from __future__ import annotations import asyncio from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import text from app.config import Settings from app.db import SessionLocal from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" LONG_QUESTION = "How is my Kubernetes cluster set up? write a long answer" NORMAL_QUESTION = "How is my Kubernetes cluster set up?" async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread (Playwright owns the test loop).""" box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int) -> ImportSummary: with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() return _run_in_thread(_import_fixtures(mock_port)) def _last_brain_text(page: Page) -> str: return page.locator(".msg.brain .bubble").last.inner_text() # --------------------------------------------------------------------------- # 1. Long answer: the final line must survive the stream # --------------------------------------------------------------------------- def test_long_answer_streams_to_completion( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: _reset_db(mock_llm) page.set_default_timeout(45_000) page.goto(app_url) page.fill("#message-input", LONG_QUESTION) page.click("#send-btn") # The mock streams ~900 words in ~8s; wait for the unique final line — # under the old 700-token cap it was cut off and never arrived. expect(page.locator(".msg.brain .bubble").last).to_contain_text( "LONG-ANSWER-END", timeout=60_000 ) full = _last_brain_text(page) # The old cap would have stopped the answer at 700 words — prove the # rendered answer ran well past it. assert len(full.split()) > 700, ( f"answer looks truncated at {len(full.split())} words" ) # First and last step both rendered (the markdown list strips the # "1." prefix — no mid-sentence cut between them either). assert "Step 1:" in full assert "Step 40:" in full # Turn settled: send button re-enabled (never-stale contract). expect(page.locator("#send-btn")).to_be_enabled() # --------------------------------------------------------------------------- # 2. Normal (short) answers are unaffected by the raised cap # --------------------------------------------------------------------------- def test_normal_answer_unaffected( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: _reset_db(mock_llm) page.set_default_timeout(30_000) page.goto(app_url) page.fill("#message-input", NORMAL_QUESTION) page.click("#send-btn") bubble = page.locator(".msg.brain .bubble").last expect(bubble).to_contain_text("Deterministic mock answer for E2E", timeout=30_000) expect(bubble).not_to_contain_text("LONG-ANSWER-END") # Grounded: the question's own document is cited as a chip. expect( page.locator(".msg.brain .source-chip", has_text="kubernetes.md") ).to_have_count(1, timeout=30_000) expect(page.locator("#send-btn")).to_be_enabled()