brain-of-reese/tests/e2e/test_long_answers.py

"""Phase 11 E2E (Playwright): long answers stream to completion.

Story: ``.agents/user_stories/long-answers.md``
Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_long_answers.py -v --no-cov

The mock LLM honors ``max_tokens`` (token ≈ word) like a real endpoint,
and emits a ~900-word deterministic answer for the "write a long answer"
trigger. Under the old hard 700-token cap the answer loses its tail
(the final line never arrives); with ``BOR_MAX_OUTPUT_TOKENS`` defaulting
to 32 768 the full answer streams to completion.

Test → story mapping (Playwright Mapping Rule):
1. ``test_long_answer_streams_to_completion`` — trigger question →
   full ~900-word answer, final line intact, >700 words rendered.
2. ``test_normal_answer_unaffected`` — a regular question still streams
   a complete short answer.
"""
from __future__ import annotations

import asyncio
from pathlib import Path
from threading import Thread
from typing import Any

from playwright.sync_api import Page, expect
from sqlalchemy import text

from app.config import Settings
from app.db import SessionLocal
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from e2e.auth_helpers import login

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
LONG_QUESTION = "How is my Kubernetes cluster set up? write a long answer"
NORMAL_QUESTION = "How is my Kubernetes cluster set up?"


async def _import_fixtures(mock_port: int) -> ImportSummary:
    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return await import_sources([FIXTURES], LLMClient(settings))


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread (Playwright owns the test loop)."""
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(mock_port: int) -> ImportSummary:
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log"))
        db.commit()
    return _run_in_thread(_import_fixtures(mock_port))


def _last_brain_text(page: Page) -> str:
    return page.locator(".msg.brain .bubble").last.inner_text()


# ---------------------------------------------------------------------------
# 1. Long answer: the final line must survive the stream
# ---------------------------------------------------------------------------


def test_long_answer_streams_to_completion(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    _reset_db(mock_llm)
    page.set_default_timeout(45_000)
    login(page, app_url, next="/")

    page.fill("#message-input", LONG_QUESTION)
    page.click("#send-btn")

    # The mock streams ~900 words in ~8s; wait for the unique final line —
    # under the old 700-token cap it was cut off and never arrived.
    expect(page.locator(".msg.brain .bubble").last).to_contain_text(
        "LONG-ANSWER-END", timeout=60_000
    )

    full = _last_brain_text(page)
    # The old cap would have stopped the answer at 700 words — prove the
    # rendered answer ran well past it.
    assert len(full.split()) > 700, (
        f"answer looks truncated at {len(full.split())} words"
    )
    # First and last step both rendered (the markdown list strips the
    # "1." prefix — no mid-sentence cut between them either).
    assert "Step 1:" in full
    assert "Step 40:" in full
    # Turn settled: send button re-enabled (never-stale contract).
    expect(page.locator("#send-btn")).to_be_enabled()


# ---------------------------------------------------------------------------
# 2. Normal (short) answers are unaffected by the raised cap
# ---------------------------------------------------------------------------


def test_normal_answer_unaffected(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")

    page.fill("#message-input", NORMAL_QUESTION)
    page.click("#send-btn")

    bubble = page.locator(".msg.brain .bubble").last
    expect(bubble).to_contain_text("Deterministic mock answer for E2E", timeout=30_000)
    expect(bubble).not_to_contain_text("LONG-ANSWER-END")
    # Grounded: the question's own document is cited as a chip.
    expect(
        page.locator(".msg.brain .source-chip", has_text="kubernetes.md")
    ).to_have_count(1, timeout=30_000)
    expect(page.locator("#send-btn")).to_be_enabled()