brain-of-reese/tests/e2e/test_chat_error_honesty.py

"""Phase 164 E2E (Playwright): chat failure honesty — a pre-stream HTTP
rejection states what ACTUALLY happened, and a full-length answer flows
through the whole chain.

Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_chat_error_honesty.py -v --no-cov

The live finding (owner report, 2026-10-07): after an answer hit the
length cap, the NEXT turn's bubble read "My answer didn't make it — the
connection dropped. Use Retry to ask again. Failed [object Object]."
The chain: the long answer rode back as ``history`` on the next
``POST /api/chat``; the old 32 000-char ``HistoryTurn.text`` cap
rejected it with a 422 whose ``detail`` is a LIST OF OBJECTS; the
unguarded ``body.detail`` assignment stringified the array to
"[object Object]" inside the connection-dropped bubble — a lie (the
server WAS reachable; it answered 422).

The phase's remedies, proven here against the REAL ``app.js`` send
path (the stubs ride the browser's own network layer via
``page.route`` — the house pattern from test_navbar_refresh /
test_archive_upload_sources — so every assert drives the shipped
pre-stream handler, serializer, and failed-turn funnel):

1. ``test_prestream_422_renders_honest_line_with_serialized_detail`` —
   (the live bug) ``POST /api/chat`` is stubbed to the EXACT FastAPI
   422 shape for a long-history rejection. The bubble reads
   ``FAILED_REQUEST_TEXT`` verbatim (NOT the connection line), the
   ``.failed-note`` carries the pydantic ``msg`` verbatim, NO
   "[object Object]" and NO rejected ``input`` text reach the DOM, the
   banner shows the phase-114 hint, Retry + Edit land, and the
   composer re-enables (§7.4).
2. ``test_non_json_500_settles_on_the_http_status_fallback`` — a 500
   with a NON-JSON body settles on the ``HTTP 500`` default string
   (the serializer's fallback path) under the same honest pre-stream
   bubble — no crash.
3. ``test_full_length_answer_history_is_accepted`` — (the D1 fix,
   REAL server, no stub) a seeded ``bor.chat.v1`` conversation whose
   last brain record carries a ~525k-char answer (the live full-cap
   size) sends a follow-up that is ACCEPTED — the response is 200, the
   mock's deterministic echo settles the turn, the saved-chat
   auto-save chain stays 2xx — where pre-phase this was the 422 that
   killed every follow-up. The locked budget trim (A4) may drop the
   giant turn WHOLE from the prompt; this suite asserts ACCEPTANCE and
   settle, never prompt content.
4. ``test_sse_error_frame_with_array_detail_serializes_into_note`` —
   (client tolerance) a valid SSE stream whose terminal ``error``
   frame carries an ARRAY ``detail``: the note serializes the ``msg``,
   the zero-frame settle keeps ``FAILED_TURN_TEXT`` byte-identically
   (a model-side failure is NOT the pre-stream line), and the
   persisted record carries the serialized detail.

The bubble texts are read from ``app.js`` itself (the phase-120
``_js_const`` convention) so the Python pins can never drift from the
shipped copy.
"""
from __future__ import annotations

import hashlib
import json
import re
import time
from collections.abc import Iterator
from datetime import UTC, datetime
from pathlib import Path
from urllib.parse import urlparse

import pytest
from playwright.sync_api import Page, Route, expect
from sqlalchemy import text
from sqlalchemy.orm import Session

from app.db import SessionLocal
from app.models import Chunk, Document
from e2e.auth_helpers import login
from tests.e2e.mock_llm import embed_text

REPO = Path(__file__).resolve().parents[2]

# ---------------------------------------------------------------------------
# Copy constants, read from app.js (no JS/Python drift — the phase-120
# convention). The unit suite pins the strings themselves once; the E2E
# asserts the PAGE renders the SAME text.
# ---------------------------------------------------------------------------


def _js_const(name: str) -> str:
    js = (REPO / "frontend" / "assets" / "app.js").read_text(encoding="utf-8")
    m = re.search(rf'const {name}\s*=\s*\n?\s*"([^"]*)"', js)
    assert m, f"const {name} not found in app.js"
    return m.group(1)


FAILED_TURN_TEXT = _js_const("FAILED_TURN_TEXT")
FAILED_REQUEST_TEXT = _js_const("FAILED_REQUEST_TEXT")


def _js_prestream_hint() -> str:
    """The pre-stream throw's banner hint (phase 164 D3) — the ONLY
    ``err.hint`` assignment in app.js that carries a string literal (the
    SSE-frame one reads ``ev.hint``), so the match is unique."""
    js = (REPO / "frontend" / "assets" / "app.js").read_text(encoding="utf-8")
    matches = re.findall(r'err\.hint =\s*\n?\s*"([^"]*)"', js)
    assert len(matches) == 1, f"expected exactly one literal err.hint, got {matches}"
    return matches[0]


PRESTREAM_HINT = _js_prestream_hint()

#: The pydantic 83 boundary rejection for HistoryTurn.text (600 000
#: since phase 164, locked A1) — the EXACT live message the note must
#: render verbatim.
PYDANTIC_MSG = "String should have at most 600000 characters"

#: A probe from the rejected 422 entry's `input` (the whole rejected
#: turn — it must NEVER reach the DOM). A 100-char run of "x" cannot
#: occur naturally in the serialized page.
INPUT_PROBE = "x" * 100

STORAGE_KEY = "bor.chat.v1"

# ---------------------------------------------------------------------------
# Assert 3 fixtures: the seeded full-length answer + the grounded echo
# question (the test_failed_turn_retry / test_llm_history seed pattern).
# ---------------------------------------------------------------------------

SEED_SOURCE = "docs"
SEED_PATH = "homelab/kubernetes.md"
KUB_CONTENT = (REPO / "tests" / "fixtures" / "docs" / SEED_PATH).read_text()

#: The seeded conversation (the live repro's shape): a question and a
#: brain answer of the FULL-CAP size — 131 072 tokens ≈ 524 300 chars
#: at ~4 chars/token (the 2026-10-07 live report). Over the OLD 32 000
#: ceiling (the pre-phase 422), well under the new 600 000.
SEED_QUESTION = "Write everything you know about my Kubernetes cluster?"
BIG_ANSWER_LEN = 525_000
BIG_ANSWER = ("A kubernetes cluster note. " * (BIG_ANSWER_LEN // 27 + 1))[
    :BIG_ANSWER_LEN
]

#: The follow-up that died with the 422 live. `echo my history` is the
#: mock's HISTORY_TRIGGER (fires whatever the honesty gate says) and
#: "kubernetes cluster" keeps it on-topic for the one-doc KB, so the
#: turn settles with the byte-stable deterministic echo.
FOLLOWUP_Q = "echo my history about my kubernetes cluster"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"


# ---------------------------------------------------------------------------
# DB seeding (the test_failed_turn_retry pattern): TRUNCATE-then-seed
# one fixture document so the follow-up's gate is deterministic.
# ---------------------------------------------------------------------------


def _seed(db: Session) -> None:
    md = Document(
        source=SEED_SOURCE,
        path=SEED_PATH,
        full_path=f"/tmp/{SEED_PATH}",
        title="Kubernetes",
        content=KUB_CONTENT,
        content_hash=hashlib.sha256(KUB_CONTENT.encode()).hexdigest(),
        indexed_at=datetime.now(UTC),
    )
    db.add(md)
    db.flush()
    db.add(
        Chunk(
            document_id=md.id,
            position=0,
            content=KUB_CONTENT,
            embedding=embed_text(KUB_CONTENT),
        )
    )


def _reset_db() -> None:
    with SessionLocal() as db:
        db.execute(
            text("TRUNCATE chunks, documents, query_log, steering_notes, kb_overview")
        )
        db.commit()
        _seed(db)
        db.commit()


@pytest.fixture(autouse=True)
def clean_chats() -> Iterator[None]:
    """The send auto-saves a ``saved_chats`` row per turn (phase 55) —
    truncate around every test (the phase-80/103/120 isolation
    pattern, including the teardown margin for the fire-and-forget
    PUT)."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE saved_chats"))
        db.commit()
    yield
    time.sleep(0.75)
    with SessionLocal() as db:
        db.execute(text("TRUNCATE saved_chats"))
        db.commit()


# ---------------------------------------------------------------------------
# Page helpers
# ---------------------------------------------------------------------------


def _submit(page: Page, question: str) -> None:
    page.fill("#message-input", question)
    page.click("#send-btn")
    expect(page.locator(".msg.user .bubble").last).to_contain_text(question)


def _stored(page: Page) -> dict:
    raw = page.evaluate(f"() => localStorage.getItem('{STORAGE_KEY}')")
    assert raw is not None, "the conversation key must exist in localStorage"
    return json.loads(raw)


def _assert_settled_composer(page: Page) -> None:
    """§7.4 never-stale: after the failure the composer is fully usable
    again (Send enabled, label back — nothing zombified)."""
    expect(page.locator("#send-btn")).to_be_enabled(timeout=30_000)
    expect(page.locator("#send-label")).to_have_text("Send", timeout=30_000)


def _assert_dom_is_honest(page: Page, *, forbidden_input_probe: str | None) -> None:
    """The [object Object] / input-leak rule, page-wide: neither the
    garbage stringification nor a probe of the rejected `input` may
    appear anywhere in the DOM."""
    html = page.content()
    assert "[object Object]" not in html
    if forbidden_input_probe:
        assert forbidden_input_probe not in html


# ---------------------------------------------------------------------------
# 1 — the live bug: a pre-stream 422 (FastAPI's EXACT validation shape)
#     renders the distinct honest line + the serialized msg, never
#     "[object Object]", never the rejected input
# ---------------------------------------------------------------------------


def _stub_long_history_422(route: Route) -> None:
    """The EXACT FastAPI 422 body for a long-history rejection (a
    ``string_too_long`` entry whose ``input`` carries the whole
    rejected answer — the payload whose array ``detail`` produced the
    live "[object Object]")."""
    route.fulfill(
        status=422,
        content_type="application/json",
        body=json.dumps(
            {
                "detail": [
                    {
                        "type": "string_too_long",
                        "loc": ["body", "history", 5, "text"],
                        "msg": PYDANTIC_MSG,
                        "input": "x" * 1000,
                    }
                ]
            }
        ),
    )


def test_prestream_422_renders_honest_line_with_serialized_detail(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    _reset_db()
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")

    page.route("**/api/chat", _stub_long_history_422)
    with page.expect_response(
        lambda r: urlparse(r.url).path == "/api/chat"
    ) as info:
        _submit(page, "How is my Kubernetes cluster set up? (phase 164 e2e)")
    # The stub answered through the browser's real network layer — this
    # is the pre-stream HTTP settle, not a transport failure.
    assert info.value.status == 422

    # The bubble: the DISTINCT honest pre-stream line (phase 164 D3) —
    # NOT the connection-dropped copy (the server answered deliberately;
    # the connection held).
    brain = page.locator(".msg.brain")
    expect(brain).to_have_count(1)
    expect(brain.locator(".bubble")).to_have_text(FAILED_REQUEST_TEXT)

    # The note: the pydantic `msg` VERBATIM (task 02's serializer), and
    # nowhere on the page the array garbage or the rejected `input`.
    note = brain.locator(".failed-note")
    expect(note).to_have_count(1)
    expect(note).to_contain_text("Failed")
    expect(note).to_contain_text(PYDANTIC_MSG)
    _assert_dom_is_honest(page, forbidden_input_probe=INPUT_PROBE)

    # The banner (role=alert): the same detail plus the phase-114 hint
    # channel (D3) — "the app answered, nothing was lost but this send".
    banner = page.locator("#kb-banner")
    expect(banner).to_have_attribute("role", "alert", timeout=30_000)
    expect(banner).to_be_visible()
    expect(banner).to_contain_text(PYDANTIC_MSG)
    expect(banner).to_contain_text(PRESTREAM_HINT)

    # Recovery affordances: Retry on the failed bubble (phase 120) +
    # Edit on the question (phase 159), and the composer fully usable
    # again (§7.4).
    expect(brain.locator(".retry-btn")).to_have_count(1)
    expect(page.locator(".msg.user .edit-btn")).to_have_count(1)
    _assert_settled_composer(page)

    # Persisted (the phase-120 contract extended to the new text):
    # what the user saw is what is stored — the new bubble line +
    # failed: true + the serialized detail in the capped `error`.
    stored = _stored(page)
    assert [m["who"] for m in stored["messages"]] == ["user", "brain"]
    failed = stored["messages"][-1]
    assert failed["text"] == FAILED_REQUEST_TEXT
    assert failed["failed"] is True
    assert failed["error"] == PYDANTIC_MSG


# ---------------------------------------------------------------------------
# 2 — the fallback: a 500 with a NON-JSON body settles on the
#     "HTTP 500" default string under the same honest pre-stream line
# ---------------------------------------------------------------------------


def _stub_html_500(route: Route) -> None:
    """An outage page: text/html where JSON was expected — the
    serializer's `res.json()` catch keeps the per-status default."""
    route.fulfill(
        status=500,
        content_type="text/html",
        body="<html><head><title>500</title></head><body>gateway trouble</body></html>",
    )


def test_non_json_500_settles_on_the_http_status_fallback(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    _reset_db()
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")

    page.route("**/api/chat", _stub_html_500)
    with page.expect_response(
        lambda r: urlparse(r.url).path == "/api/chat"
    ) as info:
        _submit(page, "How is my Kubernetes cluster set up? (phase 164 e2e 500)")
    assert info.value.status == 500

    # Still the pre-stream settle (the server answered 500) — the
    # honest request line, with the `HTTP 500` default string on the
    # note (the body was not JSON; the serializer fell back).
    brain = page.locator(".msg.brain")
    expect(brain).to_have_count(1)
    expect(brain.locator(".bubble")).to_have_text(FAILED_REQUEST_TEXT)
    note = brain.locator(".failed-note")
    expect(note).to_have_count(1)
    expect(note).to_contain_text("Failed")
    expect(note.locator(".failed-detail")).to_have_text(
        "Brain's API answered with HTTP 500."
    )
    _assert_dom_is_honest(page, forbidden_input_probe=None)

    # No crash, and every control usable again (§7.4).
    banner = page.locator("#kb-banner")
    expect(banner).to_be_visible()
    expect(banner).to_contain_text(PRESTREAM_HINT)
    expect(brain.locator(".retry-btn")).to_have_count(1)
    _assert_settled_composer(page)

    stored = _stored(page)
    failed = stored["messages"][-1]
    assert failed["text"] == FAILED_REQUEST_TEXT
    assert failed["failed"] is True
    assert failed["error"] == "Brain's API answered with HTTP 500."


# ---------------------------------------------------------------------------
# 3 — the D1 fix, REAL server: a history turn carrying a ~525k-char
#     brain answer is ACCEPTED (the live "every follow-up dies after a
#     long answer" failure mode is gone) and the turn settles
# ---------------------------------------------------------------------------


def test_full_length_answer_history_is_accepted(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    _reset_db()
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")

    assert len(BIG_ANSWER) == BIG_ANSWER_LEN
    assert len(BIG_ANSWER) > 32_000, "the repro must exceed the OLD 32 000 cap"

    # Seed the live repro's shape into bor.chat.v1 (the saved-chat
    # suites' seed pattern): question + a FULL-CAP brain answer, the
    # brain record LAST — exactly the record the next send turns into
    # `history`.
    seed = {
        "v": 1,
        "chatId": None,
        "messages": [
            {"who": "user", "text": SEED_QUESTION},
            {"who": "brain", "text": BIG_ANSWER},
        ],
    }
    page.evaluate(
        "([key, value]) => localStorage.setItem(key, value)",
        [STORAGE_KEY, json.dumps(seed)],
    )

    # Track the saved-chat chain too: the phase-55 auto-save carries
    # the SAME 525k-char record to POST/PUT /api/chats — every one of
    # those must stay 2xx (the mirror invariant, live).
    chat_saves: list[tuple[str, int]] = []
    page.on(
        "response",
        lambda r: chat_saves.append((r.request.method, r.status))
        if urlparse(r.url).path.startswith("/api/chats")
        else None,
    )

    page.reload()
    # The restore renders the seeded conversation — the giant bubble
    # proves the client actually holds the full-length answer the send
    # will put on the wire.
    expect(page.locator("#messages > .msg")).to_have_count(2)
    expect(page.locator(".msg.user .bubble")).to_have_text(SEED_QUESTION)
    restored = page.locator(".msg.brain .bubble").last.inner_text()
    assert len(restored) > 32_000, "the seeded long answer must restore"

    with page.expect_response(
        lambda r: urlparse(r.url).path == "/api/chat"
    ) as info:
        _submit(page, FOLLOWUP_Q)

    # THE fix: the follow-up of the full-length answer is ACCEPTED —
    # no 422 (pre-phase the 32 000 ceiling rejected exactly this
    # request and killed every follow-up).
    assert info.value.status == 200, (
        f"the long-history follow-up must be accepted, got {info.value.status}"
    )

    # The turn settles normally on the deterministic echo (the locked
    # budget trim may have dropped the giant turn WHOLE from the
    # prompt — A4 — so the echo's COUNT is never asserted, only the
    # settle). No failed bubble, no banner, composer usable (§7.4).
    bubble = page.locator(".msg.brain .bubble").last
    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=60_000)
    expect(bubble).to_contain_text("prior messages")
    expect(page.locator(".failed-note")).to_have_count(0)
    expect(page.locator("#kb-banner")).to_be_hidden()
    _assert_settled_composer(page)

    # The record chain held: the question's user record and the fresh
    # answer are persisted, the answer is not marked failed.
    stored = _stored(page)
    last = stored["messages"][-1]
    assert last["who"] == "brain"
    assert MOCK_ANSWER_MARKER in last["text"]
    assert last.get("failed") in (None, False)

    # The saved-chat auto-save of the SAME full-length conversation
    # was accepted too (201/200 — the ChatMessage mirror + the total
    # budget; any 4xx here is the phase-113 quiet-failure class).
    time.sleep(1.0)  # the fire-and-forget save margin
    assert chat_saves, "the auto-save never fired"
    assert all(status < 400 for _, status in chat_saves), chat_saves


# ---------------------------------------------------------------------------
# 4 — client tolerance: an SSE terminal `error` frame whose `detail` is
#     an ARRAY serializes into the note, and the zero-frame settle
#     keeps FAILED_TURN_TEXT (a model-side failure is NOT the
#     pre-stream line)
# ---------------------------------------------------------------------------


def _stub_sse_array_error(route: Route) -> None:
    """A valid SSE stream ending in an `error` frame with an ARRAY
    detail (server-built frames carry a plain string — pinned in the
    task-02 units; this is the tolerant half). The `input` again must
    never reach the DOM."""
    frame = {
        "type": "error",
        "detail": [
            {
                "type": "value_error",
                "loc": ["body"],
                "msg": "mock stream-side rejection",
                "input": "q" * 1000,
            }
        ],
    }
    route.fulfill(
        status=200,
        content_type="text/event-stream",
        body=f"data: {json.dumps(frame)}\n\n",
    )


def test_sse_error_frame_with_array_detail_serializes_into_note(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    _reset_db()
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")

    page.route("**/api/chat", _stub_sse_array_error)
    _submit(page, "How is my Kubernetes cluster set up? (phase 164 e2e sse)")

    # Zero frames + the terminal error frame: the MODEL-side settle —
    # FAILED_TURN_TEXT byte-identically (phase 120; the pre-stream
    # bubbleText is NEVER set on an SSE-frame throw), with the array
    # detail serialized into the note instead of stringified garbage.
    brain = page.locator(".msg.brain")
    expect(brain).to_have_count(1)
    expect(brain.locator(".bubble")).to_have_text(FAILED_TURN_TEXT)
    note = brain.locator(".failed-note")
    expect(note).to_have_count(1)
    expect(note).to_contain_text("Failed")
    expect(note).to_contain_text("mock stream-side rejection")
    _assert_dom_is_honest(page, forbidden_input_probe="q" * 100)

    # The persisted record: the same text the user saw + the serialized
    # detail (the phase-120 persistence contract, unchanged shape).
    stored = _stored(page)
    failed = stored["messages"][-1]
    assert failed["text"] == FAILED_TURN_TEXT
    assert failed["failed"] is True
    assert failed["error"] == "mock stream-side rejection"

    # Recovery + re-enable (§7.4).
    expect(brain.locator(".retry-btn")).to_have_count(1)
    expect(page.locator(".msg.user .edit-btn")).to_have_count(1)
    _assert_settled_composer(page)