brain-of-reese/tests/e2e/test_source_chip_quality.py

"""Phase 113 E2E (Playwright): the source-chip quality contract (TODO L5 +
L2c) — the usefulness bar + the de-emphasized related-docs row, as VISIBLE
chip counts.

Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov

Acceptance (TODO L144–146): "for a single-document question, the turn
shows one citation chip"; a weak 2nd doc renders only in the
de-emphasized related row (``.related-doc`` links, NEVER ``.source-chip``);
a deflected turn shows zero citation chips (its weak hits, if any, live in
the related row).

The fixture KB's tier shapes are deterministic under the E2E mock's
bag-of-words embeddings + the mock-calibrated bar (conftest:
``BOR_SOURCE_USEFULNESS_FLOOR=0.15``, half the 0.30 threshold — like the
lexical floor):

* **single-source question** — "What SSH aliases do I have?":
  ``ssh_aliases.txt`` is the ONLY doc whose best-chunk cosine clears the
  bar (0.352 ≥ 0.15; grounded at 0.352 ≥ 0.30) → the done frame carries
  exactly ONE cited ref; the two below-bar docs (gitlab-compose.yaml
  0.025, uptime_probe.py 0.113) ride the related tier. This IS the
  strong+weak two-tier shape on the wire (the four OBSERVED live shapes
  are unit-pinned at plan level in
  ``tests/unit/test_source_chip_quality.py`` — the fixture KB reproduces
  the same shape live, so no docstring caveat is needed).
* **deflected question** — "How do I bake sourdough bread?": best cosine
  0.109 < 0.30 and zero FTS hits → honest deflection; nothing clears the
  bar → zero cited refs; the weak hits (new-service.md 0.109,
  ssh_aliases.txt 0.050) ride the related row.
"""
from __future__ import annotations

import asyncio
from pathlib import Path
from threading import Thread
from typing import Any

from playwright.sync_api import Page, expect
from sqlalchemy import select, text

from app.config import Settings, get_settings
from app.db import SessionLocal
from app.models import QueryLog
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from e2e.auth_helpers import login

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
SINGLE_SOURCE_QUESTION = "What SSH aliases do I have?"
OFF_TOPIC = "How do I bake sourdough bread?"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"


async def _import_fixtures(mock_port: int) -> ImportSummary:
    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return await import_sources([FIXTURES], LLMClient(settings))


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread (Playwright owns the test loop)."""
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(mock_port: int, seed: bool = True) -> ImportSummary | None:
    """Truncate the KB (and query log), then optionally re-import fixtures."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log"))
        db.commit()
    if not seed:
        return None
    return _run_in_thread(_import_fixtures(mock_port))


def _ask(page: Page, message: str) -> None:
    page.fill("#message-input", message)
    page.click("#send-btn")


def test_single_source_question_shows_exactly_one_citation_chip(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The acceptance pin: a single-document question → the done bubble
    carries EXACTLY ONE ``.source-chip`` (the bar-clearing doc) and the
    below-bar docs render only in the de-emphasized ``.related-docs`` row
    (``.related-doc`` links — never ``.source-chip``), labeled
    "Nearby docs, in case:". The durable record keeps the FULL retrieval
    (LOCKED A3)."""
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated

    _ask(page, SINGLE_SOURCE_QUESTION)
    expect(page.locator(".msg.user .bubble")).to_contain_text(SINGLE_SOURCE_QUESTION)

    bubble = page.locator(".msg.brain .bubble").first
    bubble.wait_for(state="visible", timeout=30_000)
    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000)
    # Grounded: no deflected bubble at all.
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)

    # EXACTLY ONE citation chip — the bar-clearing doc (the 2nd chip of
    # the pre-phase turn demoted to the row; the acceptance criterion).
    chip = page.locator(".msg.brain .source-chip")
    expect(chip).to_have_count(1, timeout=30_000)
    expect(chip.first).to_contain_text("ssh_aliases.txt")

    # The below-bar docs ride the related row: a labeled, de-emphasized
    # list — one .related-doc link per doc (rank order, capped at
    # related_max_docs = 2), never a .source-chip.
    row = page.locator(".msg.brain .related-docs")
    expect(row).to_have_count(1)
    expect(row.first).to_have_attribute("aria-label", "Nearby docs, in case")
    expect(row.first.locator(".related-docs-label")).to_have_text("Nearby docs, in case:")
    links = page.locator(".msg.brain .related-docs .related-doc")
    expect(links).to_have_count(2, timeout=30_000)
    expect(links.nth(0)).to_contain_text("gitlab-compose.yaml")
    expect(links.nth(1)).to_contain_text("uptime_probe.py")
    expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0)
    # The related links keep the chip's /document.html href + identity.
    expect(links.first).to_have_attribute(
        "title", "docs/homelab/container_gitlab/gitlab-compose.yaml"
    )

    # Durable record: not deflected; the FULL retrieval (cited + related)
    # is logged — query_log records retrieval, not citations (LOCKED A3).
    with SessionLocal() as db:
        row_log = db.scalars(select(QueryLog)).one()
    assert row_log.question == SINGLE_SOURCE_QUESTION
    assert row_log.deflected is False
    for path in (
        "homelab/ssh/ssh_aliases.txt",
        "homelab/container_gitlab/gitlab-compose.yaml",
        "homelab/scripts/uptime_probe.py",
    ):
        assert path in row_log.sources


def test_deflected_question_shows_zero_citation_chips(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The acceptance pin: a deflected turn (known-out-of-KB) → ZERO
    ``.source-chip`` elements under the bubble; the weak hits live in the
    related row only — its links are ``.related-doc``, never
    ``.source-chip`` (L2c: "at minimum: never render them as answer
    citations"). The "Maybe try" chips and the durable record are
    unchanged."""
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated

    _ask(page, OFF_TOPIC)
    bubble = page.locator(".msg.brain.is-deflected .bubble").first
    bubble.wait_for(state="visible", timeout=30_000)
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(1)

    # Wait for the done frame to have fully rendered (the related row and
    # the suggestion chips append in it), THEN pin the absence of chips —
    # a mid-stream reading would be a false zero.
    chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip")
    expect(chips.first).to_be_visible(timeout=30_000)
    assert chips.count() >= 2, "deflection must offer 2-3 alternative chips"

    row = page.locator(".msg.brain .related-docs")
    expect(row).to_have_count(1, timeout=30_000)
    links = page.locator(".msg.brain .related-docs .related-doc")
    expect(links).to_have_count(2)  # the weak hits, rank order, capped at 2
    expect(links.nth(0)).to_contain_text("new-service.md")
    expect(links.nth(1)).to_contain_text("ssh_aliases.txt")

    # ZERO citation chips under the bubble — the weak hits are scored
    # docs, not citations (the phase-112/113 contract on the wire).
    expect(page.locator(".msg.brain .source-chip")).to_have_count(0)
    expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0)

    # Durable record: deflected, weak top score — the retrieval stays
    # logged for threshold tuning (LOCKED A3: observability unchanged).
    with SessionLocal() as db:
        row_log = db.scalars(select(QueryLog)).one()
    assert row_log.question == OFF_TOPIC
    assert row_log.deflected is True
    assert 0.0 < row_log.top_score < get_settings().relevance_threshold
    assert row_log.fts_hits == 0
    assert row_log.sources  # the weak-hit paths, for threshold tuning