brain-of-reese/tests/e2e/test_source_chip_quality.py

"""Phase 113 E2E (Playwright) — phase 118 re-targeted: the citation-surface
contract (LOCKED A4) as VISIBLE chip counts — the summary-seed contract
replaced the phase-112/113 usefulness bar: the chip row is the
SUGGESTED tier (top-5 distinct docs, NO floor) + agent reads (none on
these turns), the related row is rank 6+ (capped at
``related_max_docs`` = 2), and a deflected turn still cites nothing
(done.sources = [] — its weak hits are suggested for the durable record
but are never citation chips).

Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov

Acceptance (TODO L144–146, phase-118 shape): a grounded turn's chip row
is the suggested tier — the 5 docs the model was seeded with (its
"start here" set — the old "one chip per bar-clearing doc" is retired
with the full-text seeds; the LLM decides what the summaries earn);
a deflected turn shows zero citation chips (its weak hits, if any, live
in the de-emphasized related row — ``.related-doc`` links, NEVER
``.source-chip``; L2c: never render weak hits as answer citations).

The fixture KB's tier shapes are deterministic under the E2E mock's
bag-of-words embeddings + the fused rank walk (probe-verified, pinned
here against the wire):

* **grounded question** — "What SSH aliases do I have?": best cosine
  0.352 ≥ 0.30 → grounded; the suggested tier (top-5, NO floor) is
  ssh_aliases.txt > gitlab-compose.yaml > uptime_probe.py > kubernetes.md
  > backups.md (fused rank — the lexical-only cosine-0.0 docs rank when
  they rank: no floor filters them, LOCKED A3) and the done frame
  carries EXACTLY those five cited refs; the rank-6+ remainder
  (compose.container, static-dns.json) rides the related row, capped at
  two. (The four OBSERVED live shapes are unit-pinned at plan level in
  ``tests/unit/test_source_chip_quality.py`` — the fixture KB
  reproduces the tiered shape live, so no docstring caveat is needed.)
* **deflected question** — "How do I bake sourdough bread?": best cosine
  0.109 < 0.30 and zero FTS hits → honest deflection → zero cited refs
  (done.sources = []); the weak hits ARE suggested (no floor) for the
  durable record — new-service.md > ssh_aliases.txt > lan.network >
  uptime_probe.py > compose.container — and the rank-6+ remainder
  (backups.md, kubernetes.md) rides the related row.
"""
from __future__ import annotations

import asyncio
from pathlib import Path
from threading import Thread
from typing import Any

from playwright.sync_api import Page, expect
from sqlalchemy import select, text

from app.config import Settings, get_settings
from app.db import SessionLocal
from app.models import QueryLog
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from e2e.auth_helpers import login

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
SINGLE_SOURCE_QUESTION = "What SSH aliases do I have?"
OFF_TOPIC = "How do I bake sourdough bread?"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"


async def _import_fixtures(mock_port: int) -> ImportSummary:
    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return await import_sources([FIXTURES], LLMClient(settings))


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread (Playwright owns the test loop)."""
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(mock_port: int, seed: bool = True) -> ImportSummary | None:
    """Truncate the KB (and query log), then optionally re-import fixtures."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log"))
        db.commit()
    if not seed:
        return None
    return _run_in_thread(_import_fixtures(mock_port))


def _ask(page: Page, message: str) -> None:
    page.fill("#message-input", message)
    page.click("#send-btn")


def test_single_source_question_shows_exactly_one_citation_chip(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The acceptance pin (phase 118, A4): a grounded question → the done
    bubble carries EXACTLY the suggested tier as ``.source-chip``s (the
    five docs the model was seeded with — top-5, NO floor: the old
    usefulness-bar "one chip" is retired with the full-text seeds) and
    the rank-6+ remainder renders only in the de-emphasized
    ``.related-docs`` row (``.related-doc`` links — never
    ``.source-chip``), labeled "Nearby docs, in case:". The durable
    record keeps the FULL retrieval (LOCKED A3)."""
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated

    _ask(page, SINGLE_SOURCE_QUESTION)
    expect(page.locator(".msg.user .bubble")).to_contain_text(SINGLE_SOURCE_QUESTION)

    bubble = page.locator(".msg.brain .bubble").first
    bubble.wait_for(state="visible", timeout=30_000)
    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000)
    # Grounded: no deflected bubble at all.
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)

    # The citation surface IS the suggested tier (LOCKED A4 — top-5,
    # NO floor): five chips, in fused rank order — the model was
    # seeded with exactly these five docs' summaries.
    chip = page.locator(".msg.brain .source-chip")
    expect(chip).to_have_count(5, timeout=30_000)
    expect(chip.nth(0)).to_contain_text("ssh_aliases.txt")
    expect(chip.nth(1)).to_contain_text("gitlab-compose.yaml")
    expect(chip.nth(2)).to_contain_text("uptime_probe.py")
    expect(chip.nth(3)).to_contain_text("kubernetes.md")
    expect(chip.nth(4)).to_contain_text("backups.md")

    # The rank-6+ remainder rides the related row: a labeled,
    # de-emphasized list — one .related-doc link per doc (rank order,
    # capped at related_max_docs = 2), never a .source-chip.
    row = page.locator(".msg.brain .related-docs")
    expect(row).to_have_count(1)
    expect(row.first).to_have_attribute("aria-label", "Nearby docs, in case")
    expect(row.first.locator(".related-docs-label")).to_have_text("Nearby docs, in case:")
    links = page.locator(".msg.brain .related-docs .related-doc")
    expect(links).to_have_count(2, timeout=30_000)
    expect(links.nth(0)).to_contain_text("compose.container")
    expect(links.nth(1)).to_contain_text("static-dns.json")
    expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0)
    # The related links keep the chip's /document.html href + identity.
    expect(links.first).to_have_attribute(
        "title", "docs/homelab/quadlet/compose.container"
    )

    # Durable record: not deflected; the FULL retrieval (suggested +
    # related + read, deduped) is logged — query_log records retrieval,
    # not citations (LOCKED A3).
    with SessionLocal() as db:
        row_log = db.scalars(select(QueryLog)).one()
    assert row_log.question == SINGLE_SOURCE_QUESTION
    assert row_log.deflected is False
    # Suggested tier + related remainder, in the logged order.
    assert row_log.sources == (
        "docs/homelab/ssh/ssh_aliases.txt, "
        "docs/homelab/container_gitlab/gitlab-compose.yaml, "
        "docs/homelab/scripts/uptime_probe.py, "
        "docs/homelab/kubernetes.md, "
        "docs/homelab/backups.md, "
        "docs/homelab/quadlet/compose.container, "
        "docs/homelab/networking/static-dns.json"
    ), row_log.sources


def test_deflected_question_shows_zero_citation_chips(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The acceptance pin (phase 118, A4): a deflected turn
    (known-out-of-KB) → ZERO ``.source-chip`` elements under the bubble
    (done.sources = [] — a deflected answer cites nothing; the weak hits
    are suggested for the durable record but are scored docs, not
    citations); the rank-6+ weak hits live in the related row only —
    its links are ``.related-doc``, never ``.source-chip`` (L2c: "at
    minimum: never render them as answer citations"). The "Maybe try"
    chips are unchanged."""
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated

    _ask(page, OFF_TOPIC)
    bubble = page.locator(".msg.brain.is-deflected .bubble").first
    bubble.wait_for(state="visible", timeout=30_000)
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(1)

    # Wait for the done frame to have fully rendered (the related row and
    # the suggestion chips append in it), THEN pin the absence of chips —
    # a mid-stream reading would be a false zero.
    chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip")
    expect(chips.first).to_be_visible(timeout=30_000)
    assert chips.count() >= 2, "deflection must offer 2-3 alternative chips"

    row = page.locator(".msg.brain .related-docs")
    expect(row).to_have_count(1, timeout=30_000)
    links = page.locator(".msg.brain .related-docs .related-doc")
    # The rank-6+ remainder of the weak hits, rank order, capped at 2 —
    # the top five weak hits are suggested (no floor) for the durable
    # record, but a deflected turn cites nothing, so they never render
    # as chips.
    expect(links).to_have_count(2)
    expect(links.nth(0)).to_contain_text("backups.md")
    expect(links.nth(1)).to_contain_text("kubernetes.md")

    # ZERO citation chips under the bubble — the weak hits are scored
    # docs, not citations (the phase-112/113 contract on the wire).
    expect(page.locator(".msg.brain .source-chip")).to_have_count(0)
    expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0)

    # Durable record: deflected, weak top score — the FULL retrieval
    # (the weak hits ARE suggested — no floor — plus the rank-6+ related
    # remainder) stays logged for threshold tuning (LOCKED A3:
    # observability unchanged).
    with SessionLocal() as db:
        row_log = db.scalars(select(QueryLog)).one()
    assert row_log.question == OFF_TOPIC
    assert row_log.deflected is True
    assert 0.0 < row_log.top_score < get_settings().relevance_threshold
    assert row_log.fts_hits == 0
    assert row_log.sources == (
        "docs/deployments/new-service.md, "
        "docs/homelab/ssh/ssh_aliases.txt, "
        "docs/homelab/quadlet/lan.network, "
        "docs/homelab/scripts/uptime_probe.py, "
        "docs/homelab/quadlet/compose.container, "
        "docs/homelab/backups.md, "
        "docs/homelab/kubernetes.md"
    ), row_log.sources