brain-of-reese/tests/e2e/test_source_chip_quality.py

"""Phase 113 E2E (Playwright) — phase 118 re-targeted, phase 119
re-targeted again: the citation-surface contract (phase 119, LOCKED
A1) as VISIBLE chip counts — chips are the AGENT-READ docs only (the
retired phase-118 A4 suggested+read union is gone): neither turn on
this suite reads anything, so a grounded turn shows ZERO citation
chips (the suggested tier is seed context, not citations — the
phase-112/113 usefulness bar was retired with the full-text seeds, and
the suggested chips were retired with the phase-118 A4 union), the
related row is rank 6+ (capped at ``related_max_docs`` = 2, untouched),
and a deflected turn still cites nothing (done.sources = [] — its weak
hits are suggested for the durable record but are never citation
chips).

Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov

Acceptance (TODO L144–146, phase-119 shape): a grounded turn's chip row
is the READ set — empty on these no-read turns (the visible consequence
of the owner decision 2026-09-16: the suggested docs the model was
seeded with are context, not citations — the LLM decides what the
summaries earn, and a zero-read turn chips nothing); a deflected turn
shows zero citation chips (its weak hits, if any, live in the
de-emphasized related row — ``.related-doc`` links, NEVER
``.source-chip``; L2c: never render weak hits as answer citations).

The fixture KB's tier shapes are deterministic under the E2E mock's
bag-of-words embeddings + the fused rank walk (probe-verified, pinned
here against the wire):

* **grounded question** — "What SSH aliases do I have?": best cosine
  0.352 ≥ 0.30 → grounded; the suggested tier (top-5, NO floor) is
  ssh_aliases.txt > gitlab-compose.yaml > uptime_probe.py > backups.md
  > kubernetes.md (fused rank — the lexical-only cosine-0.0 docs rank
  when they rank: no floor filters them, LOCKED A3; phase 119, D1 —
  ``aliases`` is a stem sub-component of ``ssh_aliases.txt``, so the
  doc name-hits, its name-hit row LEADS the lexical list, and the FTS
  rank shift re-orders the fused scores — the pre-phase
  kubernetes-before-backups order is retired) and the done frame
  carries NO cited refs (phase 119 A1 — nothing was read: the
  suggested tier seeds the prompt but never chips); the rank-6+
  remainder (compose.container, static-dns.json) rides the related
  row, capped at two. (The four OBSERVED live shapes are unit-pinned at
  plan level in ``tests/unit/test_source_chip_quality.py`` — the
  fixture KB reproduces the tiered shape live, so no docstring caveat
  is needed.)
* **deflected question** — "How do I bake sourdough bread?": best cosine
  0.109 < 0.30 and zero FTS hits → honest deflection → zero cited refs
  (done.sources = []); the weak hits ARE suggested (no floor) for the
  durable record — new-service.md > ssh_aliases.txt > lan.network >
  uptime_probe.py > compose.container — and the rank-6+ remainder
  (backups.md, kubernetes.md) rides the related row.
"""
from __future__ import annotations

import asyncio
from pathlib import Path
from threading import Thread
from typing import Any

from playwright.sync_api import Page, expect
from sqlalchemy import select, text

from app.config import Settings, get_settings
from app.db import SessionLocal
from app.models import QueryLog
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from e2e.auth_helpers import login

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
SINGLE_SOURCE_QUESTION = "What SSH aliases do I have?"
OFF_TOPIC = "How do I bake sourdough bread?"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"


async def _import_fixtures(mock_port: int) -> ImportSummary:
    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return await import_sources([FIXTURES], LLMClient(settings))


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread (Playwright owns the test loop)."""
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(mock_port: int, seed: bool = True) -> ImportSummary | None:
    """Truncate the KB (and query log), then optionally re-import fixtures."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log"))
        db.commit()
    if not seed:
        return None
    return _run_in_thread(_import_fixtures(mock_port))


def _ask(page: Page, message: str) -> None:
    page.fill("#message-input", message)
    page.click("#send-btn")


def test_single_source_question_shows_exactly_one_citation_chip(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The acceptance pin (phase 119, A1): a grounded, NO-READ question →
    the done bubble carries ZERO ``.source-chip``s (the chip row is the
    agent-read docs only — nothing was read here: the five suggested
    docs the model was seeded with are context, not citations — the
    retired phase-118 A4 union is gone) and the rank-6+ remainder
    renders only in the de-emphasized ``.related-docs`` row
    (``.related-doc`` links — never ``.source-chip``), labeled "Nearby
    docs, in case:" (phase-113 behavior, untouched). The durable record
    keeps the FULL retrieval (LOCKED A3, untouched)."""
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated

    _ask(page, SINGLE_SOURCE_QUESTION)
    expect(page.locator(".msg.user .bubble")).to_contain_text(SINGLE_SOURCE_QUESTION)

    bubble = page.locator(".msg.brain .bubble").first
    bubble.wait_for(state="visible", timeout=30_000)
    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000)
    # Grounded: no deflected bubble at all.
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)

    # Phase 119 (LOCKED A1): the citation surface is the READ docs only
    # — this turn read nothing, so ZERO chips; the five suggested docs
    # (ssh_aliases.txt > gitlab-compose.yaml > uptime_probe.py >
    # backups.md > kubernetes.md — the phase-119 D1 re-rank, pinned in
    # the durable record below) seeded the prompt but never chip —
    # the retired phase-118 A4 union is gone. The suggested tier itself
    # is pinned below by the durable record (LOCKED A3, untouched).
    expect(page.locator(".msg.brain .source-chip")).to_have_count(0)

    # The rank-6+ remainder rides the related row: a labeled,
    # de-emphasized list — one .related-doc link per doc (rank order,
    # capped at related_max_docs = 2), never a .source-chip.
    row = page.locator(".msg.brain .related-docs")
    expect(row).to_have_count(1)
    expect(row.first).to_have_attribute("aria-label", "Nearby docs, in case")
    expect(row.first.locator(".related-docs-label")).to_have_text("Nearby docs, in case:")
    links = page.locator(".msg.brain .related-docs .related-doc")
    expect(links).to_have_count(2, timeout=30_000)
    expect(links.nth(0)).to_contain_text("compose.container")
    expect(links.nth(1)).to_contain_text("static-dns.json")
    expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0)
    # The related links keep the chip's /document.html href + identity.
    expect(links.first).to_have_attribute(
        "title", "docs/homelab/quadlet/compose.container"
    )

    # Durable record: not deflected; the FULL retrieval (suggested +
    # related + read, deduped) is logged — query_log records retrieval,
    # not citations (LOCKED A3).
    with SessionLocal() as db:
        row_log = db.scalars(select(QueryLog)).one()
    assert row_log.question == SINGLE_SOURCE_QUESTION
    assert row_log.deflected is False
    # Suggested tier + related remainder, in the logged order. Phase
    # 119 (D1): the stem sub-component ``aliases`` name-hits
    # ``ssh_aliases.txt`` — its name-hit row leads the lexical list,
    # the FTS rank shift re-orders the fused scores, and ``backups.md``
    # (best eff 0.016277) now out-ranks ``kubernetes.md`` (0.016036);
    # re-pinned against the post-phase walk (the record's CONTENT —
    # suggested + related + read, deduped — is LOCKED A3, untouched).
    assert row_log.sources == (
        "docs/homelab/ssh/ssh_aliases.txt, "
        "docs/homelab/container_gitlab/gitlab-compose.yaml, "
        "docs/homelab/scripts/uptime_probe.py, "
        "docs/homelab/backups.md, "
        "docs/homelab/kubernetes.md, "
        "docs/homelab/quadlet/compose.container, "
        "docs/homelab/networking/static-dns.json"
    ), row_log.sources


def test_deflected_question_shows_zero_citation_chips(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The acceptance pin (phase 118, A4): a deflected turn
    (known-out-of-KB) → ZERO ``.source-chip`` elements under the bubble
    (done.sources = [] — a deflected answer cites nothing; the weak hits
    are suggested for the durable record but are scored docs, not
    citations); the rank-6+ weak hits live in the related row only —
    its links are ``.related-doc``, never ``.source-chip`` (L2c: "at
    minimum: never render them as answer citations"). The "Maybe try"
    chips are unchanged."""
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated

    _ask(page, OFF_TOPIC)
    bubble = page.locator(".msg.brain.is-deflected .bubble").first
    bubble.wait_for(state="visible", timeout=30_000)
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(1)

    # Wait for the done frame to have fully rendered (the related row and
    # the suggestion chips append in it), THEN pin the absence of chips —
    # a mid-stream reading would be a false zero.
    chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip")
    expect(chips.first).to_be_visible(timeout=30_000)
    assert chips.count() >= 2, "deflection must offer 2-3 alternative chips"

    row = page.locator(".msg.brain .related-docs")
    expect(row).to_have_count(1, timeout=30_000)
    links = page.locator(".msg.brain .related-docs .related-doc")
    # The rank-6+ remainder of the weak hits, rank order, capped at 2 —
    # the top five weak hits are suggested (no floor) for the durable
    # record, but a deflected turn cites nothing, so they never render
    # as chips.
    expect(links).to_have_count(2)
    expect(links.nth(0)).to_contain_text("backups.md")
    expect(links.nth(1)).to_contain_text("kubernetes.md")

    # ZERO citation chips under the bubble — the weak hits are scored
    # docs, not citations (the phase-112/113 contract on the wire).
    expect(page.locator(".msg.brain .source-chip")).to_have_count(0)
    expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0)

    # Durable record: deflected, weak top score — the FULL retrieval
    # (the weak hits ARE suggested — no floor — plus the rank-6+ related
    # remainder) stays logged for threshold tuning (LOCKED A3:
    # observability unchanged).
    with SessionLocal() as db:
        row_log = db.scalars(select(QueryLog)).one()
    assert row_log.question == OFF_TOPIC
    assert row_log.deflected is True
    assert 0.0 < row_log.top_score < get_settings().relevance_threshold
    assert row_log.fts_hits == 0
    assert row_log.sources == (
        "docs/deployments/new-service.md, "
        "docs/homelab/ssh/ssh_aliases.txt, "
        "docs/homelab/quadlet/lan.network, "
        "docs/homelab/scripts/uptime_probe.py, "
        "docs/homelab/quadlet/compose.container, "
        "docs/homelab/backups.md, "
        "docs/homelab/kubernetes.md"
    ), row_log.sources