brain-of-reese/tests/e2e/test_source_chip_quality.py

"""Phase 113 E2E (Playwright) — phase 118 re-targeted, phase 119
re-targeted again: the citation-surface contract (phase 119, LOCKED
A1) as VISIBLE chip counts — chips are the AGENT-READ docs only (the
retired phase-118 A4 suggested+read union is gone): neither turn on
this suite reads anything, so a grounded turn shows ZERO citation
chips (the suggested tier is seed context, not citations — the
phase-112/113 usefulness bar was retired with the full-text seeds, and
the suggested chips were retired with the phase-118 A4 union), the
related row is rank 6+ (capped at ``related_max_docs`` = 2, untouched),
and a deflected turn still cites nothing (done.sources = [] — its weak
hits are suggested for the durable record but are never citation
chips).

Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov

Acceptance (TODO L144–146, phase-119 shape): a grounded turn's chip row
is the READ set — empty on these no-read turns (the visible consequence
of the owner decision 2026-09-16: the suggested docs the model was
seeded with are context, not citations — the LLM decides what the
summaries earn, and a zero-read turn chips nothing); a deflected turn
shows zero citation chips (its weak hits, if any, live in the
de-emphasized related row — ``.related-doc`` links, NEVER
``.source-chip``; L2c: never render weak hits as answer citations).

The fixture KB's tier shapes are deterministic under the E2E mock's
bag-of-words embeddings + the fused rank walk (probe-verified, pinned
here against the wire):

* **grounded question** — "What SSH aliases do I have?": best cosine
  0.352 ≥ 0.30 → grounded; the suggested tier (top-5, NO floor) is
  ssh_aliases.txt > gitlab-compose.yaml > uptime_probe.py > backups.md
  > kubernetes.md (fused rank — the lexical-only cosine-0.0 docs rank
  when they rank: no floor filters them, LOCKED A3; phase 119, D1 —
  ``aliases`` is a stem sub-component of ``ssh_aliases.txt``, so the
  doc name-hits, its name-hit row LEADS the lexical list, and the FTS
  rank shift re-orders the fused scores — the pre-phase
  kubernetes-before-backups order is retired) and the done frame
  carries NO cited refs (phase 119 A1 — nothing was read: the
  suggested tier seeds the prompt but never chips); the rank-6+
  remainder (compose.container, static-dns.json) rides the related
  row, capped at two. (The four OBSERVED live shapes are unit-pinned at
  plan level in ``tests/unit/test_source_chip_quality.py`` — the
  fixture KB reproduces the tiered shape live, so no docstring caveat
  is needed.)
* **deflected question** — "How do I bake sourdough bread?": best cosine
  0.109 < 0.30 and zero FTS hits → honest deflection → zero cited refs
  (done.sources = []); the weak hits ARE suggested (no floor) for the
  durable record — new-service.md > ssh_aliases.txt > lan.network >
  uptime_probe.py > compose.container — and the rank-6+ remainder
  (backups.md, kubernetes.md) rides the related row.
"""
from __future__ import annotations

import asyncio
from pathlib import Path
from threading import Thread
from typing import Any

from playwright.sync_api import Page, expect
from sqlalchemy import select, text

from app.config import Settings, get_settings
from app.db import SessionLocal
from app.models import QueryLog
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from e2e.auth_helpers import login

REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
SINGLE_SOURCE_QUESTION = "What SSH aliases do I have?"
OFF_TOPIC = "How do I bake sourdough bread?"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"


async def _import_fixtures(mock_port: int) -> ImportSummary:
    kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
    settings = Settings(**kwargs)  # pyright: ignore[reportCallIssue]
    return await import_sources([FIXTURES], LLMClient(settings))


def _run_in_thread(coro: Any) -> Any:
    """Run a coroutine on a worker thread (Playwright owns the test loop)."""
    box: dict[str, Any] = {}

    def runner() -> None:
        try:
            box["value"] = asyncio.run(coro)
        except BaseException as e:  # noqa: BLE001 — re-raised on the test thread
            box["error"] = e

    t = Thread(target=runner)
    t.start()
    t.join()
    if "error" in box:
        raise box["error"]
    return box["value"]


def _reset_db(mock_port: int, seed: bool = True) -> ImportSummary | None:
    """Truncate the KB (and query log), then optionally re-import fixtures."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE chunks, documents, query_log"))
        db.commit()
    if not seed:
        return None
    return _run_in_thread(_import_fixtures(mock_port))


def _ask(page: Page, message: str) -> None:
    page.fill("#message-input", message)
    page.click("#send-btn")


def _assert_ranked(recorded: str, groups: list[list[str]]) -> None:
    """Compare a ``query_log.sources`` string against an expected rank list
    whose trailing entries are TIE-GROUPS (order-free sets).

    WHY the groups exist: the E2E mock embedder builds real token-overlap
    vectors, so several tiny fixture documents end up on EXACTLY the same
    fused score. ``fuse`` then sorts by ``(-score, -cosine, path,
    position)``… but that only orders chunks whose SCORES differ; between
    two documents tied in both lists the vector rank itself comes out of
    ``ORDER BY embedding <=> q``, which is not a total order, so Postgres
    (a parallel seq scan handing its partial sorts back) may return the
    tied pair the other way round between two runs of the SAME fixture.
    That was this suite's flake. The rank of every block, and the exact
    SET inside it, are still pinned — what the story pins is that the
    record carries the FULL retrieval (LOCKED A3), not the luck of a tie.
    """
    docs = [d.strip() for d in recorded.split(",")]
    expected = [d for group in groups for d in group]
    assert len(docs) == len(expected), f"recorded {docs}, expected {expected}"
    at = 0
    for group in groups:
        window = docs[at : at + len(group)]
        assert sorted(window) == sorted(group), (
            f"rank block {at + 1}..{at + len(group)}: got {window}, "
            f"expected the set {sorted(group)}"
        )
        at += len(group)


def _assert_related_pair(links: object, titles: set[str]) -> None:
    """The two related-doc links carry EXACTLY these two documents, in
    either order (the same tie the record pins as a group — see
    :func:`_assert_ranked`)."""
    found = {
        links.nth(i).get_attribute("title")  # pyright: ignore[reportAttributeAccessIssue]
        for i in range(2)
    }
    assert found == titles, f"related pair is {found}, expected {titles}"


def test_single_source_question_shows_exactly_one_citation_chip(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The acceptance pin (phase 119, A1): a grounded, NO-READ question →
    the done bubble carries ZERO ``.source-chip``s (the chip row is the
    agent-read docs only — nothing was read here: the five suggested
    docs the model was seeded with are context, not citations — the
    retired phase-118 A4 union is gone) and the rank-6+ remainder
    renders only in the de-emphasized ``.related-docs`` row
    (``.related-doc`` links — never ``.source-chip``), labeled "Nearby
    docs, in case:" (phase-113 behavior, untouched). The durable record
    keeps the FULL retrieval (LOCKED A3, untouched)."""
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated

    _ask(page, SINGLE_SOURCE_QUESTION)
    expect(page.locator(".msg.user .bubble")).to_contain_text(SINGLE_SOURCE_QUESTION)

    bubble = page.locator(".msg.brain .bubble").first
    bubble.wait_for(state="visible", timeout=30_000)
    expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000)
    # Grounded: no deflected bubble at all.
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(0)

    # Phase 119 (LOCKED A1): the citation surface is the READ docs only
    # — this turn read nothing, so ZERO chips; the five suggested docs
    # (ssh_aliases.txt > gitlab-compose.yaml > uptime_probe.py >
    # backups.md > kubernetes.md — the phase-119 D1 re-rank, pinned in
    # the durable record below) seeded the prompt but never chip —
    # the retired phase-118 A4 union is gone. The suggested tier itself
    # is pinned below by the durable record (LOCKED A3, untouched).
    expect(page.locator(".msg.brain .source-chip")).to_have_count(0)

    # The rank-6+ remainder rides the related row: a labeled,
    # de-emphasized list — one .related-doc link per doc (rank order,
    # capped at related_max_docs = 2), never a .source-chip.
    row = page.locator(".msg.brain .related-docs")
    expect(row).to_have_count(1)
    expect(row.first).to_have_attribute("aria-label", "Nearby docs, in case")
    expect(row.first.locator(".related-docs-label")).to_have_text("Nearby docs, in case:")
    links = page.locator(".msg.brain .related-docs .related-doc")
    expect(links).to_have_count(2, timeout=30_000)
    expect(links.nth(0)).to_contain_text("compose.container")
    expect(links.nth(1)).to_contain_text("static-dns.json")
    expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0)
    # The related links keep the chip's /document.html href + identity.
    expect(links.first).to_have_attribute(
        "title", "docs/homelab/quadlet/compose.container"
    )

    # Durable record: not deflected; the FULL retrieval (suggested +
    # related + read, deduped) is logged — query_log records retrieval,
    # not citations (LOCKED A3).
    with SessionLocal() as db:
        row_log = db.scalars(select(QueryLog)).one()
    assert row_log.question == SINGLE_SOURCE_QUESTION
    assert row_log.deflected is False
    # Suggested tier + related remainder, in the logged order. Phase
    # 119 (D1): the stem sub-component ``aliases`` name-hits
    # ``ssh_aliases.txt`` — its name-hit row leads the lexical list,
    # the FTS rank shift re-orders the fused scores, and ``backups.md``
    # (best eff 0.016277) now out-ranks ``kubernetes.md`` (0.016036);
    # re-pinned against the post-phase walk (the record's CONTENT —
    # suggested + related + read, deduped — is LOCKED A3, untouched).
    _assert_ranked(
        row_log.sources,
        [
            ["docs/homelab/ssh/ssh_aliases.txt"],
            ["docs/homelab/container_gitlab/gitlab-compose.yaml"],
            ["docs/homelab/scripts/uptime_probe.py"],
            # The tied pair this record used to pin in an arbitrary order.
            ["docs/homelab/backups.md", "docs/homelab/kubernetes.md"],
            ["docs/homelab/quadlet/compose.container"],
            ["docs/homelab/networking/static-dns.json"],
        ],
    )


def test_deflected_question_shows_zero_citation_chips(
    page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
    """The acceptance pin (phase 118, A4): a deflected turn
    (known-out-of-KB) → ZERO ``.source-chip`` elements under the bubble
    (done.sources = [] — a deflected answer cites nothing; the weak hits
    are suggested for the durable record but are scored docs, not
    citations); the rank-6+ weak hits live in the related row only —
    its links are ``.related-doc``, never ``.source-chip`` (L2c: "at
    minimum: never render them as answer citations"). The "Maybe try"
    chips are unchanged."""
    _reset_db(mock_llm)
    page.set_default_timeout(30_000)
    login(page, app_url, next="/")  # phase 79: chat is require_user-gated

    _ask(page, OFF_TOPIC)
    bubble = page.locator(".msg.brain.is-deflected .bubble").first
    bubble.wait_for(state="visible", timeout=30_000)
    expect(page.locator(".msg.brain.is-deflected")).to_have_count(1)

    # Wait for the done frame to have fully rendered (the related row and
    # the suggestion chips append in it), THEN pin the absence of chips —
    # a mid-stream reading would be a false zero.
    chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip")
    expect(chips.first).to_be_visible(timeout=30_000)
    assert chips.count() >= 2, "deflection must offer 2-3 alternative chips"

    row = page.locator(".msg.brain .related-docs")
    expect(row).to_have_count(1, timeout=30_000)
    links = page.locator(".msg.brain .related-docs .related-doc")
    # The rank-6+ remainder of the weak hits, rank order, capped at 2 —
    # the top five weak hits are suggested (no floor) for the durable
    # record, but a deflected turn cites nothing, so they never render
    # as chips.
    expect(links).to_have_count(2)
    # The pair, as a SET: the two are tied in the fused score (see
    # :func:`_assert_ranked`), so only their presence is a contract.
    _assert_related_pair(links, {"docs/homelab/backups.md", "docs/homelab/kubernetes.md"})

    # ZERO citation chips under the bubble — the weak hits are scored
    # docs, not citations (the phase-112/113 contract on the wire).
    expect(page.locator(".msg.brain .source-chip")).to_have_count(0)
    expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0)

    # Durable record: deflected, weak top score — the FULL retrieval
    # (the weak hits ARE suggested — no floor — plus the rank-6+ related
    # remainder) stays logged for threshold tuning (LOCKED A3:
    # observability unchanged).
    with SessionLocal() as db:
        row_log = db.scalars(select(QueryLog)).one()
    assert row_log.question == OFF_TOPIC
    assert row_log.deflected is True
    assert 0.0 < row_log.top_score < get_settings().relevance_threshold
    assert row_log.fts_hits == 0
    _assert_ranked(
        row_log.sources,
        [
            ["docs/deployments/new-service.md"],
            ["docs/homelab/ssh/ssh_aliases.txt"],
            ["docs/homelab/quadlet/lan.network"],
            ["docs/homelab/scripts/uptime_probe.py"],
            ["docs/homelab/quadlet/compose.container"],
            # The related row IS this pair, and the row's order is the same
            # tie — pinned as a set below (``_assert_related_pair``).
            ["docs/homelab/backups.md", "docs/homelab/kubernetes.md"],
        ],
    )