"""Phase 113 E2E (Playwright) — phase 118 re-targeted, phase 119 re-targeted again: the citation-surface contract (phase 119, LOCKED A1) as VISIBLE chip counts — chips are the AGENT-READ docs only (the retired phase-118 A4 suggested+read union is gone): neither turn on this suite reads anything, so a grounded turn shows ZERO citation chips (the suggested tier is seed context, not citations — the phase-112/113 usefulness bar was retired with the full-text seeds, and the suggested chips were retired with the phase-118 A4 union), the related row is rank 6+ (capped at ``related_max_docs`` = 2, untouched), and a deflected turn still cites nothing (done.sources = [] — its weak hits are suggested for the durable record but are never citation chips). Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov Acceptance (TODO L144–146, phase-119 shape): a grounded turn's chip row is the READ set — empty on these no-read turns (the visible consequence of the owner decision 2026-09-16: the suggested docs the model was seeded with are context, not citations — the LLM decides what the summaries earn, and a zero-read turn chips nothing); a deflected turn shows zero citation chips (its weak hits, if any, live in the de-emphasized related row — ``.related-doc`` links, NEVER ``.source-chip``; L2c: never render weak hits as answer citations). The fixture KB's tier shapes are deterministic under the E2E mock's bag-of-words embeddings + the fused rank walk (probe-verified, pinned here against the wire): * **grounded question** — "What SSH aliases do I have?": best cosine 0.352 ≥ 0.30 → grounded; the suggested tier (top-5, NO floor) is ssh_aliases.txt > gitlab-compose.yaml > uptime_probe.py > backups.md > kubernetes.md (fused rank — the lexical-only cosine-0.0 docs rank when they rank: no floor filters them, LOCKED A3; phase 119, D1 — ``aliases`` is a stem sub-component of ``ssh_aliases.txt``, so the doc name-hits, its name-hit row LEADS the lexical list, and the FTS rank shift re-orders the fused scores — the pre-phase kubernetes-before-backups order is retired) and the done frame carries NO cited refs (phase 119 A1 — nothing was read: the suggested tier seeds the prompt but never chips); the rank-6+ remainder (compose.container, static-dns.json) rides the related row, capped at two. (The four OBSERVED live shapes are unit-pinned at plan level in ``tests/unit/test_source_chip_quality.py`` — the fixture KB reproduces the tiered shape live, so no docstring caveat is needed.) * **deflected question** — "How do I bake sourdough bread?": best cosine 0.109 < 0.30 and zero FTS hits → honest deflection → zero cited refs (done.sources = []); the weak hits ARE suggested (no floor) for the durable record — new-service.md > ssh_aliases.txt > lan.network > uptime_probe.py > compose.container — and the rank-6+ remainder (backups.md, kubernetes.md) rides the related row. """ from __future__ import annotations import asyncio from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import select, text from app.config import Settings, get_settings from app.db import SessionLocal from app.models import QueryLog from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from e2e.auth_helpers import login REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" SINGLE_SOURCE_QUESTION = "What SSH aliases do I have?" OFF_TOPIC = "How do I bake sourdough bread?" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread (Playwright owns the test loop).""" box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int, seed: bool = True) -> ImportSummary | None: """Truncate the KB (and query log), then optionally re-import fixtures.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) def _ask(page: Page, message: str) -> None: page.fill("#message-input", message) page.click("#send-btn") def _assert_ranked(recorded: str, groups: list[list[str]]) -> None: """Compare a ``query_log.sources`` string against an expected rank list whose trailing entries are TIE-GROUPS (order-free sets). WHY the groups exist: the E2E mock embedder builds real token-overlap vectors, so several tiny fixture documents end up on EXACTLY the same fused score. ``fuse`` then sorts by ``(-score, -cosine, path, position)``… but that only orders chunks whose SCORES differ; between two documents tied in both lists the vector rank itself comes out of ``ORDER BY embedding <=> q``, which is not a total order, so Postgres (a parallel seq scan handing its partial sorts back) may return the tied pair the other way round between two runs of the SAME fixture. That was this suite's flake. The rank of every block, and the exact SET inside it, are still pinned — what the story pins is that the record carries the FULL retrieval (LOCKED A3), not the luck of a tie. """ docs = [d.strip() for d in recorded.split(",")] expected = [d for group in groups for d in group] assert len(docs) == len(expected), f"recorded {docs}, expected {expected}" at = 0 for group in groups: window = docs[at : at + len(group)] assert sorted(window) == sorted(group), ( f"rank block {at + 1}..{at + len(group)}: got {window}, " f"expected the set {sorted(group)}" ) at += len(group) def _assert_related_pair(links: object, titles: set[str]) -> None: """The two related-doc links carry EXACTLY these two documents, in either order (the same tie the record pins as a group — see :func:`_assert_ranked`).""" found = { links.nth(i).get_attribute("title") # pyright: ignore[reportAttributeAccessIssue] for i in range(2) } assert found == titles, f"related pair is {found}, expected {titles}" def test_single_source_question_shows_exactly_one_citation_chip( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """The acceptance pin (phase 119, A1): a grounded, NO-READ question → the done bubble carries ZERO ``.source-chip``s (the chip row is the agent-read docs only — nothing was read here: the five suggested docs the model was seeded with are context, not citations — the retired phase-118 A4 union is gone) and the rank-6+ remainder renders only in the de-emphasized ``.related-docs`` row (``.related-doc`` links — never ``.source-chip``), labeled "Nearby docs, in case:" (phase-113 behavior, untouched). The durable record keeps the FULL retrieval (LOCKED A3, untouched).""" _reset_db(mock_llm) page.set_default_timeout(30_000) login(page, app_url, next="/") # phase 79: chat is require_user-gated _ask(page, SINGLE_SOURCE_QUESTION) expect(page.locator(".msg.user .bubble")).to_contain_text(SINGLE_SOURCE_QUESTION) bubble = page.locator(".msg.brain .bubble").first bubble.wait_for(state="visible", timeout=30_000) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) # Grounded: no deflected bubble at all. expect(page.locator(".msg.brain.is-deflected")).to_have_count(0) # Phase 119 (LOCKED A1): the citation surface is the READ docs only # — this turn read nothing, so ZERO chips; the five suggested docs # (ssh_aliases.txt > gitlab-compose.yaml > uptime_probe.py > # backups.md > kubernetes.md — the phase-119 D1 re-rank, pinned in # the durable record below) seeded the prompt but never chip — # the retired phase-118 A4 union is gone. The suggested tier itself # is pinned below by the durable record (LOCKED A3, untouched). expect(page.locator(".msg.brain .source-chip")).to_have_count(0) # The rank-6+ remainder rides the related row: a labeled, # de-emphasized list — one .related-doc link per doc (rank order, # capped at related_max_docs = 2), never a .source-chip. row = page.locator(".msg.brain .related-docs") expect(row).to_have_count(1) expect(row.first).to_have_attribute("aria-label", "Nearby docs, in case") expect(row.first.locator(".related-docs-label")).to_have_text("Nearby docs, in case:") links = page.locator(".msg.brain .related-docs .related-doc") expect(links).to_have_count(2, timeout=30_000) expect(links.nth(0)).to_contain_text("compose.container") expect(links.nth(1)).to_contain_text("static-dns.json") expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0) # The related links keep the chip's /document.html href + identity. expect(links.first).to_have_attribute( "title", "docs/homelab/quadlet/compose.container" ) # Durable record: not deflected; the FULL retrieval (suggested + # related + read, deduped) is logged — query_log records retrieval, # not citations (LOCKED A3). with SessionLocal() as db: row_log = db.scalars(select(QueryLog)).one() assert row_log.question == SINGLE_SOURCE_QUESTION assert row_log.deflected is False # Suggested tier + related remainder, in the logged order. Phase # 119 (D1): the stem sub-component ``aliases`` name-hits # ``ssh_aliases.txt`` — its name-hit row leads the lexical list, # the FTS rank shift re-orders the fused scores, and ``backups.md`` # (best eff 0.016277) now out-ranks ``kubernetes.md`` (0.016036); # re-pinned against the post-phase walk (the record's CONTENT — # suggested + related + read, deduped — is LOCKED A3, untouched). _assert_ranked( row_log.sources, [ ["docs/homelab/ssh/ssh_aliases.txt"], ["docs/homelab/container_gitlab/gitlab-compose.yaml"], ["docs/homelab/scripts/uptime_probe.py"], # The tied pair this record used to pin in an arbitrary order. ["docs/homelab/backups.md", "docs/homelab/kubernetes.md"], ["docs/homelab/quadlet/compose.container"], ["docs/homelab/networking/static-dns.json"], ], ) def test_deflected_question_shows_zero_citation_chips( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """The acceptance pin (phase 118, A4): a deflected turn (known-out-of-KB) → ZERO ``.source-chip`` elements under the bubble (done.sources = [] — a deflected answer cites nothing; the weak hits are suggested for the durable record but are scored docs, not citations); the rank-6+ weak hits live in the related row only — its links are ``.related-doc``, never ``.source-chip`` (L2c: "at minimum: never render them as answer citations"). The "Maybe try" chips are unchanged.""" _reset_db(mock_llm) page.set_default_timeout(30_000) login(page, app_url, next="/") # phase 79: chat is require_user-gated _ask(page, OFF_TOPIC) bubble = page.locator(".msg.brain.is-deflected .bubble").first bubble.wait_for(state="visible", timeout=30_000) expect(page.locator(".msg.brain.is-deflected")).to_have_count(1) # Wait for the done frame to have fully rendered (the related row and # the suggestion chips append in it), THEN pin the absence of chips — # a mid-stream reading would be a false zero. chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip") expect(chips.first).to_be_visible(timeout=30_000) assert chips.count() >= 2, "deflection must offer 2-3 alternative chips" row = page.locator(".msg.brain .related-docs") expect(row).to_have_count(1, timeout=30_000) links = page.locator(".msg.brain .related-docs .related-doc") # The rank-6+ remainder of the weak hits, rank order, capped at 2 — # the top five weak hits are suggested (no floor) for the durable # record, but a deflected turn cites nothing, so they never render # as chips. expect(links).to_have_count(2) # The pair, as a SET: the two are tied in the fused score (see # :func:`_assert_ranked`), so only their presence is a contract. _assert_related_pair(links, {"docs/homelab/backups.md", "docs/homelab/kubernetes.md"}) # ZERO citation chips under the bubble — the weak hits are scored # docs, not citations (the phase-112/113 contract on the wire). expect(page.locator(".msg.brain .source-chip")).to_have_count(0) expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0) # Durable record: deflected, weak top score — the FULL retrieval # (the weak hits ARE suggested — no floor — plus the rank-6+ related # remainder) stays logged for threshold tuning (LOCKED A3: # observability unchanged). with SessionLocal() as db: row_log = db.scalars(select(QueryLog)).one() assert row_log.question == OFF_TOPIC assert row_log.deflected is True assert 0.0 < row_log.top_score < get_settings().relevance_threshold assert row_log.fts_hits == 0 _assert_ranked( row_log.sources, [ ["docs/deployments/new-service.md"], ["docs/homelab/ssh/ssh_aliases.txt"], ["docs/homelab/quadlet/lan.network"], ["docs/homelab/scripts/uptime_probe.py"], ["docs/homelab/quadlet/compose.container"], # The related row IS this pair, and the row's order is the same # tie — pinned as a set below (``_assert_related_pair``). ["docs/homelab/backups.md", "docs/homelab/kubernetes.md"], ], )