"""Phase 113 E2E (Playwright) — phase 118 re-targeted: the citation-surface contract (LOCKED A4) as VISIBLE chip counts — the summary-seed contract replaced the phase-112/113 usefulness bar: the chip row is the SUGGESTED tier (top-5 distinct docs, NO floor) + agent reads (none on these turns), the related row is rank 6+ (capped at ``related_max_docs`` = 2), and a deflected turn still cites nothing (done.sources = [] — its weak hits are suggested for the durable record but are never citation chips). Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov Acceptance (TODO L144–146, phase-118 shape): a grounded turn's chip row is the suggested tier — the 5 docs the model was seeded with (its "start here" set — the old "one chip per bar-clearing doc" is retired with the full-text seeds; the LLM decides what the summaries earn); a deflected turn shows zero citation chips (its weak hits, if any, live in the de-emphasized related row — ``.related-doc`` links, NEVER ``.source-chip``; L2c: never render weak hits as answer citations). The fixture KB's tier shapes are deterministic under the E2E mock's bag-of-words embeddings + the fused rank walk (probe-verified, pinned here against the wire): * **grounded question** — "What SSH aliases do I have?": best cosine 0.352 ≥ 0.30 → grounded; the suggested tier (top-5, NO floor) is ssh_aliases.txt > gitlab-compose.yaml > uptime_probe.py > kubernetes.md > backups.md (fused rank — the lexical-only cosine-0.0 docs rank when they rank: no floor filters them, LOCKED A3) and the done frame carries EXACTLY those five cited refs; the rank-6+ remainder (compose.container, static-dns.json) rides the related row, capped at two. (The four OBSERVED live shapes are unit-pinned at plan level in ``tests/unit/test_source_chip_quality.py`` — the fixture KB reproduces the tiered shape live, so no docstring caveat is needed.) * **deflected question** — "How do I bake sourdough bread?": best cosine 0.109 < 0.30 and zero FTS hits → honest deflection → zero cited refs (done.sources = []); the weak hits ARE suggested (no floor) for the durable record — new-service.md > ssh_aliases.txt > lan.network > uptime_probe.py > compose.container — and the rank-6+ remainder (backups.md, kubernetes.md) rides the related row. """ from __future__ import annotations import asyncio from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import select, text from app.config import Settings, get_settings from app.db import SessionLocal from app.models import QueryLog from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from e2e.auth_helpers import login REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" SINGLE_SOURCE_QUESTION = "What SSH aliases do I have?" OFF_TOPIC = "How do I bake sourdough bread?" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread (Playwright owns the test loop).""" box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int, seed: bool = True) -> ImportSummary | None: """Truncate the KB (and query log), then optionally re-import fixtures.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) def _ask(page: Page, message: str) -> None: page.fill("#message-input", message) page.click("#send-btn") def test_single_source_question_shows_exactly_one_citation_chip( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """The acceptance pin (phase 118, A4): a grounded question → the done bubble carries EXACTLY the suggested tier as ``.source-chip``s (the five docs the model was seeded with — top-5, NO floor: the old usefulness-bar "one chip" is retired with the full-text seeds) and the rank-6+ remainder renders only in the de-emphasized ``.related-docs`` row (``.related-doc`` links — never ``.source-chip``), labeled "Nearby docs, in case:". The durable record keeps the FULL retrieval (LOCKED A3).""" _reset_db(mock_llm) page.set_default_timeout(30_000) login(page, app_url, next="/") # phase 79: chat is require_user-gated _ask(page, SINGLE_SOURCE_QUESTION) expect(page.locator(".msg.user .bubble")).to_contain_text(SINGLE_SOURCE_QUESTION) bubble = page.locator(".msg.brain .bubble").first bubble.wait_for(state="visible", timeout=30_000) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) # Grounded: no deflected bubble at all. expect(page.locator(".msg.brain.is-deflected")).to_have_count(0) # The citation surface IS the suggested tier (LOCKED A4 — top-5, # NO floor): five chips, in fused rank order — the model was # seeded with exactly these five docs' summaries. chip = page.locator(".msg.brain .source-chip") expect(chip).to_have_count(5, timeout=30_000) expect(chip.nth(0)).to_contain_text("ssh_aliases.txt") expect(chip.nth(1)).to_contain_text("gitlab-compose.yaml") expect(chip.nth(2)).to_contain_text("uptime_probe.py") expect(chip.nth(3)).to_contain_text("kubernetes.md") expect(chip.nth(4)).to_contain_text("backups.md") # The rank-6+ remainder rides the related row: a labeled, # de-emphasized list — one .related-doc link per doc (rank order, # capped at related_max_docs = 2), never a .source-chip. row = page.locator(".msg.brain .related-docs") expect(row).to_have_count(1) expect(row.first).to_have_attribute("aria-label", "Nearby docs, in case") expect(row.first.locator(".related-docs-label")).to_have_text("Nearby docs, in case:") links = page.locator(".msg.brain .related-docs .related-doc") expect(links).to_have_count(2, timeout=30_000) expect(links.nth(0)).to_contain_text("compose.container") expect(links.nth(1)).to_contain_text("static-dns.json") expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0) # The related links keep the chip's /document.html href + identity. expect(links.first).to_have_attribute( "title", "docs/homelab/quadlet/compose.container" ) # Durable record: not deflected; the FULL retrieval (suggested + # related + read, deduped) is logged — query_log records retrieval, # not citations (LOCKED A3). with SessionLocal() as db: row_log = db.scalars(select(QueryLog)).one() assert row_log.question == SINGLE_SOURCE_QUESTION assert row_log.deflected is False # Suggested tier + related remainder, in the logged order. assert row_log.sources == ( "docs/homelab/ssh/ssh_aliases.txt, " "docs/homelab/container_gitlab/gitlab-compose.yaml, " "docs/homelab/scripts/uptime_probe.py, " "docs/homelab/kubernetes.md, " "docs/homelab/backups.md, " "docs/homelab/quadlet/compose.container, " "docs/homelab/networking/static-dns.json" ), row_log.sources def test_deflected_question_shows_zero_citation_chips( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """The acceptance pin (phase 118, A4): a deflected turn (known-out-of-KB) → ZERO ``.source-chip`` elements under the bubble (done.sources = [] — a deflected answer cites nothing; the weak hits are suggested for the durable record but are scored docs, not citations); the rank-6+ weak hits live in the related row only — its links are ``.related-doc``, never ``.source-chip`` (L2c: "at minimum: never render them as answer citations"). The "Maybe try" chips are unchanged.""" _reset_db(mock_llm) page.set_default_timeout(30_000) login(page, app_url, next="/") # phase 79: chat is require_user-gated _ask(page, OFF_TOPIC) bubble = page.locator(".msg.brain.is-deflected .bubble").first bubble.wait_for(state="visible", timeout=30_000) expect(page.locator(".msg.brain.is-deflected")).to_have_count(1) # Wait for the done frame to have fully rendered (the related row and # the suggestion chips append in it), THEN pin the absence of chips — # a mid-stream reading would be a false zero. chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip") expect(chips.first).to_be_visible(timeout=30_000) assert chips.count() >= 2, "deflection must offer 2-3 alternative chips" row = page.locator(".msg.brain .related-docs") expect(row).to_have_count(1, timeout=30_000) links = page.locator(".msg.brain .related-docs .related-doc") # The rank-6+ remainder of the weak hits, rank order, capped at 2 — # the top five weak hits are suggested (no floor) for the durable # record, but a deflected turn cites nothing, so they never render # as chips. expect(links).to_have_count(2) expect(links.nth(0)).to_contain_text("backups.md") expect(links.nth(1)).to_contain_text("kubernetes.md") # ZERO citation chips under the bubble — the weak hits are scored # docs, not citations (the phase-112/113 contract on the wire). expect(page.locator(".msg.brain .source-chip")).to_have_count(0) expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0) # Durable record: deflected, weak top score — the FULL retrieval # (the weak hits ARE suggested — no floor — plus the rank-6+ related # remainder) stays logged for threshold tuning (LOCKED A3: # observability unchanged). with SessionLocal() as db: row_log = db.scalars(select(QueryLog)).one() assert row_log.question == OFF_TOPIC assert row_log.deflected is True assert 0.0 < row_log.top_score < get_settings().relevance_threshold assert row_log.fts_hits == 0 assert row_log.sources == ( "docs/deployments/new-service.md, " "docs/homelab/ssh/ssh_aliases.txt, " "docs/homelab/quadlet/lan.network, " "docs/homelab/scripts/uptime_probe.py, " "docs/homelab/quadlet/compose.container, " "docs/homelab/backups.md, " "docs/homelab/kubernetes.md" ), row_log.sources