"""Phase 113 E2E (Playwright): the source-chip quality contract (TODO L5 + L2c) — the usefulness bar + the de-emphasized related-docs row, as VISIBLE chip counts. Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_source_chip_quality.py -v --no-cov Acceptance (TODO L144–146): "for a single-document question, the turn shows one citation chip"; a weak 2nd doc renders only in the de-emphasized related row (``.related-doc`` links, NEVER ``.source-chip``); a deflected turn shows zero citation chips (its weak hits, if any, live in the related row). The fixture KB's tier shapes are deterministic under the E2E mock's bag-of-words embeddings + the mock-calibrated bar (conftest: ``BOR_SOURCE_USEFULNESS_FLOOR=0.15``, half the 0.30 threshold — like the lexical floor): * **single-source question** — "What SSH aliases do I have?": ``ssh_aliases.txt`` is the ONLY doc whose best-chunk cosine clears the bar (0.352 ≥ 0.15; grounded at 0.352 ≥ 0.30) → the done frame carries exactly ONE cited ref; the two below-bar docs (gitlab-compose.yaml 0.025, uptime_probe.py 0.113) ride the related tier. This IS the strong+weak two-tier shape on the wire (the four OBSERVED live shapes are unit-pinned at plan level in ``tests/unit/test_source_chip_quality.py`` — the fixture KB reproduces the same shape live, so no docstring caveat is needed). * **deflected question** — "How do I bake sourdough bread?": best cosine 0.109 < 0.30 and zero FTS hits → honest deflection; nothing clears the bar → zero cited refs; the weak hits (new-service.md 0.109, ssh_aliases.txt 0.050) ride the related row. """ from __future__ import annotations import asyncio from pathlib import Path from threading import Thread from typing import Any from playwright.sync_api import Page, expect from sqlalchemy import select, text from app.config import Settings, get_settings from app.db import SessionLocal from app.models import QueryLog from app.rag.importer import ImportSummary, import_sources from app.rag.llm import LLMClient from e2e.auth_helpers import login REPO = Path(__file__).resolve().parents[2] FIXTURES = REPO / "tests" / "fixtures" / "docs" SINGLE_SOURCE_QUESTION = "What SSH aliases do I have?" OFF_TOPIC = "How do I bake sourdough bread?" MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E" async def _import_fixtures(mock_port: int) -> ImportSummary: kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"} settings = Settings(**kwargs) # pyright: ignore[reportCallIssue] return await import_sources([FIXTURES], LLMClient(settings)) def _run_in_thread(coro: Any) -> Any: """Run a coroutine on a worker thread (Playwright owns the test loop).""" box: dict[str, Any] = {} def runner() -> None: try: box["value"] = asyncio.run(coro) except BaseException as e: # noqa: BLE001 — re-raised on the test thread box["error"] = e t = Thread(target=runner) t.start() t.join() if "error" in box: raise box["error"] return box["value"] def _reset_db(mock_port: int, seed: bool = True) -> ImportSummary | None: """Truncate the KB (and query log), then optionally re-import fixtures.""" with SessionLocal() as db: db.execute(text("TRUNCATE chunks, documents, query_log")) db.commit() if not seed: return None return _run_in_thread(_import_fixtures(mock_port)) def _ask(page: Page, message: str) -> None: page.fill("#message-input", message) page.click("#send-btn") def test_single_source_question_shows_exactly_one_citation_chip( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """The acceptance pin: a single-document question → the done bubble carries EXACTLY ONE ``.source-chip`` (the bar-clearing doc) and the below-bar docs render only in the de-emphasized ``.related-docs`` row (``.related-doc`` links — never ``.source-chip``), labeled "Nearby docs, in case:". The durable record keeps the FULL retrieval (LOCKED A3).""" _reset_db(mock_llm) page.set_default_timeout(30_000) login(page, app_url, next="/") # phase 79: chat is require_user-gated _ask(page, SINGLE_SOURCE_QUESTION) expect(page.locator(".msg.user .bubble")).to_contain_text(SINGLE_SOURCE_QUESTION) bubble = page.locator(".msg.brain .bubble").first bubble.wait_for(state="visible", timeout=30_000) expect(bubble).to_contain_text(MOCK_ANSWER_MARKER, timeout=30_000) # Grounded: no deflected bubble at all. expect(page.locator(".msg.brain.is-deflected")).to_have_count(0) # EXACTLY ONE citation chip — the bar-clearing doc (the 2nd chip of # the pre-phase turn demoted to the row; the acceptance criterion). chip = page.locator(".msg.brain .source-chip") expect(chip).to_have_count(1, timeout=30_000) expect(chip.first).to_contain_text("ssh_aliases.txt") # The below-bar docs ride the related row: a labeled, de-emphasized # list — one .related-doc link per doc (rank order, capped at # related_max_docs = 2), never a .source-chip. row = page.locator(".msg.brain .related-docs") expect(row).to_have_count(1) expect(row.first).to_have_attribute("aria-label", "Nearby docs, in case") expect(row.first.locator(".related-docs-label")).to_have_text("Nearby docs, in case:") links = page.locator(".msg.brain .related-docs .related-doc") expect(links).to_have_count(2, timeout=30_000) expect(links.nth(0)).to_contain_text("gitlab-compose.yaml") expect(links.nth(1)).to_contain_text("uptime_probe.py") expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0) # The related links keep the chip's /document.html href + identity. expect(links.first).to_have_attribute( "title", "docs/homelab/container_gitlab/gitlab-compose.yaml" ) # Durable record: not deflected; the FULL retrieval (cited + related) # is logged — query_log records retrieval, not citations (LOCKED A3). with SessionLocal() as db: row_log = db.scalars(select(QueryLog)).one() assert row_log.question == SINGLE_SOURCE_QUESTION assert row_log.deflected is False for path in ( "homelab/ssh/ssh_aliases.txt", "homelab/container_gitlab/gitlab-compose.yaml", "homelab/scripts/uptime_probe.py", ): assert path in row_log.sources def test_deflected_question_shows_zero_citation_chips( page: Page, app_url: str, mock_llm: int, db_ready: None ) -> None: """The acceptance pin: a deflected turn (known-out-of-KB) → ZERO ``.source-chip`` elements under the bubble; the weak hits live in the related row only — its links are ``.related-doc``, never ``.source-chip`` (L2c: "at minimum: never render them as answer citations"). The "Maybe try" chips and the durable record are unchanged.""" _reset_db(mock_llm) page.set_default_timeout(30_000) login(page, app_url, next="/") # phase 79: chat is require_user-gated _ask(page, OFF_TOPIC) bubble = page.locator(".msg.brain.is-deflected .bubble").first bubble.wait_for(state="visible", timeout=30_000) expect(page.locator(".msg.brain.is-deflected")).to_have_count(1) # Wait for the done frame to have fully rendered (the related row and # the suggestion chips append in it), THEN pin the absence of chips — # a mid-stream reading would be a false zero. chips = page.locator(".msg.brain.is-deflected .maybe-try .suggestion-chip") expect(chips.first).to_be_visible(timeout=30_000) assert chips.count() >= 2, "deflection must offer 2-3 alternative chips" row = page.locator(".msg.brain .related-docs") expect(row).to_have_count(1, timeout=30_000) links = page.locator(".msg.brain .related-docs .related-doc") expect(links).to_have_count(2) # the weak hits, rank order, capped at 2 expect(links.nth(0)).to_contain_text("new-service.md") expect(links.nth(1)).to_contain_text("ssh_aliases.txt") # ZERO citation chips under the bubble — the weak hits are scored # docs, not citations (the phase-112/113 contract on the wire). expect(page.locator(".msg.brain .source-chip")).to_have_count(0) expect(page.locator(".msg.brain .related-docs .source-chip")).to_have_count(0) # Durable record: deflected, weak top score — the retrieval stays # logged for threshold tuning (LOCKED A3: observability unchanged). with SessionLocal() as db: row_log = db.scalars(select(QueryLog)).one() assert row_log.question == OFF_TOPIC assert row_log.deflected is True assert 0.0 < row_log.top_score < get_settings().relevance_threshold assert row_log.fts_hits == 0 assert row_log.sources # the weak-hit paths, for threshold tuning