Files
brain-of-reese/tests/e2e/test_global_tuning.py
T

551 lines
22 KiB
Python

"""Phase 27 E2E (Playwright): global tuning manager — steering notes without
a chat.
The admin manages steering notes on a dedicated page (``/tuning.html``):
create, edit inline, list, and delete — **without any chat conversation**.
Notes are stored in Postgres (``steering_notes``) and read into the system
prompt of every future turn as the ``<tuning>`` section. The mock LLM
echoes the first tuning note into its answer (`` (tuning: <first note
line>)``), so prompt injection — created OR edited on this page — is
observable in the chat UI deterministically. The page's header carries the
admin-only "Tuning" nav link (``#nav-tuning`` — is-active on this page):
like the Sources link it ships hidden and ``header.js`` reveals it once
whoami says admin, so an anonymous visitor never sees it.
Story: ``.agent/user_stories/global-tuning.md``
Run in isolation (DB must be up: ``podman compose up -d db``):
uv run pytest tests/e2e/test_global_tuning.py -v --no-cov
Test → story mapping (Playwright Mapping Rule):
1. ``test_create_note_without_chat``
2. ``test_edit_note_inline``
3. ``test_delete_note``
4. ``test_edit_note_steers_answer``
5. ``test_tuning_page_a11y_and_no_cdn``
6. ``test_anonymous_cannot_manage``
"""
from __future__ import annotations
import asyncio
import re
from pathlib import Path
from threading import Thread
from typing import Any
from playwright.sync_api import Page, expect
from sqlalchemy import select, text
from app.config import Settings
from app.db import SessionLocal
from app.models import SteeringNote
from app.rag.importer import ImportSummary, import_sources
from app.rag.llm import LLMClient
from e2e.auth_helpers import login
REPO = Path(__file__).resolve().parents[2]
FIXTURES = REPO / "tests" / "fixtures" / "docs"
TUNING_URL = "/tuning.html"
QUESTION = "How is my Kubernetes cluster set up?"
MOCK_ANSWER_MARKER = "Deterministic mock answer for E2E"
NOTE = "TUNE-MARKER be concise"
EDITED_NOTE = "EDIT-MARKER assume I'm on NixOS"
SEED_NOTE = "SEED-MARKER an existing note"
# ---------------------------------------------------------------------------
# KB seeding + DB reset (same harness pattern as test_steering.py /
# test_document_viewer.py)
# ---------------------------------------------------------------------------
async def _import_fixtures(mock_port: int) -> ImportSummary:
kwargs: dict[str, Any] = {"_env_file": None, "llm_base_url": f"http://127.0.0.1:{mock_port}/v1"}
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
return await import_sources([FIXTURES], LLMClient(settings))
def _run_in_thread(coro: Any) -> Any:
"""Run a coroutine on a worker thread.
Playwright's sync API keeps an asyncio loop running on the test thread,
so ``asyncio.run`` cannot be called directly from a test body.
"""
box: dict[str, Any] = {}
def runner() -> None:
try:
box["value"] = asyncio.run(coro)
except BaseException as e: # noqa: BLE001 — re-raised on the test thread
box["error"] = e
t = Thread(target=runner)
t.start()
t.join()
if "error" in box:
raise box["error"]
return box["value"]
def _reset_db(mock_port: int, seed: bool) -> ImportSummary | None:
"""Truncate the KB (and query log + steering notes), re-import fixtures."""
with SessionLocal() as db:
db.execute(text("TRUNCATE chunks, documents, query_log, steering_notes"))
db.commit()
if not seed:
return None
return _run_in_thread(_import_fixtures(mock_port))
def _open_tuning(page: Page, app_url: str) -> None:
"""Form-login and land on /tuning.html, admin state fully wired.
The page's own admin-only "Tuning" nav link is the sync point: it
ships hidden and ``header.js`` reveals it once whoami says admin (the
exact Sources-link contract), so a visible ``#nav-tuning`` proves the
shared-header wiring on this page.
"""
login(page, app_url, next=TUNING_URL)
expect(page.locator("#nav-tuning")).to_be_visible(timeout=15_000)
expect(page.locator("#nav-sources")).to_be_visible()
expect(page.locator("#sign-out-btn")).to_be_visible()
expect(page.locator("#sign-in-link")).to_be_hidden()
# The link marks the current page.
expect(page.locator("#nav-tuning")).to_have_attribute("aria-current", "page")
expect(page.locator("#nav-tuning")).to_have_class(re.compile(r"\bis-active\b"))
def _create_note(page: Page, note: str) -> None:
"""Type *note* into the page's create form and submit it."""
page.fill("#tune-note", note)
page.click("#tune-save")
expect(page.locator("#tune-list .tuning-note-text")).to_have_text(note, timeout=15_000)
def _edit_note_inline(page: Page, new_note: str) -> None:
"""Open the row's inline edit form, replace the text, and Save."""
page.locator(".tuning-edit").click()
form = page.locator(".tuning-edit-form")
expect(form).to_be_visible()
form.locator(".tuning-edit-input").fill(new_note)
form.locator(".tune-save").click()
expect(page.locator("#tune-list .tuning-note-text")).to_have_text(new_note, timeout=15_000)
expect(page.locator(".tuning-edit-form")).to_have_count(0)
def _ask(page: Page, question: str) -> None:
"""Send one chat turn and wait until the grounded answer has fully landed."""
page.fill("#message-input", question)
page.click("#send-btn")
expect(page.locator(".msg.user .bubble").last).to_contain_text(question)
expect(page.locator(".msg.brain .bubble").last).to_contain_text(
MOCK_ANSWER_MARKER, timeout=30_000
)
expect(page.locator("#send-btn")).to_be_enabled()
expect(page.locator("#send-label")).to_have_text("Send")
# ---------------------------------------------------------------------------
# :focus-visible Tab walk (same pattern as test_responsive_polish.py) and
# WCAG 2.1 contrast (same helpers as test_dark_tech_theme.py)
# ---------------------------------------------------------------------------
def _tab_outline_walk(page: Page, max_tabs: int = 60) -> list[dict[str, str]]:
"""Real keyboard Tab walk; returns each focused element's outline."""
first_key: str | None = None
seen: list[dict[str, str]] = []
for _ in range(max_tabs):
page.keyboard.press("Tab")
info = page.evaluate(
"""() => {
const el = document.activeElement;
const cs = getComputedStyle(el);
const cls = String(el.className).split(" ")[0];
const label = (el.getAttribute("aria-label")
|| el.textContent || "").trim().slice(0, 24);
return {
key: el.tagName + "#" + (el.id || "") + "." + cls + ":" + label,
outline_style: cs.outlineStyle,
outline_width: cs.outlineWidth,
};
}"""
)
if info["key"].startswith("BODY"):
continue # focus has not entered the document yet
if first_key is None:
first_key = info["key"]
seen.append(info)
if len(seen) > 1 and info["key"] == first_key:
break # wrapped back to the first focusable
return seen
def _rgb(value: str) -> tuple[int, int, int]:
value = value.strip()
hex_match = re.match(r"^#([0-9a-f]{6})$", value, re.IGNORECASE)
if hex_match:
h = hex_match.group(1)
return int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16)
match = re.match(r"^rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)", value)
assert match, f"unparsable color: {value!r}"
return int(match.group(1)), int(match.group(2)), int(match.group(3))
def _rel_luminance(rgb: tuple[int, int, int]) -> float:
def chan(c: int) -> float:
s = c / 255
return s / 12.92 if s <= 0.04045 else ((s + 0.055) / 1.055) ** 2.4
r, g, b = (chan(c) for c in rgb)
return 0.2126 * r + 0.7152 * g + 0.0722 * b
def contrast_ratio(fg: str, bg: str) -> float:
l1, l2 = _rel_luminance(_rgb(fg)), _rel_luminance(_rgb(bg))
if l1 < l2:
l1, l2 = l2, l1
return (l1 + 0.05) / (l2 + 0.05)
def _assert_aa(pair: Any, label: str) -> None:
fg, bg = str(pair[0]), str(pair[1])
ratio = contrast_ratio(fg, bg)
assert ratio >= 4.5, f"contrast {label}: {fg} on {bg} = {ratio:.2f}:1 (< 4.5:1)"
# ---------------------------------------------------------------------------
# 1. Create a note without any chat
# ---------------------------------------------------------------------------
def test_create_note_without_chat(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_reset_db(mock_port=0, seed=False)
page.set_default_timeout(30_000)
_open_tuning(page, app_url)
# Fresh page: the empty state, no notes — and no chat anywhere in
# sight (the story's point: a note can exist before any conversation).
expect(page.locator("#tune-list .tuning-note")).to_have_count(0)
expect(page.locator("#tune-empty")).to_be_visible()
assert page.locator(".msg").count() == 0, "the tuning page carries no chat"
# Create: type + submit. The row lands in the list and the empty
# state steps aside…
_create_note(page, NOTE)
expect(page.locator("#tune-list .tuning-note")).to_have_count(1)
expect(page.locator("#tune-empty")).to_be_hidden()
# …the live region announces it, and the form is cleared (201).
expect(page.locator("#tune-announcer")).to_contain_text("Tuning note added.")
expect(page.locator("#tune-note")).to_have_value("")
# Persisted in Postgres — with zero chat turns in this whole test.
with SessionLocal() as db:
rows = db.scalars(select(SteeringNote)).all()
assert [r.note for r in rows] == [NOTE]
# ---------------------------------------------------------------------------
# 2. Edit a note inline (Cancel reverts, Save persists)
# ---------------------------------------------------------------------------
def test_edit_note_inline(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_reset_db(mock_port=0, seed=False)
page.set_default_timeout(30_000)
_open_tuning(page, app_url)
_create_note(page, NOTE)
# Open the inline edit form: the row swaps to a prefilled textarea
# with a (visually-hidden) label, while the row's text span and its
# Edit/Delete buttons step aside.
page.locator(".tuning-edit").click()
form = page.locator(".tuning-edit-form")
expect(form).to_be_visible()
edit_input = form.locator(".tuning-edit-input")
expect(edit_input).to_have_value(NOTE)
expect(form.locator("label.visually-hidden")).to_have_count(1)
expect(page.locator(".tuning-note-text")).to_be_hidden()
expect(page.locator(".tuning-note > .tuning-edit")).to_be_hidden()
expect(page.locator(".tuning-note > .tuning-delete")).to_be_hidden()
# Cancel reverts to the text span — the note is untouched.
form.locator(".tune-cancel").click()
expect(page.locator(".tuning-edit-form")).to_have_count(0)
expect(page.locator("#tune-list .tuning-note-text")).to_have_text(NOTE)
# Now for real: open, change the text, Save.
page.locator(".tuning-edit").click()
form = page.locator(".tuning-edit-form")
expect(form).to_be_visible()
form.locator(".tuning-edit-input").fill(EDITED_NOTE)
form.locator(".tune-save").click()
# The list shows the NEW text, the edit form is gone, and a "Saved"
# pill + the live region confirm the update.
expect(page.locator("#tune-list .tuning-note-text")).to_have_text(
EDITED_NOTE, timeout=15_000
)
expect(page.locator(".tuning-edit-form")).to_have_count(0)
expect(page.locator(".tuning-note-text")).to_be_visible()
expect(page.locator(".tuning-saved")).to_contain_text("Saved")
expect(page.locator("#tune-announcer")).to_contain_text("Tuning note updated.")
# Postgres: still exactly one note, with the updated text (edited in
# place — no duplicate row).
with SessionLocal() as db:
rows = db.scalars(select(SteeringNote)).all()
assert [r.note for r in rows] == [EDITED_NOTE]
# ---------------------------------------------------------------------------
# 3. Delete a note → row removed, empty state back
# ---------------------------------------------------------------------------
def test_delete_note(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_reset_db(mock_port=0, seed=False)
page.set_default_timeout(30_000)
_open_tuning(page, app_url)
_create_note(page, NOTE)
expect(page.locator("#tune-empty")).to_be_hidden()
# Delete: the row leaves the DOM and the empty state comes back.
page.locator(".tuning-delete").click()
expect(page.locator("#tune-list .tuning-note")).to_have_count(0, timeout=15_000)
expect(page.locator("#tune-empty")).to_be_visible()
expect(page.locator("#tune-announcer")).to_contain_text("Tuning note deleted.")
# Gone from Postgres.
with SessionLocal() as db:
assert db.scalars(select(SteeringNote)).all() == []
# ---------------------------------------------------------------------------
# 4. A note created + edited on /tuning.html steers the next chat answer
# ---------------------------------------------------------------------------
def test_edit_note_steers_answer(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
summary = _reset_db(mock_llm, seed=True)
assert summary is not None and summary.added == 8 # A9 formats
page.set_default_timeout(30_000)
_open_tuning(page, app_url)
# Create the note without a chat…
_create_note(page, NOTE)
# …and edit it to the final wording (inline — still no chat).
_edit_note_inline(page, EDITED_NOTE)
expect(page.locator("#tune-announcer")).to_contain_text("Tuning note updated.")
# Now to the chat via the header's "Chat" nav link (a real navigation):
# the next answer must carry the EDITED note. The mock echoes the
# first <tuning> line, so the marker proves the edited note reached
# the system prompt.
page.click('a.nav-link[href="/"]')
expect(page.locator("#message-input")).to_be_visible(timeout=15_000)
_ask(page, QUESTION)
bubble = page.locator(".msg.brain .bubble").last
expect(bubble).to_contain_text(MOCK_ANSWER_MARKER)
expect(bubble).to_contain_text(f"(tuning: {EDITED_NOTE})")
# ---------------------------------------------------------------------------
# 5. Page accessibility + no-CDN
# ---------------------------------------------------------------------------
def test_tuning_page_a11y_and_no_cdn(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_reset_db(mock_port=0, seed=False)
# One note in the DB before load so the page renders a real row (the
# row's Edit/Delete buttons are JS-built — they need a note).
with SessionLocal() as db:
db.add(SteeringNote(note=SEED_NOTE))
db.commit()
page.set_default_timeout(30_000)
_open_tuning(page, app_url)
expect(page.locator("#tune-list .tuning-note")).to_have_count(1)
page.wait_for_load_state("networkidle")
# Landmarks (PLAN §7.2).
assert page.locator("header.app-header").count() == 1, "header missing"
assert page.locator("nav[aria-label]").count() == 1, "labeled nav missing"
assert page.locator("main#main").count() == 1, "main#main missing"
assert page.locator("footer.app-footer").count() == 1, "footer missing"
# Skip link: present, first Tab lands on it, Enter moves focus to #main.
skip = page.locator('a.skip-link[href="#main"]')
assert skip.count() == 1, "skip link missing"
page.keyboard.press("Tab")
assert page.evaluate("() => document.activeElement.className") == "skip-link", (
"first Tab must land on the skip link"
)
page.keyboard.press("Enter")
assert page.evaluate("() => document.activeElement.id") == "main", (
"skip link must move focus to #main"
)
# Labeled controls: the create textarea has its (visually-hidden)
# label, every button has an accessible name, and the per-row Delete
# is labeled with the note text (Edit carries visible text).
expect(page.get_by_label("Add a global tuning note")).to_have_count(1)
unnamed = page.evaluate(
"""() => [...document.querySelectorAll("button")]
.filter((b) => !(b.getAttribute("aria-label") || b.textContent.trim()))
.length"""
)
assert unnamed == 0, f"{unnamed} button(s) without an accessible name"
delete_btn = page.locator("#tune-list .tuning-delete")
assert (delete_btn.get_attribute("aria-label") or "").startswith(
"Delete tuning note:"
), "row Delete must be labeled with the note text"
expect(page.locator("#tune-list .tuning-edit")).to_have_text("Edit")
# The live region that announces create / edit / delete.
announcer = page.locator("#tune-announcer")
assert announcer.get_attribute("role") == "status"
assert announcer.get_attribute("aria-live") == "polite"
# ≥44px touch targets on every interactive control.
for selector in (
"#tune-save",
"#tune-note",
"#nav-tuning",
".new-chat-btn",
"#sign-out-btn",
".tuning-edit",
".tuning-delete",
):
box = page.locator(selector).first.bounding_box()
assert box is not None and box["height"] >= 44, f"target too small: {selector} {box}"
# :focus-visible — a real keyboard Tab walk: every focused element
# shows a visible outline (solid, ≥2px; the design uses 3px).
page.evaluate(
"() => { if (document.activeElement instanceof HTMLElement)"
" document.activeElement.blur(); }"
)
seen = _tab_outline_walk(page)
assert len(seen) >= 6, f"expected several focusables, tabbed {len(seen)}"
for info in seen:
width_px = float(info["outline_width"].replace("px", ""))
assert info["outline_style"] == "solid" and width_px >= 2, (
f"no visible focus outline on {info['key']} "
f"({info['outline_style']} {info['outline_width']})"
)
# Dark theme: the page canvas is the phase-08 dark bg (body stays
# transparent for the background layers).
bg = page.evaluate("() => getComputedStyle(document.documentElement).backgroundColor")
assert bg == "rgb(10, 14, 23)", f"expected the dark page bg, got {bg}"
# Contrast: the live text/background pairs compute ≥ 4.5:1 (AA).
pairs = page.evaluate(
"""() => {
const cs = (sel, prop) => getComputedStyle(document.querySelector(sel))[prop];
return {
save_btn: [cs("#tune-save", "color"), cs("#tune-save", "backgroundColor")],
active_nav: [
cs(".nav-link.is-active", "color"),
cs(".nav-link.is-active", "backgroundColor"),
],
note_text: [
cs(".tuning-note-text", "color"),
cs(".tuning-panel", "backgroundColor"),
],
empty_state: [
cs("#tune-empty", "color"),
cs(".tuning-panel", "backgroundColor"),
],
};
}"""
)
_assert_aa(pairs["save_btn"], "dark ink on brand (Add note)")
_assert_aa(pairs["active_nav"], "dark ink on brand (active nav)")
_assert_aa(pairs["note_text"], "ink on surface (note text)")
_assert_aa(pairs["empty_state"], "ink-soft on surface (empty state)")
# No CDN: every script/link reference is same-origin or a data: URI.
refs = page.evaluate(
"""() => [...document.querySelectorAll("script[src], link[href]")]
.map((el) => el.src || el.href)"""
)
assert refs, "expected local asset references"
for ref in refs:
assert ref.startswith(app_url) or ref.startswith("data:"), (
f"non-local asset reference: {ref}"
)
# ---------------------------------------------------------------------------
# 6. Anonymous: no list, no PUT, no create
# ---------------------------------------------------------------------------
def test_anonymous_cannot_manage(
page: Page, app_url: str, mock_llm: int, db_ready: None
) -> None:
_reset_db(mock_port=0, seed=False)
# A note exists in the DB (the admin saved it at some point) — an
# anonymous visitor must never see it or touch it.
note = SteeringNote(note=SEED_NOTE)
with SessionLocal() as db:
db.add(note)
db.commit()
db.refresh(note)
note_id = note.id
page.set_default_timeout(30_000)
page.goto(app_url + TUNING_URL)
# Anonymous header state: Sign in visible, and the admin-only nav
# links — Sources AND this page's own Tuning link — stay hidden.
expect(page.locator("#sign-in-link")).to_be_visible()
expect(page.locator("#sign-out-btn")).to_be_hidden()
expect(page.locator("#nav-sources")).to_be_hidden()
expect(page.locator("#nav-tuning")).to_be_hidden()
# The list stays on its empty state even though a note exists…
expect(page.locator("#tune-list .tuning-note")).to_have_count(0)
expect(page.locator("#tune-empty")).to_be_visible()
# …and the API is gated: GET and a PUT on a REAL note id are 403.
get_status = page.evaluate("async () => (await fetch('/api/steering')).status")
assert get_status == 403, f"anonymous GET /api/steering → {get_status}"
put_status = page.evaluate(
"""async (id) => (await fetch('/api/steering/' + id, {
method: 'PUT',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({note: 'anonymous overwrite attempt'}),
})).status""",
str(note_id),
)
assert put_status == 403, f"anonymous PUT /api/steering/{note_id} → {put_status}"
with SessionLocal() as db:
row = db.get(SteeringNote, note_id)
assert row is not None and row.note == SEED_NOTE, "the note must stay untouched"
# The create form 403s gracefully: an inline error (role=alert) with
# the API detail, and the typed instruction survives in the textarea.
page.fill("#tune-note", "an anonymous attempt")
page.click("#tune-save")
error = page.locator("#tune-form .tuning-error")
expect(error).to_have_attribute("role", "alert")
expect(error).to_be_visible(timeout=15_000)
assert "admin only" in (error.inner_text() or "").lower()
expect(page.locator("#tune-note")).to_have_value("an anonymous attempt")
# Nothing was created: the DB still holds only the seeded note.
with SessionLocal() as db:
assert [r.note for r in db.scalars(select(SteeringNote)).all()] == [SEED_NOTE]