"""Phase 172 E2E (Playwright): the Thinking level row in the admin Settings panel — the owner's acceptance for a pi-style, operator-tunable LLM reasoning knob, proven in a real browser. Source: ``TODO.md`` L3 — "Brain of Reese needs to support thinking levels like pi does. should be a free-form text entry like chat model. Default should be 'low'. Update .env and admin settings." Run in isolation (DB must be up: ``podman compose up -d db``): uv run pytest tests/e2e/test_thinking_level.py -v --no-cov Test → contract mapping (one story, one phase, one isolated file): 1. ``test_row_renders_free_form_with_default_low`` — the row: the ``LLM (aipi)`` group band is visible (the phase-171 band contract), a row labelled **Thinking level** with the ``BOR_LLM_THINKING_LEVEL`` env-var display (the catalogue pattern) is present, its input is a **free-form text** input (``type="text"`` — NOT ``number``/``select``, the A4 contract), it shows ``low`` with the ``env`` badge, and the ``llm_chat_model`` row's input is the same ``type="text"`` control (the "like chat model" reference, in one assertion pair). 2. ``test_saved_level_persists_without_restart`` — the save (the phase-168 §7.4 Save lifecycle: dirty → save → ``DB`` badge): ``high`` is typed (the dirty hint appears), then the free-form custom value ``think-soft`` is typed once (no allowlist — A4) and saved; the effective value reads ``think-soft`` (the API catalogue AND the raw ``app_settings`` row), and a reload of ``/settings.html`` still shows ``think-soft`` with the ``DB`` badge (no restart, the phase-168 contract). 3. ``test_reset_returns_to_env_default`` — the per-row Reset (the per-row ``DELETE``): after a save, the Reset is live; clicking it lands the status line and the value reads ``low`` again with the ``env`` badge (the overlay reset, task 02's contract, in the browser). DB isolation: the shared e2e Postgres keeps ``app_settings`` (the overlay rows — a leftover row would retune every later suite's app through the conftest pin's failure mode), ``ui_settings`` (the single row the caching middleware reads for EVERY served page) and ``api_tokens`` rows across suites. An autouse fixture truncates ``app_settings`` and ``ui_settings`` and deletes the ``e2e-``-labeled tokens before AND after every test (never a TRUNCATE on ``api_tokens`` — the shared DB may hold the owner's real tokens); the module teardown pins ``app_settings`` empty. The module app opts BACK IN over the conftest's global ``BOR_SETTINGS_DB=0`` pin (``env["BOR_ SETTINGS_DB"] = "1"`` — the ``test_pictorial_pdf.py`` opt-in pattern) and pins ``BOR_LLM_THINKING_LEVEL`` (and the "like chat model" reference, ``BOR_LLM_CHAT_MODEL``) to their CODE DEFAULTS — the env var is set (so the badge reads ``env``, not ``default``) and an operator's local ``.env`` cannot leak a different level into the suite's app (the house leak-guard pattern). A module-scope ``.env`` mtime guard pins LOCKED A1: no code path the suite drives opens the env file for writing. ASSUMPTION A6 (phase-locked): the per-turn-line assertion (``thinking=``) lives in the unit/integration layer (task 04) — the mock LLM turn does not exercise the wire, and this suite proves the settings surface only. """ from __future__ import annotations import json import os import subprocess import sys from collections.abc import Iterator from typing import Any import httpx import pytest from playwright.sync_api import Page, expect from sqlalchemy import text from app.config import Settings from app.db import SessionLocal from e2e.auth_helpers import login from e2e.conftest import ( ADMIN_PASSWORD, APP_PORT, REPO, SESSION_SECRET, USE_REAL_LLM, _wait_http, ) APP_URL = f"http://127.0.0.1:{APP_PORT}" #: The save's custom free-form value (test 2) — NOT a known pi #: level (proves the A4 contract: no allowlist, the endpoint is #: the source of truth). SAVED_LEVEL = "think-soft" # --------------------------------------------------------------------------- # Per-module app env (the phase-168 tuning pattern) # --------------------------------------------------------------------------- def _boot_app() -> subprocess.Popen[bytes]: """The one module app: the shared env block + the phase-172 pins, with the overlay opted BACK IN (``BOR_SETTINGS_DB=1``).""" global _MOCK_PORT env = dict(os.environ) env.pop("DEBUGPY", None) env["BOR_ENVIRONMENT"] = "e2e" env["BOR_STATIC_DIR"] = str(REPO / "frontend") env["BOR_LLM_BASE_URL"] = ( "https://aipi.reeseapps.com/v1" if USE_REAL_LLM else f"http://127.0.0.1:{_MOCK_PORT}/v1" ) # The E2E-calibrated gate pins (the conftest pattern — the mock's # token-overlap distribution, explicit so an operator's local # .env cannot leak a different gate into the suite's app). env["BOR_RELEVANCE_THRESHOLD"] = "0.30" env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15" env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15" env["BOR_LLM_RETRY_DELAY"] = "0" env["BOR_LLM_RETRIES"] = str(Settings.model_fields["llm_retries"].default) env.setdefault( "BOR_DATABASE_URL", "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese", ) # Phase 16: admin auth must be set or create_app() refuses to boot. env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD env["BOR_SESSION_SECRET"] = SESSION_SECRET env["BOR_DOCS_REPO"] = "" env["BOR_SUGGESTIONS"] = json.dumps( Settings.model_fields["suggestions"].default ) # Leak guards: the branding base and the "like chat model" # reference row stay the CODE defaults (derived from the class # fields, never drift from app/config.py). env["BOR_APP_NAME"] = Settings.model_fields["app_name"].default env["BOR_INPUT_PLACEHOLDER"] = Settings.model_fields["input_placeholder"].default env["BOR_FOOTER_TEXT"] = Settings.model_fields["footer_text"].default env["BOR_GIT_SOURCES"] = "" env["BOR_LLM_CHAT_MODEL"] = Settings.model_fields["llm_chat_model"].default # Phase 168 (the panel suite's pattern): the module opts BACK IN # over the conftest's global BOR_SETTINGS_DB=0 pin (the # test_pictorial_pdf.py pattern) — the row's save/reset # contract needs the overlay on. env["BOR_SETTINGS_DB"] = "1" # Phase 172 (task 05): pin the thinking level to the CODE # DEFAULT — the env var is SET (so the badge reads ``env``, not # ``default``) and an operator's local .env cannot leak a # different level into the suite's app. env["BOR_LLM_THINKING_LEVEL"] = Settings.model_fields[ "llm_thinking_level" ].default return subprocess.Popen( [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"], cwd=REPO, env=env, ) def _stop_app(proc: subprocess.Popen[bytes]) -> None: proc.terminate() try: proc.wait(timeout=10) except subprocess.TimeoutExpired: proc.kill() @pytest.fixture(scope="module") def app_server(mock_llm: int) -> Iterator[str]: """The real app under test — per-module env: the thinking level pinned to the code default ``low`` and ``BOR_SETTINGS_DB=1`` (the module opts back in over the conftest pin).""" global _MOCK_PORT _MOCK_PORT = mock_llm proc = _boot_app() try: _wait_http(f"{APP_URL}/api/health") yield APP_URL finally: _stop_app(proc) @pytest.fixture(scope="module") def app_url(app_server: str) -> str: return app_server # --------------------------------------------------------------------------- # DB isolation + helpers # --------------------------------------------------------------------------- def _clean_settings_state() -> None: """Fresh settings + theme + token state per test: truncate ``app_settings`` (a leftover row is the cross-suite poison this phase's overlay contract exists to prevent — the new key's rows are wiped between tests) and the single-row ``ui_settings`` (the middleware reads it for EVERY page), and delete this suite's issued tokens (label-scoped on ``e2e-`` — never a TRUNCATE: the shared DB may hold the owner's real tokens).""" with SessionLocal() as db: db.execute(text("TRUNCATE app_settings")) db.execute(text("TRUNCATE ui_settings")) db.execute(text("DELETE FROM api_tokens WHERE label LIKE 'e2e-%'")) db.commit() @pytest.fixture(autouse=True) def _clean(db_ready: None) -> Iterator[None]: _clean_settings_state() yield _clean_settings_state() # The module's final state: app_settings EMPTY (the cross-suite # poison guard — the next suite's app must boot with no # overrides). with SessionLocal() as db: assert db.execute(text("SELECT count(*) FROM app_settings")).scalar() == 0 @pytest.fixture(scope="module", autouse=True) def _env_never_written() -> Iterator[None]: """LOCKED A1, mtime-pinned: no code path the suite drives opens ``.env`` for writing (the file is gitignored — a ``git status`` check cannot see it, so ``stat``'s mtime is the assertion).""" env_file = REPO / ".env" before = env_file.stat().st_mtime_ns if env_file.exists() else None yield after = env_file.stat().st_mtime_ns if env_file.exists() else None assert before == after, "the app under test must never write .env" def _cookies(page: Page) -> dict[str, str]: """The session cookies the browser context holds (the test's API side sees exactly what that browser sees).""" return { c["name"]: c["value"] for c in page.context.cookies() if "name" in c and "value" in c } def _admin_client(app_url: str) -> httpx.Client: """The API side of the form login (the 204 + the signed session cookie in the jar).""" client = httpx.Client(timeout=60.0) r = client.post(f"{app_url}/api/login", json={"password": ADMIN_PASSWORD}) assert r.status_code == 204, f"admin login → {r.status_code}: {r.text[:120]}" return client def _catalog_item(client: httpx.Client, app_url: str, key: str) -> dict[str, Any]: r = client.get(f"{app_url}/api/admin/settings", timeout=30.0) assert r.status_code == 200, r.text[:200] for item in r.json()["items"]: if item["key"] == key: return item raise AssertionError(f"{key} missing from the catalogue") # --------------------------------------------------------------------------- # 1. The row renders free-form with default low (the "like chat # model" reference, in one assertion pair) # --------------------------------------------------------------------------- def test_row_renders_free_form_with_default_low(page: Page, app_url: str, db_ready: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/settings.html") expect(page.locator("#settings-content")).to_be_visible(timeout=15_000) expect(page.locator("#settings-disabled")).to_be_hidden() # the overlay is on # The LLM (aipi) group band is visible (the phase-171 band # contract — the row sits in it)… expect(page.locator("tr.settings-group-row", has_text="LLM (aipi)")).to_be_visible() # …and the row is present with its label… row = page.locator('tr[data-key="llm_thinking_level"]') expect(row).to_be_visible() expect(row.locator(".cell-clip")).to_have_text("Thinking level") # …and the env-var display (the catalogue pattern). expect(row.locator(".settings-env")).to_have_text("BOR_LLM_THINKING_LEVEL") # The input is a FREE-FORM TEXT control (A4: type="text" — NOT # number/select; no allowlist, the endpoint is the source of # truth). assert row.locator(".settings-control").get_attribute("type") == "text" # The "like chat model" reference: the llm_chat_model row's # input is the SAME type="text" control (one assertion pair). thinking_type = row.locator(".settings-control").get_attribute("type") chat_type = page.locator( 'tr[data-key="llm_chat_model"] .settings-control' ).get_attribute("type") assert thinking_type == chat_type == "text" # The default: the env-pinned ``low`` with the env badge (the # env var is set — no app_settings row, no code-default badge). expect(row.locator(".settings-control")).to_have_value("low") expect(row.locator(".settings-src")).to_have_text("env") # Nothing is saved yet: the row's Reset is disabled (no saved # override — the phase-168 render contract). expect(row.locator(".settings-reset")).to_be_disabled() # --------------------------------------------------------------------------- # 2. The save: dirty → save → DB badge, the effective value is the # free-form custom, and it survives a reload WITHOUT a restart # --------------------------------------------------------------------------- def test_saved_level_persists_without_restart(page: Page, app_url: str, db_ready: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/settings.html") expect(page.locator("#settings-content")).to_be_visible(timeout=15_000) client = _admin_client(app_url) # The §7.4 Save lifecycle: type ``high`` — the dirty hint # appears (one unsaved change on the row)… row = page.locator('tr[data-key="llm_thinking_level"]') row.locator(".settings-control").fill("high") expect(page.locator("#settings-dirty")).to_have_text("1 unsaved changes") # …then type the free-form custom value once (A4 — NOT a known # pi level: no allowlist, the endpoint is the source of truth). row.locator(".settings-control").fill(SAVED_LEVEL) expect(page.locator("#settings-save")).to_be_enabled() # Save: the status line, then the badge flips to DB (the # re-GET settled the row). page.click("#settings-save") expect(page.locator("#settings-status")).to_contain_text( "Saved 1 settings.", timeout=30_000 ) expect(row.locator(".settings-src")).to_have_text("DB") # The effective value reads ``think-soft`` — the API catalogue # (what the LLM layer reads per request via get_settings)… item = _catalog_item(client, app_url, "llm_thinking_level") assert item["source"] == "db" and item["value"] == SAVED_LEVEL # …and the raw app_settings row is stored RAW (the overlay's # D1 env-native string form). with SessionLocal() as db: raw = db.execute( text("SELECT value FROM app_settings WHERE key = 'llm_thinking_level'") ).scalar_one() assert raw == SAVED_LEVEL, f"the row must be stored raw (got {raw!r})" # No restart: the DB value survives a reload of /settings.html # (the panel re-reads the settled server state) with the DB # badge still on. page.goto(app_url + "/settings.html") expect(page.locator("#settings-content")).to_be_visible(timeout=15_000) expect( page.locator('tr[data-key="llm_thinking_level"] .settings-control') ).to_have_value(SAVED_LEVEL, timeout=15_000) expect( page.locator('tr[data-key="llm_thinking_level"] .settings-src') ).to_have_text("DB") # --------------------------------------------------------------------------- # 3. The Reset: control goes back to the environment (the per-row # DELETE — task 02's contract, in the browser) # --------------------------------------------------------------------------- def test_reset_returns_to_env_default(page: Page, app_url: str, db_ready: None) -> None: page.set_default_timeout(30_000) login(page, app_url, next="/settings.html") expect(page.locator("#settings-content")).to_be_visible(timeout=15_000) client = _admin_client(app_url) # Establish a saved override first (the Reset is disabled with # nothing to reset — the phase-168 render contract): save # ``high`` so the row's Reset goes live… row = page.locator('tr[data-key="llm_thinking_level"]') row.locator(".settings-control").fill("high") page.click("#settings-save") expect(page.locator("#settings-status")).to_contain_text( "Saved 1 settings.", timeout=30_000 ) expect(row.locator(".settings-src")).to_have_text("DB") expect(row.locator(".settings-reset")).to_be_enabled() # The per-row Reset (the per-row DELETE): the status line… row.locator(".settings-reset").click() expect(page.locator("#settings-status")).to_contain_text( "BOR_LLM_THINKING_LEVEL is back to the environment value.", timeout=30_000 ) # …the value reads ``low`` again with the env badge (the env # var is set — not the code-default badge)… expect(row.locator(".settings-control")).to_have_value("low") expect(row.locator(".settings-src")).to_have_text("env") # …and the server agrees: the override is gone and the # effective value is the env value again. item = _catalog_item(client, app_url, "llm_thinking_level") assert item["source"] == "env" and item["value"] == "low" with SessionLocal() as db: assert ( db.execute( text("SELECT count(*) FROM app_settings WHERE key = 'llm_thinking_level'") ).scalar() == 0 ), "the DELETE must remove the row"