brain-of-reese/tests/e2e/test_thinking_level.py

"""Phase 172 E2E (Playwright): the Thinking level row in the admin Settings
panel — the owner's acceptance for a pi-style, operator-tunable LLM
reasoning knob, proven in a real browser.

Source: ``TODO.md`` L3 — "Brain of Reese needs to support thinking levels
like pi does. should be a free-form text entry like chat model. Default
should be 'low'. Update .env and admin settings."

Run in isolation (DB must be up: ``podman compose up -d db``):

    uv run pytest tests/e2e/test_thinking_level.py -v --no-cov

Test → contract mapping (one story, one phase, one isolated file):

1. ``test_row_renders_free_form_with_default_low`` — the row: the
   ``LLM (aipi)`` group band is visible (the phase-171 band
   contract), a row labelled **Thinking level** with the
   ``BOR_LLM_THINKING_LEVEL`` env-var display (the catalogue
   pattern) is present, its input is a **free-form text** input
   (``type="text"`` — NOT ``number``/``select``, the A4
   contract), it shows ``low`` with the ``env`` badge, and the
   ``llm_chat_model`` row's input is the same ``type="text"``
   control (the "like chat model" reference, in one assertion
   pair).
2. ``test_saved_level_persists_without_restart`` — the save
   (the phase-168 §7.4 Save lifecycle: dirty → save → ``DB``
   badge): ``high`` is typed (the dirty hint appears), then the
   free-form custom value ``think-soft`` is typed once (no
   allowlist — A4) and saved; the effective value reads
   ``think-soft`` (the API catalogue AND the raw
   ``app_settings`` row), and a reload of ``/settings.html``
   still shows ``think-soft`` with the ``DB`` badge (no
   restart, the phase-168 contract).
3. ``test_reset_returns_to_env_default`` — the per-row Reset
   (the per-row ``DELETE``): after a save, the Reset is live;
   clicking it lands the status line and the value reads
   ``low`` again with the ``env`` badge (the overlay reset,
   task 02's contract, in the browser).

DB isolation: the shared e2e Postgres keeps ``app_settings``
(the overlay rows — a leftover row would retune every later
suite's app through the conftest pin's failure mode),
``ui_settings`` (the single row the caching middleware reads
for EVERY served page) and ``api_tokens`` rows across suites.
An autouse fixture truncates ``app_settings`` and
``ui_settings`` and deletes the ``e2e-``-labeled tokens before
AND after every test (never a TRUNCATE on ``api_tokens`` — the
shared DB may hold the owner's real tokens); the module teardown
pins ``app_settings`` empty. The module app opts BACK IN over
the conftest's global ``BOR_SETTINGS_DB=0`` pin (``env["BOR_
SETTINGS_DB"] = "1"`` — the ``test_pictorial_pdf.py`` opt-in
pattern) and pins ``BOR_LLM_THINKING_LEVEL`` (and the
"like chat model" reference, ``BOR_LLM_CHAT_MODEL``) to their
CODE DEFAULTS — the env var is set (so the badge reads ``env``,
not ``default``) and an operator's local ``.env`` cannot leak a
different level into the suite's app (the house leak-guard
pattern). A module-scope ``.env`` mtime guard pins LOCKED A1:
no code path the suite drives opens the env file for writing.

ASSUMPTION A6 (phase-locked): the per-turn-line assertion
(``thinking=<value>``) lives in the unit/integration layer
(task 04) — the mock LLM turn does not exercise the wire, and
this suite proves the settings surface only.
"""
from __future__ import annotations

import json
import os
import subprocess
import sys
from collections.abc import Iterator
from typing import Any

import httpx
import pytest
from playwright.sync_api import Page, expect
from sqlalchemy import text

from app.config import Settings
from app.db import SessionLocal
from e2e.auth_helpers import login
from e2e.conftest import (
    ADMIN_PASSWORD,
    APP_PORT,
    REPO,
    SESSION_SECRET,
    USE_REAL_LLM,
    _wait_http,
)

APP_URL = f"http://127.0.0.1:{APP_PORT}"

#: The save's custom free-form value (test 2) — NOT a known pi
#: level (proves the A4 contract: no allowlist, the endpoint is
#: the source of truth).
SAVED_LEVEL = "think-soft"


# ---------------------------------------------------------------------------
# Per-module app env (the phase-168 tuning pattern)
# ---------------------------------------------------------------------------


def _boot_app() -> subprocess.Popen[bytes]:
    """The one module app: the shared env block + the phase-172 pins,
    with the overlay opted BACK IN (``BOR_SETTINGS_DB=1``)."""
    global _MOCK_PORT
    env = dict(os.environ)
    env.pop("DEBUGPY", None)
    env["BOR_ENVIRONMENT"] = "e2e"
    env["BOR_STATIC_DIR"] = str(REPO / "frontend")
    env["BOR_LLM_BASE_URL"] = (
        "https://aipi.reeseapps.com/v1"
        if USE_REAL_LLM
        else f"http://127.0.0.1:{_MOCK_PORT}/v1"
    )
    # The E2E-calibrated gate pins (the conftest pattern — the mock's
    # token-overlap distribution, explicit so an operator's local
    # .env cannot leak a different gate into the suite's app).
    env["BOR_RELEVANCE_THRESHOLD"] = "0.30"
    env["BOR_LEXICAL_SUPPORT_FLOOR"] = "0.15"
    env["BOR_SOURCE_USEFULNESS_FLOOR"] = "0.15"
    env["BOR_LLM_RETRY_DELAY"] = "0"
    env["BOR_LLM_RETRIES"] = str(Settings.model_fields["llm_retries"].default)
    env.setdefault(
        "BOR_DATABASE_URL",
        "postgresql+psycopg://reese:reese@localhost:5432/brain_of_reese",
    )
    # Phase 16: admin auth must be set or create_app() refuses to boot.
    env["BOR_ADMIN_PASSWORD"] = ADMIN_PASSWORD
    env["BOR_SESSION_SECRET"] = SESSION_SECRET
    env["BOR_DOCS_REPO"] = ""
    env["BOR_SUGGESTIONS"] = json.dumps(
        Settings.model_fields["suggestions"].default
    )
    # Leak guards: the branding base and the "like chat model"
    # reference row stay the CODE defaults (derived from the class
    # fields, never drift from app/config.py).
    env["BOR_APP_NAME"] = Settings.model_fields["app_name"].default
    env["BOR_INPUT_PLACEHOLDER"] = Settings.model_fields["input_placeholder"].default
    env["BOR_FOOTER_TEXT"] = Settings.model_fields["footer_text"].default
    env["BOR_GIT_SOURCES"] = ""
    env["BOR_LLM_CHAT_MODEL"] = Settings.model_fields["llm_chat_model"].default
    # Phase 168 (the panel suite's pattern): the module opts BACK IN
    # over the conftest's global BOR_SETTINGS_DB=0 pin (the
    # test_pictorial_pdf.py pattern) — the row's save/reset
    # contract needs the overlay on.
    env["BOR_SETTINGS_DB"] = "1"
    # Phase 172 (task 05): pin the thinking level to the CODE
    # DEFAULT — the env var is SET (so the badge reads ``env``, not
    # ``default``) and an operator's local .env cannot leak a
    # different level into the suite's app.
    env["BOR_LLM_THINKING_LEVEL"] = Settings.model_fields[
        "llm_thinking_level"
    ].default
    return subprocess.Popen(
        [sys.executable, "-m", "uvicorn", "app.main:app",
         "--host", "127.0.0.1", "--port", str(APP_PORT), "--log-level", "warning"],
        cwd=REPO,
        env=env,
    )


def _stop_app(proc: subprocess.Popen[bytes]) -> None:
    proc.terminate()
    try:
        proc.wait(timeout=10)
    except subprocess.TimeoutExpired:
        proc.kill()


@pytest.fixture(scope="module")
def app_server(mock_llm: int) -> Iterator[str]:
    """The real app under test — per-module env: the thinking level
    pinned to the code default ``low`` and ``BOR_SETTINGS_DB=1``
    (the module opts back in over the conftest pin)."""
    global _MOCK_PORT
    _MOCK_PORT = mock_llm
    proc = _boot_app()
    try:
        _wait_http(f"{APP_URL}/api/health")
        yield APP_URL
    finally:
        _stop_app(proc)


@pytest.fixture(scope="module")
def app_url(app_server: str) -> str:
    return app_server


# ---------------------------------------------------------------------------
# DB isolation + helpers
# ---------------------------------------------------------------------------


def _clean_settings_state() -> None:
    """Fresh settings + theme + token state per test: truncate
    ``app_settings`` (a leftover row is the cross-suite poison this
    phase's overlay contract exists to prevent — the new key's rows
    are wiped between tests) and the single-row ``ui_settings``
    (the middleware reads it for EVERY page), and delete this
    suite's issued tokens (label-scoped on ``e2e-`` — never a
    TRUNCATE: the shared DB may hold the owner's real tokens)."""
    with SessionLocal() as db:
        db.execute(text("TRUNCATE app_settings"))
        db.execute(text("TRUNCATE ui_settings"))
        db.execute(text("DELETE FROM api_tokens WHERE label LIKE 'e2e-%'"))
        db.commit()


@pytest.fixture(autouse=True)
def _clean(db_ready: None) -> Iterator[None]:
    _clean_settings_state()
    yield
    _clean_settings_state()
    # The module's final state: app_settings EMPTY (the cross-suite
    # poison guard — the next suite's app must boot with no
    # overrides).
    with SessionLocal() as db:
        assert db.execute(text("SELECT count(*) FROM app_settings")).scalar() == 0


@pytest.fixture(scope="module", autouse=True)
def _env_never_written() -> Iterator[None]:
    """LOCKED A1, mtime-pinned: no code path the suite drives opens
    ``.env`` for writing (the file is gitignored — a ``git status``
    check cannot see it, so ``stat``'s mtime is the assertion)."""
    env_file = REPO / ".env"
    before = env_file.stat().st_mtime_ns if env_file.exists() else None
    yield
    after = env_file.stat().st_mtime_ns if env_file.exists() else None
    assert before == after, "the app under test must never write .env"


def _cookies(page: Page) -> dict[str, str]:
    """The session cookies the browser context holds (the test's API
    side sees exactly what that browser sees)."""
    return {
        c["name"]: c["value"]
        for c in page.context.cookies()
        if "name" in c and "value" in c
    }


def _admin_client(app_url: str) -> httpx.Client:
    """The API side of the form login (the 204 + the signed session
    cookie in the jar)."""
    client = httpx.Client(timeout=60.0)
    r = client.post(f"{app_url}/api/login", json={"password": ADMIN_PASSWORD})
    assert r.status_code == 204, f"admin login → {r.status_code}: {r.text[:120]}"
    return client


def _catalog_item(client: httpx.Client, app_url: str, key: str) -> dict[str, Any]:
    r = client.get(f"{app_url}/api/admin/settings", timeout=30.0)
    assert r.status_code == 200, r.text[:200]
    for item in r.json()["items"]:
        if item["key"] == key:
            return item
    raise AssertionError(f"{key} missing from the catalogue")


# ---------------------------------------------------------------------------
# 1. The row renders free-form with default low (the "like chat
#    model" reference, in one assertion pair)
# ---------------------------------------------------------------------------


def test_row_renders_free_form_with_default_low(page: Page, app_url: str, db_ready: None) -> None:
    page.set_default_timeout(30_000)
    login(page, app_url, next="/settings.html")
    expect(page.locator("#settings-content")).to_be_visible(timeout=15_000)
    expect(page.locator("#settings-disabled")).to_be_hidden()  # the overlay is on

    # The LLM (aipi) group band is visible (the phase-171 band
    # contract — the row sits in it)…
    expect(page.locator("tr.settings-group-row", has_text="LLM (aipi)")).to_be_visible()
    # …and the row is present with its label…
    row = page.locator('tr[data-key="llm_thinking_level"]')
    expect(row).to_be_visible()
    expect(row.locator(".cell-clip")).to_have_text("Thinking level")
    # …and the env-var display (the catalogue pattern).
    expect(row.locator(".settings-env")).to_have_text("BOR_LLM_THINKING_LEVEL")

    # The input is a FREE-FORM TEXT control (A4: type="text" — NOT
    # number/select; no allowlist, the endpoint is the source of
    # truth).
    assert row.locator(".settings-control").get_attribute("type") == "text"
    # The "like chat model" reference: the llm_chat_model row's
    # input is the SAME type="text" control (one assertion pair).
    thinking_type = row.locator(".settings-control").get_attribute("type")
    chat_type = page.locator(
        'tr[data-key="llm_chat_model"] .settings-control'
    ).get_attribute("type")
    assert thinking_type == chat_type == "text"

    # The default: the env-pinned ``low`` with the env badge (the
    # env var is set — no app_settings row, no code-default badge).
    expect(row.locator(".settings-control")).to_have_value("low")
    expect(row.locator(".settings-src")).to_have_text("env")
    # Nothing is saved yet: the row's Reset is disabled (no saved
    # override — the phase-168 render contract).
    expect(row.locator(".settings-reset")).to_be_disabled()


# ---------------------------------------------------------------------------
# 2. The save: dirty → save → DB badge, the effective value is the
#    free-form custom, and it survives a reload WITHOUT a restart
# ---------------------------------------------------------------------------


def test_saved_level_persists_without_restart(page: Page, app_url: str, db_ready: None) -> None:
    page.set_default_timeout(30_000)
    login(page, app_url, next="/settings.html")
    expect(page.locator("#settings-content")).to_be_visible(timeout=15_000)
    client = _admin_client(app_url)

    # The §7.4 Save lifecycle: type ``high`` — the dirty hint
    # appears (one unsaved change on the row)…
    row = page.locator('tr[data-key="llm_thinking_level"]')
    row.locator(".settings-control").fill("high")
    expect(page.locator("#settings-dirty")).to_have_text("1 unsaved changes")
    # …then type the free-form custom value once (A4 — NOT a known
    # pi level: no allowlist, the endpoint is the source of truth).
    row.locator(".settings-control").fill(SAVED_LEVEL)
    expect(page.locator("#settings-save")).to_be_enabled()

    # Save: the status line, then the badge flips to DB (the
    # re-GET settled the row).
    page.click("#settings-save")
    expect(page.locator("#settings-status")).to_contain_text(
        "Saved 1 settings.", timeout=30_000
    )
    expect(row.locator(".settings-src")).to_have_text("DB")

    # The effective value reads ``think-soft`` — the API catalogue
    # (what the LLM layer reads per request via get_settings)…
    item = _catalog_item(client, app_url, "llm_thinking_level")
    assert item["source"] == "db" and item["value"] == SAVED_LEVEL
    # …and the raw app_settings row is stored RAW (the overlay's
    # D1 env-native string form).
    with SessionLocal() as db:
        raw = db.execute(
            text("SELECT value FROM app_settings WHERE key = 'llm_thinking_level'")
        ).scalar_one()
    assert raw == SAVED_LEVEL, f"the row must be stored raw (got {raw!r})"

    # No restart: the DB value survives a reload of /settings.html
    # (the panel re-reads the settled server state) with the DB
    # badge still on.
    page.goto(app_url + "/settings.html")
    expect(page.locator("#settings-content")).to_be_visible(timeout=15_000)
    expect(
        page.locator('tr[data-key="llm_thinking_level"] .settings-control')
    ).to_have_value(SAVED_LEVEL, timeout=15_000)
    expect(
        page.locator('tr[data-key="llm_thinking_level"] .settings-src')
    ).to_have_text("DB")


# ---------------------------------------------------------------------------
# 3. The Reset: control goes back to the environment (the per-row
#    DELETE — task 02's contract, in the browser)
# ---------------------------------------------------------------------------


def test_reset_returns_to_env_default(page: Page, app_url: str, db_ready: None) -> None:
    page.set_default_timeout(30_000)
    login(page, app_url, next="/settings.html")
    expect(page.locator("#settings-content")).to_be_visible(timeout=15_000)
    client = _admin_client(app_url)

    # Establish a saved override first (the Reset is disabled with
    # nothing to reset — the phase-168 render contract): save
    # ``high`` so the row's Reset goes live…
    row = page.locator('tr[data-key="llm_thinking_level"]')
    row.locator(".settings-control").fill("high")
    page.click("#settings-save")
    expect(page.locator("#settings-status")).to_contain_text(
        "Saved 1 settings.", timeout=30_000
    )
    expect(row.locator(".settings-src")).to_have_text("DB")
    expect(row.locator(".settings-reset")).to_be_enabled()

    # The per-row Reset (the per-row DELETE): the status line…
    row.locator(".settings-reset").click()
    expect(page.locator("#settings-status")).to_contain_text(
        "BOR_LLM_THINKING_LEVEL is back to the environment value.", timeout=30_000
    )
    # …the value reads ``low`` again with the env badge (the env
    # var is set — not the code-default badge)…
    expect(row.locator(".settings-control")).to_have_value("low")
    expect(row.locator(".settings-src")).to_have_text("env")
    # …and the server agrees: the override is gone and the
    # effective value is the env value again.
    item = _catalog_item(client, app_url, "llm_thinking_level")
    assert item["source"] == "env" and item["value"] == "low"
    with SessionLocal() as db:
        assert (
            db.execute(
                text("SELECT count(*) FROM app_settings WHERE key = 'llm_thinking_level'")
            ).scalar()
            == 0
        ), "the DELETE must remove the row"