"""The DB-over-env settings overlay (phase 168, task 02). ``TODO.md`` L3: "…save it to the database. The database value should override the env var." :func:`resolve_settings` layers the ``app_settings`` rows (phase 168 task 01: bare ``Settings`` field name → env-native string) over the environment and hands every existing consumer one valid ``Settings`` — through :func:`app.config.get_settings` (which delegates here, lazily, under its unchanged ``lru_cache``). No consumer changes, no restart (D2). The kill switch is env-only: ``BOR_SETTINGS_DB=0`` (read straight from ``os.environ`` per call — it is not a ``Settings`` field, so the escape hatch cannot disable itself). Fail loud on write, fail soft on read (LOCKED A5, D5): a write validates the FULL candidate set (task 04); a read that finds an unusable row (hand-edited, or a validator changed in a later release) skips THAT key, logs ``settings override ignored key=… reason=…`` and applies the rest — a DB value must never brick boot, because a bricked app cannot be un-bricked from the panel. The overlay's engine is a lazily-created ``NullPool`` engine built from ``Settings().database_url`` (never the app's real pool): importing ``app.db`` from here would be a cycle (``app.db`` → ``app.config`` at import time), and the pool whose size this table can change must not be the one that reads the table. ``NullPool`` = one connection per resolution, none held. A missing ``app_settings`` table (fresh deployment, migrations not applied) means "no overrides" + one DEBUG line per process (K3 — a pre-migration deployment is a normal state); any other DB failure is one WARNING per process naming the exception class ("settings overrides unavailable — env values in effect"). Both "once per process" guards are module-level flags, reset by :func:`reset_state_for_tests`. """ from __future__ import annotations import json import logging import os from collections.abc import Callable, Mapping from typing import Any, get_origin from pydantic import ValidationError from sqlalchemy import create_engine, text from sqlalchemy.engine import Engine from sqlalchemy.exc import ProgrammingError, SQLAlchemyError from sqlalchemy.pool import NullPool from app.config import Settings from app.core.settings_catalog import CATALOG logger = logging.getLogger("app.settings") #: ``BOR_SETTINGS_DB`` — env-only, read per call from ``os.environ`` so #: tests can flip it; unset/``1`` = on, the house bool off-spellings #: are off. It is NOT a ``Settings`` field, so the escape hatch cannot #: disable itself from the database (D6). _DISABLED_SPELLOS = frozenset({"0", "false", "no", "off"}) _engine: Engine | None = None _warned_db_failure = False _debugged_missing_table = False _invalidators: list[Callable[[], None]] = [] #: The one ``list[str]`` field (derived — a new list field would join #: automatically); every other field is a scalar or CSV string (K2). LIST_FIELDS: frozenset[str] = frozenset( name for name, field in Settings.model_fields.items() if get_origin(field.annotation) in (list, tuple) ) def enabled() -> bool: """The ``BOR_SETTINGS_DB`` kill switch — read per call (D6).""" value = os.environ.get("BOR_SETTINGS_DB", "1") return value.strip().lower() not in _DISABLED_SPELLOS def _get_engine() -> Engine: """The overlay's lazily-created ``NullPool`` engine (see module docstring — never the app's real pool: import cycle + the pool whose size this table can change must not read the table).""" global _engine if _engine is None: _engine = create_engine( Settings().database_url, poolclass=NullPool, pool_pre_ping=True ) return _engine def load_overrides() -> dict[str, str]: """``SELECT key, value FROM app_settings ORDER BY key`` — fail-soft. A missing table (pre-migration deployment) is one DEBUG line per process (K3); any other DB failure is one WARNING per process naming the exception class. Both return ``{}`` — the env values stay in effect, and the NEXT resolution retries (a boot-time miss is never a permanently cached env-only view — the ``create_app`` prime, task 02 step 4). """ global _warned_db_failure, _debugged_missing_table try: with _get_engine().connect() as conn: rows = conn.execute(text("SELECT key, value FROM app_settings ORDER BY key")) return {key: value for key, value in rows} except ProgrammingError: # Undefined table (psycopg ``UndefinedTable`` → SQLAlchemy # ProgrammingError): the pre-migration deployment — DEBUG (K3). if not _debugged_missing_table: _debugged_missing_table = True logger.debug( "app_settings table missing — no DB overrides in effect " "(pre-migration deployment, phase 168 K3)" ) return {} except (SQLAlchemyError, OSError) as exc: if not _warned_db_failure: _warned_db_failure = True logger.warning( "settings overrides unavailable — env values in effect (%s)", type(exc).__name__, ) return {} def _coerce(field_name: str, raw: str) -> Any: """Coerce one env-native string row value for the candidate set. The one ``list[str]`` field (``suggestions``) is ``json.loads``-ed — it recovers exactly what the env spelling (a JSON array, K2) wrote. Every other field is the raw string: pydantic-settings lax-coerces ``str`` → ``int`` / ``float`` / ``bool``, and init kwargs rank ABOVE env in pydantic-settings — precisely "the database value overrides the env var" (D2). A malformed ``suggestions`` JSON raises :class:`ValueError`, which :func:`merge` turns into a per-key skip (LOCKED A5: one bad row costs one setting, never the whole table). The return type is ``Any`` on purpose — the value is whatever pydantic will lax-coerce, decided in exactly one place (the ``Settings`` validators). """ if field_name in LIST_FIELDS: try: return json.loads(raw) except json.JSONDecodeError as exc: raise ValueError(f"not a JSON array ({exc.msg})") from exc return raw def merge( rows: Mapping[str, str], ) -> tuple[Settings, list[tuple[str, str]]]: """Layer ``rows`` over the env base (``Settings()``) and validate. Returns ``(settings, skipped)`` — ``skipped`` is the ``(key, reason)`` pairs the read-side fail-soft (LOCKED A5) dropped so one unusable row costs one setting, not the whole table. Dropped keys: an orphan row (not a ``Settings`` field — the setting was removed in a later release), a LOCKED A3 key (a hand-inserted ``database_url`` / ``embedding_dim`` row must never hijack the engine / the fixed 768-dim table), a malformed ``suggestions`` JSON, and any value the real ``Settings`` validators reject (retried per field — ``error["loc"][0]``). Each dropped key is logged once at ERROR (``settings override ignored key=… reason=…``). """ kwargs: dict[str, Any] = {} skipped: list[tuple[str, str]] = [] for key, raw in rows.items(): if key not in Settings.model_fields: skipped.append((key, "unknown setting (orphan row from a removed field)")) continue if not CATALOG[key].overridable: skipped.append((key, "not DB-overridable (LOCKED A3 — read-only)")) continue try: kwargs[key] = _coerce(key, raw) except ValueError as exc: skipped.append((key, str(exc))) while True: try: settings = Settings(**kwargs) break except ValidationError as exc: err = exc.errors()[0] bad = str(err["loc"][0]) if bad in kwargs: # The offending row is an override — drop it and retry # (LOCKED A5: one bad row costs one setting). kwargs.pop(bad) skipped.append((bad, err["msg"])) continue # The validator blames a field with NO override — its value # comes from env/.env and collides with a cross-field # override (e.g. the env lexical_support_floor above a # DB-overridden relevance_threshold). The env base alone is # always valid (the app booted with it pre-phase), so try # dropping the override(s) until the candidate set is valid # again — one unresolvable collision costs the colliding # row, the rest of the table stands. resolved = False for candidate in list(kwargs): trial = dict(kwargs) trial.pop(candidate) try: settings = Settings(**trial) except ValidationError: continue skipped.append( ( candidate, f"cross-field collision with env/base {bad}: {err['msg']} " "— override skipped, env value for the other field stands", ) ) kwargs = trial resolved = True break if not resolved: # Nothing to drop (no overrides at all) or no single drop # heals the set: the env base alone would boot (pre-phase # proof), so return it — overrides skipped, boot intact. skipped.append( ( bad, f"unresolvable cross-field collision: {err['msg']} — " "all overrides skipped, env values in effect", ) ) settings = Settings() break for key, reason in skipped: logger.error("settings override ignored key=%s reason=%s", key, reason) return settings, skipped def resolve_settings() -> Settings: """The whole overlay: env base + kill switch + DB rows. Never calls :func:`app.config.get_settings` (recursion guard — that resolver delegates HERE): ``Settings()`` is the env-only entry point, and the kill switch short-circuits before any DB access. """ if not enabled(): return Settings() settings, _skipped = merge(load_overrides()) return settings def register_invalidator(fn: Callable[[], None]) -> None: """Register a runtime reset hook (task 03: the chat LLM client + semaphore). Hooks run on every :func:`app.config.invalidate_settings` — after the cache clear — so the next resolution rebuilds the singletons from the fresh settings.""" _invalidators.append(fn) def run_invalidators() -> None: """Run the registered reset hooks, in order. A hook that raises is logged, never propagated (a failed reset must not undo a successful save — LOCKED A5's "the save stands").""" for fn in list(_invalidators): try: fn() except Exception: logger.exception( "settings invalidator failed — the save stands, this " "runtime reset did not run" ) def reset_state_for_tests() -> None: """Clear the module engine + the once-per-process flags + the hook list (the unit suites flip the kill switch and stub the engine).""" global _engine, _warned_db_failure, _debugged_missing_table _engine = None _warned_db_failure = False _debugged_missing_table = False _invalidators.clear()