"""Phase 168 (task 03): the end-to-end proof of "no restart", in-process. After ``invalidate_settings()`` (the save-side wrapper), the NEXT request behaves under the new value — pinned for the three settings that are NOT covered by the cache clear alone or that prove it end to end: the retrieval honesty gate (``relevance_threshold`` — the per-use read, ``app/api/chat.py``'s ``plan_turn`` gate), the shared ``LLMClient`` (rebuilt: model + timeout from the new ``Settings`` snapshot, ``app/api/chat.py::get_llm``), and the chat concurrency semaphore (``chat_max_concurrent``). Plus the kill-switch counterpart (``BOR_SETTINGS_DB=0`` with rows present → env values are back) and the ``scripts/``-consumer surface (``import_extension_set``), and the phase-172 thinking level (``llm_chat_thinking_level`` — the value the LLM layer reads per request via ``get_settings()``; the off switch is live-settable, A2) and its phase-178 summary twin (task 02: ``llm_summary_thinking_level``, default ``""`` = off, A2), and the phase-182 verbatim-display family (task 01: ``answer_text`` — the master switch — plus the two ``show_text_max_*`` budgets, whose inert ``0`` values and reset-to-default resolve live with no restart). The rows are inserted directly through the ``db`` session (the admin API is task 04); the save itself is ``invalidate_settings()`` — exactly what task 04's PUT/DELETE call after the write. Real Postgres (``podman compose up -d db``); no LLM involved. Requires: podman compose up -d db """ from __future__ import annotations import uuid from collections.abc import Iterator from datetime import UTC, datetime import pytest from sqlalchemy import text from sqlalchemy.orm import Session from app.api import chat as chat_api from app.config import get_settings, invalidate_settings from app.core import settings_overlay from app.core.settings_overlay import register_invalidator from app.models import Document from app.rag.retriever import RetrievedChunk @pytest.fixture(autouse=True) def _clean_state( db: Session, monkeypatch: pytest.MonkeyPatch ) -> Iterator[None]: """A clean ``app_settings`` table + a fresh overlay/hook slate. ``reset_state_for_tests()`` clears the process-wide hook list (a prior suite's test may have wiped the hook ``app.main`` registered at import — re-wire what ``create_app()`` wires: the chat runtime reset), then the table is emptied and the switch is ON for the test. Teardown deletes the rows and invalidates (runs the reset hook — a test-built ``LLMClient``/semaphore snapshot never leaks into a later suite) + clears the cache; ``monkeypatch`` then restores ``BOR_SETTINGS_DB`` and the next resolution is the conftest-pinned env-only view. """ settings_overlay.reset_state_for_tests() register_invalidator(chat_api.reset_runtime_caches) db.execute(text("DELETE FROM app_settings")) db.commit() monkeypatch.setenv("BOR_SETTINGS_DB", "1") invalidate_settings() yield db.execute(text("DELETE FROM app_settings")) db.commit() invalidate_settings() get_settings.cache_clear() def _save(db: Session, key: str, value: str) -> None: """One upserted override row (the panel's Save, minus the API).""" db.execute( text( "INSERT INTO app_settings (key, value) VALUES (:k, :v)" " ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value" ), {"k": key, "v": value}, ) db.commit() def _doc(title: str) -> Document: return Document( id=uuid.uuid4(), source="Homelab", path=f"{title.lower().replace(' ', '-')}.md", full_path="/tmp/doc.md", title=title, content=f"{title} content body for the gate pin", summary=None, content_hash="0" * 64, created_at=datetime(2024, 6, 15, 12, 0, 0, tzinfo=UTC), ) def _chunk(doc: Document, score: float, cosine: float) -> RetrievedChunk: """A non-FTS candidate: *cosine* is the honesty-gate input.""" return RetrievedChunk( chunk_id=uuid.uuid4(), position=0, content=doc.content[:32], score=score, document=doc, cosine=cosine, ) # --------------------------------------------------------------------------- # the per-use read: the A8 gate behaves under the saved threshold # --------------------------------------------------------------------------- def test_relevance_threshold_db_value_bites_in_the_gate(db: Session) -> None: base = get_settings().relevance_threshold if base >= 0.95: pytest.skip("env threshold too close to 1.0 to bracket a DB override") # A DB value over the base; a cosine BETWEEN them — grounded under # the base, deflected under the DB value (no FTS: fts_hits = 0). db_value = min(base + 0.2, 0.9) cosine = (base + db_value) / 2 chunk = _chunk(_doc("Gate Pin"), score=0.1, cosine=cosine) assert chat_api.plan_turn([chunk], get_settings()).deflected is False _save(db, "relevance_threshold", str(db_value)) invalidate_settings() assert get_settings().relevance_threshold == db_value # the SAME chunk, the SAME function, the overlay-fed settings — # the gate now deflects (no process restart in between) assert chat_api.plan_turn([chunk], get_settings()).deflected is True # --------------------------------------------------------------------------- # the LLM client snapshot: Save swaps the shared LLMClient # --------------------------------------------------------------------------- def test_llm_client_rebuilt_from_saved_values(db: Session) -> None: _save(db, "llm_chat_model", "gpt-test") _save(db, "llm_timeout", "7") invalidate_settings() llm = chat_api.get_llm() # the rebuilt client holds the DB-overridden snapshot… assert llm.settings.llm_chat_model == "gpt-test" assert llm.settings.llm_timeout == 7.0 # …and its AsyncOpenAI transport was built from it (base_url + # the timeout's connect value — the way llm.py's __init__ builds # it; the openai client enforces a trailing slash on base_url) assert llm._client.base_url == ( # pyright: ignore[reportPrivateUsage] llm.settings.llm_base_url.rstrip("/") + "/" ) assert llm._client._client.timeout.connect == 7.0 # pyright: ignore[reportPrivateUsage] def test_llm_client_repoints_on_the_next_save(db: Session) -> None: _save(db, "llm_chat_model", "gpt-first") invalidate_settings() assert chat_api.get_llm().settings.llm_chat_model == "gpt-first" # the NEXT save re-points the singleton (not just the first) _save(db, "llm_chat_model", "gpt-second") invalidate_settings() llm = chat_api.get_llm() assert llm.settings.llm_chat_model == "gpt-second" # --------------------------------------------------------------------------- # the chat thinking level: save/reset resolve the live value # (phase 178, task 01 — the phase-172 test, renamed) # --------------------------------------------------------------------------- def test_llm_chat_thinking_level_live_apply_no_restart( db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 178 (task 01): the chat thinking level follows the overlay live — a save (the PUT's write, minus the API: ``_save`` + ``invalidate_settings()`` is exactly what task 04's PUT/DELETE call after the write) resolves ``get_settings().llm_chat_thinking_level`` WITHOUT a restart (the proof the LLM layer — task 03 reads the setting per request via ``get_settings()`` — picks the override up live), the EMPTY value resolves live too (the off switch is live-settable, A2/A4), and a reset (the DELETE) returns the env value. """ # A deterministic base (machine-independent, the kill-switch test's # idiom): the code default, no row in the table. monkeypatch.delenv("BOR_LLM_CHAT_THINKING_LEVEL", raising=False) invalidate_settings() assert get_settings().llm_chat_thinking_level == "low" _save(db, "llm_chat_thinking_level", "high") invalidate_settings() assert get_settings().llm_chat_thinking_level == "high" # NO restart # The off switch is live-settable too (A2/A4): the empty value # resolves through the overlay, not stripped or coerced. _save(db, "llm_chat_thinking_level", "") invalidate_settings() assert get_settings().llm_chat_thinking_level == "" # The reset returns the env value (no process restart in between — # the LLM layer re-reads per request, task 03). db.execute( text("DELETE FROM app_settings WHERE key = 'llm_chat_thinking_level'") ) db.commit() invalidate_settings() assert get_settings().llm_chat_thinking_level == "low" # --------------------------------------------------------------------------- # the summary thinking level: save/reset resolve the live value # (phase 178, task 02 — the chat twin's summary counterpart) # --------------------------------------------------------------------------- def test_llm_summary_thinking_level_live_apply_no_restart( db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 178 (task 02): the summary thinking level follows the overlay live — a save (the PUT's write, minus the API: ``_save`` + ``invalidate_settings()`` is exactly what task 04's PUT/DELETE call after the write) resolves ``get_settings().llm_summary_thinking_level`` WITHOUT a restart (the proof the LLM layer — task 03 reads the setting per request via ``get_settings()`` — picks the override up live), the EMPTY value resolves live too (the off switch is live-settable, A2/A4), and a reset (the DELETE) returns the env value (``""`` — the code default; the chat twin's reset returns ``"low"``, the two levels are independent). """ # A deterministic base (machine-independent, the kill-switch test's # idiom): the code default, no row in the table. monkeypatch.delenv("BOR_LLM_SUMMARY_THINKING_LEVEL", raising=False) invalidate_settings() assert get_settings().llm_summary_thinking_level == "" _save(db, "llm_summary_thinking_level", "high") invalidate_settings() assert get_settings().llm_summary_thinking_level == "high" # NO restart # The off switch is live-settable too (A2/A4): the empty value # resolves through the overlay, not stripped or coerced. _save(db, "llm_summary_thinking_level", "") invalidate_settings() assert get_settings().llm_summary_thinking_level == "" # The reset returns the env value (no process restart in between — # the LLM layer re-reads per request, task 03). db.execute( text("DELETE FROM app_settings WHERE key = 'llm_summary_thinking_level'") ) db.commit() invalidate_settings() assert get_settings().llm_summary_thinking_level == "" # --------------------------------------------------------------------------- # the verbatim-display switch + budgets: save/reset resolve the live values # (phase 182, task 01) # --------------------------------------------------------------------------- def test_verbatim_display_settings_live_apply_no_restart( db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: """Phase 182 (task 01, LOCKED A8): the verbatim-display master switch and its two budgets follow the overlay live — a save (the PUT's write, minus the API: ``_save`` + ``invalidate_settings()`` is exactly what the PUT/DELETE call after the write) resolves ``get_settings().answer_text`` / ``.show_text_max_chars`` / ``.show_text_max_blocks`` WITHOUT a restart (the chat request path reads them per turn via ``get_settings()``), the ``0`` values resolve live as INERT (not an error, not coerced to the default), the bool accepts the env-native ``false``/``0`` spellings, and a reset (the DELETE) returns the env default. """ # A deterministic base (machine-independent, the kill-switch test's # idiom): the code defaults, no row in the table. for var in ( "BOR_ANSWER_TEXT", "BOR_SHOW_TEXT_MAX_CHARS", "BOR_SHOW_TEXT_MAX_BLOCKS", ): monkeypatch.delenv(var, raising=False) invalidate_settings() assert get_settings().answer_text is True assert get_settings().show_text_max_chars == 12_000 assert get_settings().show_text_max_blocks == 5 # Save: every value resolves through the overlay, no restart. _save(db, "answer_text", "false") _save(db, "show_text_max_chars", "2500") _save(db, "show_text_max_blocks", "1") invalidate_settings() assert get_settings().answer_text is False assert get_settings().show_text_max_chars == 2500 assert get_settings().show_text_max_blocks == 1 # ``0`` is the inert state (the field is not treated as "unset"), and # the bool's other env-native spelling resolves to True. _save(db, "answer_text", "1") _save(db, "show_text_max_chars", "0") _save(db, "show_text_max_blocks", "0") invalidate_settings() assert get_settings().answer_text is True assert get_settings().show_text_max_chars == 0 assert get_settings().show_text_max_blocks == 0 # The reset returns the env defaults (no process restart in between). db.execute( text( "DELETE FROM app_settings WHERE key IN " "('answer_text', 'show_text_max_chars', 'show_text_max_blocks')" ) ) db.commit() invalidate_settings() assert get_settings().answer_text is True assert get_settings().show_text_max_chars == 12_000 assert get_settings().show_text_max_blocks == 5 # --------------------------------------------------------------------------- # the chat concurrency semaphore follows the saved cap # --------------------------------------------------------------------------- def test_chat_semaphore_follows_saved_cap(db: Session) -> None: base_cap = get_settings().chat_max_concurrent first = 3 if base_cap != 3 else 5 _save(db, "chat_max_concurrent", str(first)) invalidate_settings() assert get_settings().chat_max_concurrent == first sem = chat_api._get_chat_semaphore() assert sem._value == first # pyright: ignore[reportPrivateUsage] # the next save re-points the semaphore too second = 5 if first == 3 else 3 _save(db, "chat_max_concurrent", str(second)) invalidate_settings() rebuilt = chat_api._get_chat_semaphore() assert rebuilt is not sem assert rebuilt._value == second # pyright: ignore[reportPrivateUsage] # --------------------------------------------------------------------------- # the kill switch: rows present, switch off → the env values are back # --------------------------------------------------------------------------- def test_kill_switch_returns_env_values( db: Session, monkeypatch: pytest.MonkeyPatch ) -> None: base = get_settings().input_placeholder _save(db, "input_placeholder", "DB-WINS-PLACEHOLDER") invalidate_settings() assert get_settings().input_placeholder == "DB-WINS-PLACEHOLDER" monkeypatch.setenv("BOR_SETTINGS_DB", "0") invalidate_settings() assert get_settings().input_placeholder == base # --------------------------------------------------------------------------- # the scripts/ consumer: the importer reads the same overlay # --------------------------------------------------------------------------- def test_import_extensions_override_reaches_script_consumer(db: Session) -> None: base = get_settings().import_extension_set # a REAL change — guard against a dev .env that already carries # exactly this set (a no-op would still pass the final assert) if base == {".md", ".txt", ".csv"}: saved, expect = "md,txt,csv,ini", {".md", ".txt", ".csv", ".ini"} else: saved, expect = "md,txt,csv", {".md", ".txt", ".csv"} _save(db, "import_extensions", saved) invalidate_settings() assert get_settings().import_extension_set == expect