phase: 172_thinking_level
All verification complete. Final report: **Phase 172 final verification pass — all tasks (01–05) were already complete; verified the whole phase, no code fixes needed.** - **Verified implementation:** `Settings.llm_thinking_level: str = "low"` + `CATALOG` row (`LLM (aipi)` / "Thinking level", plain: not secret/restart/overridable-exempt) + `.env.example` line under `BOR_LLM_CHAT_MODEL` + README table row; `_thinking_extra_body()` → `reasoning_effort` via `extra_body` on `chat()` + `chat_stream()` when trimmed-non-empty, key absent (byte-identical) when empty, embeddings untouched; per-turn line gains `thinking=<value>` (`-` when empty) before `thinking_chars`. - **Tests/lint/coverage (all green):** `uv run pytest` → 4084 passed · `uv run pytest --cov=app --cov-report=term-missing` → 4084 passed, `app/` **99%** (>90%) · `uv run ruff check .` → All checks passed · `uv run pyright` → 0 errors, 0 warnings. - **Completion criteria:** (1) `pytest -k "config and thinking"` 4 passed + `.env.example` grep ✓ (2) live `GET /api/admin/settings` curl shows the row (str, `low`, `default`) + integration PUT/DELETE round-trip ✓ (3) 9 mocked-client unit tests assert `extra_body` present/absent ✓ (4) log-line tests assert `low`/`high`/`-`/whitespace ✓ (5) suite + coverage + lint ✓ (6) `test_thinking_level.py` 3/3 in isolation; `test_admin_settings_panel.py` 12/12; `test_admin_settings_layout.py` 5/5 ✓ (7) atomic commit + phase move = the harness's job (task files already in `complete/`; I ran no git index/history commands). - **Documentation:** README.md reviewed in full — verified consistent (env table complete vs `.env.example`, catalog restart/non-overridable claims re-verified at runtime); AGENTS.md reviewed in full — verified consistent (no rule affected, A5 = no UI change). No changes needed. **Flag (not fixable here):** PLAN.md §9's per-turn-line spec predates the phase-172 `thinking=` field; PLAN.md is off-limits per protocol. - **Next pending phase:** none — `todo/` contains only phase 172 (the harness moves it on this pass's success).
This commit is contained in:
@@ -1324,6 +1324,124 @@ def test_chat_retry_delay_is_flat_never_backoff(monkeypatch: pytest.MonkeyPatch)
|
||||
assert sleeps == [1.25, 1.25, 1.25]
|
||||
|
||||
|
||||
# ---------- thinking level passthrough (phase 172, task 03) ----------
|
||||
|
||||
|
||||
def test_chat_sends_reasoning_effort_extra_body_when_level_set() -> None:
|
||||
"""Phase 172 (A1): a non-empty level rides the one-shot request as
|
||||
the top-level ``reasoning_effort`` parameter via the SDK's
|
||||
``extra_body``."""
|
||||
llm, completions = _make_chat_client(
|
||||
_FakeCompletion("ok"), llm_thinking_level="low"
|
||||
)
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["extra_body"] == {"reasoning_effort": "low"}
|
||||
|
||||
|
||||
def test_chat_default_level_low_is_always_sent() -> None:
|
||||
"""The default (``low``) is always sent (A2): a client built without
|
||||
naming the setting still carries the parameter."""
|
||||
llm, completions = _make_chat_client(_FakeCompletion("ok"))
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["extra_body"] == {"reasoning_effort": "low"}
|
||||
|
||||
|
||||
def test_chat_trims_the_level_before_sending() -> None:
|
||||
"""TRIM: a hand-edited ``.env`` value with trailing whitespace
|
||||
(``"low "``) is sent trimmed — the raw stored string never reaches
|
||||
the wire with surrounding whitespace."""
|
||||
llm, completions = _make_chat_client(
|
||||
_FakeCompletion("ok"), llm_thinking_level="low "
|
||||
)
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
assert completions.chat_kwargs is not None
|
||||
assert completions.chat_kwargs["extra_body"] == {"reasoning_effort": "low"}
|
||||
|
||||
|
||||
def test_chat_empty_level_absent_from_kwargs_byte_identical() -> None:
|
||||
"""Phase 172 (A2): a whitespace value is the off switch —
|
||||
``extra_body`` is ABSENT (never None, never ``"off"``) and the
|
||||
request kwargs keep the pre-phase-172 shape."""
|
||||
llm, completions = _make_chat_client(
|
||||
_FakeCompletion("ok"), llm_thinking_level=" "
|
||||
)
|
||||
asyncio.run(llm.chat([{"role": "user", "content": "q"}]))
|
||||
assert completions.chat_kwargs is not None
|
||||
assert "extra_body" not in completions.chat_kwargs
|
||||
assert set(completions.chat_kwargs) == {
|
||||
"model",
|
||||
"messages",
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"stream",
|
||||
}
|
||||
|
||||
|
||||
def test_chat_stream_sends_reasoning_effort_extra_body_when_level_set() -> None:
|
||||
"""Phase 172 (A1): the chat turn rides the parameter too."""
|
||||
llm, completions = _make_stream_client(
|
||||
[_chunk("x")], llm_thinking_level="high"
|
||||
)
|
||||
asyncio.run(_collect(llm, [{"role": "user", "content": "q"}]))
|
||||
assert completions.kwargs is not None
|
||||
assert completions.kwargs["extra_body"] == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
def test_chat_stream_trims_the_level_before_sending() -> None:
|
||||
llm, completions = _make_stream_client(
|
||||
[_chunk("x")], llm_thinking_level=" high "
|
||||
)
|
||||
asyncio.run(_collect(llm, [{"role": "user", "content": "q"}]))
|
||||
assert completions.kwargs is not None
|
||||
assert completions.kwargs["extra_body"] == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
def test_chat_stream_empty_level_absent_from_kwargs_byte_identical() -> None:
|
||||
"""Phase 172 (A2): the stream's off switch — ``extra_body`` absent
|
||||
(never None) and the kwargs dict keeps the pre-phase-172 shape."""
|
||||
llm, completions = _make_stream_client(
|
||||
[_chunk("x")], llm_thinking_level=""
|
||||
)
|
||||
asyncio.run(_collect(llm, [{"role": "user", "content": "q"}]))
|
||||
assert completions.kwargs is not None
|
||||
assert "extra_body" not in completions.kwargs
|
||||
assert set(completions.kwargs) == {
|
||||
"model",
|
||||
"messages",
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"stream",
|
||||
}
|
||||
|
||||
|
||||
def test_chat_stream_whitespace_level_is_the_off_switch() -> None:
|
||||
"""Whitespace-only (``" "``) is empty for wire purposes (A2)."""
|
||||
llm, completions = _make_stream_client(
|
||||
[_chunk("x")], llm_thinking_level=" "
|
||||
)
|
||||
asyncio.run(_collect(llm, [{"role": "user", "content": "q"}]))
|
||||
assert completions.kwargs is not None
|
||||
assert "extra_body" not in completions.kwargs
|
||||
|
||||
|
||||
def test_embeddings_body_never_carries_reasoning_effort() -> None:
|
||||
"""Phase 172 (A3): the embeddings path is untouched — with the level
|
||||
set, every POST body stays the minimal ``{model, input}`` payload;
|
||||
``reasoning_effort`` never appears in it."""
|
||||
service = _FakeEmbeddingsService()
|
||||
llm, http = _make_client(service, llm_thinking_level="high")
|
||||
asyncio.run(llm.embed(["a", "b"]))
|
||||
vec = asyncio.run(llm.embed_one("c"))
|
||||
assert len(vec) == 768
|
||||
assert http.bodies # the requests actually happened
|
||||
for body in http.bodies:
|
||||
assert set(body) == {"model", "input"}
|
||||
assert "reasoning_effort" not in body
|
||||
assert "extra_body" not in body
|
||||
|
||||
|
||||
# ---------- chat_stream_retried (phase 67, task 01) ----------
|
||||
|
||||
_RETRY_MSGS: list[dict[str, str]] = [{"role": "user", "content": "q"}]
|
||||
|
||||
Reference in New Issue
Block a user