fix(rag): lift chat output cap to 32768 tokens — long answers no longer cut off

This commit is contained in:
2026-08-22 11:30:19 -04:00
parent 6ec6181c7b
commit 0da5275eeb
9 changed files with 295 additions and 5 deletions
+8
View File
@@ -34,6 +34,8 @@ def test_defaults_match_locked_decisions(monkeypatch: pytest.MonkeyPatch) -> Non
assert s.hybrid_lexical_candidates >= 1
assert s.rrf_k >= 1
assert s.top_n_docs >= 1
# Owner instruction 2026-08-22: answers may run up to 32 768 tokens.
assert s.max_output_tokens == 32_768
assert len(s.suggestions) >= 3
# A9 (revised): the import scope covers the seven A9 formats.
assert s.import_extension_set == {
@@ -49,6 +51,12 @@ def test_env_override(monkeypatch) -> None:
assert s.llm_chat_model == "juggernaut"
def test_max_output_tokens_env_override(monkeypatch) -> None:
monkeypatch.setenv("BOR_MAX_OUTPUT_TOKENS", "1234")
s = _settings()
assert s.max_output_tokens == 1234
def test_import_extensions_env_override_is_a_csv_list(monkeypatch) -> None:
monkeypatch.setenv("BOR_IMPORT_EXTENSIONS", "md,yml")
s = _settings()
+17 -3
View File
@@ -273,11 +273,13 @@ class _FakeCompletions:
def _make_stream_client(
chunks: list | None = None, fail: Exception | None = None
chunks: list | None = None,
fail: Exception | None = None,
**settings_kwargs: Any,
) -> tuple[LLMClient, _FakeCompletions]:
completions = _FakeCompletions(chunks, fail)
fake_openai = SimpleNamespace(chat=SimpleNamespace(completions=completions))
llm = LLMClient(_settings())
llm = LLMClient(_settings(**settings_kwargs))
llm._client = fake_openai # pyright: ignore[reportAttributeAccessIssue]
return llm, completions
@@ -302,10 +304,22 @@ def test_chat_stream_uses_locked_generation_params() -> None:
assert completions.kwargs["model"] == "turbo"
assert completions.kwargs["stream"] is True
assert completions.kwargs["temperature"] == 0.4
assert completions.kwargs["max_tokens"] == 700
# Phase 11: the old hard 700-token cap is gone — answers may run up to
# BOR_MAX_OUTPUT_TOKENS (default 32 768) so they are not cut off.
assert completions.kwargs["max_tokens"] == 32_768
assert completions.kwargs["messages"] == messages
def test_chat_stream_max_tokens_comes_from_settings() -> None:
"""The output cap is operator-configurable, not a client constant."""
llm, completions = _make_stream_client(
[_chunk("x")], max_output_tokens=1234 # pyright: ignore[reportArgumentType]
)
asyncio.run(_collect(llm, [{"role": "user", "content": "q"}]))
assert completions.kwargs is not None
assert completions.kwargs["max_tokens"] == 1234
def test_chat_stream_skips_empty_deltas_and_choiceless_chunks() -> None:
llm, _ = _make_stream_client([_chunk("a"), _chunk(empty=True), _chunk(None), _chunk("b")])
assert asyncio.run(_collect(llm, [{"role": "user", "content": "q"}])) == ["a", "b"]