fix(rag): lift chat output cap to 32768 tokens — long answers no longer cut off

This commit is contained in:
2026-08-22 11:30:19 -04:00
parent 6ec6181c7b
commit 0da5275eeb
9 changed files with 295 additions and 5 deletions
+17 -3
View File
@@ -273,11 +273,13 @@ class _FakeCompletions:
def _make_stream_client(
chunks: list | None = None, fail: Exception | None = None
chunks: list | None = None,
fail: Exception | None = None,
**settings_kwargs: Any,
) -> tuple[LLMClient, _FakeCompletions]:
completions = _FakeCompletions(chunks, fail)
fake_openai = SimpleNamespace(chat=SimpleNamespace(completions=completions))
llm = LLMClient(_settings())
llm = LLMClient(_settings(**settings_kwargs))
llm._client = fake_openai # pyright: ignore[reportAttributeAccessIssue]
return llm, completions
@@ -302,10 +304,22 @@ def test_chat_stream_uses_locked_generation_params() -> None:
assert completions.kwargs["model"] == "turbo"
assert completions.kwargs["stream"] is True
assert completions.kwargs["temperature"] == 0.4
assert completions.kwargs["max_tokens"] == 700
# Phase 11: the old hard 700-token cap is gone — answers may run up to
# BOR_MAX_OUTPUT_TOKENS (default 32 768) so they are not cut off.
assert completions.kwargs["max_tokens"] == 32_768
assert completions.kwargs["messages"] == messages
def test_chat_stream_max_tokens_comes_from_settings() -> None:
"""The output cap is operator-configurable, not a client constant."""
llm, completions = _make_stream_client(
[_chunk("x")], max_output_tokens=1234 # pyright: ignore[reportArgumentType]
)
asyncio.run(_collect(llm, [{"role": "user", "content": "q"}]))
assert completions.kwargs is not None
assert completions.kwargs["max_tokens"] == 1234
def test_chat_stream_skips_empty_deltas_and_choiceless_chunks() -> None:
llm, _ = _make_stream_client([_chunk("a"), _chunk(empty=True), _chunk(None), _chunk("b")])
assert asyncio.run(_collect(llm, [{"role": "user", "content": "q"}])) == ["a", "b"]