fix(rag): lift chat output cap to 32768 tokens — long answers no longer cut off

This commit is contained in:
2026-08-22 11:30:19 -04:00
parent 6ec6181c7b
commit 0da5275eeb
9 changed files with 295 additions and 5 deletions
+5 -1
View File
@@ -175,6 +175,10 @@ class LLMClient:
non-empty ``delta.content`` pieces. Any failure (network, HTTP,
malformed stream) surfaces as :class:`LLMError` so the API layer can
turn it into an SSE ``error`` event instead of a hung request.
Answers are allowed up to ``BOR_MAX_OUTPUT_TOKENS`` (default 32 768)
output tokens — the old hard 700-token cap cut long answers off
mid-sentence (owner report 2026-08-22).
"""
try:
# ``{role, content}`` dicts are exactly what the message params
@@ -183,7 +187,7 @@ class LLMClient:
model=self.settings.llm_chat_model,
messages=cast("list[ChatCompletionMessageParam]", messages),
temperature=0.4,
max_tokens=700,
max_tokens=self.settings.max_output_tokens,
stream=True,
)
async for chunk in stream: