fix(rag): lift chat output cap to 32768 tokens — long answers no longer cut off
This commit is contained in:
@@ -51,6 +51,10 @@ class Settings(BaseSettings):
|
||||
# AND no candidate chunk matches the question lexically (see A8).
|
||||
relevance_threshold: float = 0.62
|
||||
max_context_chars: int = 24_000
|
||||
#: Maximum output tokens a chat answer may use (owner instruction
|
||||
#: 2026-08-22: answers must run to their natural end — the old hard
|
||||
#: 700-token cap cut long answers off mid-sentence).
|
||||
max_output_tokens: int = 32_768
|
||||
chunk_target_chars: int = 2_000
|
||||
chunk_overlap_chars: int = 200
|
||||
embed_batch_size: int = 16
|
||||
|
||||
+5
-1
@@ -175,6 +175,10 @@ class LLMClient:
|
||||
non-empty ``delta.content`` pieces. Any failure (network, HTTP,
|
||||
malformed stream) surfaces as :class:`LLMError` so the API layer can
|
||||
turn it into an SSE ``error`` event instead of a hung request.
|
||||
|
||||
Answers are allowed up to ``BOR_MAX_OUTPUT_TOKENS`` (default 32 768)
|
||||
output tokens — the old hard 700-token cap cut long answers off
|
||||
mid-sentence (owner report 2026-08-22).
|
||||
"""
|
||||
try:
|
||||
# ``{role, content}`` dicts are exactly what the message params
|
||||
@@ -183,7 +187,7 @@ class LLMClient:
|
||||
model=self.settings.llm_chat_model,
|
||||
messages=cast("list[ChatCompletionMessageParam]", messages),
|
||||
temperature=0.4,
|
||||
max_tokens=700,
|
||||
max_tokens=self.settings.max_output_tokens,
|
||||
stream=True,
|
||||
)
|
||||
async for chunk in stream:
|
||||
|
||||
Reference in New Issue
Block a user