fix(rag): lift chat output cap to 32768 tokens — long answers no longer cut off
This commit is contained in:
+41
-1
@@ -9,10 +9,16 @@ Implements just enough of the aipi surface:
|
||||
unrelated ones score low and trigger honest deflection.
|
||||
* ``POST /v1/chat/completions`` — streaming (SSE) or not. The content keys
|
||||
off markers in the system prompt:
|
||||
- user message containing ``write a long answer`` -> a ~900-word
|
||||
deterministic numbered answer (long-answers story, phase 11)
|
||||
- ``DEFLECT_MODE`` -> honest "I haven't done anything like that" answer
|
||||
- otherwise -> upbeat answer quoting the provided document context
|
||||
- user message containing ``pretend to think slowly`` -> 3s warm-up delay
|
||||
(used by the loading-feedback story).
|
||||
|
||||
``max_tokens`` is honored deterministically (token ≈ whitespace word),
|
||||
like a real endpoint: an answer longer than the cap is truncated. This
|
||||
is what makes the phase-11 truncation regression observable.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -63,9 +69,31 @@ def _context(body: dict[str, Any]) -> str:
|
||||
return max((m.get("content", "") for m in msgs), key=len)
|
||||
|
||||
|
||||
LONG_ANSWER_TRIGGER = "write a long answer"
|
||||
#: ~920 words — comfortably past the old hard 700-token cap (where the
|
||||
#: tail would be cut) yet short enough to stream in ~8s at the mock's
|
||||
#: per-chunk pacing.
|
||||
LONG_ANSWER_LINES = 40
|
||||
LONG_ANSWER_END = "LONG-ANSWER-END"
|
||||
|
||||
|
||||
def long_answer() -> str:
|
||||
"""~900-word deterministic walkthrough (phase 11): numbered steps plus
|
||||
a unique final line that must survive the stream untruncated."""
|
||||
lines = [
|
||||
f"{i}. Step {i}: configure node-{i} with the homelab defaults and "
|
||||
f"verify that step {i} of the long walkthrough is complete before moving on."
|
||||
for i in range(1, LONG_ANSWER_LINES + 1)
|
||||
]
|
||||
lines.append(LONG_ANSWER_END)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def compose_answer(body: dict[str, Any]) -> str:
|
||||
system = _system(body)
|
||||
user = _user(body)
|
||||
if LONG_ANSWER_TRIGGER in user.lower():
|
||||
return long_answer()
|
||||
if "DEFLECT_MODE" in system:
|
||||
return (
|
||||
"Ah — I haven't done anything like that, so I don't want to make stuff up! "
|
||||
@@ -163,9 +191,21 @@ def json_dumps(obj: dict[str, Any]) -> str:
|
||||
return json.dumps(obj)
|
||||
|
||||
|
||||
def _apply_max_tokens(answer: str, max_tokens: Any) -> str:
|
||||
"""Deterministic stand-in for the endpoint's output cap: one token ≈
|
||||
one whitespace-separated word. Answers within the cap pass through
|
||||
byte-identical, so existing (short) answers are unaffected."""
|
||||
if not isinstance(max_tokens, int) or max_tokens <= 0:
|
||||
return answer
|
||||
words = answer.split()
|
||||
if len(words) <= max_tokens:
|
||||
return answer
|
||||
return " ".join(words[:max_tokens])
|
||||
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
def chat_completions(body: dict[str, Any]) -> Any:
|
||||
answer = compose_answer(body)
|
||||
answer = _apply_max_tokens(compose_answer(body), body.get("max_tokens"))
|
||||
delay = 3.0 if "pretend to think slowly" in _user(body) else 0.0
|
||||
|
||||
if not body.get("stream"):
|
||||
|
||||
Reference in New Issue
Block a user