add configurable llm timeout
This commit is contained in:
@@ -19,6 +19,7 @@ BOR_LLM_BASE_URL=https://aipi.reeseapps.com/v1
|
||||
BOR_LLM_API_KEY= # falls back to $AIPI_KEY, then "not-needed"
|
||||
BOR_LLM_CHAT_MODEL=turbo
|
||||
# BOR_LLM_RETRIES=3 # retry a dead LLM request before the first token lands (phase 67); 0 = off
|
||||
# BOR_LLM_TIMEOUT=300 # HTTP timeout for LLM API calls, seconds (default 120)
|
||||
# BOR_LLM_RETRY_DELAY=5 # seconds between LLM retries (phase 67)
|
||||
BOR_LLM_EMBED_MODEL=embed
|
||||
BOR_LLM_SUMMARY_MODEL=lite # one-shot completions: document summaries (phase 30), KB overview (phase 31)
|
||||
|
||||
@@ -80,6 +80,11 @@ class Settings(BaseSettings):
|
||||
#: Flat seconds to wait between attempts (phase 67,
|
||||
#: ``BOR_LLM_RETRY_DELAY``); the TODO-locked 5 s, no backoff.
|
||||
llm_retry_delay: float = 5.0
|
||||
#: HTTP timeout in seconds for LLM API calls (chat + embeddings).
|
||||
#: Increase when long prompt processing or slow models exceed the
|
||||
#: default 120 s (``BOR_LLM_TIMEOUT``; ``0`` = use the OpenAI SDK
|
||||
#: default, which is platform-dependent).
|
||||
llm_timeout: float = 120.0
|
||||
# --- Chat history (phase 74, TODO L4: prior turns + prior thinking) ---
|
||||
#: Newest client-provided history turns kept per ``POST /api/chat``
|
||||
#: (phase 74, ``BOR_HISTORY_MAX_TURNS``): the request's ``history``
|
||||
|
||||
+1
-1
@@ -186,7 +186,7 @@ class LLMClient:
|
||||
self._client = AsyncOpenAI(
|
||||
base_url=self.settings.llm_base_url,
|
||||
api_key=self.settings.effective_api_key,
|
||||
timeout=120.0,
|
||||
timeout=self.settings.llm_timeout,
|
||||
)
|
||||
|
||||
async def _post_embeddings(self, texts: list[str]) -> list[list[float]]:
|
||||
|
||||
Reference in New Issue
Block a user