update ai stack with turbo

This commit is contained in:
2026-05-25 21:40:56 -04:00
parent a56402c2cc
commit b531e75413
8 changed files with 74 additions and 27 deletions
@@ -21,17 +21,17 @@ Exec=--port 8002 \
-n 32768 \
--temp 0.7 \
--top-p 0.8 \
--min-p 0.0 \
--top-k 20 \
--repeat-penalty 1.0 \
--presence-penalty 1.5 \
--reasoning-budget 0 \
--min-p 0.00 \
--perf \
--n-gpu-layers all \
-fa on \
--spec-type draft-mtp --spec-draft-n-max 2 \
--jinja \
-m /models/qwen3.6-35b-a3b/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
--mmproj /models/qwen3.6-35b-a3b/mmproj-F16.gguf \
--chat-template-kwargs '{"enable_thinking": false}' \
--chat-template-kwargs '{"preserve_thinking": false, "enable_thinking": false}' \
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
--alias instruct
# Health Check
@@ -41,7 +41,8 @@ HealthRetries=3
HealthStartPeriod=10s
HealthTimeout=30s
HealthOnFailure=kill
EnvironmentFile=/home/ai/.llama-api/keys.env
# EnvironmentFile=/home/ai/.llama-api/keys.env
[Service]
Restart=always