update ai stack with turbo
This commit is contained in:
@@ -21,17 +21,17 @@ Exec=--port 8002 \
|
||||
-n 32768 \
|
||||
--temp 0.7 \
|
||||
--top-p 0.8 \
|
||||
--min-p 0.0 \
|
||||
--top-k 20 \
|
||||
--repeat-penalty 1.0 \
|
||||
--presence-penalty 1.5 \
|
||||
--reasoning-budget 0 \
|
||||
--min-p 0.00 \
|
||||
--perf \
|
||||
--n-gpu-layers all \
|
||||
-fa on \
|
||||
--spec-type draft-mtp --spec-draft-n-max 2 \
|
||||
--jinja \
|
||||
-m /models/qwen3.6-35b-a3b/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
|
||||
--mmproj /models/qwen3.6-35b-a3b/mmproj-F16.gguf \
|
||||
--chat-template-kwargs '{"enable_thinking": false}' \
|
||||
--chat-template-kwargs '{"preserve_thinking": false, "enable_thinking": false}' \
|
||||
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
|
||||
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
|
||||
--alias instruct
|
||||
|
||||
# Health Check
|
||||
@@ -41,7 +41,8 @@ HealthRetries=3
|
||||
HealthStartPeriod=10s
|
||||
HealthTimeout=30s
|
||||
HealthOnFailure=kill
|
||||
EnvironmentFile=/home/ai/.llama-api/keys.env
|
||||
|
||||
# EnvironmentFile=/home/ai/.llama-api/keys.env
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
|
||||
Reference in New Issue
Block a user