update ai stack with turbo

2026-05-25 21:40:56 -04:00
parent a56402c2cc
commit b531e75413
8 changed files with 74 additions and 27 deletions
@@ -0,0 +1,57 @@
+[Unit]
+Description=A Llama CPP Server Running a Reasoning Model
+
+[Container]
+# Shared AI internal pod without internet access
+Pod=ai-internal.pod
+
+# Image is built locally via podman build
+# latest-mtp is for mtp testing
+# latest is main branch
+Image=localhost/llama-cpp-vulkan:latest
+
+# Downloaded models volume
+Volume=/home/ai/models/text:/models:z
+
+# GPU Device
+AddDevice=/dev/kfd
+AddDevice=/dev/dri
+
+# Server command
+Exec=--port 8003 \
+    -c 262144 \
+    -n 32768 \
+    --temp 0.6 \
+    --top-p 0.95 \
+    --top-k 20 \
+    --min-p 0.0 \
+    --presence-penalty 0.0 \
+    --repeat-penalty 1.0 \
+    --reasoning-budget 4096 \
+    --perf \
+    --n-gpu-layers all \
+    -fa on \
+    --spec-type draft-mtp --spec-draft-n-max 2 \
+    --jinja \
+    --chat-template-kwargs '{"preserve_thinking": false}' \
+    --chat-template-file /models/qwen3.6-35b-a3b-mtp/chat_template.jinja \
+    -m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
+    --mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
+    --alias tool
+
+# Health Check
+HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8003/health || exit 1
+HealthInterval=10s
+HealthRetries=3
+HealthStartPeriod=10s
+HealthTimeout=30s
+HealthOnFailure=kill
+
+[Service]
+Restart=always
+# Extend Timeout to allow time to pull the image
+TimeoutStartSec=900
+
+[Install]
+# Start by default on boot
+WantedBy=multi-user.target default.target