57 lines
1.3 KiB
INI
57 lines
1.3 KiB
INI
[Unit]
|
|
Description=A Llama CPP Server Running a Reasoning Model
|
|
|
|
[Container]
|
|
# Shared AI internal pod without internet access
|
|
Pod=ai-internal.pod
|
|
|
|
# Image is built locally via podman build
|
|
# latest-mtp is for mtp testing
|
|
# latest is main branch
|
|
Image=localhost/llama-cpp-vulkan:latest
|
|
|
|
# Downloaded models volume
|
|
Volume=/home/ai/models/text:/models:z
|
|
|
|
# GPU Device
|
|
AddDevice=/dev/kfd
|
|
AddDevice=/dev/dri
|
|
|
|
# Server command
|
|
Exec=--port 8000 \
|
|
-c 262144 \
|
|
-n 32768 \
|
|
--temp 0.6 \
|
|
--top-p 0.95 \
|
|
--top-k 20 \
|
|
--min-p 0.0 \
|
|
--presence-penalty 0.0 \
|
|
--repeat-penalty 1.0 \
|
|
--reasoning-budget 4096 \
|
|
--perf \
|
|
--n-gpu-layers all \
|
|
-fa on \
|
|
--spec-type draft-mtp --spec-draft-n-max 2 \
|
|
--jinja \
|
|
--chat-template-kwargs '{"preserve_thinking": true}' \
|
|
--chat-template-file /models/qwen3.6-35b-a3b-mtp/chat_template.jinja \
|
|
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
|
|
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
|
|
--alias turbo
|
|
|
|
# Health Check
|
|
HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8000/health || exit 1
|
|
HealthInterval=10s
|
|
HealthRetries=3
|
|
HealthStartPeriod=10s
|
|
HealthTimeout=30s
|
|
HealthOnFailure=kill
|
|
|
|
[Service]
|
|
Restart=always
|
|
# Extend Timeout to allow time to pull the image
|
|
TimeoutStartSec=900
|
|
|
|
[Install]
|
|
# Start by default on boot
|
|
WantedBy=multi-user.target default.target |