checkpoint commit

active/software_ai_stack/llama-instruct.container

@@ -1,8 +1,8 @@
 [Unit]
-Description=A Llama CPP Server Running GPT OSS 120b
+Description=A Llama CPP Server Running a Non-Reasoning Model
 
 [Container]
-# Shared AI internal pod
+# Shared AI internal pod without internet access
 Pod=ai-internal.pod
 
 # Image is built locally via podman build
@@ -17,29 +17,31 @@ AddDevice=/dev/dri
 
 # Server command
 Exec=--port 8002 \
-    -c 64000 \
-    --perf \
-    -v \
-    --top-k 20 \
-    --top-p 0.8 \
-    --min-p 0 \
-    --presence-penalty 1.5 \
-    --repeat-penalty 1 \
+    -c 262144 \
+    -n 32768 \
     --temp 0.7 \
+    --top-p 0.8 \
+    --min-p 0.0 \
+    --top-k 20 \
+    --repeat-penalty 1.0 \
+    --presence-penalty 1.5 \
+    --reasoning-budget 0 \
+    --perf \
     --n-gpu-layers all \
     --jinja \
+    -m /models/qwen3.6-35b-a3b/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
+    --mmproj /models/qwen3.6-35b-a3b/mmproj-F16.gguf \
     --chat-template-kwargs '{"enable_thinking": false}' \
-    -m /models/qwen3.5-35b-a3b/Qwen3.5-35B-A3B-Q8_0.gguf \
-    --mmproj /models/qwen3.5-35b-a3b/mmproj-F16.gguf \
     --alias instruct
 
 # Health Check
-HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8000/health || exit 1
+HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8002/health || exit 1
 HealthInterval=10s
 HealthRetries=3
 HealthStartPeriod=10s
 HealthTimeout=30s
 HealthOnFailure=kill
+EnvironmentFile=/home/ai/.llama-api/keys.env
 
 [Service]
 Restart=always
@@ -48,4 +50,4 @@ TimeoutStartSec=900
 
 [Install]
 # Start by default on boot
-WantedBy=multi-user.target default.target
+WantedBy=multi-user.target default.target