services/homelab
1
1
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

update instruct and think containers

Browse Source
This commit is contained in:
Reese Wells
2026-04-16 16:05:52 -04:00
parent 66f9304cc6
commit e43c534ceb
2 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
active/software_ai_stack/llama-instruct.container
View File

@@ -17,7 +17,7 @@ AddDevice=/dev/dri
# Server command # Server command
Exec=--port 8002 \ Exec=--port 8002 \
-c 16000 \ -c 64000 \
--perf \ --perf \
-v \ -v \
--top-k 20 \ --top-k 20 \

11
active/software_ai_stack/llama-think.container
View File

@@ -17,12 +17,17 @@ AddDevice=/dev/dri
# Server command # Server command
Exec=--port 8000 \ Exec=--port 8000 \
-c 64000 \ -c 128000 \
--top-k 64 \
--top-p 0.95 \
--temp 1.0 \
--perf \ --perf \
-v \
--n-gpu-layers all \ --n-gpu-layers all \
--jinja \ --jinja \
--models-max 1 \ -m /models/gemma-4-26b-a4b/gemma-4-26B-A4B-it-UD-Q8_K_XL.gguf \
--models-dir /models --mmproj /models/gemma-4-26b-a4b/mmproj-BF16.gguf \
--alias think
# Health Check # Health Check
HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8000/health || exit 1 HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8000/health || exit 1