services/homelab
2
1
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

update ai stack with turbo

Browse Source
This commit is contained in:
ducoterra
2026-05-25 21:40:56 -04:00
parent a56402c2cc
commit b531e75413
8 changed files with 74 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
active/software_ai_stack/ai-internal.pod
+1 -1
View File
@@ -6,7 +6,7 @@ PublishPort=8000:8000/tcp
PublishPort=8001:8001/tcp
# llama.cpp instruct
PublishPort=8002:8002/tcp
# llama.cpp code
# llama.cpp tool
PublishPort=8003:8003/tcp
# stable-diffusion.cpp gen
PublishPort=1234:1234/tcp
active/software_ai_stack/ai_stack.md
+32
View File
@@ -31,7 +31,10 @@
- [Nomic Embedding](#nomic-embedding)
- [llama.cpp](#llamacpp)
- [stable-diffusion.cpp](#stable-diffusioncpp)
- [STT whisper.cpp](#stt-whispercpp)
- [TTS coqui-tts](#tts-coqui-tts)
- [open-webui](#open-webui)
- [Upgrade Open Webui](#upgrade-open-webui)
- [lite-llm](#lite-llm)
- [Install Services with Quadlets](#install-services-with-quadlets)
- [API Keys](#api-keys)
@@ -526,6 +529,28 @@ localhost/stable-diffusion-cpp:latest \
-p "A normal business meeting. People discuss business for 2 seconds. Suddenly, a horde of furries carrying assault rifles bursts into the room and causes a panic. Hatsune Miku leads the charge screaming in rage."
```
## STT whisper.cpp
Whisper stt service.
```bash
podman run -it --rm -p "8081:8080" \
-v /home/ai/models/stt:/models:z \
--device /dev/kfd \
--device /dev/dri \
ghcr.io/ggml-org/whisper.cpp:main-vulkan \
"whisper-server \
--host 0.0.0.0 \
-m /models/ggml-large-v3-turbo/ggml-large-v3-turbo-q5_0.bin \
--convert"
```
## TTS coqui-tts
```bash
podman run --rm -it -p 5002:5002 --entrypoint /bin/bash ghcr.io/coqui-ai/tts-cpu
```
## open-webui
```bash
@@ -552,6 +577,13 @@ Use the following connections:
| stable-diffusion.cpp | <http://host.containers.internal:1234/v1> |
| stable-diffusion.cpp edit | <http://host.containers.internal:1235/v1> |
### Upgrade Open Webui
```bash
podman image pull ghcr.io/open-webui/open-webui:main
systemctl --user restart open-webui.service
```
## lite-llm
<https://docs.litellm.ai/docs/proxy/configs>
active/software_ai_stack/install_ai_instruct_stack.yaml
+1 -1
View File
@@ -15,7 +15,7 @@
- ai-internal.pod
- llama-embed.container
- llama-instruct.container
- llama-code.container
- llama-tool.container
- name: Reload and start the ai-internal-pod service
ansible.builtin.systemd_service:
state: restarted
active/software_ai_stack/install_ai_think_stack.yaml → active/software_ai_stack/install_ai_turbo_stack.yaml
+1 -1
View File
@@ -13,7 +13,7 @@
loop:
- ai-internal.network
- ai-internal.pod
- llama-think.container
- llama-turbo.container
- name: Reload and start the ai-internal-pod service
ansible.builtin.systemd_service:
state: restarted
active/software_ai_stack/llama-embed.container
+2 -1
View File
@@ -33,7 +33,8 @@ HealthRetries=3
HealthStartPeriod=10s
HealthTimeout=30s
HealthOnFailure=kill
EnvironmentFile=/home/ai/.llama-api/keys.env
# EnvironmentFile=/home/ai/.llama-api/keys.env
[Service]
Restart=always
active/software_ai_stack/llama-instruct.container
+8 -7
View File
@@ -21,17 +21,17 @@ Exec=--port 8002 \
-n 32768 \
--temp 0.7 \
--top-p 0.8 \
--min-p 0.0 \
--top-k 20 \
--repeat-penalty 1.0 \
--presence-penalty 1.5 \
--reasoning-budget 0 \
--min-p 0.00 \
--perf \
--n-gpu-layers all \
-fa on \
--spec-type draft-mtp --spec-draft-n-max 2 \
--jinja \
-m /models/qwen3.6-35b-a3b/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
--mmproj /models/qwen3.6-35b-a3b/mmproj-F16.gguf \
--chat-template-kwargs '{"enable_thinking": false}' \
--chat-template-kwargs '{"preserve_thinking": false, "enable_thinking": false}' \
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
--alias instruct
# Health Check
@@ -41,7 +41,8 @@ HealthRetries=3
HealthStartPeriod=10s
HealthTimeout=30s
HealthOnFailure=kill
EnvironmentFile=/home/ai/.llama-api/keys.env
# EnvironmentFile=/home/ai/.llama-api/keys.env
[Service]
Restart=always
active/software_ai_stack/llama-code.container → active/software_ai_stack/llama-tool.container
+18 -10
View File
@@ -1,11 +1,13 @@
[Unit]
Description=A Llama CPP Server Running a Coding Model
Description=A Llama CPP Server Running a Reasoning Model
[Container]
# Shared AI internal pod without internet access
Pod=ai-internal.pod
# Image is built locally via podman build
# latest-mtp is for mtp testing
# latest is main branch
Image=localhost/llama-cpp-vulkan:latest
# Downloaded models volume
@@ -17,17 +19,25 @@ AddDevice=/dev/dri
# Server command
Exec=--port 8003 \
-c 256000 \
-n 65536 \
--temp 0.7 \
--top-p 0.8 \
-c 262144 \
-n 32768 \
--temp 0.6 \
--top-p 0.95 \
--top-k 20 \
--repeat-penalty 1.05 \
--min-p 0.0 \
--presence-penalty 0.0 \
--repeat-penalty 1.0 \
--reasoning-budget 4096 \
--perf \
--n-gpu-layers all \
-fa on \
--spec-type draft-mtp --spec-draft-n-max 2 \
--jinja \
-m /models/qwen3-coder-30b-a3b/Qwen3-Coder-30B-A3B-Instruct-Q5_K_M.gguf \
--alias code
--chat-template-kwargs '{"preserve_thinking": false}' \
--chat-template-file /models/qwen3.6-35b-a3b-mtp/chat_template.jinja \
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
--alias tool
# Health Check
HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8003/health || exit 1
@@ -37,8 +47,6 @@ HealthStartPeriod=10s
HealthTimeout=30s
HealthOnFailure=kill
EnvironmentFile=/home/ai/.llama-api/keys.env
[Service]
Restart=always
# Extend Timeout to allow time to pull the image
active/software_ai_stack/llama-think.container → active/software_ai_stack/llama-turbo.container
+11 -6
View File
@@ -6,6 +6,8 @@ Description=A Llama CPP Server Running a Reasoning Model
Pod=ai-internal.pod
# Image is built locally via podman build
# latest-mtp is for mtp testing
# latest is main branch
Image=localhost/llama-cpp-vulkan:latest
# Downloaded models volume
@@ -19,20 +21,23 @@ AddDevice=/dev/dri
Exec=--port 8000 \
-c 262144 \
-n 32768 \
--temp 0.7 \
--temp 0.6 \
--top-p 0.95 \
--top-k 20 \
--min-p 0.0 \
--presence-penalty 0.0 \
--repeat-penalty 1.0 \
--reasoning-budget 5000 \
-fa on \
--reasoning-budget 4096 \
--perf \
--n-gpu-layers all \
-fa on \
--spec-type draft-mtp --spec-draft-n-max 2 \
--jinja \
-m /models/qwen3.6-35b-a3b/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
--mmproj /models/qwen3.6-35b-a3b/mmproj-F16.gguf \
--alias think
--chat-template-kwargs '{"preserve_thinking": true}' \
--chat-template-file /models/qwen3.6-35b-a3b-mtp/chat_template.jinja \
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
--alias turbo
# Health Check
HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8000/health || exit 1