diff --git a/active/software_ai_stack/ai-internal.pod b/active/software_ai_stack/ai-internal.pod index 136791b..afc4492 100644 --- a/active/software_ai_stack/ai-internal.pod +++ b/active/software_ai_stack/ai-internal.pod @@ -6,7 +6,7 @@ PublishPort=8000:8000/tcp PublishPort=8001:8001/tcp # llama.cpp instruct PublishPort=8002:8002/tcp -# llama.cpp code +# llama.cpp tool PublishPort=8003:8003/tcp # stable-diffusion.cpp gen PublishPort=1234:1234/tcp diff --git a/active/software_ai_stack/ai_stack.md b/active/software_ai_stack/ai_stack.md index 5fb821e..9ecb659 100644 --- a/active/software_ai_stack/ai_stack.md +++ b/active/software_ai_stack/ai_stack.md @@ -31,7 +31,10 @@ - [Nomic Embedding](#nomic-embedding) - [llama.cpp](#llamacpp) - [stable-diffusion.cpp](#stable-diffusioncpp) + - [STT whisper.cpp](#stt-whispercpp) + - [TTS coqui-tts](#tts-coqui-tts) - [open-webui](#open-webui) + - [Upgrade Open Webui](#upgrade-open-webui) - [lite-llm](#lite-llm) - [Install Services with Quadlets](#install-services-with-quadlets) - [API Keys](#api-keys) @@ -526,6 +529,28 @@ localhost/stable-diffusion-cpp:latest \ -p "A normal business meeting. People discuss business for 2 seconds. Suddenly, a horde of furries carrying assault rifles bursts into the room and causes a panic. Hatsune Miku leads the charge screaming in rage." ``` +## STT whisper.cpp + +Whisper stt service. + +```bash +podman run -it --rm -p "8081:8080" \ +-v /home/ai/models/stt:/models:z \ +--device /dev/kfd \ +--device /dev/dri \ +ghcr.io/ggml-org/whisper.cpp:main-vulkan \ +"whisper-server \ +--host 0.0.0.0 \ +-m /models/ggml-large-v3-turbo/ggml-large-v3-turbo-q5_0.bin \ +--convert" +``` + +## TTS coqui-tts + +```bash +podman run --rm -it -p 5002:5002 --entrypoint /bin/bash ghcr.io/coqui-ai/tts-cpu +``` + ## open-webui ```bash @@ -552,6 +577,13 @@ Use the following connections: | stable-diffusion.cpp | | | stable-diffusion.cpp edit | | +### Upgrade Open Webui + +```bash +podman image pull ghcr.io/open-webui/open-webui:main +systemctl --user restart open-webui.service +``` + ## lite-llm diff --git a/active/software_ai_stack/install_ai_instruct_stack.yaml b/active/software_ai_stack/install_ai_instruct_stack.yaml index e530b91..56d275c 100644 --- a/active/software_ai_stack/install_ai_instruct_stack.yaml +++ b/active/software_ai_stack/install_ai_instruct_stack.yaml @@ -15,7 +15,7 @@ - ai-internal.pod - llama-embed.container - llama-instruct.container - - llama-code.container + - llama-tool.container - name: Reload and start the ai-internal-pod service ansible.builtin.systemd_service: state: restarted diff --git a/active/software_ai_stack/install_ai_think_stack.yaml b/active/software_ai_stack/install_ai_turbo_stack.yaml similarity index 95% rename from active/software_ai_stack/install_ai_think_stack.yaml rename to active/software_ai_stack/install_ai_turbo_stack.yaml index dd9da70..07b45c0 100644 --- a/active/software_ai_stack/install_ai_think_stack.yaml +++ b/active/software_ai_stack/install_ai_turbo_stack.yaml @@ -13,7 +13,7 @@ loop: - ai-internal.network - ai-internal.pod - - llama-think.container + - llama-turbo.container - name: Reload and start the ai-internal-pod service ansible.builtin.systemd_service: state: restarted diff --git a/active/software_ai_stack/llama-embed.container b/active/software_ai_stack/llama-embed.container index afbb330..91510de 100644 --- a/active/software_ai_stack/llama-embed.container +++ b/active/software_ai_stack/llama-embed.container @@ -33,7 +33,8 @@ HealthRetries=3 HealthStartPeriod=10s HealthTimeout=30s HealthOnFailure=kill -EnvironmentFile=/home/ai/.llama-api/keys.env + +# EnvironmentFile=/home/ai/.llama-api/keys.env [Service] Restart=always diff --git a/active/software_ai_stack/llama-instruct.container b/active/software_ai_stack/llama-instruct.container index d34cbf0..6f063ef 100644 --- a/active/software_ai_stack/llama-instruct.container +++ b/active/software_ai_stack/llama-instruct.container @@ -21,17 +21,17 @@ Exec=--port 8002 \ -n 32768 \ --temp 0.7 \ --top-p 0.8 \ - --min-p 0.0 \ --top-k 20 \ - --repeat-penalty 1.0 \ --presence-penalty 1.5 \ - --reasoning-budget 0 \ + --min-p 0.00 \ --perf \ --n-gpu-layers all \ + -fa on \ + --spec-type draft-mtp --spec-draft-n-max 2 \ --jinja \ - -m /models/qwen3.6-35b-a3b/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \ - --mmproj /models/qwen3.6-35b-a3b/mmproj-F16.gguf \ - --chat-template-kwargs '{"enable_thinking": false}' \ + --chat-template-kwargs '{"preserve_thinking": false, "enable_thinking": false}' \ + -m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \ + --mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \ --alias instruct # Health Check @@ -41,7 +41,8 @@ HealthRetries=3 HealthStartPeriod=10s HealthTimeout=30s HealthOnFailure=kill -EnvironmentFile=/home/ai/.llama-api/keys.env + +# EnvironmentFile=/home/ai/.llama-api/keys.env [Service] Restart=always diff --git a/active/software_ai_stack/llama-code.container b/active/software_ai_stack/llama-tool.container similarity index 54% rename from active/software_ai_stack/llama-code.container rename to active/software_ai_stack/llama-tool.container index 2e74c55..241da93 100644 --- a/active/software_ai_stack/llama-code.container +++ b/active/software_ai_stack/llama-tool.container @@ -1,11 +1,13 @@ [Unit] -Description=A Llama CPP Server Running a Coding Model +Description=A Llama CPP Server Running a Reasoning Model [Container] # Shared AI internal pod without internet access Pod=ai-internal.pod # Image is built locally via podman build +# latest-mtp is for mtp testing +# latest is main branch Image=localhost/llama-cpp-vulkan:latest # Downloaded models volume @@ -17,17 +19,25 @@ AddDevice=/dev/dri # Server command Exec=--port 8003 \ - -c 256000 \ - -n 65536 \ - --temp 0.7 \ - --top-p 0.8 \ + -c 262144 \ + -n 32768 \ + --temp 0.6 \ + --top-p 0.95 \ --top-k 20 \ - --repeat-penalty 1.05 \ + --min-p 0.0 \ + --presence-penalty 0.0 \ + --repeat-penalty 1.0 \ + --reasoning-budget 4096 \ --perf \ --n-gpu-layers all \ + -fa on \ + --spec-type draft-mtp --spec-draft-n-max 2 \ --jinja \ - -m /models/qwen3-coder-30b-a3b/Qwen3-Coder-30B-A3B-Instruct-Q5_K_M.gguf \ - --alias code + --chat-template-kwargs '{"preserve_thinking": false}' \ + --chat-template-file /models/qwen3.6-35b-a3b-mtp/chat_template.jinja \ + -m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \ + --mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \ + --alias tool # Health Check HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8003/health || exit 1 @@ -37,8 +47,6 @@ HealthStartPeriod=10s HealthTimeout=30s HealthOnFailure=kill -EnvironmentFile=/home/ai/.llama-api/keys.env - [Service] Restart=always # Extend Timeout to allow time to pull the image diff --git a/active/software_ai_stack/llama-think.container b/active/software_ai_stack/llama-turbo.container similarity index 68% rename from active/software_ai_stack/llama-think.container rename to active/software_ai_stack/llama-turbo.container index 9472eb4..496c042 100644 --- a/active/software_ai_stack/llama-think.container +++ b/active/software_ai_stack/llama-turbo.container @@ -6,6 +6,8 @@ Description=A Llama CPP Server Running a Reasoning Model Pod=ai-internal.pod # Image is built locally via podman build +# latest-mtp is for mtp testing +# latest is main branch Image=localhost/llama-cpp-vulkan:latest # Downloaded models volume @@ -19,20 +21,23 @@ AddDevice=/dev/dri Exec=--port 8000 \ -c 262144 \ -n 32768 \ - --temp 0.7 \ + --temp 0.6 \ --top-p 0.95 \ --top-k 20 \ --min-p 0.0 \ --presence-penalty 0.0 \ --repeat-penalty 1.0 \ - --reasoning-budget 5000 \ - -fa on \ + --reasoning-budget 4096 \ --perf \ --n-gpu-layers all \ + -fa on \ + --spec-type draft-mtp --spec-draft-n-max 2 \ --jinja \ - -m /models/qwen3.6-35b-a3b/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \ - --mmproj /models/qwen3.6-35b-a3b/mmproj-F16.gguf \ - --alias think + --chat-template-kwargs '{"preserve_thinking": true}' \ + --chat-template-file /models/qwen3.6-35b-a3b-mtp/chat_template.jinja \ + -m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \ + --mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \ + --alias turbo # Health Check HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8000/health || exit 1