various updates
This commit is contained in:
@@ -1,2 +0,0 @@
|
||||
[Network]
|
||||
Internal=true
|
||||
@@ -1,14 +0,0 @@
|
||||
[Pod]
|
||||
Network=ai-internal.network
|
||||
# llama.cpp server
|
||||
PublishPort=8000:8000/tcp
|
||||
# llama.cpp embed
|
||||
PublishPort=8001:8001/tcp
|
||||
# llama.cpp instruct
|
||||
PublishPort=8002:8002/tcp
|
||||
# llama.cpp tool
|
||||
PublishPort=8003:8003/tcp
|
||||
# stable-diffusion.cpp gen
|
||||
PublishPort=1234:1234/tcp
|
||||
# stable-diffusion.cpp edit
|
||||
PublishPort=1235:1235/tcp
|
||||
@@ -3,10 +3,6 @@
|
||||
- [Self Hosted AI Stack](#self-hosted-ai-stack)
|
||||
- [Notes](#notes)
|
||||
- [Podman Volume Locations](#podman-volume-locations)
|
||||
- [List of Internal Links](#list-of-internal-links)
|
||||
- [Quick Install](#quick-install)
|
||||
- [Text Stack](#text-stack)
|
||||
- [Image Stack](#image-stack)
|
||||
- [Setup](#setup)
|
||||
- [Create the AI user](#create-the-ai-user)
|
||||
- [Helper aliases](#helper-aliases)
|
||||
@@ -36,16 +32,7 @@
|
||||
- [open-webui](#open-webui)
|
||||
- [Upgrade Open Webui](#upgrade-open-webui)
|
||||
- [lite-llm](#lite-llm)
|
||||
- [Install Services with Quadlets](#install-services-with-quadlets)
|
||||
- [API Keys](#api-keys)
|
||||
- [Internal and External Pods](#internal-and-external-pods)
|
||||
- [Llama CPP Server (Port 8000)](#llama-cpp-server-port-8000)
|
||||
- [Llama CPP Embedding Server (Port 8001)](#llama-cpp-embedding-server-port-8001)
|
||||
- [Llama CPP Instruct Server (Port 8002)](#llama-cpp-instruct-server-port-8002)
|
||||
- [Stable Diffusion CPP (Port 1234 and 1235)](#stable-diffusion-cpp-port-1234-and-1235)
|
||||
- [Open Webui (Port 8080)](#open-webui-port-8080)
|
||||
- [Install the update script](#install-the-update-script)
|
||||
- [Install Guest Open Webui with Start/Stop Services](#install-guest-open-webui-with-startstop-services)
|
||||
- [Install the update script](#install-the-update-script)
|
||||
- [Benchmark Results](#benchmark-results)
|
||||
- [Testing with Curl](#testing-with-curl)
|
||||
- [OpenAI API](#openai-api)
|
||||
@@ -77,33 +64,6 @@ hf-download ()
|
||||
|
||||
`~/.local/share/containers/storage/volumes/`
|
||||
|
||||
### List of Internal Links
|
||||
|
||||
- llama-cpp
|
||||
- llama-embed
|
||||
- llama-instruct
|
||||
- image-gen
|
||||
- image-edit
|
||||
- openwebui
|
||||
|
||||
## Quick Install
|
||||
|
||||
### Text Stack
|
||||
|
||||
```bash
|
||||
ansible-playbook \
|
||||
-i ansible/inventory.yaml \
|
||||
active/software_ai_stack/install_ai_text_stack.yaml
|
||||
```
|
||||
|
||||
### Image Stack
|
||||
|
||||
```bash
|
||||
ansible-playbook \
|
||||
-i ansible/inventory.yaml \
|
||||
active/software_ai_stack/install_ai_image_stack.yaml
|
||||
```
|
||||
|
||||
## Setup
|
||||
|
||||
### Create the AI user
|
||||
@@ -186,8 +146,7 @@ rsync -av --progress /home/ai/models/ /srv/models/
|
||||
In my completely subjective opinion: 5 bit quant is usually the sweet spot for
|
||||
unsloth models. Q5_K_S is usually just fine.
|
||||
|
||||
I usually download the F16 mmproj files. This is also completely subjective.
|
||||
BF16 is fine. F32 is overkill.
|
||||
I usually download the BF16 mmproj files. This is also completely subjective.
|
||||
|
||||
#### Text models
|
||||
|
||||
@@ -391,20 +350,43 @@ podman run \
|
||||
--name llama-server-demo \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
-v /home/ai/models/text:/models:z \
|
||||
-v $(pwd):/models:z \
|
||||
-p 8010:8000 \
|
||||
--ipc host \
|
||||
localhost/llama-cpp-vulkan:latest \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
-c 128000 \
|
||||
--perf \
|
||||
--n-gpu-layers all \
|
||||
--jinja \
|
||||
--models-max 1 \
|
||||
--models-dir /models \
|
||||
--chat-template-kwargs '{"enable_thinking": false}' \
|
||||
-m /models/qwen3.5-35b-a3b
|
||||
--model /models/gemma-4-E4B-it-qat-UD-Q4_K_XL.gguf \
|
||||
--mmproj /models/mmproj-F16.gguf \
|
||||
--temp 1.0 \
|
||||
--top-p 0.95 \
|
||||
--top-k 64 \
|
||||
--chat-template-kwargs '{"enable_thinking":true}'
|
||||
|
||||
podman run \
|
||||
--rm \
|
||||
--name llama-server-demo \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
-v $(pwd):/models:z \
|
||||
-p 8010:8000 \
|
||||
--ipc host \
|
||||
localhost/llama-cpp-vulkan:latest \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--perf \
|
||||
--n-gpu-layers all \
|
||||
--jinja \
|
||||
--model /models/Qwen3.5-9B-UD-Q4_K_XL.gguf \
|
||||
--mmproj /models/mmproj-BF16.gguf \
|
||||
--temp 0.6 \
|
||||
--top-p 0.95 \
|
||||
--top-k 20 \
|
||||
--min-p 0.00 \
|
||||
--chat-template-kwargs '{"enable_thinking":true}'
|
||||
```
|
||||
|
||||
Embedding models
|
||||
@@ -595,96 +577,7 @@ podman run \
|
||||
-p 4000:4000
|
||||
```
|
||||
|
||||
## Install Services with Quadlets
|
||||
|
||||
### API Keys
|
||||
|
||||
```bash
|
||||
mkdir -p /home/ai/.llama-api
|
||||
touch /home/ai/.llama-api/keys.env
|
||||
chmod 600 /home/ai/.llama-api/keys.env
|
||||
vim /home/ai/.llama-api/keys.env
|
||||
|
||||
LLAMA_API_KEY=
|
||||
|
||||
# Generate keys and append to file, then comma separate the keys
|
||||
openssl rand -base64 48 >> keys.env
|
||||
openssl rand -base64 48 >> keys.env
|
||||
openssl rand -base64 48 >> keys.env
|
||||
```
|
||||
|
||||
### Internal and External Pods
|
||||
|
||||
These will be used to restrict internet access to our llama.cpp and
|
||||
stable-diffusion.cpp services while allowing the frontend services to
|
||||
communicate with those containers.
|
||||
|
||||
```bash
|
||||
scp -r active/software_ai_stack/ai-internal.* deskwork-ai:.config/containers/systemd/
|
||||
ssh deskwork-ai
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user start ai-internal-pod.service
|
||||
```
|
||||
|
||||
### Llama CPP Server (Port 8000)
|
||||
|
||||
Installs the llama.cpp server to run our text models.
|
||||
|
||||
```bash
|
||||
scp -r active/software_ai_stack/llama-think.container deskwork-ai:.config/containers/systemd/
|
||||
ssh deskwork-ai
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user restart ai-internal-pod.service
|
||||
```
|
||||
|
||||
### Llama CPP Embedding Server (Port 8001)
|
||||
|
||||
Installs the llama.cpp server to run our embedding models
|
||||
|
||||
```bash
|
||||
scp -r active/software_ai_stack/llama-embed.container deskwork-ai:.config/containers/systemd/
|
||||
ssh deskwork-ai
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user restart ai-internal-pod.service
|
||||
```
|
||||
|
||||
### Llama CPP Instruct Server (Port 8002)
|
||||
|
||||
Installs the llama.cpp server to run a constant instruct (no thinking) model for quick replies
|
||||
|
||||
```bash
|
||||
scp -r active/software_ai_stack/llama-instruct.container deskwork-ai:.config/containers/systemd/
|
||||
ssh deskwork-ai
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user restart ai-internal-pod.service
|
||||
```
|
||||
|
||||
### Stable Diffusion CPP (Port 1234 and 1235)
|
||||
|
||||
Installs the stable-diffusion.cpp server to run our image models.
|
||||
|
||||
```bash
|
||||
scp -r active/software_ai_stack/quadlets_stable_diffusion/* deskwork-ai:.config/containers/systemd/
|
||||
ssh deskwork-ai
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user restart ai-internal-pod.service
|
||||
```
|
||||
|
||||
### Open Webui (Port 8080)
|
||||
|
||||
Installs the open webui frontend.
|
||||
|
||||
```bash
|
||||
scp -r active/software_ai_stack/quadlets_openwebui/* deskwork-ai:.config/containers/systemd/
|
||||
ssh deskwork-ai
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user restart ai-external-pod.service
|
||||
```
|
||||
|
||||
Note, all services will be available at `host.containers.internal`. So llama.cpp
|
||||
will be up at `http://host.containers.internal:8000`.
|
||||
|
||||
### Install the update script
|
||||
## Install the update script
|
||||
|
||||
```bash
|
||||
# 1. Builds the latest llama.cpp and stable-diffusion.cpp
|
||||
@@ -696,18 +589,6 @@ chmod +x update-script.sh
|
||||
./update-script.sh
|
||||
```
|
||||
|
||||
### Install Guest Open Webui with Start/Stop Services
|
||||
|
||||
Optionally install a guest openwebui service.
|
||||
|
||||
```bash
|
||||
scp -r active/software_ai_stack/systemd/. deskwork-ai:.config/systemd/user/
|
||||
ssh deskwork-ai
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable open-webui-guest-start.timer
|
||||
systemctl --user enable open-webui-guest-stop.timer
|
||||
```
|
||||
|
||||
## Benchmark Results
|
||||
|
||||
Benchmarks are run with [unsloth gpt-oss-20b Q8_0](https://huggingface.co/unsloth/gpt-oss-20b-GGUF/blob/main/gpt-oss-20b-Q8_0.gguf)
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
- name: Create Deskwork AI Stack
|
||||
hosts: deskwork-ai
|
||||
tasks:
|
||||
- name: Create /home/ai/.config/containers/systemd
|
||||
ansible.builtin.file:
|
||||
path: /home/ai/.config/containers/systemd
|
||||
state: directory
|
||||
mode: "0755"
|
||||
- name: Copy Quadlets
|
||||
template:
|
||||
src: "{{ item }}"
|
||||
dest: "/home/ai/.config/containers/systemd/{{ item }}"
|
||||
loop:
|
||||
- ai-internal.network
|
||||
- ai-internal.pod
|
||||
- stable-diffusion-gen-server.container
|
||||
- stable-diffusion-edit-server.container
|
||||
- name: Reload and start the ai-internal-pod service
|
||||
ansible.builtin.systemd_service:
|
||||
state: restarted
|
||||
name: ai-internal-pod.service
|
||||
daemon_reload: true
|
||||
scope: user
|
||||
@@ -1,24 +0,0 @@
|
||||
- name: Create Deskwork AI Stack
|
||||
hosts: deskwork-ai
|
||||
tasks:
|
||||
- name: Create /home/ai/.config/containers/systemd
|
||||
ansible.builtin.file:
|
||||
path: /home/ai/.config/containers/systemd
|
||||
state: directory
|
||||
mode: "0755"
|
||||
- name: Copy Quadlets
|
||||
template:
|
||||
src: "{{ item }}"
|
||||
dest: "/home/ai/.config/containers/systemd/{{ item }}"
|
||||
loop:
|
||||
- ai-internal.network
|
||||
- ai-internal.pod
|
||||
- llama-embed.container
|
||||
- llama-instruct.container
|
||||
- llama-tool.container
|
||||
- name: Reload and start the ai-internal-pod service
|
||||
ansible.builtin.systemd_service:
|
||||
state: restarted
|
||||
name: ai-internal-pod.service
|
||||
daemon_reload: true
|
||||
scope: user
|
||||
@@ -1,22 +0,0 @@
|
||||
- name: Create Deskwork AI Stack
|
||||
hosts: driveripper-ai
|
||||
tasks:
|
||||
- name: Create /home/ai/.config/containers/systemd
|
||||
ansible.builtin.file:
|
||||
path: /home/ai/.config/containers/systemd
|
||||
state: directory
|
||||
mode: "0755"
|
||||
- name: Copy Quadlets
|
||||
template:
|
||||
src: "{{ item }}"
|
||||
dest: "/home/ai/.config/containers/systemd/{{ item }}"
|
||||
loop:
|
||||
- ai-internal.network
|
||||
- ai-internal.pod
|
||||
- llama-turbo.container
|
||||
- name: Reload and start the ai-internal-pod service
|
||||
ansible.builtin.systemd_service:
|
||||
state: restarted
|
||||
name: ai-internal-pod.service
|
||||
daemon_reload: true
|
||||
scope: user
|
||||
@@ -1,46 +0,0 @@
|
||||
[Unit]
|
||||
Description=A Llama CPP Server For Embedding Models
|
||||
|
||||
[Container]
|
||||
# Shared AI internal pod without internet access
|
||||
Pod=ai-internal.pod
|
||||
|
||||
# Image is built locally via podman build
|
||||
Image=localhost/llama-cpp-vulkan:latest
|
||||
|
||||
# Downloaded models volume
|
||||
Volume=/home/ai/models/embedding:/models:z
|
||||
|
||||
# GPU Device
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
|
||||
# Server command
|
||||
Exec=--port 8001 \
|
||||
-c 0 \
|
||||
-b 1024 \
|
||||
-ub 1024 \
|
||||
--perf \
|
||||
--n-gpu-layers all \
|
||||
--embedding \
|
||||
-m /models/emebeddinggemma-300m/embeddinggemma-300M-BF16.gguf \
|
||||
--alias embed
|
||||
|
||||
# Health Check
|
||||
HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8001/health || exit 1
|
||||
HealthInterval=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=10s
|
||||
HealthTimeout=30s
|
||||
HealthOnFailure=kill
|
||||
|
||||
# EnvironmentFile=/home/ai/.llama-api/keys.env
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
# Extend Timeout to allow time to pull the image
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
# Start by default on boot
|
||||
WantedBy=multi-user.target default.target
|
||||
@@ -1,54 +0,0 @@
|
||||
[Unit]
|
||||
Description=A Llama CPP Server Running a Non-Reasoning Model
|
||||
|
||||
[Container]
|
||||
# Shared AI internal pod without internet access
|
||||
Pod=ai-internal.pod
|
||||
|
||||
# Image is built locally via podman build
|
||||
Image=localhost/llama-cpp-vulkan:latest
|
||||
|
||||
# Downloaded models volume
|
||||
Volume=/home/ai/models/text:/models:z
|
||||
|
||||
# GPU Device
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
|
||||
# Server command
|
||||
Exec=--port 8002 \
|
||||
-c 262144 \
|
||||
-n 32768 \
|
||||
--temp 0.7 \
|
||||
--top-p 0.8 \
|
||||
--top-k 20 \
|
||||
--presence-penalty 1.5 \
|
||||
--min-p 0.00 \
|
||||
--perf \
|
||||
--n-gpu-layers all \
|
||||
-fa on \
|
||||
--spec-type draft-mtp --spec-draft-n-max 2 \
|
||||
--jinja \
|
||||
--chat-template-kwargs '{"preserve_thinking": false, "enable_thinking": false}' \
|
||||
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
|
||||
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
|
||||
--alias instruct
|
||||
|
||||
# Health Check
|
||||
HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8002/health || exit 1
|
||||
HealthInterval=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=10s
|
||||
HealthTimeout=30s
|
||||
HealthOnFailure=kill
|
||||
|
||||
# EnvironmentFile=/home/ai/.llama-api/keys.env
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
# Extend Timeout to allow time to pull the image
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
# Start by default on boot
|
||||
WantedBy=multi-user.target default.target
|
||||
@@ -1,57 +0,0 @@
|
||||
[Unit]
|
||||
Description=A Llama CPP Server Running a Reasoning Model
|
||||
|
||||
[Container]
|
||||
# Shared AI internal pod without internet access
|
||||
Pod=ai-internal.pod
|
||||
|
||||
# Image is built locally via podman build
|
||||
# latest-mtp is for mtp testing
|
||||
# latest is main branch
|
||||
Image=localhost/llama-cpp-vulkan:latest
|
||||
|
||||
# Downloaded models volume
|
||||
Volume=/home/ai/models/text:/models:z
|
||||
|
||||
# GPU Device
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
|
||||
# Server command
|
||||
Exec=--port 8003 \
|
||||
-c 262144 \
|
||||
-n 32768 \
|
||||
--temp 0.6 \
|
||||
--top-p 0.95 \
|
||||
--top-k 20 \
|
||||
--min-p 0.0 \
|
||||
--presence-penalty 0.0 \
|
||||
--repeat-penalty 1.0 \
|
||||
--reasoning-budget 4096 \
|
||||
--perf \
|
||||
--n-gpu-layers all \
|
||||
-fa on \
|
||||
--spec-type draft-mtp --spec-draft-n-max 2 \
|
||||
--jinja \
|
||||
--chat-template-kwargs '{"preserve_thinking": false}' \
|
||||
--chat-template-file /models/qwen3.6-35b-a3b-mtp/chat_template.jinja \
|
||||
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
|
||||
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
|
||||
--alias tool
|
||||
|
||||
# Health Check
|
||||
HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8003/health || exit 1
|
||||
HealthInterval=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=10s
|
||||
HealthTimeout=30s
|
||||
HealthOnFailure=kill
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
# Extend Timeout to allow time to pull the image
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
# Start by default on boot
|
||||
WantedBy=multi-user.target default.target
|
||||
@@ -1,57 +0,0 @@
|
||||
[Unit]
|
||||
Description=A Llama CPP Server Running a Reasoning Model
|
||||
|
||||
[Container]
|
||||
# Shared AI internal pod without internet access
|
||||
Pod=ai-internal.pod
|
||||
|
||||
# Image is built locally via podman build
|
||||
# latest-mtp is for mtp testing
|
||||
# latest is main branch
|
||||
Image=localhost/llama-cpp-vulkan:latest
|
||||
|
||||
# Downloaded models volume
|
||||
Volume=/home/ai/models/text:/models:z
|
||||
|
||||
# GPU Device
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
|
||||
# Server command
|
||||
Exec=--port 8000 \
|
||||
-c 262144 \
|
||||
-n 32768 \
|
||||
--temp 0.6 \
|
||||
--top-p 0.95 \
|
||||
--top-k 20 \
|
||||
--min-p 0.0 \
|
||||
--presence-penalty 0.0 \
|
||||
--repeat-penalty 1.0 \
|
||||
--reasoning-budget 4096 \
|
||||
--perf \
|
||||
--n-gpu-layers all \
|
||||
-fa on \
|
||||
--spec-type draft-mtp --spec-draft-n-max 2 \
|
||||
--jinja \
|
||||
--chat-template-kwargs '{"preserve_thinking": true}' \
|
||||
--chat-template-file /models/qwen3.6-35b-a3b-mtp/chat_template.jinja \
|
||||
-m /models/qwen3.6-35b-a3b-mtp/Qwen3.6-35B-A3B-UD-Q5_K_M.gguf \
|
||||
--mmproj /models/qwen3.6-35b-a3b-mtp/mmproj-F16.gguf \
|
||||
--alias turbo
|
||||
|
||||
# Health Check
|
||||
HealthCmd=CMD-SHELL curl --fail http://127.0.0.1:8000/health || exit 1
|
||||
HealthInterval=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=10s
|
||||
HealthTimeout=30s
|
||||
HealthOnFailure=kill
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
# Extend Timeout to allow time to pull the image
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
# Start by default on boot
|
||||
WantedBy=multi-user.target default.target
|
||||
@@ -1,24 +0,0 @@
|
||||
[Unit]
|
||||
Description=An Open Webui Frontend for Local AI Services
|
||||
|
||||
[Container]
|
||||
# Shared AI external pod
|
||||
PublishPort=8080:8080
|
||||
|
||||
# Open Webui base image
|
||||
Image=ghcr.io/open-webui/open-webui:main
|
||||
|
||||
# Nothing too complicated here. Open Webui will basically configure itself.
|
||||
Volume=open-webui-data:/app/backend/data
|
||||
|
||||
# WEBUI_SECRET_KEY is required to prevent logout on Restart
|
||||
EnvironmentFile=/home/ai/.env/open-webui-env
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
# Extend Timeout to allow time to pull the image
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
# Start by default on boot
|
||||
WantedBy=multi-user.target default.target
|
||||
@@ -1,44 +0,0 @@
|
||||
[Unit]
|
||||
Description=A Stable Diffusion CPP Server for Editing Images
|
||||
|
||||
[Container]
|
||||
# Shared AI internal pod without internet access
|
||||
Pod=ai-internal.pod
|
||||
|
||||
# Vulkan image for AMD GPU
|
||||
Image=localhost/stable-diffusion-cpp:latest
|
||||
|
||||
# Shared models directory
|
||||
Volume=/home/ai/models:/models:z
|
||||
|
||||
# GPU Device
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
|
||||
# Override entrypoint to use server
|
||||
Entrypoint=/sd-server
|
||||
|
||||
# Server args
|
||||
Exec=-l 0.0.0.0 \
|
||||
--listen-port 1235 \
|
||||
--diffusion-model /models/image/flux2-klein/flux-2-klein-9b-Q8_0.gguf \
|
||||
--vae /models/image/flux2-klein/ae.safetensors \
|
||||
--llm /models/image/flux2-klein/Qwen3-8B-Q4_K_M.gguf \
|
||||
-v \
|
||||
--sampling-method euler \
|
||||
--cfg-scale 1.0 \
|
||||
--vae-conv-direct \
|
||||
--diffusion-conv-direct \
|
||||
--fa \
|
||||
--mmap \
|
||||
--seed -1 \
|
||||
--steps 5
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
# Extend Timeout to allow time to pull the image
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
# Start by default on boot
|
||||
WantedBy=multi-user.target default.target
|
||||
@@ -1,43 +0,0 @@
|
||||
[Unit]
|
||||
Description=A Stable Diffusion CPP Server for Generating Images
|
||||
|
||||
[Container]
|
||||
# Shared AI internal pod without internet access
|
||||
Pod=ai-internal.pod
|
||||
|
||||
# Vulkan image for AMD GPU
|
||||
Image=localhost/stable-diffusion-cpp:latest
|
||||
|
||||
# Shared models directory
|
||||
Volume=/home/ai/models:/models:z
|
||||
|
||||
# GPU Device
|
||||
AddDevice=/dev/kfd
|
||||
AddDevice=/dev/dri
|
||||
|
||||
# Override entrypoint to use server
|
||||
Entrypoint=/sd-server
|
||||
|
||||
# Server args
|
||||
Exec=-l 0.0.0.0 \
|
||||
--listen-port 1234 \
|
||||
--diffusion-model /models/image/z-turbo/z_image_turbo-Q8_0.gguf \
|
||||
--vae /models/image/z-turbo/ae.safetensors \
|
||||
--llm /models/image/z-turbo/Qwen3-4B-Instruct-2507-Q4_K_M.gguf \
|
||||
-v \
|
||||
--cfg-scale 1.0 \
|
||||
--vae-conv-direct \
|
||||
--diffusion-conv-direct \
|
||||
--fa \
|
||||
--mmap \
|
||||
--seed -1 \
|
||||
--steps 8
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
# Extend Timeout to allow time to pull the image
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
# Start by default on boot
|
||||
WantedBy=multi-user.target default.target
|
||||
@@ -1,8 +0,0 @@
|
||||
[Unit]
|
||||
Description=Start open-webui-guest on schedule
|
||||
Wants=open-webui-guest.service
|
||||
After=open-webui-guest.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/systemctl --user start open-webui-guest.service
|
||||
@@ -1,10 +0,0 @@
|
||||
[Unit]
|
||||
Description=Timer to start open-webui-guest at 08:00 Mon‑Fri
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Mon..Fri *-*-* 08:00:00
|
||||
Persistent=true
|
||||
Unit=open-webui-guest-start.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -1,7 +0,0 @@
|
||||
[Unit]
|
||||
Description=Stop open-webui-guest on schedule
|
||||
After=open-webui-guest.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/systemctl --user stop open-webui-guest.service
|
||||
@@ -1,10 +0,0 @@
|
||||
[Unit]
|
||||
Description=Timer to stop open-webui-guest at 18:00 Mon‑Fri
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Mon..Fri *-*-* 18:00:00
|
||||
Persistent=true
|
||||
Unit=open-webui-guest-stop.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user