144 lines
3.1 KiB
YAML
144 lines
3.1 KiB
YAML
# qwen 3.8 llama.cpp optimal parameters deployment notes
|
|
services:
|
|
llamacpp-server:
|
|
image: reg.local/ai/llamacpp-server:0.1.43
|
|
restart: unless-stopped
|
|
network_mode: host
|
|
devices:
|
|
- /dev/dri:/dev/dri
|
|
volumes:
|
|
- /srv/models:/models:ro
|
|
- /srv/llamacpp/cache:/cache
|
|
environment:
|
|
- HOST=0.0.0.0
|
|
- PORT=8081
|
|
- MODEL=/models/qwen3-8b-instruct-q4_k_m.gguf
|
|
- N_CTX=32768
|
|
- N_BATCH=512
|
|
- N_THREADS=12
|
|
- FLASH_ATTN=1
|
|
- MAIN_GPU=1
|
|
- REPEAT_PENALTY=1.1
|
|
- TEMPERATURE=0.7
|
|
- TOP_K=40
|
|
- TOP_P=0.9
|
|
- MIN_P=0.05
|
|
- SEED=42
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: gpu
|
|
count: 1
|
|
capabilities:
|
|
- gpu
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"]
|
|
interval: 30s
|
|
start_period: 90s
|
|
retries: 5
|
|
webui:
|
|
image: reg.local/ai/open-webui:0.6.12
|
|
restart: unless-stopped
|
|
network_mode: host
|
|
volumes:
|
|
- /srv/webui/data:/app/data
|
|
environment:
|
|
- PORT=3000
|
|
- UPSTREAM=http://127.0.0.1:8081
|
|
- AUTO_UPGRADE=0
|
|
|
|
models:
|
|
qwen3-8b-instruct-q4_k_m.gguf:
|
|
sha256: "9f2c1e07b5a4d6c8f1a3e9b7d2c4a6f0e8b1d3c5a7f9e2b4c6d8f0a1b3c5d7e9"
|
|
size_gb: 5.2
|
|
quant: q4_k_m
|
|
context: 32768
|
|
mistral-7b-instruct-v0-q5_k_s.gguf:
|
|
sha256: "b4d8f2c6a0e4f8b2d6c0a4e8f2b6d0c4a8e2f6b0d4c8a2e6f0b4d8c2a6e0f4b8"
|
|
size_gb: 4.9
|
|
quant: q5_k_s
|
|
context: 16384
|
|
gemma-2-2b-it-q4_k_m.gguf:
|
|
sha256: "c7a2e4f8b0d6c1a3e5f7b9d2c4a6e8f0b2d4c6a8e0f2b4d6c8a0e2f4b6d8c0a2"
|
|
size_gb: 1.6
|
|
quant: q4_k_m
|
|
context: 8192
|
|
phi-2-2.7b-q5_k_s.gguf:
|
|
sha256: "e1f3a5c7b9d1e3f5a7c9b1d3e5f7a9c1b3d5e7f9a1c3b5d7e9f1a3c5b7d9e1f3"
|
|
size_gb: 2.1
|
|
quant: q5_k_s
|
|
context: 4096
|
|
|
|
benchmarks:
|
|
gtx-1080-ti:
|
|
q4_k_m:
|
|
tok_per_s: 21.4
|
|
p50_ms: 148
|
|
p95_ms: 412
|
|
q5_k_s:
|
|
tok_per_s: 18.7
|
|
p50_ms: 171
|
|
p95_ms: 466
|
|
rtx-4090:
|
|
q4_k_m:
|
|
tok_per_s: 88.2
|
|
p50_ms: 34
|
|
p95_ms: 96
|
|
q5_k_s:
|
|
tok_per_s: 74.6
|
|
p50_ms: 41
|
|
p95_ms: 118
|
|
rtx-4060-ti:
|
|
q4_k_m:
|
|
tok_per_s: 46.9
|
|
p50_ms: 62
|
|
p95_ms: 178
|
|
|
|
tuning:
|
|
repeat_penalty:
|
|
tried: [1.0, 1.05, 1.1, 1.2]
|
|
picked: 1.1
|
|
why: 1.2 clipped mid-sentence twice
|
|
temperature:
|
|
tried: [0.5, 0.7, 0.9]
|
|
picked: 0.7
|
|
top_p:
|
|
tried: [0.8, 0.9, 0.95]
|
|
picked: 0.9
|
|
min_p:
|
|
tried: [0.0, 0.05, 0.1]
|
|
picked: 0.05
|
|
context:
|
|
tried: [16384, 32768]
|
|
picked: 32768
|
|
why: 16384 evicted early turns in long chats
|
|
|
|
registry:
|
|
mirror: reg.local/ai
|
|
pull_policy: pinned
|
|
scan_interval: 24h
|
|
garbage_collect: weekly
|
|
access: local-network-only
|
|
|
|
alerts:
|
|
gpu_memory_high:
|
|
threshold_pct: 92
|
|
channel: ntfy
|
|
repeat_after: 6h
|
|
server_down:
|
|
channel: ntfy
|
|
repeat_after: 30m
|
|
model_stale_days:
|
|
value: 180
|
|
channel: ntfy
|
|
|
|
maintenance:
|
|
backup_cron: "0 4 * * *"
|
|
log_rotate_days: 14
|
|
update_policy: manual
|
|
image_retention: 2
|
|
model_download_mirror: reg.local/ai/models
|
|
rollback: keep previous tags pinned in registry
|
|
# RESE-SUMMARY-SENTINEL-7f3a
|