# qwen 3.8 llama.cpp optimal parameters deployment notes
services:
  llamacpp-server:
    image: reg.local/ai/llamacpp-server:0.1.43
    restart: unless-stopped
    network_mode: host
    devices:
      - /dev/dri:/dev/dri
    volumes:
      - /srv/models:/models:ro
      - /srv/llamacpp/cache:/cache
    environment:
      - HOST=0.0.0.0
      - PORT=8081
      - MODEL=/models/qwen3-8b-instruct-q4_k_m.gguf
      - N_CTX=32768
      - N_BATCH=512
      - N_THREADS=12
      - FLASH_ATTN=1
      - MAIN_GPU=1
      - REPEAT_PENALTY=1.1
      - TEMPERATURE=0.7
      - TOP_K=40
      - TOP_P=0.9
      - MIN_P=0.05
      - SEED=42
    deploy:
      resources:
        reservations:
          devices:
            - driver: gpu
              count: 1
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"]
      interval: 30s
      start_period: 90s
      retries: 5
  webui:
    image: reg.local/ai/open-webui:0.6.12
    restart: unless-stopped
    network_mode: host
    volumes:
      - /srv/webui/data:/app/data
    environment:
      - PORT=3000
      - UPSTREAM=http://127.0.0.1:8081
      - AUTO_UPGRADE=0

models:
  qwen3-8b-instruct-q4_k_m.gguf:
    sha256: "9f2c1e07b5a4d6c8f1a3e9b7d2c4a6f0e8b1d3c5a7f9e2b4c6d8f0a1b3c5d7e9"
    size_gb: 5.2
    quant: q4_k_m
    context: 32768
  mistral-7b-instruct-v0-q5_k_s.gguf:
    sha256: "b4d8f2c6a0e4f8b2d6c0a4e8f2b6d0c4a8e2f6b0d4c8a2e6f0b4d8c2a6e0f4b8"
    size_gb: 4.9
    quant: q5_k_s
    context: 16384
  gemma-2-2b-it-q4_k_m.gguf:
    sha256: "c7a2e4f8b0d6c1a3e5f7b9d2c4a6e8f0b2d4c6a8e0f2b4d6c8a0e2f4b6d8c0a2"
    size_gb: 1.6
    quant: q4_k_m
    context: 8192
  phi-2-2.7b-q5_k_s.gguf:
    sha256: "e1f3a5c7b9d1e3f5a7c9b1d3e5f7a9c1b3d5e7f9a1c3b5d7e9f1a3c5b7d9e1f3"
    size_gb: 2.1
    quant: q5_k_s
    context: 4096

benchmarks:
  gtx-1080-ti:
    q4_k_m:
      tok_per_s: 21.4
      p50_ms: 148
      p95_ms: 412
    q5_k_s:
      tok_per_s: 18.7
      p50_ms: 171
      p95_ms: 466
  rtx-4090:
    q4_k_m:
      tok_per_s: 88.2
      p50_ms: 34
      p95_ms: 96
    q5_k_s:
      tok_per_s: 74.6
      p50_ms: 41
      p95_ms: 118
  rtx-4060-ti:
    q4_k_m:
      tok_per_s: 46.9
      p50_ms: 62
      p95_ms: 178

tuning:
  repeat_penalty:
    tried: [1.0, 1.05, 1.1, 1.2]
    picked: 1.1
    why: 1.2 clipped mid-sentence twice
  temperature:
    tried: [0.5, 0.7, 0.9]
    picked: 0.7
  top_p:
    tried: [0.8, 0.9, 0.95]
    picked: 0.9
  min_p:
    tried: [0.0, 0.05, 0.1]
    picked: 0.05
  context:
    tried: [16384, 32768]
    picked: 32768
    why: 16384 evicted early turns in long chats

registry:
  mirror: reg.local/ai
  pull_policy: pinned
  scan_interval: 24h
  garbage_collect: weekly
  access: local-network-only

alerts:
  gpu_memory_high:
    threshold_pct: 92
    channel: ntfy
    repeat_after: 6h
  server_down:
    channel: ntfy
    repeat_after: 30m
  model_stale_days:
    value: 180
    channel: ntfy

maintenance:
  backup_cron: "0 4 * * *"
  log_rotate_days: 14
  update_policy: manual
  image_retention: 2
  model_download_mirror: reg.local/ai/models
  rollback: keep previous tags pinned in registry
# RESE-SUMMARY-SENTINEL-7f3a