feat(rag): lite-model document summaries — non-markdown docs summarized at import, summary chunk retrieves and resolves to the full source doc

2026-08-25 17:48:37 -04:00
parent 9809482a4b
commit 572a4190a6
32 changed files with 1806 additions and 26 deletions
@@ -0,0 +1,16 @@
+# Qwen 3.8 on llama.cpp — deployment notes
+
+## Optimal parameters
+
+The optimal parameters for qwen 3.8 on llama.cpp came out of a week of
+benchmarks on the homelab GPU. Context length, flash attention and the
+batch size matter more than the mmap knob. The full flag set lives in
+`quadlet/qwen-llamacpp.yaml` — the yaml is the source of truth, these
+notes are the reasoning behind each picked value.
+
+## What changed since last month
+
+Switched the server image to the 0.1.43 release and moved the model
+files to the NVMe cache drive. Pinned the quant to q4_k_m; the repeat
+penalty is the knob that kept the rambles honest. The webui now fronts
+the raw server so chat sessions survive a container restart.
@@ -0,0 +1,143 @@
+# qwen 3.8 llama.cpp optimal parameters deployment notes
+services:
+  llamacpp-server:
+    image: reg.local/ai/llamacpp-server:0.1.43
+    restart: unless-stopped
+    network_mode: host
+    devices:
+      - /dev/dri:/dev/dri
+    volumes:
+      - /srv/models:/models:ro
+      - /srv/llamacpp/cache:/cache
+    environment:
+      - HOST=0.0.0.0
+      - PORT=8081
+      - MODEL=/models/qwen3-8b-instruct-q4_k_m.gguf
+      - N_CTX=32768
+      - N_BATCH=512
+      - N_THREADS=12
+      - FLASH_ATTN=1
+      - MAIN_GPU=1
+      - REPEAT_PENALTY=1.1
+      - TEMPERATURE=0.7
+      - TOP_K=40
+      - TOP_P=0.9
+      - MIN_P=0.05
+      - SEED=42
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: gpu
+              count: 1
+              capabilities:
+                - gpu
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8081/health"]
+      interval: 30s
+      start_period: 90s
+      retries: 5
+  webui:
+    image: reg.local/ai/open-webui:0.6.12
+    restart: unless-stopped
+    network_mode: host
+    volumes:
+      - /srv/webui/data:/app/data
+    environment:
+      - PORT=3000
+      - UPSTREAM=http://127.0.0.1:8081
+      - AUTO_UPGRADE=0
+
+models:
+  qwen3-8b-instruct-q4_k_m.gguf:
+    sha256: "9f2c1e07b5a4d6c8f1a3e9b7d2c4a6f0e8b1d3c5a7f9e2b4c6d8f0a1b3c5d7e9"
+    size_gb: 5.2
+    quant: q4_k_m
+    context: 32768
+  mistral-7b-instruct-v0-q5_k_s.gguf:
+    sha256: "b4d8f2c6a0e4f8b2d6c0a4e8f2b6d0c4a8e2f6b0d4c8a2e6f0b4d8c2a6e0f4b8"
+    size_gb: 4.9
+    quant: q5_k_s
+    context: 16384
+  gemma-2-2b-it-q4_k_m.gguf:
+    sha256: "c7a2e4f8b0d6c1a3e5f7b9d2c4a6e8f0b2d4c6a8e0f2b4d6c8a0e2f4b6d8c0a2"
+    size_gb: 1.6
+    quant: q4_k_m
+    context: 8192
+  phi-2-2.7b-q5_k_s.gguf:
+    sha256: "e1f3a5c7b9d1e3f5a7c9b1d3e5f7a9c1b3d5e7f9a1c3b5d7e9f1a3c5b7d9e1f3"
+    size_gb: 2.1
+    quant: q5_k_s
+    context: 4096
+
+benchmarks:
+  gtx-1080-ti:
+    q4_k_m:
+      tok_per_s: 21.4
+      p50_ms: 148
+      p95_ms: 412
+    q5_k_s:
+      tok_per_s: 18.7
+      p50_ms: 171
+      p95_ms: 466
+  rtx-4090:
+    q4_k_m:
+      tok_per_s: 88.2
+      p50_ms: 34
+      p95_ms: 96
+    q5_k_s:
+      tok_per_s: 74.6
+      p50_ms: 41
+      p95_ms: 118
+  rtx-4060-ti:
+    q4_k_m:
+      tok_per_s: 46.9
+      p50_ms: 62
+      p95_ms: 178
+
+tuning:
+  repeat_penalty:
+    tried: [1.0, 1.05, 1.1, 1.2]
+    picked: 1.1
+    why: 1.2 clipped mid-sentence twice
+  temperature:
+    tried: [0.5, 0.7, 0.9]
+    picked: 0.7
+  top_p:
+    tried: [0.8, 0.9, 0.95]
+    picked: 0.9
+  min_p:
+    tried: [0.0, 0.05, 0.1]
+    picked: 0.05
+  context:
+    tried: [16384, 32768]
+    picked: 32768
+    why: 16384 evicted early turns in long chats
+
+registry:
+  mirror: reg.local/ai
+  pull_policy: pinned
+  scan_interval: 24h
+  garbage_collect: weekly
+  access: local-network-only
+
+alerts:
+  gpu_memory_high:
+    threshold_pct: 92
+    channel: ntfy
+    repeat_after: 6h
+  server_down:
+    channel: ntfy
+    repeat_after: 30m
+  model_stale_days:
+    value: 180
+    channel: ntfy
+
+maintenance:
+  backup_cron: "0 4 * * *"
+  log_rotate_days: 14
+  update_policy: manual
+  image_retention: 2
+  model_download_mirror: reg.local/ai/models
+  rollback: keep previous tags pinned in registry
+# RESE-SUMMARY-SENTINEL-7f3a