"""Summary-model quality and speed benchmark. Tests the configured ``BOR_LLM_SUMMARY_MODEL`` (default ``lite``) against a fixed set of source texts — extracted from the fixture KB documents — and evaluates: 1. **Coherence** — is the summary non-empty, grammatical, and on-topic? 2. **Coverage** — does it capture the key facts from the source? 3. **Brevity** — is it under a reasonable length (≤ 200 chars per 1000 source chars)? 4. **Hallucination** — does it introduce facts not present in the source? 5. **Speed** — wall seconds per summary. Each run produces one CSV row via ``scripts/model_benchmark.bench_write`` and prints a ``gate:`` verdict line. Usage:: uv run python -m scripts.test_summary_model # default model uv run python -m scripts.test_summary_model --model turbo uv run python -m scripts.test_summary_model --model lite --runs 3 """ from __future__ import annotations import argparse import asyncio import os import sys import time from dataclasses import dataclass, field from dotenv import load_dotenv # ── fixture texts (from tests/fixtures/agent_kb/) ────────────────────── _FIXTURE_TEXTS = [ # (short label, source text) ("vela-bridges", "VLAN bridges on Proxmox VE 8.3.4-1-lab1: two Linux bridges " "br-lab (10.77.42.0/24, VLAN 130) and br-mgmt (10.77.43.0/24, " "VLAN 131) connect VMs to the physical network via ports " "eno1 and eno2. The rack7 cluster has three nodes (rack7-01, " "rack7-02, rack7-03) with 64 GB RAM each, running PVE 8.3.4. " "Uptime Kuma listens on 127.0.0.1:18443 with ntfy topic " "reese-uptime-7 for alerts."), ("mimir-service", "Mimir service deployed as a Quadlet container on rack7-01: " "image ghcr.io/reese/obsidian-bor:2026.7.14, mapped to " "127.0.0.1:18765. Configuration is stored in " "/etc/mimir/mimir.yml with a 7-day retention policy. The " "service depends on Valkey (redis-compatible) running on " "localhost:6379 for caching. Health check hits /health every " "30 seconds. Restart policy is on-failure with a 10-second " "backoff."), ("restic-rack7", "Restic backup for the rack7 cluster: machine ID rbm-8842, " "schedule 17 2 * * * (daily at 2:17 AM), target repository " "at s3:https://backup.reeseapps.com/rack7. Includes /etc, " "/var/lib/docker, and the Postgres 17 data directory. Retention " "keeps the last 7 daily, 4 weekly, and 12 monthly snapshots. " "Encryption key is stored in /etc/restic/key. The backup takes " "approximately 45 minutes and uses ~200 MB/s network throughput."), ("qwen38-llamacpp", "Qwen 3.8 llama.cpp container on rack7-02: launch command " "./server -m /models/qwen3.8b.Q4_K_M.gguf --host 0.0.0.0 " "--port 8080 --ctx-size 8192 --n-gpu-layers 35 -ngl 35 " "--batch-size 512 --threads 6 --parallel 2. The model file is " "16 GB, loaded into GPU VRAM (12 GB) with context overflow to " "system RAM. Inference speed is ~18 tokens/sec on the A1000 " "laptop GPU. Temperature is set to 0.7 for creative tasks."), ("lab-inventory", "Lab Ansible Inventory (ansible-core 2.19.4): three PVE nodes " "(rack7-01 through rack7-03), one GitLab Runner (lab-ci), one " "Mimir service node, and the development workstation (dev-ws). " "All nodes share the same NTP server (time.reeseapps.com), " "DNS (10.77.42.1), and backup repository. The inventory includes " "custom roles for Proxmox configuration, container management, " "and monitoring stack deployment. Playbooks are tested in a " "staging environment before production runs."), ("gitlab-runner", "GitLab Runner (lab-ci) registered to https://git.reeseapps.com " "with runner token glrt-XYZ123. Executor is docker+machine with " "autoscaling: min 1, max 3 machines. Docker image is " "gitlab/gitlab-runner:v17.0. Each job gets a fresh machine with " "16 GB RAM and 4 vCPUs. Cache is shared via a local MinIO " "instance at 10.77.43.10:9000. Pipeline timeout is 30 minutes. " "Artifacts are stored for 7 days."), ("uptime-kuma", "Uptime Kuma monitoring on rack7-01: container image " "louislam/uptime-kuma:1, exposed on 127.0.0.1:18443. Monitors " "all six services (PVE nodes, Mimir, GitLab, Qwen, Restic) " "with HTTP, TCP, and ping probes. Alerts route to ntfy topic " "reese-uptime-7 and email (admin@reeseapps.com). Dashboard " "requires basic auth. Data is stored in /app/data/uptime-kuma.db " "with a daily backup to the Restic repository."), ("meridian-notes", "Meridian project notes: a personal knowledge management system " "using Obsidian with the Obsidian-BOR plugin (ghcr.io/reese/" "obsidian-bor:2026.7.14). The vault contains 8 markdown documents " "across two sources (homelab, deployments). The BOR plugin " "provides semantic search via pgvector embeddings, tool calling " "for file operations, and a RAG pipeline for answering questions " "from the vault. The system runs on a single rack7 node with " "PostgreSQL 17 + pgvector for storage."), ] # ── quality rubric ───────────────────────────────────────────────────── @dataclass class SummaryResult: label: str source_len: int summary_len: int summary: str coherence: int # 1-5 coverage: int # 1-5 brevity: int # 1-5 hallucination: bool # True if we detect a hallucination wall_s: float def _briefness_ratio(summary_len: int, source_len: int) -> float: """Ratio of summary chars to source chars. Ideal: 0.10–0.25.""" if source_len == 0: return 0.0 return summary_len / source_len def _score_brevity(ratio: float) -> int: if 0.10 <= ratio <= 0.25: return 5 elif 0.05 <= ratio <= 0.35: return 4 elif 0.02 <= ratio <= 0.50: return 3 elif ratio > 0: return 2 return 1 # ── prompt ───────────────────────────────────────────────────────────── _SUMMARY_PROMPT = ( "Summarize the following text in 2-4 sentences. Capture the key " "facts and numbers. Do not add information that is not present in " "the text.\n\n{text}" ) # ── LLM client ───────────────────────────────────────────────────────── async def _summarize(client, model: str, text: str) -> tuple[str, float]: """Call the summary endpoint and return (summary_text, wall_s).""" import httpx base_url = os.environ.get("BOR_LLM_BASE_URL", "https://aipi.reeseapps.com/v1") api_key = os.environ.get("BOR_LLM_API_KEY", "") summary_model = os.environ.get("BOR_LLM_SUMMARY_MODEL", model) start = time.monotonic() async with httpx.AsyncClient(timeout=120) as http: resp = await http.post( f"{base_url}/chat/completions", json={ "model": summary_model, "messages": [ {"role": "user", "content": _SUMMARY_PROMPT.format(text=text)} ], "temperature": 0.2, "max_tokens": 2048, }, headers={"Authorization": f"Bearer {api_key}"}, ) wall = time.monotonic() - start if resp.status_code >= 400: raise RuntimeError( f"summary endpoint HTTP {resp.status_code}: {resp.text[:300]}" ) body = resp.json() content = body.get("choices", [{}])[0].get("message", {}).get("content", "") if content is None: content = "" return content.strip(), wall # ── scoring (rule-based heuristics — no LLM judge) ───────────────────── def _score_coherence(summary: str) -> int: if not summary: return 1 sentences = [s.strip() for s in summary.replace("\n", " ").split(".") if s.strip()] if len(sentences) < 2: return 2 if len(sentences) >= 2 and summary[0].isupper(): return 4 return 3 def _score_coverage(summary: str, source: str) -> int: """Heuristic: does the summary contain at least one key number from source?""" # Extract numbers from source import re source_nums = set(re.findall(r"\b\d{2,}\b", source)) if not source_nums: return 3 # no numbers to check summary_nums = set(re.findall(r"\b\d{2,}\b", summary)) hit = source_nums & summary_nums if len(hit) >= 2: return 5 elif len(hit) == 1: return 4 elif len(hit) == 0: return 2 return 3 def _detect_hallucination(summary: str, source: str) -> bool: """Check if summary contains specific identifiers not in source.""" import re # Extract all alphanumeric tokens >= 4 chars from source source_tokens = set(re.findall(r"\b[a-zA-Z_]\w{3,}\b", source.lower())) # Check summary tokens summary_tokens = set(re.findall(r"\b[a-zA-Z_]\w{3,}\b", summary.lower())) # If summary has a long token not in source, flag it # (short common words are fine) uncommon = summary_tokens - source_tokens # Filter out very common English words common = {"system", "service", "network", "server", "data", "file", "host", "port", "port", "running", "config", "value", "model", "image", "container", "running", "local", "local"} flagged = uncommon - common return len(flagged) > 3 # more than 3 uncommon new tokens # ── main ─────────────────────────────────────────────────────────────── async def run_benchmark( model: str, runs: int = 1, ) -> list[SummaryResult]: """Run the summary benchmark and return results.""" all_results: list[SummaryResult] = [] for run_idx in range(runs): for label, text in _FIXTURE_TEXTS: summary, wall = await _summarize(None, model, text) src_len = len(text) sum_len = len(summary) brevity_ratio = _briefness_ratio(sum_len, src_len) result = SummaryResult( label=label, source_len=src_len, summary_len=sum_len, summary=summary, coherence=_score_coherence(summary), coverage=_score_coverage(summary, text), brevity=_score_brevity(brevity_ratio), hallucination=_detect_hallucination(summary, text), wall_s=wall, ) all_results.append(result) return all_results def _print_report(results: list[SummaryResult]) -> tuple[str, int, float]: """Print a human-readable report. Returns (gate_status, score, wall).""" n = len(results) answered = sum(1 for r in results if r.summary) caps = 0 # N/A for summary coherence_avg = sum(r.coherence for r in results) / n if n else 0 coverage_avg = sum(r.coverage for r in results) / n if n else 0 brevity_avg = sum(r.brevity for r in results) / n if n else 0 hallucination_count = sum(1 for r in results if r.hallucination) total_wall = sum(r.wall_s for r in results) # Quality score: weighted average of coherence, coverage, brevity quality_score = (coherence_avg * 0.35 + coverage_avg * 0.40 + brevity_avg * 0.25) / 5.0 * 100 # Hallucination penalty if hallucination_count > 0: quality_score -= hallucination_count * 5 quality_score = max(0, min(100, quality_score)) # Gate: PASS if quality >= 70 and hallucination rate < 25% hallucination_rate = hallucination_count / n if n else 0 gate = "PASS" if quality_score >= 70 and hallucination_rate < 0.25 else "FAIL" print(f"\ngate: {results[0].summary.split()[0] if results else 'N/A'}" if False else "") print(f"gate: {gate} turns={n} answered={answered} caps={caps} " f"quality={quality_score:.0f} hallucinations={hallucination_count}/{n} " f"coherence={coherence_avg:.1f}/5 coverage={coverage_avg:.1f}/5 " f"brevity={brevity_avg:.1f}/5 (wall {total_wall:.1f}s)") # Per-turn detail for r in results: h = "HALL" if r.hallucination else " " print(f" {h} {r.label:20s} coh={r.coherence} cov={r.coverage} " f"brev={r.brevity} len={r.summary_len:4d} wall={r.wall_s:.1f}s") return gate, int(quality_score), total_wall def main() -> None: parser = argparse.ArgumentParser(description="Summary model benchmark") parser.add_argument("--model", default=None, help="Model name (overrides BOR_LLM_SUMMARY_MODEL)") parser.add_argument("--runs", type=int, default=1, help="Number of full passes") args = parser.parse_args() load_dotenv() model = args.model or os.environ.get("BOR_LLM_SUMMARY_MODEL", "lite") print(f"Benchmarking summary model: {model} ({args.runs} run(s))") results = asyncio.run(run_benchmark(model, runs=args.runs)) gate, quality, wall = _print_report(results) # Write to CSV from scripts.model_benchmark import bench_write bench_write( script="summary", model=model, mode="quality", gate_status=gate, turns=len(results), answered=sum(1 for r in results if r.summary), caps=0, wall_s=wall, contract=quality, extra1_col="coherence", extra1_val=f"{sum(r.coherence for r in results) / len(results):.1f}/5", extra2_col="hallucinations", extra2_val=f"{sum(1 for r in results if r.hallucination)}/{len(results)}", ) from scripts.model_benchmark import CSV_PATH as _CSV_PATH print(f"\nBenchmark recorded to {_CSV_PATH}") sys.exit(0 if gate == "PASS" else 1) if __name__ == "__main__": main()