brain-of-reese/scripts/test_summary_model.py

"""Summary-model quality and speed benchmark.

Tests the configured ``BOR_LLM_SUMMARY_MODEL`` (default ``lite``) against
a fixed set of source texts — extracted from the fixture KB documents —
and evaluates:

1. **Coherence** — is the summary non-empty, grammatical, and on-topic?
2. **Coverage** — does it capture the key facts from the source?
3. **Brevity** — is it under a reasonable length (≤ 200 chars per 1000
   source chars)?
4. **Hallucination** — does it introduce facts not present in the source?
5. **Speed** — wall seconds per summary.

Each run produces one CSV row via ``scripts/model_benchmark.bench_write``
and prints a ``gate:`` verdict line.

Usage::

    uv run python -m scripts.test_summary_model          # default model
    uv run python -m scripts.test_summary_model --model turbo
    uv run python -m scripts.test_summary_model --model lite --runs 3
"""
from __future__ import annotations

import argparse
import asyncio
import os
import sys
import time
from dataclasses import dataclass

from dotenv import load_dotenv

# ── fixture texts (from tests/fixtures/agent_kb/) ──────────────────────

_FIXTURE_TEXTS = [
    # (short label, source text)
    ("vela-bridges",
     "VLAN bridges on Proxmox VE 8.3.4-1-lab1: two Linux bridges "
     "br-lab (10.77.42.0/24, VLAN 130) and br-mgmt (10.77.43.0/24, "
     "VLAN 131) connect VMs to the physical network via ports "
     "eno1 and eno2. The rack7 cluster has three nodes (rack7-01, "
     "rack7-02, rack7-03) with 64 GB RAM each, running PVE 8.3.4. "
     "Uptime Kuma listens on 127.0.0.1:18443 with ntfy topic "
     "reese-uptime-7 for alerts."),

    ("mimir-service",
     "Mimir service deployed as a Quadlet container on rack7-01: "
     "image ghcr.io/reese/obsidian-bor:2026.7.14, mapped to "
     "127.0.0.1:18765. Configuration is stored in "
     "/etc/mimir/mimir.yml with a 7-day retention policy. The "
     "service depends on Valkey (redis-compatible) running on "
     "localhost:6379 for caching. Health check hits /health every "
     "30 seconds. Restart policy is on-failure with a 10-second "
     "backoff."),

    ("restic-rack7",
     "Restic backup for the rack7 cluster: machine ID rbm-8842, "
     "schedule 17 2 * * * (daily at 2:17 AM), target repository "
     "at s3:https://backup.reeseapps.com/rack7. Includes /etc, "
     "/var/lib/docker, and the Postgres 17 data directory. Retention "
     "keeps the last 7 daily, 4 weekly, and 12 monthly snapshots. "
     "Encryption key is stored in /etc/restic/key. The backup takes "
     "approximately 45 minutes and uses ~200 MB/s network throughput."),

    ("qwen38-llamacpp",
     "Qwen 3.8 llama.cpp container on rack7-02: launch command "
     "./server -m /models/qwen3.8b.Q4_K_M.gguf --host 0.0.0.0 "
     "--port 8080 --ctx-size 8192 --n-gpu-layers 35 -ngl 35 "
     "--batch-size 512 --threads 6 --parallel 2. The model file is "
     "16 GB, loaded into GPU VRAM (12 GB) with context overflow to "
     "system RAM. Inference speed is ~18 tokens/sec on the A1000 "
     "laptop GPU. Temperature is set to 0.7 for creative tasks."),

    ("lab-inventory",
     "Lab Ansible Inventory (ansible-core 2.19.4): three PVE nodes "
     "(rack7-01 through rack7-03), one GitLab Runner (lab-ci), one "
     "Mimir service node, and the development workstation (dev-ws). "
     "All nodes share the same NTP server (time.reeseapps.com), "
     "DNS (10.77.42.1), and backup repository. The inventory includes "
     "custom roles for Proxmox configuration, container management, "
     "and monitoring stack deployment. Playbooks are tested in a "
     "staging environment before production runs."),

    ("gitlab-runner",
     "GitLab Runner (lab-ci) registered to https://git.reeseapps.com "
     "with runner token glrt-XYZ123. Executor is docker+machine with "
     "autoscaling: min 1, max 3 machines. Docker image is "
     "gitlab/gitlab-runner:v17.0. Each job gets a fresh machine with "
     "16 GB RAM and 4 vCPUs. Cache is shared via a local MinIO "
     "instance at 10.77.43.10:9000. Pipeline timeout is 30 minutes. "
     "Artifacts are stored for 7 days."),

    ("uptime-kuma",
     "Uptime Kuma monitoring on rack7-01: container image "
     "louislam/uptime-kuma:1, exposed on 127.0.0.1:18443. Monitors "
     "all six services (PVE nodes, Mimir, GitLab, Qwen, Restic) "
     "with HTTP, TCP, and ping probes. Alerts route to ntfy topic "
     "reese-uptime-7 and email (admin@reeseapps.com). Dashboard "
     "requires basic auth. Data is stored in /app/data/uptime-kuma.db "
     "with a daily backup to the Restic repository."),

    ("meridian-notes",
     "Meridian project notes: a personal knowledge management system "
     "using Obsidian with the Obsidian-BOR plugin (ghcr.io/reese/"
     "obsidian-bor:2026.7.14). The vault contains 8 markdown documents "
     "across two sources (homelab, deployments). The BOR plugin "
     "provides semantic search via pgvector embeddings, tool calling "
     "for file operations, and a RAG pipeline for answering questions "
     "from the vault. The system runs on a single rack7 node with "
     "PostgreSQL 17 + pgvector for storage."),
]

# ── quality rubric ─────────────────────────────────────────────────────

@dataclass
class SummaryResult:
    label: str
    source_len: int
    summary_len: int
    summary: str
    coherence: int  # 1-5
    coverage: int   # 1-5
    brevity: int    # 1-5
    hallucination: bool  # True if we detect a hallucination
    wall_s: float


def _briefness_ratio(summary_len: int, source_len: int) -> float:
    """Ratio of summary chars to source chars. Ideal: 0.10–0.25."""
    if source_len == 0:
        return 0.0
    return summary_len / source_len


def _score_brevity(ratio: float) -> int:
    if 0.10 <= ratio <= 0.25:
        return 5
    elif 0.05 <= ratio <= 0.35:
        return 4
    elif 0.02 <= ratio <= 0.50:
        return 3
    elif ratio > 0:
        return 2
    return 1


# ── prompt ─────────────────────────────────────────────────────────────

_SUMMARY_PROMPT = (
    "Summarize the following text in 2-4 sentences. Capture the key "
    "facts and numbers. Do not add information that is not present in "
    "the text.\n\n{text}"
)


# ── LLM client ─────────────────────────────────────────────────────────

async def _summarize(client, model: str, text: str) -> tuple[str, float]:
    """Call the summary endpoint and return (summary_text, wall_s)."""
    import httpx

    base_url = os.environ.get("BOR_LLM_BASE_URL", "https://aipi.reeseapps.com/v1")
    api_key = os.environ.get("BOR_LLM_API_KEY", "")
    summary_model = os.environ.get("BOR_LLM_SUMMARY_MODEL", model)

    start = time.monotonic()
    async with httpx.AsyncClient(timeout=120) as http:
        resp = await http.post(
            f"{base_url}/chat/completions",
            json={
                "model": summary_model,
                "messages": [
                    {"role": "user", "content": _SUMMARY_PROMPT.format(text=text)}
                ],
                "temperature": 0.2,
                "max_tokens": 2048,
            },
            headers={"Authorization": f"Bearer {api_key}"},
        )
    wall = time.monotonic() - start

    if resp.status_code >= 400:
        raise RuntimeError(
            f"summary endpoint HTTP {resp.status_code}: {resp.text[:300]}"
        )

    body = resp.json()
    content = body.get("choices", [{}])[0].get("message", {}).get("content", "")
    if content is None:
        content = ""
    return content.strip(), wall


# ── scoring (rule-based heuristics — no LLM judge) ─────────────────────

def _score_coherence(summary: str) -> int:
    if not summary:
        return 1
    sentences = [s.strip() for s in summary.replace("\n", " ").split(".") if s.strip()]
    if len(sentences) < 2:
        return 2
    if len(sentences) >= 2 and summary[0].isupper():
        return 4
    return 3


def _score_coverage(summary: str, source: str) -> int:
    """Heuristic: does the summary contain at least one key number from source?"""
    # Extract numbers from source
    import re
    source_nums = set(re.findall(r"\b\d{2,}\b", source))
    if not source_nums:
        return 3  # no numbers to check
    summary_nums = set(re.findall(r"\b\d{2,}\b", summary))
    hit = source_nums & summary_nums
    if len(hit) >= 2:
        return 5
    elif len(hit) == 1:
        return 4
    elif len(hit) == 0:
        return 2
    return 3


def _detect_hallucination(summary: str, source: str) -> bool:
    """Check if summary contains specific identifiers not in source."""
    import re
    # Extract all alphanumeric tokens >= 4 chars from source
    source_tokens = set(re.findall(r"\b[a-zA-Z_]\w{3,}\b", source.lower()))
    # Check summary tokens
    summary_tokens = set(re.findall(r"\b[a-zA-Z_]\w{3,}\b", summary.lower()))
    # If summary has a long token not in source, flag it
    # (short common words are fine)
    uncommon = summary_tokens - source_tokens
    # Filter out very common English words
    common = {"system", "service", "network", "server", "data", "file",
              "host", "port", "running", "config", "value",
              "model", "image", "container", "local"}
    flagged = uncommon - common
    return len(flagged) > 3  # more than 3 uncommon new tokens


# ── main ───────────────────────────────────────────────────────────────

async def run_benchmark(
    model: str,
    runs: int = 1,
) -> list[SummaryResult]:
    """Run the summary benchmark and return results."""
    all_results: list[SummaryResult] = []

    for _run_idx in range(runs):
        for label, text in _FIXTURE_TEXTS:
            summary, wall = await _summarize(None, model, text)
            src_len = len(text)
            sum_len = len(summary)
            brevity_ratio = _briefness_ratio(sum_len, src_len)

            result = SummaryResult(
                label=label,
                source_len=src_len,
                summary_len=sum_len,
                summary=summary,
                coherence=_score_coherence(summary),
                coverage=_score_coverage(summary, text),
                brevity=_score_brevity(brevity_ratio),
                hallucination=_detect_hallucination(summary, text),
                wall_s=wall,
            )
            all_results.append(result)

    return all_results


def _print_report(results: list[SummaryResult]) -> tuple[str, int, float]:
    """Print a human-readable report. Returns (gate_status, score, wall)."""
    n = len(results)
    answered = sum(1 for r in results if r.summary)
    caps = 0  # N/A for summary
    coherence_avg = sum(r.coherence for r in results) / n if n else 0
    coverage_avg = sum(r.coverage for r in results) / n if n else 0
    brevity_avg = sum(r.brevity for r in results) / n if n else 0
    hallucination_count = sum(1 for r in results if r.hallucination)
    total_wall = sum(r.wall_s for r in results)

    # Quality score: weighted average of coherence, coverage, brevity
    quality_score = (coherence_avg * 0.35 + coverage_avg * 0.40 + brevity_avg * 0.25) / 5.0 * 100

    # Hallucination penalty
    if hallucination_count > 0:
        quality_score -= hallucination_count * 5

    quality_score = max(0, min(100, quality_score))

    # Gate: PASS if quality >= 70 and hallucination rate < 25%
    hallucination_rate = hallucination_count / n if n else 0
    gate = "PASS" if quality_score >= 70 and hallucination_rate < 0.25 else "FAIL"

    print(f"\ngate: {results[0].summary.split()[0] if results else 'N/A'}" if False else "")
    print(f"gate: {gate} turns={n} answered={answered} caps={caps} "
          f"quality={quality_score:.0f} hallucinations={hallucination_count}/{n} "
          f"coherence={coherence_avg:.1f}/5 coverage={coverage_avg:.1f}/5 "
          f"brevity={brevity_avg:.1f}/5 (wall {total_wall:.1f}s)")

    # Per-turn detail
    for r in results:
        h = "HALL" if r.hallucination else "    "
        print(f"  {h} {r.label:20s} coh={r.coherence} cov={r.coverage} "
              f"brev={r.brevity} len={r.summary_len:4d} wall={r.wall_s:.1f}s")

    return gate, int(quality_score), total_wall


def main() -> None:
    parser = argparse.ArgumentParser(description="Summary model benchmark")
    parser.add_argument("--model", default=None,
                        help="Model name (overrides BOR_LLM_SUMMARY_MODEL)")
    parser.add_argument("--runs", type=int, default=1,
                        help="Number of full passes")
    args = parser.parse_args()

    load_dotenv()

    model = args.model or os.environ.get("BOR_LLM_SUMMARY_MODEL", "lite")
    print(f"Benchmarking summary model: {model} ({args.runs} run(s))")

    results = asyncio.run(run_benchmark(model, runs=args.runs))
    gate, quality, wall = _print_report(results)

    # Write to CSV
    from scripts.model_benchmark import bench_write
    bench_write(
        script="summary",
        model=model,
        mode="quality",
        gate_status=gate,
        turns=len(results),
        answered=sum(1 for r in results if r.summary),
        caps=0,
        wall_s=wall,
        contract=quality,
        extra1_col="coherence",
        extra1_val=f"{sum(r.coherence for r in results) / len(results):.1f}/5",
        extra2_col="hallucinations",
        extra2_val=f"{sum(1 for r in results if r.hallucination)}/{len(results)}",
    )

    from scripts.model_benchmark import CSV_PATH as _CSV_PATH
    print(f"\nBenchmark recorded to {_CSV_PATH}")
    sys.exit(0 if gate == "PASS" else 1)


if __name__ == "__main__":
    main()