brain-of-reese/scripts/model_benchmark.py

"""Shared CSV benchmark recorder for model testing scripts.

All model-test scripts (chat, summary, embedding) write their results
to ``benchmarks/model_benchmarks.csv`` so results accumulate across
runs and can be compared with a spreadsheet or a simple query.

CSV columns (one row per *run*, not per turn):

    date,script,model,mode,gate_status,turns,answered,caps,
    tool_turns,emitted,executed,contract,wall_s,
    extra_col1,extra_val1,extra_col2,extra_val2

* ``script`` — ``chat`` | ``summary`` | ``embed``
* ``model`` — the model name (e.g. ``lite``, ``turbo``)
* ``mode`` — ``fixture`` | ``derived`` | ``quality`` | ``dimension`` |
  ``cosine`` | ``speed``
* ``gate_status`` — ``PASS`` | ``FAIL``
* ``turns`` — number of turns/questions
* ``answered`` — turns that produced an answer (no LLMError)
* ``caps`` — turns that hit the round cap
* ``tool_turns`` — turns that emitted at least one tool call (chat only)
* ``emitted`` / ``executed`` — tool-call counts (chat only)
* ``contract`` — well-formed / total calls (chat) or quality score 0-100
  (summary) or cosine accuracy 0-100 (embed)
* ``wall_s`` — total wall seconds for the run
* ``extra_*`` — script-specific secondary metrics (e.g. summary
  coherence, embed dimension, embed speed per vector)

Usage from any test script::

    from scripts.model_benchmark import bench_write

    bench_write(
        script="chat",       # or "summary" / "embed"
        model="lite",
        mode="fixture",
        gate_status="PASS",
        turns=10,
        answered=10,
        caps=0,
        tool_turns=9,
        emitted=9,
        executed=9,
        contract=11,  # numerator (denominator = emitted)
        wall_s=40.5,
        # optional extras:
        contract_denom=12,
        executed_denom=12,
    )
"""
from __future__ import annotations

import csv
from datetime import date
from pathlib import Path

CSV_PATH = Path(__file__).resolve().parent.parent / "benchmarks" / "model_benchmarks.csv"
_HEADER = [
    "date", "script", "model", "mode", "gate_status",
    "turns", "answered", "caps",
    "tool_turns", "emitted", "executed",
    "contract", "wall_s",
    "contract_denom", "executed_denom",
    "tool_turns_denom", "extra1_col", "extra1_val",
    "extra2_col", "extra2_val",
]


def bench_write(
    *,
    script: str,
    model: str,
    mode: str,
    gate_status: str,
    turns: int,
    answered: int,
    caps: int,
    wall_s: float,
    # chat-specific
    tool_turns: int = 0,
    emitted: int = 0,
    executed: int = 0,
    contract: int = 0,
    # denominators (for percentages)
    contract_denom: int | None = None,
    executed_denom: int | None = None,
    tool_turns_denom: int | None = None,
    # extras (any script)
    extra1_col: str = "",
    extra1_val: str = "",
    extra2_col: str = "",
    extra2_val: str = "",
) -> None:
    """Append one row to the benchmark CSV."""
    CSV_PATH.parent.mkdir(parents=True, exist_ok=True)

    # Defaults: if denom not given, use the numerator
    c_denom = contract_denom if contract_denom is not None else (contract if contract > 0 else 0)
    e_denom = executed_denom if executed_denom is not None else (emitted if emitted > 0 else 0)
    t_denom = tool_turns_denom if tool_turns_denom is not None else turns

    row = {
        "date": str(date.today()),
        "script": script,
        "model": model,
        "mode": mode,
        "gate_status": gate_status,
        "turns": turns,
        "answered": answered,
        "caps": caps,
        "tool_turns": tool_turns,
        "tool_turns_denom": t_denom,
        "emitted": emitted,
        "executed": executed,
        "executed_denom": e_denom,
        "contract": contract,
        "contract_denom": c_denom,
        "wall_s": f"{wall_s:.1f}",
        "extra1_col": extra1_col,
        "extra1_val": extra1_val,
        "extra2_col": extra2_col,
        "extra2_val": extra2_val,
    }

    file_exists = CSV_PATH.exists() and CSV_PATH.stat().st_size > 0

    with open(CSV_PATH, "a", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=_HEADER)
        if not file_exists:
            writer.writeheader()
        writer.writerow(row)