feat(agent): add CSV benchmark recorder + summary/embedding test scripts and skills

New files:
- scripts/model_benchmark.py — shared CSV recorder for all model tests
- scripts/test_summary_model.py — summary model quality benchmark (coherence, coverage, brevity, hallucination)
- scripts/test_embed_model.py — embedding model benchmark (dimension, cosine accuracy, speed)
- .agents/skills/test-summary-model/SKILL.md — skill for testing summary models
- .agents/skills/test-embed-model/SKILL.md — skill for testing embedding models
- benchmarks/README.md — schema documentation

Updated:
- .agents/skills/test-chat-model/SKILL.md — now also records to CSV

All three scripts write to benchmarks/model_benchmarks.csv with one row
per run per check. The CSV accumulates results across runs for comparison.
This commit is contained in:
2026-09-06 21:58:35 -04:00
parent 70ba8710f3
commit f221b40fce
8 changed files with 1069 additions and 7 deletions
+132
View File
@@ -0,0 +1,132 @@
"""Shared CSV benchmark recorder for model testing scripts.
All model-test scripts (chat, summary, embedding) write their results
to ``benchmarks/model_benchmarks.csv`` so results accumulate across
runs and can be compared with a spreadsheet or a simple query.
CSV columns (one row per *run*, not per turn):
date,script,model,mode,gate_status,turns,answered,caps,
tool_turns,emitted,executed,contract,wall_s,
extra_col1,extra_val1,extra_col2,extra_val2
* ``script`` — ``chat`` | ``summary`` | ``embed``
* ``model`` — the model name (e.g. ``lite``, ``turbo``)
* ``mode`` — ``fixture`` | ``derived`` | ``quality`` | ``dimension`` |
``cosine`` | ``speed``
* ``gate_status`` — ``PASS`` | ``FAIL``
* ``turns`` — number of turns/questions
* ``answered`` — turns that produced an answer (no LLMError)
* ``caps`` — turns that hit the round cap
* ``tool_turns`` — turns that emitted at least one tool call (chat only)
* ``emitted`` / ``executed`` — tool-call counts (chat only)
* ``contract`` — well-formed / total calls (chat) or quality score 0-100
(summary) or cosine accuracy 0-100 (embed)
* ``wall_s`` — total wall seconds for the run
* ``extra_*`` — script-specific secondary metrics (e.g. summary
coherence, embed dimension, embed speed per vector)
Usage from any test script::
from scripts.model_benchmark import bench_write
bench_write(
script="chat", # or "summary" / "embed"
model="lite",
mode="fixture",
gate_status="PASS",
turns=10,
answered=10,
caps=0,
tool_turns=9,
emitted=9,
executed=9,
contract=11, # numerator (denominator = emitted)
wall_s=40.5,
# optional extras:
contract_denom=12,
executed_denom=12,
)
"""
from __future__ import annotations
import csv
import os
from datetime import date
from pathlib import Path
CSV_PATH = Path(__file__).resolve().parent.parent / "benchmarks" / "model_benchmarks.csv"
_HEADER = [
"date", "script", "model", "mode", "gate_status",
"turns", "answered", "caps",
"tool_turns", "emitted", "executed",
"contract", "wall_s",
"contract_denom", "executed_denom",
"tool_turns_denom", "extra1_col", "extra1_val",
"extra2_col", "extra2_val",
]
def bench_write(
*,
script: str,
model: str,
mode: str,
gate_status: str,
turns: int,
answered: int,
caps: int,
wall_s: float,
# chat-specific
tool_turns: int = 0,
emitted: int = 0,
executed: int = 0,
contract: int = 0,
# denominators (for percentages)
contract_denom: int | None = None,
executed_denom: int | None = None,
tool_turns_denom: int | None = None,
# extras (any script)
extra1_col: str = "",
extra1_val: str = "",
extra2_col: str = "",
extra2_val: str = "",
) -> None:
"""Append one row to the benchmark CSV."""
CSV_PATH.parent.mkdir(parents=True, exist_ok=True)
# Defaults: if denom not given, use the numerator
c_denom = contract_denom if contract_denom is not None else (contract if contract > 0 else 0)
e_denom = executed_denom if executed_denom is not None else (emitted if emitted > 0 else 0)
t_denom = tool_turns_denom if tool_turns_denom is not None else turns
row = {
"date": str(date.today()),
"script": script,
"model": model,
"mode": mode,
"gate_status": gate_status,
"turns": turns,
"answered": answered,
"caps": caps,
"tool_turns": tool_turns,
"tool_turns_denom": t_denom,
"emitted": emitted,
"executed": executed,
"executed_denom": e_denom,
"contract": contract,
"contract_denom": c_denom,
"wall_s": f"{wall_s:.1f}",
"extra1_col": extra1_col,
"extra1_val": extra1_val,
"extra2_col": extra2_col,
"extra2_val": extra2_val,
}
file_exists = CSV_PATH.exists() and CSV_PATH.stat().st_size > 0
with open(CSV_PATH, "a", newline="") as f:
writer = csv.DictWriter(f, fieldnames=_HEADER)
if not file_exists:
writer.writeheader()
writer.writerow(row)