feat(agent): add CSV benchmark recorder + summary/embedding test scripts and skills
New files: - scripts/model_benchmark.py — shared CSV recorder for all model tests - scripts/test_summary_model.py — summary model quality benchmark (coherence, coverage, brevity, hallucination) - scripts/test_embed_model.py — embedding model benchmark (dimension, cosine accuracy, speed) - .agents/skills/test-summary-model/SKILL.md — skill for testing summary models - .agents/skills/test-embed-model/SKILL.md — skill for testing embedding models - benchmarks/README.md — schema documentation Updated: - .agents/skills/test-chat-model/SKILL.md — now also records to CSV All three scripts write to benchmarks/model_benchmarks.csv with one row per run per check. The CSV accumulates results across runs for comparison.
This commit is contained in:
@@ -0,0 +1,132 @@
|
||||
"""Shared CSV benchmark recorder for model testing scripts.
|
||||
|
||||
All model-test scripts (chat, summary, embedding) write their results
|
||||
to ``benchmarks/model_benchmarks.csv`` so results accumulate across
|
||||
runs and can be compared with a spreadsheet or a simple query.
|
||||
|
||||
CSV columns (one row per *run*, not per turn):
|
||||
|
||||
date,script,model,mode,gate_status,turns,answered,caps,
|
||||
tool_turns,emitted,executed,contract,wall_s,
|
||||
extra_col1,extra_val1,extra_col2,extra_val2
|
||||
|
||||
* ``script`` — ``chat`` | ``summary`` | ``embed``
|
||||
* ``model`` — the model name (e.g. ``lite``, ``turbo``)
|
||||
* ``mode`` — ``fixture`` | ``derived`` | ``quality`` | ``dimension`` |
|
||||
``cosine`` | ``speed``
|
||||
* ``gate_status`` — ``PASS`` | ``FAIL``
|
||||
* ``turns`` — number of turns/questions
|
||||
* ``answered`` — turns that produced an answer (no LLMError)
|
||||
* ``caps`` — turns that hit the round cap
|
||||
* ``tool_turns`` — turns that emitted at least one tool call (chat only)
|
||||
* ``emitted`` / ``executed`` — tool-call counts (chat only)
|
||||
* ``contract`` — well-formed / total calls (chat) or quality score 0-100
|
||||
(summary) or cosine accuracy 0-100 (embed)
|
||||
* ``wall_s`` — total wall seconds for the run
|
||||
* ``extra_*`` — script-specific secondary metrics (e.g. summary
|
||||
coherence, embed dimension, embed speed per vector)
|
||||
|
||||
Usage from any test script::
|
||||
|
||||
from scripts.model_benchmark import bench_write
|
||||
|
||||
bench_write(
|
||||
script="chat", # or "summary" / "embed"
|
||||
model="lite",
|
||||
mode="fixture",
|
||||
gate_status="PASS",
|
||||
turns=10,
|
||||
answered=10,
|
||||
caps=0,
|
||||
tool_turns=9,
|
||||
emitted=9,
|
||||
executed=9,
|
||||
contract=11, # numerator (denominator = emitted)
|
||||
wall_s=40.5,
|
||||
# optional extras:
|
||||
contract_denom=12,
|
||||
executed_denom=12,
|
||||
)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import os
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
CSV_PATH = Path(__file__).resolve().parent.parent / "benchmarks" / "model_benchmarks.csv"
|
||||
_HEADER = [
|
||||
"date", "script", "model", "mode", "gate_status",
|
||||
"turns", "answered", "caps",
|
||||
"tool_turns", "emitted", "executed",
|
||||
"contract", "wall_s",
|
||||
"contract_denom", "executed_denom",
|
||||
"tool_turns_denom", "extra1_col", "extra1_val",
|
||||
"extra2_col", "extra2_val",
|
||||
]
|
||||
|
||||
|
||||
def bench_write(
|
||||
*,
|
||||
script: str,
|
||||
model: str,
|
||||
mode: str,
|
||||
gate_status: str,
|
||||
turns: int,
|
||||
answered: int,
|
||||
caps: int,
|
||||
wall_s: float,
|
||||
# chat-specific
|
||||
tool_turns: int = 0,
|
||||
emitted: int = 0,
|
||||
executed: int = 0,
|
||||
contract: int = 0,
|
||||
# denominators (for percentages)
|
||||
contract_denom: int | None = None,
|
||||
executed_denom: int | None = None,
|
||||
tool_turns_denom: int | None = None,
|
||||
# extras (any script)
|
||||
extra1_col: str = "",
|
||||
extra1_val: str = "",
|
||||
extra2_col: str = "",
|
||||
extra2_val: str = "",
|
||||
) -> None:
|
||||
"""Append one row to the benchmark CSV."""
|
||||
CSV_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Defaults: if denom not given, use the numerator
|
||||
c_denom = contract_denom if contract_denom is not None else (contract if contract > 0 else 0)
|
||||
e_denom = executed_denom if executed_denom is not None else (emitted if emitted > 0 else 0)
|
||||
t_denom = tool_turns_denom if tool_turns_denom is not None else turns
|
||||
|
||||
row = {
|
||||
"date": str(date.today()),
|
||||
"script": script,
|
||||
"model": model,
|
||||
"mode": mode,
|
||||
"gate_status": gate_status,
|
||||
"turns": turns,
|
||||
"answered": answered,
|
||||
"caps": caps,
|
||||
"tool_turns": tool_turns,
|
||||
"tool_turns_denom": t_denom,
|
||||
"emitted": emitted,
|
||||
"executed": executed,
|
||||
"executed_denom": e_denom,
|
||||
"contract": contract,
|
||||
"contract_denom": c_denom,
|
||||
"wall_s": f"{wall_s:.1f}",
|
||||
"extra1_col": extra1_col,
|
||||
"extra1_val": extra1_val,
|
||||
"extra2_col": extra2_col,
|
||||
"extra2_val": extra2_val,
|
||||
}
|
||||
|
||||
file_exists = CSV_PATH.exists() and CSV_PATH.stat().st_size > 0
|
||||
|
||||
with open(CSV_PATH, "a", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=_HEADER)
|
||||
if not file_exists:
|
||||
writer.writeheader()
|
||||
writer.writerow(row)
|
||||
Reference in New Issue
Block a user