ducoterra
f221b40fce
New files: - scripts/model_benchmark.py — shared CSV recorder for all model tests - scripts/test_summary_model.py — summary model quality benchmark (coherence, coverage, brevity, hallucination) - scripts/test_embed_model.py — embedding model benchmark (dimension, cosine accuracy, speed) - .agents/skills/test-summary-model/SKILL.md — skill for testing summary models - .agents/skills/test-embed-model/SKILL.md — skill for testing embedding models - benchmarks/README.md — schema documentation Updated: - .agents/skills/test-chat-model/SKILL.md — now also records to CSV All three scripts write to benchmarks/model_benchmarks.csv with one row per run per check. The CSV accumulates results across runs for comparison.
133 lines
4.1 KiB
Python
133 lines
4.1 KiB
Python
"""Shared CSV benchmark recorder for model testing scripts.
|
|
|
|
All model-test scripts (chat, summary, embedding) write their results
|
|
to ``benchmarks/model_benchmarks.csv`` so results accumulate across
|
|
runs and can be compared with a spreadsheet or a simple query.
|
|
|
|
CSV columns (one row per *run*, not per turn):
|
|
|
|
date,script,model,mode,gate_status,turns,answered,caps,
|
|
tool_turns,emitted,executed,contract,wall_s,
|
|
extra_col1,extra_val1,extra_col2,extra_val2
|
|
|
|
* ``script`` — ``chat`` | ``summary`` | ``embed``
|
|
* ``model`` — the model name (e.g. ``lite``, ``turbo``)
|
|
* ``mode`` — ``fixture`` | ``derived`` | ``quality`` | ``dimension`` |
|
|
``cosine`` | ``speed``
|
|
* ``gate_status`` — ``PASS`` | ``FAIL``
|
|
* ``turns`` — number of turns/questions
|
|
* ``answered`` — turns that produced an answer (no LLMError)
|
|
* ``caps`` — turns that hit the round cap
|
|
* ``tool_turns`` — turns that emitted at least one tool call (chat only)
|
|
* ``emitted`` / ``executed`` — tool-call counts (chat only)
|
|
* ``contract`` — well-formed / total calls (chat) or quality score 0-100
|
|
(summary) or cosine accuracy 0-100 (embed)
|
|
* ``wall_s`` — total wall seconds for the run
|
|
* ``extra_*`` — script-specific secondary metrics (e.g. summary
|
|
coherence, embed dimension, embed speed per vector)
|
|
|
|
Usage from any test script::
|
|
|
|
from scripts.model_benchmark import bench_write
|
|
|
|
bench_write(
|
|
script="chat", # or "summary" / "embed"
|
|
model="lite",
|
|
mode="fixture",
|
|
gate_status="PASS",
|
|
turns=10,
|
|
answered=10,
|
|
caps=0,
|
|
tool_turns=9,
|
|
emitted=9,
|
|
executed=9,
|
|
contract=11, # numerator (denominator = emitted)
|
|
wall_s=40.5,
|
|
# optional extras:
|
|
contract_denom=12,
|
|
executed_denom=12,
|
|
)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import os
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
CSV_PATH = Path(__file__).resolve().parent.parent / "benchmarks" / "model_benchmarks.csv"
|
|
_HEADER = [
|
|
"date", "script", "model", "mode", "gate_status",
|
|
"turns", "answered", "caps",
|
|
"tool_turns", "emitted", "executed",
|
|
"contract", "wall_s",
|
|
"contract_denom", "executed_denom",
|
|
"tool_turns_denom", "extra1_col", "extra1_val",
|
|
"extra2_col", "extra2_val",
|
|
]
|
|
|
|
|
|
def bench_write(
|
|
*,
|
|
script: str,
|
|
model: str,
|
|
mode: str,
|
|
gate_status: str,
|
|
turns: int,
|
|
answered: int,
|
|
caps: int,
|
|
wall_s: float,
|
|
# chat-specific
|
|
tool_turns: int = 0,
|
|
emitted: int = 0,
|
|
executed: int = 0,
|
|
contract: int = 0,
|
|
# denominators (for percentages)
|
|
contract_denom: int | None = None,
|
|
executed_denom: int | None = None,
|
|
tool_turns_denom: int | None = None,
|
|
# extras (any script)
|
|
extra1_col: str = "",
|
|
extra1_val: str = "",
|
|
extra2_col: str = "",
|
|
extra2_val: str = "",
|
|
) -> None:
|
|
"""Append one row to the benchmark CSV."""
|
|
CSV_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Defaults: if denom not given, use the numerator
|
|
c_denom = contract_denom if contract_denom is not None else (contract if contract > 0 else 0)
|
|
e_denom = executed_denom if executed_denom is not None else (emitted if emitted > 0 else 0)
|
|
t_denom = tool_turns_denom if tool_turns_denom is not None else turns
|
|
|
|
row = {
|
|
"date": str(date.today()),
|
|
"script": script,
|
|
"model": model,
|
|
"mode": mode,
|
|
"gate_status": gate_status,
|
|
"turns": turns,
|
|
"answered": answered,
|
|
"caps": caps,
|
|
"tool_turns": tool_turns,
|
|
"tool_turns_denom": t_denom,
|
|
"emitted": emitted,
|
|
"executed": executed,
|
|
"executed_denom": e_denom,
|
|
"contract": contract,
|
|
"contract_denom": c_denom,
|
|
"wall_s": f"{wall_s:.1f}",
|
|
"extra1_col": extra1_col,
|
|
"extra1_val": extra1_val,
|
|
"extra2_col": extra2_col,
|
|
"extra2_val": extra2_val,
|
|
}
|
|
|
|
file_exists = CSV_PATH.exists() and CSV_PATH.stat().st_size > 0
|
|
|
|
with open(CSV_PATH, "a", newline="") as f:
|
|
writer = csv.DictWriter(f, fieldnames=_HEADER)
|
|
if not file_exists:
|
|
writer.writeheader()
|
|
writer.writerow(row)
|