ducoterra
c4bb374ccc
Three dashboards served by one FastAPI app with a SQLite history store
(raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana,
no CDN, no build step. Nothing is fetched from the internet at runtime.
* system: a stdlib-only agent (agent/host_agent.py) reports /proc and
/sys counters; the dashboard turns them into rates over a labelled
window, so every number says how it was measured.
* llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house
exposition parser. Measured delta/elapsed throughput beats a native
per-request gauge, and each value carries its provenance.
* ups: a NUT wire-protocol client plus health checks that name what
they looked at. Commands are gated by NUT_ALLOW_ACTIONS,
DASH_READ_ONLY and a DANGER confirmation, re-checked server-side.
Missing values render as n/a rather than 0, and an unconfigured source
answers 404 {"configured": false} without dialling anything, so "not
deployed here" never looks like an outage.
253 tests at 94% coverage against captured real fixtures; ruff and
pyright clean.
LLM-Generated: true
268 lines
12 KiB
Python
268 lines
12 KiB
Python
"""LLM engine parsing: real captures, provenance on every number, absent never 0.
|
|
|
|
The engines do not export the names people expect, so these tests are pinned to captures of
|
|
the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentation.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
import pytest
|
|
|
|
from dashboards.rates import RateEstimator
|
|
from dashboards.sources.llm import (
|
|
engine_result,
|
|
fleet_totals,
|
|
parse_gufo_metrics,
|
|
parse_vllm_metrics,
|
|
series_names,
|
|
)
|
|
|
|
|
|
def bump_metric(text: str, name: str, delta: float) -> str:
|
|
"""Return the exposition with one counter increased — a scrape `dt` later."""
|
|
pattern = re.compile(rf"^({re.escape(name)}(?:\{{[^}}]*\}})?) (\S+)$", re.M)
|
|
|
|
def replace(match: re.Match[str]) -> str:
|
|
return f"{match.group(1)} {float(match.group(2)) + delta}"
|
|
|
|
bumped, count = pattern.subn(replace, text)
|
|
assert count, f"{name} not present in the capture"
|
|
return bumped
|
|
|
|
|
|
# ---- gufo / llama.cpp ----------------------------------------------------- #
|
|
|
|
def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
assert metrics["tokens_generated_total"]["value"] == 877_336.0
|
|
assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0
|
|
assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0
|
|
assert metrics["requests_processing"]["value"] == 0.0
|
|
assert metrics["device_lost_total"]["value"] == 0.0
|
|
assert metrics["kv_cache_usage_ratio"]["value"] == 0.0
|
|
|
|
|
|
def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None:
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
accept = metrics["spec_accept_rate"]["value"]
|
|
assert accept == pytest.approx(578_514 / 789_028)
|
|
assert str(metrics["spec_accept_rate"]["source"]).startswith("computed")
|
|
|
|
|
|
def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None:
|
|
"""gufo's prompt counter excludes cache hits; the lifetime total must not."""
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
total = metrics["tokens_total"]["value"]
|
|
assert total == 1_350_666 + 124_529_105 + 877_336
|
|
|
|
|
|
def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None:
|
|
"""llamacpp:predicted_tokens_seconds is ONE request's last pass — never the headline."""
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
now = 1_000.0
|
|
first = parse_gufo_metrics(text, rates, now)
|
|
assert "decode_tokens_per_second" not in first or first["decode_tokens_per_second"]["value"] is None
|
|
|
|
text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0)
|
|
metrics = parse_gufo_metrics(text, rates, now + 10.0)
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == pytest.approx(40.0)
|
|
assert decode["window"] == pytest.approx(10.0)
|
|
assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"]
|
|
# the native gauge is still there, labelled, because it answers a different question
|
|
assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)
|
|
|
|
|
|
def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None:
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
text = bump_metric(text, "llamacpp:prompt_tokens_total", 1_000.0)
|
|
metrics = parse_gufo_metrics(text, rates, 1_010.0)
|
|
assert metrics["prompt_tokens_per_second"]["value"] == pytest.approx(100.0)
|
|
|
|
|
|
def test_gufo_counter_reset_produces_n_a_not_a_spike(metrics_text) -> None:
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=60)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
restarted = bump_metric(text, "llamacpp:tokens_predicted_total", -800_000.0)
|
|
metrics = parse_gufo_metrics(restarted, rates, 1_010.0)
|
|
assert metrics.get("decode_tokens_per_second", {}).get("value") is None
|
|
|
|
|
|
def test_absent_gufo_metrics_are_none_with_no_source() -> None:
|
|
metrics = parse_gufo_metrics("# nothing here\n")
|
|
assert metrics["requests_processing"]["value"] is None
|
|
assert metrics["requests_processing"]["source"] is None
|
|
assert metrics.get("spec_accept_rate", {}).get("value") is None
|
|
|
|
|
|
# ---- vLLM ----------------------------------------------------------------- #
|
|
|
|
def test_vllm_counters_and_gauges(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
assert metrics["prompt_tokens_total"]["value"] == pytest.approx(1.8389909e7)
|
|
assert metrics["generation_tokens_total"]["value"] == pytest.approx(611_163.0)
|
|
usage = metrics["gpu_cache_usage"]["value"]
|
|
assert usage is None or 0.0 <= usage <= 1.0, "a ratio summed over label sets would exceed 1"
|
|
assert metrics["num_requests_running"]["value"] is not None
|
|
assert metrics["num_preemptions_total"]["value"] is not None
|
|
|
|
|
|
def test_vllm_latency_histograms(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
ttft = metrics["time_to_first_token_seconds"]
|
|
assert ttft and ttft["count"] > 0 and ttft["mean"] > 0
|
|
assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"]
|
|
assert metrics["time_per_output_token_seconds"]["mean"] > 0
|
|
|
|
|
|
def test_vllm_per_request_speed_is_the_reciprocal_of_tpot(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
tpot = metrics["time_per_output_token_seconds"]["mean"]
|
|
per_request = metrics["per_request_decode_tokens_per_second"]
|
|
assert per_request["value"] == pytest.approx(1.0 / tpot)
|
|
assert per_request["derived"] is True
|
|
|
|
|
|
def test_vllm_prefix_cache_hit_ratio(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
ratio = metrics["prefix_cache_hit_ratio"]["value"]
|
|
assert ratio is not None and 0.0 <= ratio <= 1.0
|
|
assert "hits" in metrics["prefix_cache_hit_ratio"]["source"]
|
|
|
|
|
|
def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) -> None:
|
|
text = metrics_text("vllm")
|
|
rates = RateEstimator(span=10)
|
|
parse_vllm_metrics(text, rates, 2_000.0)
|
|
text = bump_metric(text, "vllm:generation_tokens_total", 250.0)
|
|
metrics = parse_vllm_metrics(text, rates, 2_010.0)
|
|
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0)
|
|
|
|
|
|
def test_vllm_waiting_requests_are_broken_down_by_reason(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
reasons = metrics["waiting_by_reason"]["value"]
|
|
assert isinstance(reasons, dict)
|
|
|
|
|
|
def test_vllm_empty_exposition_is_all_none() -> None:
|
|
metrics = parse_vllm_metrics("")
|
|
assert metrics["num_requests_running"]["value"] is None
|
|
assert metrics["time_to_first_token_seconds"] is None
|
|
assert metrics["per_request_decode_tokens_per_second"]["value"] is None
|
|
|
|
|
|
# ---- Result assembly ------------------------------------------------------ #
|
|
|
|
def test_engine_result_payload_and_samples(metrics_text) -> None:
|
|
rates = RateEstimator(span=10)
|
|
result = engine_result("gufo", "http://gufo.internal:8731", metrics_text("gufo"), rates,
|
|
now=3_000.0)
|
|
payload = result.payload
|
|
assert payload["engine"] == "gufo"
|
|
assert payload["upstream"] == "http://gufo.internal:8731"
|
|
assert payload["series_count"] > 10
|
|
assert payload["unparsed"] == 0
|
|
assert payload["sessions"]["detail"], "a missing /slots must be explained, not blank"
|
|
for sample in result.samples:
|
|
assert sample.source == "llm" and sample.host == "gufo"
|
|
assert sample.metric.startswith("llm.")
|
|
assert sample.value == sample.value and sample.value is not None # no NaN
|
|
|
|
|
|
def test_engine_result_charts_a_measured_rate_after_the_second_poll(metrics_text) -> None:
|
|
rates = RateEstimator(span=10)
|
|
engine_result("gufo", "http://g:1", metrics_text("gufo"), rates, now=3_000.0)
|
|
bumped = bump_metric(metrics_text("gufo"), "llamacpp:tokens_predicted_total", 100.0)
|
|
result = engine_result("gufo", "http://g:1", bumped, rates, now=3_010.0)
|
|
metrics = result.payload["metrics"]
|
|
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(10.0)
|
|
assert any(s.metric == "llm.decode_tokens_per_second" for s in result.samples)
|
|
|
|
|
|
def test_sessions_pass_through_slots_for_gufo() -> None:
|
|
slots = [{"id": 0, "idle": False, "n_prompt": 12, "n_predict": 40}]
|
|
result = engine_result("gufo", "http://g:1", "", RateEstimator(), slots=slots, now=1.0)
|
|
sessions = result.payload["sessions"]
|
|
assert sessions["list"] == slots
|
|
assert sessions["source"] == "gufo /slots"
|
|
|
|
|
|
def test_sessions_say_vllm_has_no_per_session_view() -> None:
|
|
result = engine_result("vllm", "http://v:1", "", RateEstimator(), now=1.0)
|
|
sessions = result.payload["sessions"]
|
|
assert sessions["list"] == []
|
|
assert "vLLM" in sessions["detail"]
|
|
|
|
|
|
# ---- fleet totals --------------------------------------------------------- #
|
|
|
|
def _engine(name: str, **values) -> dict:
|
|
metrics = {key: {"value": value} for key, value in values.items()}
|
|
return {"engine": name, "metrics": metrics}
|
|
|
|
|
|
def test_fleet_totals_sum_only_engines_that_answered() -> None:
|
|
engines = [
|
|
_engine("gufo", decode_tokens_per_second=40.0, prompt_tokens_per_second=10.0,
|
|
kv_cache_usage_ratio=0.25, requests_processing=1.0, requests_deferred=0.0,
|
|
tokens_total=100.0),
|
|
_engine("vllm", decode_tokens_per_second=10.0, prompt_tokens_per_second=5.0,
|
|
gpu_cache_usage=0.75, num_requests_running=2.0, num_requests_waiting=3.0,
|
|
tokens_total=200.0),
|
|
{}, # an engine whose poll failed
|
|
]
|
|
totals = fleet_totals(engines)
|
|
assert totals["engines_total"] == 3
|
|
assert totals["engines_live"] == 2
|
|
assert totals["decode_tokens_per_second"] == 50.0
|
|
assert totals["prompt_tokens_per_second"] == 15.0
|
|
assert totals["requests_active"] == 3.0
|
|
assert totals["requests_waiting"] == 3.0
|
|
assert totals["kv_cache_usage_max"] == 0.75, "cache pressure is a max, never a sum"
|
|
assert totals["tokens_total"] == 300.0
|
|
|
|
|
|
def test_fleet_totals_report_the_gufo_vllm_seam() -> None:
|
|
totals = fleet_totals([_engine("gufo"), _engine("vllm")])
|
|
assert totals["seams"], "the two engines count prompt tokens differently; say so"
|
|
|
|
|
|
def test_fleet_totals_of_nothing_is_none_not_zero() -> None:
|
|
totals = fleet_totals([])
|
|
assert totals["decode_tokens_per_second"] is None
|
|
assert totals["tokens_total"] is None
|
|
assert totals["engines_live"] == 0
|
|
|
|
|
|
def test_series_names_are_the_parse_catalog() -> None:
|
|
assert "tokens_generated_total" in series_names("gufo")
|
|
assert "generation_tokens_total" in series_names("vllm")
|
|
assert "generation_tokens_total" not in series_names("gufo")
|
|
|
|
|
|
def test_gufo_a_measured_zero_is_not_replaced_by_the_gauge(metrics_text) -> None:
|
|
"""`measured or gauge` made an idle engine advertise its last request's speed."""
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
metrics = parse_gufo_metrics(text, rates, 1_010.0) # counters unchanged: truly idle
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == 0.0
|
|
assert "predicted_tokens_seconds" not in str(decode["source"]), "native gauge must not leak in"
|
|
|
|
|
|
def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None:
|
|
text = metrics_text("vllm")
|
|
rates = RateEstimator(span=10)
|
|
parse_vllm_metrics(text, rates, 2_000.0)
|
|
metrics = parse_vllm_metrics(text, rates, 2_010.0)
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == 0.0
|
|
assert "running/tpot" not in str(decode["source"])
|