ducoterra
48bc92b9af
Build and Push Container / build-and-push (push) Successful in 38s
Single-file FastAPI proxy plus a no-CDN vanilla-JS page for watching a gufo
server and a vLLM server side by side: one Prometheus parser (class Prom) with
per-engine field maps, measured Δcounter/Δt rates for engines that export no
rate gauge, and strict read-only GET-only routing. Engines are opt-in via
*_BASE_URL — unset means no panel, no poll, and a 404 {"configured": false}
without dialling.
Includes the container image and CI so the dashboard can be deployed as an
image rather than a checkout:
- Containerfile: uv-locked two-stage build on python:3.14-slim, non-root
(uid 10001), binds 8090 (8000 is vLLM's port in this fleet), healthcheck on
/api/config so a dead engine never marks the dashboard unhealthy. No frontend
build stage — static/index.html is bundler-free by design and is copied as-is.
- .gitea/workflows/build-and-push.yml: publishes
gitea.reeseapps.com/Experimental/llm-monitor on main + release tags.
114 tests, no network and no real engines required.
LLM-Generated: true
230 lines
9.9 KiB
Python
230 lines
9.9 KiB
Python
"""Ground truth: the payloads captured from the DEPLOYED engines.
|
|
|
|
Neither engine exports the series names people expect, and gufo's `/slots` is a bare
|
|
JSON array. These fixtures are verbatim captures (gufo on deskwork-ai:8731, vLLM on
|
|
driveripper-ai:8000); if a field stops mapping, the engine renamed something and this
|
|
test tells you which one instead of the dashboard quietly showing `n/a` in production.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app import RateEstimator, parse_gufo_metrics, parse_vllm_metrics
|
|
|
|
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def gufo_text() -> str:
|
|
return (FIXTURES / "gufo_metrics_real.txt").read_text()
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def vllm_text() -> str:
|
|
return (FIXTURES / "vllm_metrics_real.txt").read_text()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# gufo
|
|
# --------------------------------------------------------------------------- #
|
|
def test_gufo_real_names_are_the_primary_aliases(gufo_text: str) -> None:
|
|
m = parse_gufo_metrics(gufo_text)
|
|
assert m["tokens_prompt_total"]["source"] == "llamacpp:prompt_tokens_total"
|
|
assert m["tokens_generated_total"]["source"] == "llamacpp:tokens_predicted_total"
|
|
# gufo names its speed gauges *_seconds while the unit is tokens/second
|
|
assert m["prompt_tokens_per_second_native"]["source"] == "llamacpp:prompt_tokens_seconds"
|
|
assert m["decode_tokens_per_second_native"]["source"] == "llamacpp:predicted_tokens_seconds"
|
|
# parsed without an estimator, both throughput cells degrade to those one-request gauges
|
|
assert m["prompt_tokens_per_second"]["source"] == "llamacpp:prompt_tokens_seconds"
|
|
# parsed without an estimator, the throughput cell degrades to that one-request gauge
|
|
assert m["decode_tokens_per_second"]["source"] == "llamacpp:predicted_tokens_seconds"
|
|
|
|
|
|
def test_gufo_aggregate_decode_rate_resolves_on_the_real_counter(gufo_text: str) -> None:
|
|
"""The Δ-rates key off the RESOLVED counter names, so they must work on gufo's real
|
|
names (`tokens_predicted_total`, `prompt_tokens_total`) and land in the same ballpark as
|
|
gufo's own busy-time counter (`tokens_predicted_seconds_total`).
|
|
"""
|
|
rates = RateEstimator()
|
|
parse_gufo_metrics(gufo_text, rates, now=0.0)
|
|
bumped = gufo_text.replace(
|
|
"llamacpp:tokens_predicted_total 173091", "llamacpp:tokens_predicted_total 173191"
|
|
).replace(
|
|
# +1000 generated and +1000 prompt tokens over 2s -> 500 tok/s of each
|
|
"llamacpp:prompt_tokens_total 374263", "llamacpp:prompt_tokens_total 375263"
|
|
)
|
|
assert bumped != gufo_text # the capture moved the counter; re-check this test
|
|
m = parse_gufo_metrics(bumped, rates, now=2.0)
|
|
d = m["decode_tokens_per_second"]
|
|
assert d["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt"
|
|
assert d["value"] == pytest.approx(50.0)
|
|
# sanity: the lifetime mean from the two cumulative counters is the same order of
|
|
# magnitude, so the windowed aggregate is not quietly summing something else
|
|
busy = float(re.search(r"tokens_predicted_seconds_total ([0-9.]+)", gufo_text).group(1))
|
|
assert 5.0 < (173091.0 / busy) < 100.0
|
|
|
|
# prompt throughput from the same two scrapes: gufo's prompt counter EXCLUDES cache hits,
|
|
# so this is newly-computed prefill work across all slots, not prompt tokens accepted
|
|
p = m["prompt_tokens_per_second"]
|
|
assert p["source"] == "computed:Δllamacpp:prompt_tokens_total/Δt"
|
|
assert p["value"] == pytest.approx(500.0)
|
|
assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(820.2841172907755)
|
|
|
|
|
|
def test_gufo_every_banner_and_strip_field_resolves(gufo_text: str) -> None:
|
|
m = parse_gufo_metrics(gufo_text)
|
|
for key in (
|
|
"requests_processing",
|
|
"requests_deferred",
|
|
"kv_cache_usage_ratio",
|
|
"tokens_prompt_total",
|
|
"tokens_generated_total",
|
|
"prompt_tokens_per_second",
|
|
"decode_tokens_per_second",
|
|
"tokens_total",
|
|
"n_tokens_max",
|
|
"device_lost_total",
|
|
):
|
|
assert m[key]["value"] is not None, f"gufo {key} stopped resolving"
|
|
assert 0.0 <= m["kv_cache_usage_ratio"]["value"] <= 1.0
|
|
|
|
|
|
def test_gufo_totals_include_cache_hits_which_are_a_separate_counter(gufo_text: str) -> None:
|
|
m = parse_gufo_metrics(gufo_text)
|
|
prompt = m["tokens_prompt_total"]["value"]
|
|
cached = m["tokens_prompt_cached_total"]["value"]
|
|
generated = m["tokens_generated_total"]["value"]
|
|
assert m["tokens_total"]["value"] == pytest.approx(prompt + cached + generated)
|
|
# 29.6M cached vs 0.37M computed on the live box — cache hits dominate
|
|
assert m["prompt_cache_hit_ratio"]["value"] == pytest.approx(cached / (prompt + cached))
|
|
assert 0.0 < m["prompt_cache_hit_ratio"]["value"] < 1.0
|
|
|
|
|
|
def test_gufo_speculative_decoding_accept_rate(gufo_text: str) -> None:
|
|
m = parse_gufo_metrics(gufo_text)
|
|
accept = m["spec_accept_rate"]
|
|
assert accept["derived"] is True
|
|
assert 0.0 <= accept["value"] <= 1.0
|
|
|
|
|
|
def test_gufo_device_lost_is_zero_on_a_healthy_box(gufo_text: str) -> None:
|
|
assert parse_gufo_metrics(gufo_text)["device_lost_total"]["value"] == 0.0
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# gufo /slots
|
|
# --------------------------------------------------------------------------- #
|
|
def test_gufo_slots_is_a_bare_array_not_an_object() -> None:
|
|
"""The UI must not assume {"slots": [...]} — the deployed engine returns a list."""
|
|
data = json.loads((FIXTURES / "gufo_slots_real.json").read_text())
|
|
assert isinstance(data, list)
|
|
assert len(data) >= 1
|
|
assert all(isinstance(slot, dict) and "id" in slot for slot in data)
|
|
|
|
|
|
def test_gufo_slot_fields_the_cards_depend_on() -> None:
|
|
data = json.loads((FIXTURES / "gufo_slots_real.json").read_text())
|
|
slot = data[0]
|
|
for key in (
|
|
"id",
|
|
"state",
|
|
"is_processing",
|
|
"id_task",
|
|
"n_prompt_tokens",
|
|
"n_prompt_tokens_cache",
|
|
"n_ctx",
|
|
"next_token",
|
|
"model",
|
|
):
|
|
assert key in slot, f"gufo slot no longer reports {key}"
|
|
assert slot["next_token"], "next_token[] should still be present, with n_decoded/n_remain"
|
|
assert "n_decoded" in slot["next_token"][0]
|
|
# an idle slot reports -1 for "nothing pending", which is not a count
|
|
if not slot["is_processing"]:
|
|
assert slot["id_task"] == -1
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# vLLM
|
|
# --------------------------------------------------------------------------- #
|
|
def test_vllm_real_names_are_the_primary_aliases(vllm_text: str) -> None:
|
|
m = parse_vllm_metrics(vllm_text)
|
|
assert m["gpu_cache_usage"]["source"] == "vllm:kv_cache_usage_perc" # not gpu_cache_usage
|
|
assert m["generation_tokens_total"]["source"] == "vllm:generation_tokens_total"
|
|
assert m["time_per_output_token_seconds"]["source"] == "vllm:inter_token_latency_seconds"
|
|
|
|
|
|
def test_vllm_every_banner_and_card_field_resolves(vllm_text: str) -> None:
|
|
m = parse_vllm_metrics(vllm_text)
|
|
for key in (
|
|
"num_requests_running",
|
|
"num_requests_waiting",
|
|
"gpu_cache_usage",
|
|
"prompt_tokens_total",
|
|
"generation_tokens_total",
|
|
"tokens_total",
|
|
"waiting_capacity",
|
|
"prefix_cache_hit_ratio",
|
|
"num_preemptions_total",
|
|
):
|
|
assert m[key]["value"] is not None, f"vLLM {key} stopped resolving"
|
|
for key in (
|
|
"time_to_first_token_seconds__mean",
|
|
"time_per_output_token_seconds__mean",
|
|
"request_queue_time_seconds__mean",
|
|
):
|
|
assert m[key]["value"] is not None, f"vLLM {key} stopped resolving"
|
|
assert 0.0 <= m["gpu_cache_usage"]["value"] <= 1.0
|
|
assert 0.0 <= m["prefix_cache_hit_ratio"]["value"] <= 1.0
|
|
|
|
|
|
def test_vllm_waiting_reasons_from_the_live_labels(vllm_text: str) -> None:
|
|
# the deployed build tags reasons as capacity/deferred, not defragmentation
|
|
m = parse_vllm_metrics(vllm_text)
|
|
assert "capacity" in m["waiting_by_reason"]["value"]
|
|
|
|
|
|
def test_vllm_histograms_survive_engine_and_model_labels(vllm_text: str) -> None:
|
|
m = parse_vllm_metrics(vllm_text)
|
|
ttft = m["time_to_first_token_seconds"]
|
|
assert ttft["count"] > 0 and ttft["mean"] > 0
|
|
# single engine label set here, so an interpolated p95 is legitimate
|
|
assert ttft["p95"] is None or ttft["p95"] >= ttft["mean"]
|
|
|
|
|
|
def test_vllm_prompt_tok_s_is_answered_even_without_a_rate_gauge(vllm_text: str) -> None:
|
|
# this build exports no throughput series at all, so prompt tok/s comes from the
|
|
# prefill histograms; it must not render n/a on a healthy, busy engine
|
|
m = parse_vllm_metrics(vllm_text)
|
|
p = m["prompt_tokens_per_second"]
|
|
assert p["value"] is not None, "prompt tok/s regressed to n/a"
|
|
assert p["derived"] is True
|
|
assert p["source"] == "computed:mean(request_prompt_tokens)/mean(request_prefill_time_seconds)"
|
|
assert p["value"] > 1000 # live box sits around 4.7k tok/s; single digits mean a bug
|
|
assert m["prefill_tokens_per_second_lifetime"]["value"] == pytest.approx(p["value"])
|
|
|
|
|
|
def test_vllm_prompt_tok_s_switches_to_measured_rate_once_scraping(vllm_text: str) -> None:
|
|
from app import RateEstimator
|
|
|
|
rates = RateEstimator()
|
|
parse_vllm_metrics(vllm_text, rates, now=1000.0)
|
|
m = parse_vllm_metrics(vllm_text, rates, now=1001.0) # same counters -> zero throughput
|
|
p = m["prompt_tokens_per_second"]
|
|
assert p["source"] == "computed:Δvllm:prompt_tokens_total/Δt"
|
|
assert p["window"] == pytest.approx(1.0)
|
|
# unchanged capture -> genuinely zero throughput in that window, not a missing gauge
|
|
assert p["value"] == 0.0
|
|
|
|
|
|
def test_vllm_decode_throughput_is_derived_on_this_build(vllm_text: str) -> None:
|
|
# no vllm:*throughput gauge exists on the deployed version -> computed from TPOT
|
|
m = parse_vllm_metrics(vllm_text)
|
|
assert m["decode_tokens_per_second"]["derived"] is True
|
|
assert m["decode_tokens_per_second"]["source"] == "computed:running/tpot"
|