ducoterra
48bc92b9af
Build and Push Container / build-and-push (push) Successful in 38s
Single-file FastAPI proxy plus a no-CDN vanilla-JS page for watching a gufo
server and a vLLM server side by side: one Prometheus parser (class Prom) with
per-engine field maps, measured Δcounter/Δt rates for engines that export no
rate gauge, and strict read-only GET-only routing. Engines are opt-in via
*_BASE_URL — unset means no panel, no poll, and a 404 {"configured": false}
without dialling.
Includes the container image and CI so the dashboard can be deployed as an
image rather than a checkout:
- Containerfile: uv-locked two-stage build on python:3.14-slim, non-root
(uid 10001), binds 8090 (8000 is vLLM's port in this fleet), healthcheck on
/api/config so a dead engine never marks the dashboard unhealthy. No frontend
build stage — static/index.html is bundler-free by design and is copied as-is.
- .gitea/workflows/build-and-push.yml: publishes
gitea.reeseapps.com/Experimental/llm-monitor on main + release tags.
114 tests, no network and no real engines required.
LLM-Generated: true
300 lines
13 KiB
Python
300 lines
13 KiB
Python
"""Derived throughput: Δcounter/Δt and the lifetime-average fallback.
|
|
|
|
The deployed vLLM exports no rate gauge of any kind (verified: zero series with a
|
|
throughput/rate name), so prompt tok/s can only be *computed*. These tests pin the
|
|
precedence — native gauge > measured Δcounter > estimate > n/a — and the counter-reset
|
|
handling, because a naive delta shows up as a giant negative spike after an engine
|
|
restart.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from app import RateEstimator, parse_gufo_metrics, parse_vllm_metrics
|
|
|
|
# gufo's generation-speed gauge is ONE request's last decode pass; the counter is every
|
|
# slot's tokens. 30 tok/s on the gauge with 60 tokens landing per second = two slots.
|
|
GUFO_SCRAPE = """
|
|
llamacpp:requests_processing 2
|
|
llamacpp:tokens_predicted_total 1000
|
|
llamacpp:predicted_tokens_seconds 30
|
|
"""
|
|
|
|
COUNTERS = """
|
|
vllm:num_requests_running{engine="0"} 2
|
|
vllm:prompt_tokens_total{engine="0"} 1000
|
|
vllm:generation_tokens_total{engine="0"} 400
|
|
vllm:inter_token_latency_seconds_sum 0.04
|
|
vllm:inter_token_latency_seconds_count 2
|
|
"""
|
|
|
|
|
|
def test_first_scrape_has_nothing_to_diff_against() -> None:
|
|
rates = RateEstimator()
|
|
assert rates.rate("k", 100.0, now=10.0) == (None, None)
|
|
|
|
|
|
def test_second_scrape_measures_the_delta() -> None:
|
|
rates = RateEstimator()
|
|
rates.rate("k", 100.0, now=10.0)
|
|
rate, window = rates.rate("k", 130.0, now=11.0)
|
|
assert rate == pytest.approx(30.0)
|
|
assert window == pytest.approx(1.0)
|
|
|
|
|
|
def test_counter_reset_yields_none_not_a_negative_spike() -> None:
|
|
rates = RateEstimator()
|
|
rates.rate("k", 5000.0, now=10.0)
|
|
assert rates.rate("k", 12.0, now=11.0) == (None, None) # engine restarted
|
|
# and it recovers on the following scrape
|
|
assert rates.rate("k", 22.0, now=12.0)[0] == pytest.approx(10.0)
|
|
|
|
|
|
def test_absent_counter_and_zero_window_are_none() -> None:
|
|
rates = RateEstimator()
|
|
assert rates.rate("k", None, now=1.0) == (None, None)
|
|
rates.rate("k", 1.0, now=5.0)
|
|
assert rates.rate("k", 2.0, now=5.0) == (None, None) # dt == 0
|
|
assert rates.rate("other", 1.0, now=6.0) == (None, None) # unknown key
|
|
|
|
|
|
def test_rate_smooths_over_the_span_once_enough_samples_exist() -> None:
|
|
# a bare 1s delta jitters (prompts arrive in bursts); the span averages it out
|
|
rates = RateEstimator(span=10.0)
|
|
rates.rate("k", 0.0, now=0.0) # outside the span by t=12
|
|
rates.rate("k", 100.0, now=5.0)
|
|
rate, window = rates.rate("k", 400.0, now=12.0)
|
|
assert window == pytest.approx(7.0) # measured from the oldest in-span sample
|
|
assert rate == pytest.approx(300.0 / 7.0)
|
|
|
|
|
|
def test_only_two_in_window_samples_still_produce_a_shorter_rate() -> None:
|
|
rates = RateEstimator(span=10.0)
|
|
rates.rate("k", 0.0, now=0.0)
|
|
rates.rate("k", 100.0, now=20.0) # first sample is long gone
|
|
rate, window = rates.rate("k", 150.0, now=21.0)
|
|
assert window == pytest.approx(1.0)
|
|
assert rate == pytest.approx(50.0)
|
|
|
|
|
|
def test_history_is_bounded() -> None:
|
|
rates = RateEstimator(span=1.0, max_samples=8)
|
|
for i in range(50):
|
|
rates.rate("k", float(i), now=float(i))
|
|
assert len(rates._samples["k"]) <= 8
|
|
|
|
|
|
def test_keys_are_independent() -> None:
|
|
rates = RateEstimator()
|
|
rates.rate("a", 0.0, now=0.0)
|
|
rates.rate("b", 0.0, now=0.0)
|
|
assert rates.rate("a", 10.0, now=1.0)[0] == pytest.approx(10.0)
|
|
assert rates.rate("b", 4.0, now=1.0)[0] == pytest.approx(4.0)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# gufo aggregate decode rate: the counter adds slots, the gauge cannot
|
|
# --------------------------------------------------------------------------- #
|
|
def test_gufo_decode_rate_adds_every_slot_generating_at_once() -> None:
|
|
"""The whole point: two slots at 30 tok/s must read 60, not 30.
|
|
|
|
gufo's `predicted_tokens_seconds` is a single unlabeled gauge holding one request's
|
|
last decode pass, so it can never sum concurrency. `tokens_predicted_total` advances
|
|
once per token from every slot, so Δ/Δt is the server-aggregate.
|
|
"""
|
|
rates = RateEstimator()
|
|
parse_gufo_metrics(GUFO_SCRAPE, rates, now=100.0)
|
|
m = parse_gufo_metrics(GUFO_SCRAPE.replace("1000", "1060"), rates, now=101.0)
|
|
d = m["decode_tokens_per_second"]
|
|
assert d["value"] == pytest.approx(60.0)
|
|
assert d["derived"] is True
|
|
assert d["window"] == pytest.approx(1.0)
|
|
assert d["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt"
|
|
# the single-request gauge stays exposed, untouched, as the native reference
|
|
assert m["decode_tokens_per_second_native"]["value"] == pytest.approx(30.0)
|
|
|
|
|
|
def test_gufo_first_scrape_falls_back_to_the_gauge_not_na() -> None:
|
|
# nothing to diff against yet, but the scrape can still answer
|
|
m = parse_gufo_metrics(GUFO_SCRAPE, RateEstimator(), now=10.0)
|
|
assert m["decode_tokens_per_second"] == {
|
|
"value": 30.0,
|
|
"source": "llamacpp:predicted_tokens_seconds",
|
|
"derived": False,
|
|
}
|
|
|
|
|
|
def test_gufo_idle_reads_zero_not_the_stale_gauge() -> None:
|
|
# an engine that generated nothing in the window is at 0 tok/s; the gauge would
|
|
# happily keep showing the last session's speed forever
|
|
rates = RateEstimator()
|
|
parse_gufo_metrics(GUFO_SCRAPE, rates, now=0.0)
|
|
m = parse_gufo_metrics(GUFO_SCRAPE, rates, now=1.0)
|
|
assert m["decode_tokens_per_second"]["value"] == 0.0
|
|
assert m["decode_tokens_per_second_native"]["value"] == pytest.approx(30.0)
|
|
|
|
|
|
def test_gufo_counter_reset_falls_back_instead_of_going_negative() -> None:
|
|
rates = RateEstimator()
|
|
parse_gufo_metrics(GUFO_SCRAPE, rates, now=0.0)
|
|
# engine restarted: the counter dropped. Never -995 tok/s.
|
|
m = parse_gufo_metrics(GUFO_SCRAPE.replace("1000", "5"), rates, now=1.0)
|
|
assert m["decode_tokens_per_second"]["value"] == pytest.approx(30.0)
|
|
assert m["decode_tokens_per_second"]["derived"] is False
|
|
# and it recovers on the following scrape
|
|
m = parse_gufo_metrics(GUFO_SCRAPE.replace("1000", "25"), rates, now=2.0)
|
|
assert m["decode_tokens_per_second"]["value"] == pytest.approx(20.0)
|
|
|
|
|
|
def test_gufo_alias_rename_starts_a_fresh_window_instead_of_a_fake_rate() -> None:
|
|
# the estimator keys off the RESOLVED series name, so a renamed counter cannot diff
|
|
# against a different series' history (and a flapping alias cannot invent throughput)
|
|
rates = RateEstimator()
|
|
parse_gufo_metrics("llamacpp:tokens_generated_total 1000\n", rates, now=0.0)
|
|
m = parse_gufo_metrics("llamacpp:tokens_predicted_total 1000\n", rates, now=1.0)
|
|
assert m["decode_tokens_per_second"]["value"] is None
|
|
|
|
|
|
def test_gufo_prompt_rate_is_measured_and_adds_slots_too() -> None:
|
|
"""Prefill gets the same treatment as decode, so a cross-engine prompt sum is additive.
|
|
|
|
gufo's `prompt_tokens_seconds` is one request's last prefill pass; the counter is every
|
|
slot's newly-computed prompt tokens.
|
|
"""
|
|
text = "llamacpp:prompt_tokens_total 1000\nllamacpp:prompt_tokens_seconds 700\n"
|
|
rates = RateEstimator()
|
|
parse_gufo_metrics(text, rates, now=0.0)
|
|
m = parse_gufo_metrics(text.replace("1000", "3400"), rates, now=2.0)
|
|
p = m["prompt_tokens_per_second"]
|
|
assert p["value"] == pytest.approx(1200.0)
|
|
assert p["source"] == "computed:Δllamacpp:prompt_tokens_total/Δt"
|
|
assert p["derived"] is True and p["window"] == pytest.approx(2.0)
|
|
# the one-request gauge stays exposed next to it
|
|
assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(700.0)
|
|
|
|
|
|
def test_gufo_prompt_first_scrape_falls_back_to_the_gauge() -> None:
|
|
text = "llamacpp:prompt_tokens_total 1000\nllamacpp:prompt_tokens_seconds 700\n"
|
|
m = parse_gufo_metrics(text, RateEstimator(), now=5.0)
|
|
assert m["prompt_tokens_per_second"] == {
|
|
"value": 700.0,
|
|
"source": "llamacpp:prompt_tokens_seconds",
|
|
"derived": False,
|
|
}
|
|
|
|
|
|
def test_gufo_prompt_and_decode_keep_separate_windows() -> None:
|
|
# one estimator, but the two counters must not read each other's history
|
|
rates = RateEstimator()
|
|
base = "llamacpp:prompt_tokens_total 1000\nllamacpp:tokens_predicted_total 500\n"
|
|
parse_gufo_metrics(base, rates, now=0.0)
|
|
m = parse_gufo_metrics(base.replace("1000", "3000").replace("500", "550"), rates, now=2.0)
|
|
assert m["prompt_tokens_per_second"]["value"] == pytest.approx(1000.0)
|
|
assert m["decode_tokens_per_second"]["value"] == pytest.approx(25.0)
|
|
|
|
|
|
def test_gufo_prompt_idle_reads_zero_not_the_stale_gauge() -> None:
|
|
text = "llamacpp:prompt_tokens_total 1000\nllamacpp:prompt_tokens_seconds 700\n"
|
|
rates = RateEstimator()
|
|
parse_gufo_metrics(text, rates, now=0.0)
|
|
m = parse_gufo_metrics(text, rates, now=1.0)
|
|
assert m["prompt_tokens_per_second"]["value"] == 0.0
|
|
assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(700.0)
|
|
|
|
|
|
def test_gufo_parse_without_rates_stays_pure() -> None:
|
|
assert parse_gufo_metrics(GUFO_SCRAPE) == parse_gufo_metrics(GUFO_SCRAPE)
|
|
assert parse_gufo_metrics(GUFO_SCRAPE)["decode_tokens_per_second"]["derived"] is False
|
|
|
|
|
|
def test_both_engines_share_one_estimator_without_cross_reading() -> None:
|
|
# RATES is one object for both engines; the series names keep the histories apart
|
|
rates = RateEstimator()
|
|
parse_gufo_metrics(GUFO_SCRAPE, rates, now=0.0)
|
|
parse_vllm_metrics(COUNTERS, rates, now=0.0)
|
|
g = parse_gufo_metrics(GUFO_SCRAPE.replace("1000", "1040"), rates, now=1.0)
|
|
v = parse_vllm_metrics(COUNTERS.replace("400", "410"), rates, now=1.0)
|
|
assert g["decode_tokens_per_second"]["value"] == pytest.approx(40.0)
|
|
assert v["decode_tokens_per_second"]["source"] == "computed:Δvllm:generation_tokens_total/Δt"
|
|
assert v["decode_tokens_per_second"]["value"] == pytest.approx(10.0)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# parser integration
|
|
# --------------------------------------------------------------------------- #
|
|
def test_prompt_and_decode_use_measured_deltas_when_rates_are_supplied() -> None:
|
|
rates = RateEstimator()
|
|
parse_vllm_metrics(COUNTERS, rates, now=100.0) # seeds the previous sample
|
|
m = parse_vllm_metrics(
|
|
COUNTERS.replace("1000", "1500").replace("400", "450"), rates, now=101.0
|
|
)
|
|
p = m["prompt_tokens_per_second"]
|
|
d = m["decode_tokens_per_second"]
|
|
assert p["derived"] is True and p["window"] == pytest.approx(1.0)
|
|
assert p["value"] == pytest.approx(500.0)
|
|
assert p["source"] == "computed:Δvllm:prompt_tokens_total/Δt"
|
|
assert d["value"] == pytest.approx(50.0)
|
|
assert d["source"] == "computed:Δvllm:generation_tokens_total/Δt"
|
|
|
|
|
|
def test_first_scrape_falls_back_to_the_lifetime_average() -> None:
|
|
# no previous sample to diff against, but prefill time + prompt length still answer
|
|
text = (
|
|
"vllm:prompt_tokens_total 1000\n"
|
|
"vllm:request_prompt_tokens_sum 8000\nvllm:request_prompt_tokens_count 2\n"
|
|
"vllm:request_prefill_time_seconds_sum 1.6\nvllm:request_prefill_time_seconds_count 2\n"
|
|
)
|
|
m = parse_vllm_metrics(text, RateEstimator(), now=10.0)
|
|
p = m["prompt_tokens_per_second"]
|
|
assert p["value"] == pytest.approx(5000.0) # 4000 tok mean / 0.8 s mean
|
|
assert p["source"] == "computed:mean(request_prompt_tokens)/mean(request_prefill_time_seconds)"
|
|
assert m["prefill_tokens_per_second_lifetime"]["value"] == pytest.approx(5000.0)
|
|
|
|
|
|
def test_measured_delta_beats_the_lifetime_average() -> None:
|
|
rates = RateEstimator()
|
|
text = (
|
|
"vllm:prompt_tokens_total 1000\n"
|
|
"vllm:request_prompt_tokens_sum 8000\nvllm:request_prompt_tokens_count 2\n"
|
|
"vllm:request_prefill_time_seconds_sum 1.6\nvllm:request_prefill_time_seconds_count 2\n"
|
|
)
|
|
parse_vllm_metrics(text, rates, now=0.0)
|
|
m = parse_vllm_metrics(text.replace("1000", "1200"), rates, now=1.0)
|
|
assert m["prompt_tokens_per_second"]["source"].startswith("computed:Δ")
|
|
assert m["prompt_tokens_per_second"]["value"] == pytest.approx(200.0)
|
|
|
|
|
|
def test_native_gauge_still_wins_over_any_derivation() -> None:
|
|
m = parse_vllm_metrics(
|
|
"vllm:prompt_throughput 1234.5\nvllm:prompt_tokens_total 10\n",
|
|
RateEstimator(),
|
|
now=1.0,
|
|
)
|
|
assert m["prompt_tokens_per_second"] == {
|
|
"value": 1234.5,
|
|
"source": "vllm:prompt_throughput",
|
|
"derived": False,
|
|
}
|
|
|
|
|
|
def test_without_rates_the_parse_stays_pure() -> None:
|
|
# parse(text) alone must not depend on wall-clock or hidden state
|
|
a = parse_vllm_metrics(COUNTERS)
|
|
b = parse_vllm_metrics(COUNTERS)
|
|
assert a == b
|
|
assert a["prompt_tokens_per_second"]["value"] is None # no prefill series here
|
|
assert a["prefill_tokens_per_second_lifetime"]["value"] is None
|
|
# decode still answers via TPOT, which is stateless
|
|
assert a["decode_tokens_per_second"]["source"] == "computed:running/tpot"
|
|
|
|
|
|
def test_prompt_tok_s_is_never_a_silent_zero() -> None:
|
|
"""The whole point: an engine that IS working must not render n/a because it lacks
|
|
a gauge, and an idle engine must show 0 rather than a stale estimate."""
|
|
rates = RateEstimator()
|
|
text = "vllm:prompt_tokens_total 500\n"
|
|
parse_vllm_metrics(text, rates, now=0.0)
|
|
m = parse_vllm_metrics(text, rates, now=1.0)
|
|
assert m["prompt_tokens_per_second"]["value"] == 0.0 # genuinely idle, not unknown
|