"""The hand-rolled Prometheus parser, tested against real engine expositions. This parser exists so the app has no Prometheus dependency for the two upstreams that already speak the format. That makes it load-bearing on text we do not control: llama.cpp emits names with colons, vLLM emits label sets and histograms, and both are free to change them. Fixtures are captured verbatim (scripts/capture-fixtures.sh --with-llm). """ from __future__ import annotations import math from dashboards.prom import Prom def test_parses_counters_and_gauges_from_a_real_capture(metrics_text) -> None: prom = Prom(metrics_text("gufo")) assert prom.scalar(["llamacpp:tokens_predicted_total"]) == 877_336.0 assert prom.scalar(["llamacpp:kv_cache_usage_ratio"]) == 0.0 assert prom.parse_errors == 0 def test_help_and_type_comments_are_not_samples(metrics_text) -> None: prom = Prom(metrics_text("gufo")) assert not any(name.startswith("#") for name in prom.series) assert "# HELP llamacpp:prompt_tokens_total" not in prom.series def test_labels_are_parsed_and_filterable(metrics_text) -> None: prom = Prom(metrics_text("vllm")) running = prom.values(["vllm:num_requests_running"]) assert running, "vLLM exposes num_requests_running" by_model = prom.by_label(["vllm:generation_tokens_total"], "model_name") assert by_model and all(model for model in by_model) def test_aggregations(metrics_text) -> None: prom = Prom(metrics_text("vllm")) total = prom.scalar(["vllm:prompt_tokens_total"], agg="sum") assert total is not None and total > 1e6 # ratios must never be summed across label sets — a 3-way sum would read as 300% used usage = prom.scalar(["vllm:kv_cache_usage_perc"], agg="max") assert usage is not None and 0.0 <= usage <= 1.0 def test_missing_series_is_none_not_zero(metrics_text) -> None: prom = Prom(metrics_text("gufo")) assert prom.scalar(["vllm:does_not_exist"]) is None assert prom.values(["vllm:does_not_exist"]) == [] assert prom.by_label(["nope"], "model_name") == {} def test_histogram_mean_and_p95(metrics_text) -> None: hist = Prom(metrics_text("vllm")).histogram(["vllm:time_to_first_token_seconds"]) assert hist is not None assert hist["count"] and hist["count"] > 0 assert hist["mean"] and hist["mean"] > 0 assert hist["p95"] is not None assert hist["p95"] >= hist["mean"], "p95 below the mean means the interpolation is wrong" assert math.isfinite(hist["p95"]) def test_histogram_resolves_through_suffixes(metrics_text) -> None: prom = Prom(metrics_text("vllm")) # the bare name never appears in the exposition; only _bucket/_sum/_count do assert "vllm:time_to_first_token_seconds" not in prom.series assert prom.resolve_hist(["vllm:time_to_first_token_seconds"]) == "vllm:time_to_first_token_seconds" assert prom.resolve(["vllm:time_to_first_token_seconds"]) is None def test_resolve_returns_the_first_present_name() -> None: prom = Prom('a_total 5\nb_total 7\n') assert prom.resolve(["missing_total", "b_total", "a_total"]) == "b_total" assert prom.resolve(["nope"]) is None def test_label_match_narrows_the_series() -> None: prom = Prom('x{engine="0"} 1\nx{engine="1"} 2\n') assert prom.scalar(["x"], agg="sum") == 3 assert prom.scalar(["x"], label_match={"engine": "1"}) == 2 assert prom.scalar(["x"], label_match={"engine": "9"}) is None def test_malformed_lines_are_counted_not_fatal() -> None: prom = Prom("good_metric 1\nthis is not prometheus\nalso_bad{unclosed 2\n") assert prom.scalar(["good_metric"]) == 1.0 assert prom.parse_errors >= 1 def test_empty_exposition_is_harmless() -> None: prom = Prom("") assert prom.series == {} assert prom.scalar(["anything"]) is None assert prom.histogram(["anything"]) is None def test_quantile_ignores_non_finite_values() -> None: prom = Prom('m_sum 10\ncount 2\nm{le="1"} NaN\n') assert prom.scalar(["m"], label_match={"le": "1"}) is None def test_multiple_histogram_label_sets_do_not_invent_a_p95(metrics_text) -> None: """Mixing per-DP-rank buckets into one cumulative curve would fabricate a percentile.""" two = ( 'h_sum{dp="0"} 10\nh_sum{dp="1"} 10\n' 'h_count{dp="0"} 2\nh_count{dp="1"} 2\n' 'h_bucket{dp="0",le="1"} 1\nh_bucket{dp="0",le="+Inf"} 2\n' 'h_bucket{dp="1",le="1"} 1\nh_bucket{dp="1",le="+Inf"} 2\n' ) hist = Prom(two).histogram(["h"]) assert hist is not None assert hist["mean"] == 5.0 assert hist["p95"] is None, "p95 across mixed label sets is a made-up number"