"""Prometheus text parser + per-engine field maps. This is the code most likely to silently rot: an engine renames a series and the dashboard must degrade to `n/a`, not crash and not misreport. The aggregation rules (sum / max / avg) are asserted explicitly because getting them wrong is how a cache ratio ends up over 100%. """ from __future__ import annotations import math import pytest from app import Prom, parse_gufo_metrics, parse_vllm_metrics from tests.conftest import GUFO_FIXTURE, VLLM_FIXTURE # --------------------------------------------------------------------------- # # Prom # --------------------------------------------------------------------------- # def test_ignores_comments_help_and_type_lines() -> None: p = Prom("# HELP a hi\n# TYPE a gauge\na 1\n\n") assert list(p.series) == ["a"] assert p.parse_errors == 0 def test_label_block_with_brace_and_escape() -> None: p = Prom(r'a{x="a}b\"c"} 5') assert p.series["a"][0][0] == {"x": 'a}b"c'} assert p.series["a"][0][1] == 5.0 def test_tolerates_space_before_labels_and_trailing_comment() -> None: p = Prom('a {m="x"} 1 # a comment\nb 2 1700000000000\n') assert p.scalar(["a"]) == 1.0 assert p.scalar(["b"]) == 2.0 def test_counts_garbage_instead_of_raising() -> None: p = Prom("this is not prometheus\nalso bad x y z\n") assert p.series == {} assert p.parse_errors == 2 def test_non_finite_values_are_dropped_from_aggregates() -> None: p = Prom("a 1\na nan\na +Inf\na 3\n") assert p.scalar(["a"]) == 4.0 # sum of the finite ones only assert p.scalar(["a"], "max") == 3.0 assert p.scalar(["a"], "avg") == 2.0 def test_absent_series_is_none_never_zero() -> None: p = Prom("other 1\n") assert p.scalar(["nope"]) is None assert p.resolve(["nope", "other"]) == "other" def test_alias_resolution_prefers_first_present() -> None: p = Prom("llama:requests_processing 7\n") assert p.resolve(["llamacpp:requests_processing", "llama:requests_processing"]) == ( "llama:requests_processing" ) def test_by_label_grouping() -> None: p = Prom('w{reason="capacity"} 2\nw{reason="capacity"} 3\nw{reason="other"} 1\n') assert p.by_label(["w"], "reason") == {"capacity": 5.0, "other": 1.0} # --------------------------------------------------------------------------- # # Histograms # --------------------------------------------------------------------------- # def test_histogram_mean_and_p95() -> None: p = Prom('b_bucket{le="1"} 1\nb_bucket{le="10"} 10\nb_bucket{le="+Inf"} 10\nb_sum 40\nb_count 10\n') h = p.histogram(["b"]) assert h is not None assert h["mean"] == 4.0 # 95th of 10 observations is the 9.5th, interpolated inside the (1,10] bucket assert h["p95"] == pytest.approx(9.5) def test_histogram_p95_when_mass_exceeds_the_last_finite_bucket() -> None: # almost everything above the last real bound: report that bound, never +Inf p = Prom('b_bucket{le="1"} 5\nb_bucket{le="10"} 9\nb_bucket{le="+Inf"} 10\nb_sum 40\nb_count 10\n') assert p.histogram(["b"])["p95"] == pytest.approx(10.0) def test_histogram_resolves_without_a_bare_series_line() -> None: # real expositions never emit a series named after the histogram itself p = Prom('vllm:x_sum 6\nvllm:x_count 3\nvllm:x_bucket{le="+Inf"} 3\n') assert p.histogram(["vllm:x"])["mean"] == 2.0 def test_histogram_p95_suppressed_across_label_sets() -> None: p = Prom( 'c_sum{dp="0"} 10\nc_sum{dp="1"} 20\n' 'c_count{dp="0"} 2\nc_count{dp="1"} 3\n' 'c_bucket{dp="0",le="1"} 2\nc_bucket{dp="1",le="1"} 3\n' ) h = p.histogram(["c"]) assert h["mean"] == 6.0 # still aggregated assert h["p95"] is None # but blending label sets would invent a percentile def test_histogram_missing_is_none() -> None: assert Prom("unrelated 1\n").histogram(["b"]) is None def test_quantile_on_empty_buckets() -> None: from app import _quantile assert _quantile([], 0, 0.95) is None # --------------------------------------------------------------------------- # # gufo field map # --------------------------------------------------------------------------- # def test_gufo_field_map() -> None: m = parse_gufo_metrics(GUFO_FIXTURE) assert m["requests_processing"] == {"value": 2.0, "source": "llamacpp:requests_processing", "derived": False} assert m["requests_deferred"]["value"] == 1.0 assert m["kv_cache_usage_ratio"]["value"] == pytest.approx(0.42) assert m["tokens_prompt_total"]["value"] == 12345.0 assert m["tokens_generated_total"]["value"] == 6789.0 assert m["prompt_tokens_per_second"]["value"] == pytest.approx(1900.5) assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(1900.5) # no rates injected -> the cell falls back to the scraped single-request gauge assert m["decode_tokens_per_second"]["value"] == pytest.approx(31.5) assert m["decode_tokens_per_second"]["source"] == "llamacpp:decode_tokens_per_second" assert m["decode_tokens_per_second_native"]["value"] == pytest.approx(31.5) assert m["tokens_total"] == { "value": 19134.0, # no cached series in this fixture -> prompt + generated "source": "computed:prompt+cached+generated", "derived": False, } def test_gufo_total_counts_cache_hits_once() -> None: # gufo's prompt_tokens_total EXCLUDES cache hits, so a naive sum under-reports m = parse_gufo_metrics( "llamacpp:prompt_tokens_total 100\n" "llamacpp:prompt_tokens_cached_total 900\n" "llamacpp:tokens_predicted_total 50\n" ) assert m["tokens_total"]["value"] == 1050.0 assert m["prompt_cache_hit_ratio"]["value"] == pytest.approx(0.9) def test_gufo_spec_accept_rate_absent_without_the_series() -> None: assert parse_gufo_metrics(GUFO_FIXTURE)["spec_accept_rate"]["value"] is None def test_gufo_spec_accept_rate_from_real_names() -> None: m = parse_gufo_metrics( "llamacpp:spec_decode_num_drafts_total 100\n" "llamacpp:spec_decode_num_draft_tokens_total 400\n" "llamacpp:spec_decode_num_accepted_tokens_total 200\n" ) assert m["spec_accept_rate"]["value"] == pytest.approx(0.5) assert m["spec_accept_rate"]["derived"] is True def test_gufo_kv_ratio_uses_max_not_sum() -> None: # two contexts at 0.4 and 0.5 must report 0.5, never 0.9 m = parse_gufo_metrics("llamacpp:kv_cache_usage_ratio 0.4\nllamacpp:kv_cache_usage_ratio 0.5\n") assert m["kv_cache_usage_ratio"]["value"] == pytest.approx(0.5) assert m["kv_cache_usage_ratio"]["value"] <= 1.0 def test_gufo_empty_exposition_is_all_none() -> None: m = parse_gufo_metrics("") assert all(cell["value"] is None for cell in m.values()) assert m["tokens_total"]["value"] is None # --------------------------------------------------------------------------- # # vLLM field map # --------------------------------------------------------------------------- # def test_vllm_field_map() -> None: m = parse_vllm_metrics(VLLM_FIXTURE) assert m["num_requests_running"]["value"] == 3.0 assert m["num_requests_waiting"]["value"] == 5.0 assert m["waiting_by_reason"]["value"] == {"capacity": 5.0, "defragmentation": 0.0} assert m["waiting_capacity"]["value"] == 5.0 assert m["gpu_cache_usage"]["value"] == pytest.approx(0.61) assert m["prompt_tokens_total"]["value"] == 1_000_000.0 assert m["generation_tokens_total"]["value"] == 250_000.0 assert m["time_to_first_token_seconds"]["mean"] == pytest.approx(1.0) assert m["time_to_first_token_seconds__mean"]["value"] == pytest.approx(1.0) assert m["request_queue_time_seconds"] is None # absent on this server -> n/a def test_vllm_decode_throughput_derived_from_tpot() -> None: m = parse_vllm_metrics(VLLM_FIXTURE) tpot = m["time_per_output_token_seconds"]["mean"] assert m["decode_tokens_per_second"]["derived"] is True assert m["decode_tokens_per_second"]["source"] == "computed:running/tpot" assert m["decode_tokens_per_second"]["value"] == pytest.approx(3 / tpot) assert m["per_request_decode_tokens_per_second"]["value"] == pytest.approx(1 / tpot) def test_vllm_native_throughput_wins_over_derivation() -> None: m = parse_vllm_metrics( 'vllm:num_requests_running{model_name="m"} 2\n' 'vllm:generation_throughput{model_name="m"} 77.5\n' "vllm:time_per_output_token_seconds_sum 1.0\n" "vllm:time_per_output_token_seconds_count 100\n" ) assert m["decode_tokens_per_second"] == { "value": 77.5, "source": "vllm:generation_throughput", "derived": False, } # per-request figure is still useful next to a native aggregate assert m["per_request_decode_tokens_per_second"]["value"] == pytest.approx(100.0) def test_vllm_inter_token_latency_alias() -> None: m = parse_vllm_metrics( "vllm:inter_token_latency_seconds_sum 4\nvllm:inter_token_latency_seconds_count 8\n" ) assert m["time_per_output_token_seconds"]["source"] == "vllm:inter_token_latency_seconds" assert m["time_per_output_token_seconds"]["mean"] == pytest.approx(0.5) def test_partial_exposition_totals_do_not_raise() -> None: """Regression: `None + value` used to 500 the endpoint when one of the two token counters was missing — a renamed series must shrink the total, not break it.""" assert parse_vllm_metrics("vllm:prompt_tokens_total 100\n")["tokens_total"]["value"] == 100.0 assert parse_vllm_metrics("vllm:generation_tokens_total 7\n")["tokens_total"]["value"] == 7.0 def test_vllm_empty_exposition_is_all_none() -> None: m = parse_vllm_metrics("# nothing here\nunrelated_metric_total 1\n") for key in ( "num_requests_running", "num_requests_waiting", "gpu_cache_usage", "prompt_tokens_total", "decode_tokens_per_second", "tokens_total", ): assert m[key]["value"] is None, key assert m["time_to_first_token_seconds"] is None assert m["waiting_by_reason"]["value"] == {} def test_values_are_finite_or_none() -> None: for cell in parse_vllm_metrics(VLLM_FIXTURE).values(): if isinstance(cell, dict) and "value" in cell and isinstance(cell["value"], float): assert math.isfinite(cell["value"])