initial import: read-only gufo + vLLM dashboard
Build and Push Container / build-and-push (push) Successful in 38s
Build and Push Container / build-and-push (push) Successful in 38s
Single-file FastAPI proxy plus a no-CDN vanilla-JS page for watching a gufo
server and a vLLM server side by side: one Prometheus parser (class Prom) with
per-engine field maps, measured Δcounter/Δt rates for engines that export no
rate gauge, and strict read-only GET-only routing. Engines are opt-in via
*_BASE_URL — unset means no panel, no poll, and a 404 {"configured": false}
without dialling.
Includes the container image and CI so the dashboard can be deployed as an
image rather than a checkout:
- Containerfile: uv-locked two-stage build on python:3.14-slim, non-root
(uid 10001), binds 8090 (8000 is vLLM's port in this fleet), healthcheck on
/api/config so a dead engine never marks the dashboard unhealthy. No frontend
build stage — static/index.html is bundler-free by design and is copied as-is.
- .gitea/workflows/build-and-push.yml: publishes
gitea.reeseapps.com/Experimental/llm-monitor on main + release tags.
114 tests, no network and no real engines required.
LLM-Generated: true
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
"""Test fixtures: a fake upstream (httpx-shaped) so no sockets are involved.
|
||||
|
||||
The app only ever calls `client.get(...)`, so a stub with an async `get` is a
|
||||
complete substitute for httpx.AsyncClient — which keeps the suite network-free,
|
||||
deterministic, and able to simulate outages exactly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app as app_module
|
||||
|
||||
GUFO_FIXTURE = """# HELP llamacpp:requests_processing how many requests are processing
|
||||
# TYPE llamacpp:requests_processing gauge
|
||||
llama:requests_processing 99
|
||||
llamacpp:requests_processing 2
|
||||
# TYPE llamacpp:requests_deferred gauge
|
||||
llamacpp:requests_deferred 1
|
||||
# TYPE llamacpp:kv_cache_usage_ratio gauge
|
||||
llamacpp:kv_cache_usage_ratio 0.42
|
||||
# TYPE llamacpp:tokens_prompt_total counter
|
||||
llamacpp:tokens_prompt_total{model="turbo"} 12345
|
||||
# TYPE llamacpp:tokens_generated_total counter
|
||||
llamacpp:tokens_generated_total{model="turbo"} 6789
|
||||
# TYPE llamacpp:prompt_tokens_per_second gauge
|
||||
llamacpp:prompt_tokens_per_second 1900.5
|
||||
# TYPE llamacpp:decode_tokens_per_second gauge
|
||||
llamacpp:decode_tokens_per_second 31.5
|
||||
"""
|
||||
|
||||
VLLM_FIXTURE = """# TYPE vllm:num_requests_running gauge
|
||||
vllm:num_requests_running{model_name="juggernaut"} 3
|
||||
# TYPE vllm:num_requests_waiting gauge
|
||||
vllm:num_requests_waiting{model_name="juggernaut"} 5
|
||||
# TYPE vllm:num_requests_waiting_by_reason gauge
|
||||
vllm:num_requests_waiting_by_reason{model_name="juggernaut",reason="capacity"} 5
|
||||
vllm:num_requests_waiting_by_reason{model_name="juggernaut",reason="defragmentation"} 0
|
||||
# TYPE vllm:gpu_cache_usage gauge
|
||||
vllm:gpu_cache_usage{model_name="juggernaut"} 0.61
|
||||
# TYPE vllm:prompt_tokens_total counter
|
||||
vllm:prompt_tokens_total{model_name="juggernaut"} 1000000
|
||||
# TYPE vllm:generation_tokens_total counter
|
||||
vllm:generation_tokens_total{model_name="juggernaut"} 250000
|
||||
# TYPE vllm:time_to_first_token_seconds histogram
|
||||
vllm:time_to_first_token_seconds_bucket{le="0.1",model_name="juggernaut"} 2
|
||||
vllm:time_to_first_token_seconds_bucket{le="1",model_name="juggernaut"} 25
|
||||
vllm:time_to_first_token_seconds_bucket{le="+Inf",model_name="juggernaut"} 30
|
||||
vllm:time_to_first_token_seconds_sum{model_name="juggernaut"} 30.0
|
||||
vllm:time_to_first_token_seconds_count{model_name="juggernaut"} 30
|
||||
# TYPE vllm:time_per_output_token_seconds histogram
|
||||
vllm:time_per_output_token_seconds_bucket{le="0.01",model_name="juggernaut"} 1500
|
||||
vllm:time_per_output_token_seconds_bucket{le="+Inf",model_name="juggernaut"} 3000
|
||||
vllm:time_per_output_token_seconds_sum{model_name="juggernaut"} 2.4
|
||||
vllm:time_per_output_token_seconds_count{model_name="juggernaut"} 3000
|
||||
"""
|
||||
|
||||
SLOTS_FIXTURE = (
|
||||
'{"slots":[{"id":0,"state":1,"id_task":12,"n_prompt_tokens":48,'
|
||||
'"n_prompt_tokens_cache":40,"n_prompt_tokens_processed":48,'
|
||||
'"next_token":[{"id":29871,"text":" ","prob":0.98,"n_decoded":3,"n_remain":127}],'
|
||||
'"n_remain":128}]}'
|
||||
)
|
||||
|
||||
|
||||
def response(
|
||||
status: int = 200,
|
||||
content: bytes = b"",
|
||||
content_type: str = "text/plain; version=0.0.4",
|
||||
) -> httpx.Response:
|
||||
# A real httpx.Response rather than a mock: .text / .content / .reason_phrase
|
||||
# then behave exactly as app.fetch() expects. Attaching a request is mandatory
|
||||
# before a body is read.
|
||||
return httpx.Response(
|
||||
status,
|
||||
content=content,
|
||||
headers={"content-type": content_type},
|
||||
request=httpx.Request("GET", "http://upstream.test"),
|
||||
)
|
||||
|
||||
|
||||
class FakeUpstream:
|
||||
"""Stands in for httpx.AsyncClient and records what was requested."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.routes: dict[str, object] = {}
|
||||
self.calls: list[tuple[str, dict[str, str]]] = []
|
||||
|
||||
def route(self, url: str, result: object) -> None:
|
||||
self.routes[url] = result
|
||||
|
||||
async def get(self, url: str, headers: dict[str, str] | None = None) -> httpx.Response:
|
||||
self.calls.append((url, dict(headers or {})))
|
||||
result = self.routes.get(url)
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
if result is None:
|
||||
return response(404, b'{"detail":"Not Found"}', "application/json")
|
||||
return result # type: ignore[return-value]
|
||||
|
||||
@property
|
||||
def requested_urls(self) -> list[str]:
|
||||
return [url for url, _headers in self.calls]
|
||||
|
||||
@property
|
||||
def methods(self) -> set[str]:
|
||||
# the only recorded verb is GET; anything else would not reach the stub
|
||||
return {"GET"} if self.calls else set()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upstream(monkeypatch) -> FakeUpstream:
|
||||
fake = FakeUpstream()
|
||||
monkeypatch.setattr(app_module, "GUFO_BASE_URL", "http://gufo.test")
|
||||
monkeypatch.setattr(app_module, "VLLM_BASE_URL", "http://vllm.test")
|
||||
monkeypatch.setattr(app_module, "GUFO_API_KEY", "")
|
||||
monkeypatch.setattr(app_module, "VLLM_API_KEY", "")
|
||||
return fake
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(upstream: FakeUpstream):
|
||||
"""TestClient without `with` => lifespan skipped, so we inject the stub client."""
|
||||
test_client = TestClient(app_module.app)
|
||||
app_module.app.state.client = upstream
|
||||
return test_client
|
||||
Vendored
+49
@@ -0,0 +1,49 @@
|
||||
# Captured verbatim from http://deskwork.reeselink.com:8731/metrics (gufo 0.16.0, Qwen3.8 Flash Next,
|
||||
# --sessions 2, --speculative mtp). Re-capture after an engine upgrade; see tests/test_real_contracts.py.
|
||||
# Values are point-in-time and drift between captures — assert on names/sources, not exact numbers.
|
||||
|
||||
# HELP gufo_device_lost_total Confirmed GPU context losses
|
||||
# TYPE gufo_device_lost_total counter
|
||||
gufo_device_lost_total 0
|
||||
# HELP llamacpp:prompt_tokens_total Total prompt tokens processed, excluding cache hits
|
||||
# TYPE llamacpp:prompt_tokens_total counter
|
||||
llamacpp:prompt_tokens_total 374263
|
||||
# HELP llamacpp:prompt_tokens_cached_total Total prompt tokens reused from cache
|
||||
# TYPE llamacpp:prompt_tokens_cached_total counter
|
||||
llamacpp:prompt_tokens_cached_total 29650554
|
||||
# HELP llamacpp:prompt_seconds_total Prompt process time
|
||||
# TYPE llamacpp:prompt_seconds_total counter
|
||||
llamacpp:prompt_seconds_total 341.91072852000008
|
||||
# HELP llamacpp:tokens_predicted_total Total tokens generated
|
||||
# TYPE llamacpp:tokens_predicted_total counter
|
||||
llamacpp:tokens_predicted_total 173091
|
||||
# HELP llamacpp:tokens_predicted_seconds_total Predict process time
|
||||
# TYPE llamacpp:tokens_predicted_seconds_total counter
|
||||
llamacpp:tokens_predicted_seconds_total 5458.0845807790029
|
||||
# HELP llamacpp:n_tokens_max Largest observed n_tokens.
|
||||
# TYPE llamacpp:n_tokens_max counter
|
||||
llamacpp:n_tokens_max 252622
|
||||
# HELP llamacpp:spec_decode_num_drafts_total Total speculative verification rounds
|
||||
# TYPE llamacpp:spec_decode_num_drafts_total counter
|
||||
llamacpp:spec_decode_num_drafts_total 54515
|
||||
# HELP llamacpp:spec_decode_num_draft_tokens_total Total draft tokens proposed
|
||||
# TYPE llamacpp:spec_decode_num_draft_tokens_total counter
|
||||
llamacpp:spec_decode_num_draft_tokens_total 158803
|
||||
# HELP llamacpp:spec_decode_num_accepted_tokens_total Total draft tokens accepted
|
||||
# TYPE llamacpp:spec_decode_num_accepted_tokens_total counter
|
||||
llamacpp:spec_decode_num_accepted_tokens_total 117552
|
||||
# HELP llamacpp:prompt_tokens_seconds Prompt processing speed in tokens per second
|
||||
# TYPE llamacpp:prompt_tokens_seconds gauge
|
||||
llamacpp:prompt_tokens_seconds 820.2841172907755
|
||||
# HELP llamacpp:predicted_tokens_seconds Generation speed in tokens per second
|
||||
# TYPE llamacpp:predicted_tokens_seconds gauge
|
||||
llamacpp:predicted_tokens_seconds 43.015060732721352
|
||||
# HELP llamacpp:requests_processing Number of admitted requests, including cache preparation
|
||||
# TYPE llamacpp:requests_processing gauge
|
||||
llamacpp:requests_processing 0
|
||||
# HELP llamacpp:requests_deferred Number of requests waiting for a session
|
||||
# TYPE llamacpp:requests_deferred gauge
|
||||
llamacpp:requests_deferred 0
|
||||
# HELP llamacpp:kv_cache_usage_ratio In-flight prompt and generated tokens over sessions times context; excludes retained cache
|
||||
# TYPE llamacpp:kv_cache_usage_ratio gauge
|
||||
llamacpp:kv_cache_usage_ratio 0
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
[{"id":0,"n_ctx":262144,"speculative":true,"is_processing":false,"id_task":-1,"task_id":-1,"state":0,"n_prompt_tokens":0,"n_prompt_tokens_cache":0,"n_prompt_tokens_processed":0,"prompt":"","next_token":[{"has_next_token":false,"has_new_line":false,"n_remain":-1,"n_decoded":0}],"model":"turbo"},{"id":1,"n_ctx":262144,"speculative":true,"is_processing":false,"id_task":-1,"task_id":-1,"state":0,"n_prompt_tokens":0,"n_prompt_tokens_cache":0,"n_prompt_tokens_processed":0,"prompt":"","next_token":[{"has_next_token":false,"has_new_line":false,"n_remain":-1,"n_decoded":0}],"model":"turbo"}]
|
||||
Vendored
+163
@@ -0,0 +1,163 @@
|
||||
# Captured verbatim from http://driveripper.reeselink.com:8000/metrics (vllm-radiance).
|
||||
# Trimmed only by dropping series the dashboard does not read; every kept line is real.
|
||||
process_cpu_seconds_total 240.62
|
||||
# HELP vllm:num_requests_running Number of requests in model execution batches.
|
||||
# TYPE vllm:num_requests_running gauge
|
||||
vllm:num_requests_running{engine="0",model_name="juggernaut"} 2.0
|
||||
# HELP vllm:num_requests_waiting Number of requests waiting to be processed.
|
||||
# TYPE vllm:num_requests_waiting gauge
|
||||
vllm:num_requests_waiting{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:num_requests_waiting_by_reason Number of waiting requests by reason. Reason labels: 'capacity' = waiting for scheduling capacity; 'deferred' = deferred by transient constraints (LoRA budget, KV transfer, blocked status). Sum of all reasons equals vllm:num_requests_waiting.
|
||||
# TYPE vllm:num_requests_waiting_by_reason gauge
|
||||
vllm:num_requests_waiting_by_reason{engine="0",model_name="juggernaut",reason="capacity"} 0.0
|
||||
vllm:num_requests_waiting_by_reason{engine="0",model_name="juggernaut",reason="deferred"} 0.0
|
||||
# HELP vllm:kv_cache_usage_perc KV-cache usage. 1 means 100 percent usage.
|
||||
# TYPE vllm:kv_cache_usage_perc gauge
|
||||
vllm:kv_cache_usage_perc{engine="0",model_name="juggernaut"} 0.39790575916230364
|
||||
# HELP vllm:prefix_cache_queries_total Prefix cache queries, in terms of number of queried tokens.
|
||||
# TYPE vllm:prefix_cache_queries_total counter
|
||||
vllm:prefix_cache_queries_total{engine="0",model_name="juggernaut"} 8.582355e+06
|
||||
# HELP vllm:prefix_cache_hits_total Prefix cache hits, in terms of number of cached tokens.
|
||||
# TYPE vllm:prefix_cache_hits_total counter
|
||||
vllm:prefix_cache_hits_total{engine="0",model_name="juggernaut"} 4.464096e+06
|
||||
# HELP vllm:num_preemptions_total Cumulative number of preemption from the engine.
|
||||
# TYPE vllm:num_preemptions_total counter
|
||||
vllm:num_preemptions_total{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:prompt_tokens_total Number of prefill tokens processed.
|
||||
# TYPE vllm:prompt_tokens_total counter
|
||||
vllm:prompt_tokens_total{engine="0",model_name="juggernaut"} 8.582355e+06
|
||||
# HELP vllm:generation_tokens_total Number of generation tokens processed.
|
||||
# TYPE vllm:generation_tokens_total counter
|
||||
vllm:generation_tokens_total{engine="0",model_name="juggernaut"} 339918.0
|
||||
vllm:request_success_total{engine="0",finished_reason="stop",model_name="juggernaut"} 2145.0
|
||||
vllm:request_success_total{engine="0",finished_reason="length",model_name="juggernaut"} 0.0
|
||||
vllm:request_success_total{engine="0",finished_reason="abort",model_name="juggernaut"} 0.0
|
||||
vllm:request_success_total{engine="0",finished_reason="error",model_name="juggernaut"} 0.0
|
||||
vllm:request_success_total{engine="0",finished_reason="repetition",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:time_to_first_token_seconds Histogram of time to first token in seconds.
|
||||
# TYPE vllm:time_to_first_token_seconds histogram
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.001",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.005",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.01",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.02",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.04",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.06",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.08",model_name="juggernaut"} 3.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.1",model_name="juggernaut"} 3.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.25",model_name="juggernaut"} 341.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 724.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.75",model_name="juggernaut"} 1076.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 1291.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 2114.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 2148.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="7.5",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="80.0",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="160.0",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="640.0",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="2560.0",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_count{engine="0",model_name="juggernaut"} 2149.0
|
||||
vllm:time_to_first_token_seconds_sum{engine="0",model_name="juggernaut"} 1861.7778627872467
|
||||
# HELP vllm:inter_token_latency_seconds Histogram of inter-token latency in seconds.
|
||||
# TYPE vllm:inter_token_latency_seconds histogram
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.01",model_name="juggernaut"} 0.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.025",model_name="juggernaut"} 0.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.05",model_name="juggernaut"} 327550.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.075",model_name="juggernaut"} 337018.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.1",model_name="juggernaut"} 337051.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.15",model_name="juggernaut"} 337141.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.2",model_name="juggernaut"} 337229.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 337313.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.4",model_name="juggernaut"} 337368.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 337430.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.75",model_name="juggernaut"} 337631.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 337651.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="7.5",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="80.0",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_count{engine="0",model_name="juggernaut"} 337769.0
|
||||
vllm:inter_token_latency_seconds_sum{engine="0",model_name="juggernaut"} 12194.350635488052
|
||||
# HELP vllm:request_queue_time_seconds Histogram of time spent in WAITING phase for request.
|
||||
# TYPE vllm:request_queue_time_seconds histogram
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 2143.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 2143.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 2143.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 2143.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_count{engine="0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_queue_time_seconds_sum{engine="0",model_name="juggernaut"} 2.707464350562077
|
||||
vllm:cache_config_info{_block_size_resolved="True",block_size="1568",cache_dtype="fp8",enable_mamba_fine_grained_prefix_cache="False",enable_prefix_caching="True",engine="0",gpu_memory_utilization="0.95",is_attention_free="False",kv_cache_dtype_skip_layers="[]",kv_cache_layout="None",kv_cache_max_concurrency="2.1333333333333333",kv_cache_memory_bytes="None",kv_cache_size_tokens="279620",kv_offloading_backend="native",kv_offloading_size="None",kv_sharing_fast_prefill="False",mamba_block_size="1568",mamba_cache_dtype="auto",mamba_cache_mode="align",mamba_page_size_padded="None",mamba_ssm_cache_dtype="float32",num_cpu_blocks="None",num_gpu_blocks="192",num_gpu_blocks_override="None",prefix_cache_retention_interval="0",prefix_caching_hash_algo="sha256",prefix_match_unit="None",replayssm_buffer_len="16",skip_page_size_padded="None",sliding_window="None",use_kda_recoverssm="False",use_replayssm="False",user_specified_block_size="False",user_specified_mamba_block_size="False"} 1.0
|
||||
|
||||
# prefill series (used for the lifetime prompt-speed derivation)
|
||||
# HELP vllm:request_prompt_tokens Number of prefill tokens processed.
|
||||
# TYPE vllm:request_prompt_tokens histogram
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 5.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 5.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 20.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 71.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 361.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 638.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 1134.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 1943.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 2026.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 2072.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 2112.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prompt_tokens_count{engine="0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prompt_tokens_sum{engine="0",model_name="juggernaut"} 8.441975e+06
|
||||
# HELP vllm:request_prefill_time_seconds Histogram of time spent in PREFILL phase for request.
|
||||
# TYPE vllm:request_prefill_time_seconds histogram
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 472.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 742.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 1138.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 1318.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 1976.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 2078.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 2117.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 2144.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_count{engine="0",model_name="juggernaut"} 2145.0
|
||||
vllm:request_prefill_time_seconds_sum{engine="0",model_name="juggernaut"} 1796.7229758803733
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Prometheus text parser + per-engine field maps.
|
||||
|
||||
This is the code most likely to silently rot: an engine renames a series and the
|
||||
dashboard must degrade to `n/a`, not crash and not misreport. The aggregation
|
||||
rules (sum / max / avg) are asserted explicitly because getting them wrong is
|
||||
how a cache ratio ends up over 100%.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from app import Prom, parse_gufo_metrics, parse_vllm_metrics
|
||||
from tests.conftest import GUFO_FIXTURE, VLLM_FIXTURE
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Prom
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_ignores_comments_help_and_type_lines() -> None:
|
||||
p = Prom("# HELP a hi\n# TYPE a gauge\na 1\n\n")
|
||||
assert list(p.series) == ["a"]
|
||||
assert p.parse_errors == 0
|
||||
|
||||
|
||||
def test_label_block_with_brace_and_escape() -> None:
|
||||
p = Prom(r'a{x="a}b\"c"} 5')
|
||||
assert p.series["a"][0][0] == {"x": 'a}b"c'}
|
||||
assert p.series["a"][0][1] == 5.0
|
||||
|
||||
|
||||
def test_tolerates_space_before_labels_and_trailing_comment() -> None:
|
||||
p = Prom('a {m="x"} 1 # a comment\nb 2 1700000000000\n')
|
||||
assert p.scalar(["a"]) == 1.0
|
||||
assert p.scalar(["b"]) == 2.0
|
||||
|
||||
|
||||
def test_counts_garbage_instead_of_raising() -> None:
|
||||
p = Prom("this is not prometheus\nalso bad x y z\n")
|
||||
assert p.series == {}
|
||||
assert p.parse_errors == 2
|
||||
|
||||
|
||||
def test_non_finite_values_are_dropped_from_aggregates() -> None:
|
||||
p = Prom("a 1\na nan\na +Inf\na 3\n")
|
||||
assert p.scalar(["a"]) == 4.0 # sum of the finite ones only
|
||||
assert p.scalar(["a"], "max") == 3.0
|
||||
assert p.scalar(["a"], "avg") == 2.0
|
||||
|
||||
|
||||
def test_absent_series_is_none_never_zero() -> None:
|
||||
p = Prom("other 1\n")
|
||||
assert p.scalar(["nope"]) is None
|
||||
assert p.resolve(["nope", "other"]) == "other"
|
||||
|
||||
|
||||
def test_alias_resolution_prefers_first_present() -> None:
|
||||
p = Prom("llama:requests_processing 7\n")
|
||||
assert p.resolve(["llamacpp:requests_processing", "llama:requests_processing"]) == (
|
||||
"llama:requests_processing"
|
||||
)
|
||||
|
||||
|
||||
def test_by_label_grouping() -> None:
|
||||
p = Prom('w{reason="capacity"} 2\nw{reason="capacity"} 3\nw{reason="other"} 1\n')
|
||||
assert p.by_label(["w"], "reason") == {"capacity": 5.0, "other": 1.0}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Histograms
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_histogram_mean_and_p95() -> None:
|
||||
p = Prom('b_bucket{le="1"} 1\nb_bucket{le="10"} 10\nb_bucket{le="+Inf"} 10\nb_sum 40\nb_count 10\n')
|
||||
h = p.histogram(["b"])
|
||||
assert h is not None
|
||||
assert h["mean"] == 4.0
|
||||
# 95th of 10 observations is the 9.5th, interpolated inside the (1,10] bucket
|
||||
assert h["p95"] == pytest.approx(9.5)
|
||||
|
||||
|
||||
def test_histogram_p95_when_mass_exceeds_the_last_finite_bucket() -> None:
|
||||
# almost everything above the last real bound: report that bound, never +Inf
|
||||
p = Prom('b_bucket{le="1"} 5\nb_bucket{le="10"} 9\nb_bucket{le="+Inf"} 10\nb_sum 40\nb_count 10\n')
|
||||
assert p.histogram(["b"])["p95"] == pytest.approx(10.0)
|
||||
|
||||
|
||||
def test_histogram_resolves_without_a_bare_series_line() -> None:
|
||||
# real expositions never emit a series named after the histogram itself
|
||||
p = Prom('vllm:x_sum 6\nvllm:x_count 3\nvllm:x_bucket{le="+Inf"} 3\n')
|
||||
assert p.histogram(["vllm:x"])["mean"] == 2.0
|
||||
|
||||
|
||||
def test_histogram_p95_suppressed_across_label_sets() -> None:
|
||||
p = Prom(
|
||||
'c_sum{dp="0"} 10\nc_sum{dp="1"} 20\n'
|
||||
'c_count{dp="0"} 2\nc_count{dp="1"} 3\n'
|
||||
'c_bucket{dp="0",le="1"} 2\nc_bucket{dp="1",le="1"} 3\n'
|
||||
)
|
||||
h = p.histogram(["c"])
|
||||
assert h["mean"] == 6.0 # still aggregated
|
||||
assert h["p95"] is None # but blending label sets would invent a percentile
|
||||
|
||||
|
||||
def test_histogram_missing_is_none() -> None:
|
||||
assert Prom("unrelated 1\n").histogram(["b"]) is None
|
||||
|
||||
|
||||
def test_quantile_on_empty_buckets() -> None:
|
||||
from app import _quantile
|
||||
|
||||
assert _quantile([], 0, 0.95) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# gufo field map
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_gufo_field_map() -> None:
|
||||
m = parse_gufo_metrics(GUFO_FIXTURE)
|
||||
assert m["requests_processing"] == {"value": 2.0, "source": "llamacpp:requests_processing", "derived": False}
|
||||
assert m["requests_deferred"]["value"] == 1.0
|
||||
assert m["kv_cache_usage_ratio"]["value"] == pytest.approx(0.42)
|
||||
assert m["tokens_prompt_total"]["value"] == 12345.0
|
||||
assert m["tokens_generated_total"]["value"] == 6789.0
|
||||
assert m["prompt_tokens_per_second"]["value"] == pytest.approx(1900.5)
|
||||
assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(1900.5)
|
||||
# no rates injected -> the cell falls back to the scraped single-request gauge
|
||||
assert m["decode_tokens_per_second"]["value"] == pytest.approx(31.5)
|
||||
assert m["decode_tokens_per_second"]["source"] == "llamacpp:decode_tokens_per_second"
|
||||
assert m["decode_tokens_per_second_native"]["value"] == pytest.approx(31.5)
|
||||
assert m["tokens_total"] == {
|
||||
"value": 19134.0, # no cached series in this fixture -> prompt + generated
|
||||
"source": "computed:prompt+cached+generated",
|
||||
"derived": False,
|
||||
}
|
||||
|
||||
|
||||
def test_gufo_total_counts_cache_hits_once() -> None:
|
||||
# gufo's prompt_tokens_total EXCLUDES cache hits, so a naive sum under-reports
|
||||
m = parse_gufo_metrics(
|
||||
"llamacpp:prompt_tokens_total 100\n"
|
||||
"llamacpp:prompt_tokens_cached_total 900\n"
|
||||
"llamacpp:tokens_predicted_total 50\n"
|
||||
)
|
||||
assert m["tokens_total"]["value"] == 1050.0
|
||||
assert m["prompt_cache_hit_ratio"]["value"] == pytest.approx(0.9)
|
||||
|
||||
|
||||
def test_gufo_spec_accept_rate_absent_without_the_series() -> None:
|
||||
assert parse_gufo_metrics(GUFO_FIXTURE)["spec_accept_rate"]["value"] is None
|
||||
|
||||
|
||||
def test_gufo_spec_accept_rate_from_real_names() -> None:
|
||||
m = parse_gufo_metrics(
|
||||
"llamacpp:spec_decode_num_drafts_total 100\n"
|
||||
"llamacpp:spec_decode_num_draft_tokens_total 400\n"
|
||||
"llamacpp:spec_decode_num_accepted_tokens_total 200\n"
|
||||
)
|
||||
assert m["spec_accept_rate"]["value"] == pytest.approx(0.5)
|
||||
assert m["spec_accept_rate"]["derived"] is True
|
||||
|
||||
|
||||
def test_gufo_kv_ratio_uses_max_not_sum() -> None:
|
||||
# two contexts at 0.4 and 0.5 must report 0.5, never 0.9
|
||||
m = parse_gufo_metrics("llamacpp:kv_cache_usage_ratio 0.4\nllamacpp:kv_cache_usage_ratio 0.5\n")
|
||||
assert m["kv_cache_usage_ratio"]["value"] == pytest.approx(0.5)
|
||||
assert m["kv_cache_usage_ratio"]["value"] <= 1.0
|
||||
|
||||
|
||||
def test_gufo_empty_exposition_is_all_none() -> None:
|
||||
m = parse_gufo_metrics("")
|
||||
assert all(cell["value"] is None for cell in m.values())
|
||||
assert m["tokens_total"]["value"] is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# vLLM field map
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_vllm_field_map() -> None:
|
||||
m = parse_vllm_metrics(VLLM_FIXTURE)
|
||||
assert m["num_requests_running"]["value"] == 3.0
|
||||
assert m["num_requests_waiting"]["value"] == 5.0
|
||||
assert m["waiting_by_reason"]["value"] == {"capacity": 5.0, "defragmentation": 0.0}
|
||||
assert m["waiting_capacity"]["value"] == 5.0
|
||||
assert m["gpu_cache_usage"]["value"] == pytest.approx(0.61)
|
||||
assert m["prompt_tokens_total"]["value"] == 1_000_000.0
|
||||
assert m["generation_tokens_total"]["value"] == 250_000.0
|
||||
assert m["time_to_first_token_seconds"]["mean"] == pytest.approx(1.0)
|
||||
assert m["time_to_first_token_seconds__mean"]["value"] == pytest.approx(1.0)
|
||||
assert m["request_queue_time_seconds"] is None # absent on this server -> n/a
|
||||
|
||||
|
||||
def test_vllm_decode_throughput_derived_from_tpot() -> None:
|
||||
m = parse_vllm_metrics(VLLM_FIXTURE)
|
||||
tpot = m["time_per_output_token_seconds"]["mean"]
|
||||
assert m["decode_tokens_per_second"]["derived"] is True
|
||||
assert m["decode_tokens_per_second"]["source"] == "computed:running/tpot"
|
||||
assert m["decode_tokens_per_second"]["value"] == pytest.approx(3 / tpot)
|
||||
assert m["per_request_decode_tokens_per_second"]["value"] == pytest.approx(1 / tpot)
|
||||
|
||||
|
||||
def test_vllm_native_throughput_wins_over_derivation() -> None:
|
||||
m = parse_vllm_metrics(
|
||||
'vllm:num_requests_running{model_name="m"} 2\n'
|
||||
'vllm:generation_throughput{model_name="m"} 77.5\n'
|
||||
"vllm:time_per_output_token_seconds_sum 1.0\n"
|
||||
"vllm:time_per_output_token_seconds_count 100\n"
|
||||
)
|
||||
assert m["decode_tokens_per_second"] == {
|
||||
"value": 77.5,
|
||||
"source": "vllm:generation_throughput",
|
||||
"derived": False,
|
||||
}
|
||||
# per-request figure is still useful next to a native aggregate
|
||||
assert m["per_request_decode_tokens_per_second"]["value"] == pytest.approx(100.0)
|
||||
|
||||
|
||||
def test_vllm_inter_token_latency_alias() -> None:
|
||||
m = parse_vllm_metrics(
|
||||
"vllm:inter_token_latency_seconds_sum 4\nvllm:inter_token_latency_seconds_count 8\n"
|
||||
)
|
||||
assert m["time_per_output_token_seconds"]["source"] == "vllm:inter_token_latency_seconds"
|
||||
assert m["time_per_output_token_seconds"]["mean"] == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_partial_exposition_totals_do_not_raise() -> None:
|
||||
"""Regression: `None + value` used to 500 the endpoint when one of the two token
|
||||
counters was missing — a renamed series must shrink the total, not break it."""
|
||||
assert parse_vllm_metrics("vllm:prompt_tokens_total 100\n")["tokens_total"]["value"] == 100.0
|
||||
assert parse_vllm_metrics("vllm:generation_tokens_total 7\n")["tokens_total"]["value"] == 7.0
|
||||
|
||||
|
||||
def test_vllm_empty_exposition_is_all_none() -> None:
|
||||
m = parse_vllm_metrics("# nothing here\nunrelated_metric_total 1\n")
|
||||
for key in (
|
||||
"num_requests_running",
|
||||
"num_requests_waiting",
|
||||
"gpu_cache_usage",
|
||||
"prompt_tokens_total",
|
||||
"decode_tokens_per_second",
|
||||
"tokens_total",
|
||||
):
|
||||
assert m[key]["value"] is None, key
|
||||
assert m["time_to_first_token_seconds"] is None
|
||||
assert m["waiting_by_reason"]["value"] == {}
|
||||
|
||||
|
||||
def test_values_are_finite_or_none() -> None:
|
||||
for cell in parse_vllm_metrics(VLLM_FIXTURE).values():
|
||||
if isinstance(cell, dict) and "value" in cell and isinstance(cell["value"], float):
|
||||
assert math.isfinite(cell["value"])
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Proxy routes: read-only guarantee, upstream failure translation, JSON shape.
|
||||
|
||||
Nothing here touches a socket — see conftest.FakeUpstream. The point is to pin
|
||||
the contract the browser depends on: what passes through, what becomes JSON, and
|
||||
exactly how an engine outage shows up.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import app as app_module
|
||||
from tests.conftest import GUFO_FIXTURE, SLOTS_FIXTURE, VLLM_FIXTURE, response
|
||||
|
||||
API_PATHS = ["/api/gufo/slots", "/api/gufo/metrics", "/api/vllm/metrics", "/api/config"]
|
||||
GUFO_PATHS = ["/api/gufo/slots", "/api/gufo/metrics"]
|
||||
VLLM_PATHS = ["/api/vllm/metrics"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# read-only by construction
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("method", ["post", "put", "patch", "delete"])
|
||||
@pytest.mark.parametrize("path", API_PATHS)
|
||||
def test_no_writing_verbs(client, method: str, path: str) -> None:
|
||||
assert getattr(client, method)(path).status_code == 405
|
||||
|
||||
|
||||
def test_only_get_reaches_upstream(client, upstream) -> None:
|
||||
upstream.route("http://gufo.test/slots", response(200, SLOTS_FIXTURE.encode()))
|
||||
client.get("/api/gufo/slots")
|
||||
upstream.route("http://gufo.test/metrics", response(200, GUFO_FIXTURE.encode()))
|
||||
client.get("/api/gufo/metrics")
|
||||
assert upstream.methods == {"GET"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# /api/gufo/slots — passthrough
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_slots_pass_through_byte_for_byte(client, upstream) -> None:
|
||||
upstream.route(
|
||||
"http://gufo.test/slots", response(200, SLOTS_FIXTURE.encode(), "application/json")
|
||||
)
|
||||
r = client.get("/api/gufo/slots")
|
||||
assert r.status_code == 200
|
||||
assert r.content.decode() == SLOTS_FIXTURE # not re-serialised, not reformatted
|
||||
assert r.headers["content-type"].startswith("application/json")
|
||||
assert "no-store" in r.headers["cache-control"]
|
||||
assert r.json()["slots"][0]["id_task"] == 12
|
||||
|
||||
|
||||
def test_slots_accepts_a_bare_array_payload(client, upstream) -> None:
|
||||
"""The deployed gufo returns `[{...}]`, not `{"slots": [...]}`. The proxy must not
|
||||
care — it forwards whatever JSON the engine sent, unchanged."""
|
||||
bare = '[{"id":0,"is_processing":true,"id_task":7,"next_token":[{"n_decoded":3,"n_remain":9}]}]'
|
||||
upstream.route("http://gufo.test/slots", response(200, bare.encode(), "application/json"))
|
||||
r = client.get("/api/gufo/slots")
|
||||
assert r.status_code == 200
|
||||
assert r.json() == json.loads(bare)
|
||||
assert isinstance(r.json(), list)
|
||||
|
||||
|
||||
def test_upstream_content_type_is_passed_through(client, upstream) -> None:
|
||||
# whatever gufo claims, the browser gets the same thing — no re-typing
|
||||
upstream.route(
|
||||
"http://gufo.test/slots",
|
||||
response(200, SLOTS_FIXTURE.encode(), "application/stream+json"),
|
||||
)
|
||||
assert client.get("/api/gufo/slots").headers["content-type"] == "application/stream+json"
|
||||
|
||||
|
||||
def test_slots_falls_back_to_the_alias(client, upstream) -> None:
|
||||
# /slots 404s (the stub's default for unrouted URLs), the alias answers
|
||||
upstream.route("http://gufo.test/v1/slots", response(200, SLOTS_FIXTURE.encode(), "application/json"))
|
||||
r = client.get("/api/gufo/slots")
|
||||
assert r.status_code == 200
|
||||
assert upstream.requested_urls == ["http://gufo.test/slots", "http://gufo.test/v1/slots"]
|
||||
|
||||
|
||||
def test_slots_reports_missing_endpoints(client, upstream) -> None:
|
||||
r = client.get("/api/gufo/slots")
|
||||
assert r.status_code == 502
|
||||
assert "/slots" in r.json()["detail"] and "/v1/slots" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_slots_does_not_swallow_a_server_error(client, upstream) -> None:
|
||||
upstream.route("http://gufo.test/slots", response(500, b"boom"))
|
||||
r = client.get("/api/gufo/slots")
|
||||
assert r.status_code == 502 # engine is unhealthy: that is a bad gateway, not a fallback
|
||||
assert "HTTP 500" in r.json()["detail"]
|
||||
assert upstream.requested_urls == ["http://gufo.test/slots"] # no pointless alias retry
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# /metrics — Prometheus -> JSON
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_gufo_metrics_endpoint(client, upstream) -> None:
|
||||
upstream.route("http://gufo.test/metrics", response(200, GUFO_FIXTURE.encode()))
|
||||
body = client.get("/api/gufo/metrics").json()
|
||||
assert body["ok"] is True
|
||||
assert body["engine"] == "gufo"
|
||||
assert body["upstream"] == "http://gufo.test/metrics"
|
||||
assert body["series"] == 8 # 7 llamacpp:* series + the legacy llama:* one
|
||||
assert body["unparsed"] == 0
|
||||
assert body["metrics"]["requests_processing"]["value"] == 2.0
|
||||
assert body["scraped_at"] > 0
|
||||
|
||||
|
||||
def test_vllm_metrics_endpoint(client, upstream) -> None:
|
||||
upstream.route("http://vllm.test/metrics", response(200, VLLM_FIXTURE.encode()))
|
||||
m = client.get("/api/vllm/metrics").json()["metrics"]
|
||||
assert m["num_requests_running"]["value"] == 3.0
|
||||
assert m["waiting_capacity"]["value"] == 5.0
|
||||
assert m["time_to_first_token_seconds"]["mean"] == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_garbage_exposition_is_ok_with_no_series(client, upstream) -> None:
|
||||
upstream.route("http://vllm.test/metrics", response(200, b"not prometheus at all\n"))
|
||||
body = client.get("/api/vllm/metrics").json()
|
||||
assert body["ok"] is True
|
||||
assert body["series"] == 0 and body["unparsed"] == 1
|
||||
assert body["metrics"]["num_requests_running"]["value"] is None # renders as n/a
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# engine outage translation (the per-engine health contract)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_unreachable_engine_is_a_502_with_a_readable_detail(client, upstream) -> None:
|
||||
upstream.route("http://vllm.test/metrics", httpx.ConnectError("connection refused"))
|
||||
r = client.get("/api/vllm/metrics")
|
||||
assert r.status_code == 502
|
||||
assert r.json()["ok"] is False
|
||||
assert "unreachable" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_slow_engine_is_a_504(client, upstream) -> None:
|
||||
upstream.route("http://gufo.test/metrics", httpx.TimeoutException("too slow"))
|
||||
r = client.get("/api/gufo/metrics")
|
||||
assert r.status_code == 504
|
||||
assert "timed out" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_one_engine_being_down_does_not_affect_the_other(client, upstream) -> None:
|
||||
upstream.route("http://gufo.test/slots", response(200, SLOTS_FIXTURE.encode(), "application/json"))
|
||||
upstream.route("http://vllm.test/metrics", httpx.ConnectError("connection refused"))
|
||||
assert client.get("/api/vllm/metrics").status_code == 502
|
||||
assert client.get("/api/gufo/slots").status_code == 200
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# auth + config
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_no_authorization_header_without_keys(client, upstream) -> None:
|
||||
upstream.route("http://gufo.test/slots", response(200, b"{}", "application/json"))
|
||||
upstream.route("http://vllm.test/metrics", response(200, b"", "text/plain"))
|
||||
client.get("/api/gufo/slots")
|
||||
client.get("/api/vllm/metrics")
|
||||
assert all("authorization" not in headers for _url, headers in upstream.calls)
|
||||
|
||||
|
||||
def test_bearer_tokens_are_per_engine(client, upstream, monkeypatch) -> None:
|
||||
monkeypatch.setattr(app_module, "GUFO_API_KEY", "sk-gufo")
|
||||
monkeypatch.setattr(app_module, "VLLM_API_KEY", "sk-vllm")
|
||||
upstream.route("http://gufo.test/slots", response(200, b"{}", "application/json"))
|
||||
upstream.route("http://vllm.test/metrics", response(200, b"", "text/plain"))
|
||||
client.get("/api/gufo/slots")
|
||||
client.get("/api/vllm/metrics")
|
||||
assert upstream.calls[0][1]["Authorization"] == "Bearer sk-gufo"
|
||||
assert upstream.calls[1][1]["Authorization"] == "Bearer sk-vllm"
|
||||
|
||||
|
||||
def test_config_never_leaks_keys(client, monkeypatch) -> None:
|
||||
monkeypatch.setattr(app_module, "GUFO_API_KEY", "sk-super-secret")
|
||||
r = client.get("/api/config")
|
||||
assert "sk-super-secret" not in r.text
|
||||
body = r.json()
|
||||
assert body["gufo_auth"] is True and body["vllm_auth"] is False
|
||||
assert body["poll_ms"] == app_module.POLL_MS
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# opt-in engines: an unset *_BASE_URL means this deployment does not run that server
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize(
|
||||
"raw, expected",
|
||||
[
|
||||
("http://box:1", "http://box:1"),
|
||||
("http://box:1/", "http://box:1"), # never a doubled //metrics
|
||||
(" http://box:1 ", "http://box:1"),
|
||||
("", ""), # set-but-empty is 'not configured', not 'http:///metrics'
|
||||
(" ", ""),
|
||||
],
|
||||
)
|
||||
def test_base_url_env_parsing(monkeypatch, raw: str, expected: str) -> None:
|
||||
monkeypatch.setenv("TEST_ENGINE_BASE_URL", raw)
|
||||
assert app_module._base_url_from_env("TEST_ENGINE_BASE_URL") == expected
|
||||
# and unset means the same as blank: no default host is invented
|
||||
monkeypatch.delenv("TEST_ENGINE_BASE_URL")
|
||||
assert app_module._base_url_from_env("TEST_ENGINE_BASE_URL") == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", GUFO_PATHS + VLLM_PATHS)
|
||||
def test_unset_base_url_is_404_not_configured(client, upstream, monkeypatch, path: str) -> None:
|
||||
"""A server nobody runs must not look like a server that is down: no upstream call,
|
||||
no 502, and a detail that names the env var to set."""
|
||||
engine = "gufo" if "gufo" in path else "vllm"
|
||||
monkeypatch.setattr(app_module, f"{engine.upper()}_BASE_URL", "")
|
||||
r = client.get(path)
|
||||
assert r.status_code == 404
|
||||
body = r.json()
|
||||
assert body["ok"] is False
|
||||
assert body["configured"] is False
|
||||
assert body["engine"] == engine
|
||||
assert f"{engine.upper()}_BASE_URL" in body["detail"]
|
||||
assert "no such route" not in body["detail"] # distinct from an unknown path
|
||||
assert upstream.requested_urls == [] # nothing was dialled
|
||||
|
||||
|
||||
def test_one_configured_engine_is_unaffected_by_the_other_being_absent(
|
||||
client, upstream, monkeypatch
|
||||
) -> None:
|
||||
"""The vLLM-only recipient of this repo gets a working dashboard, not a half-broken one."""
|
||||
monkeypatch.setattr(app_module, "GUFO_BASE_URL", "")
|
||||
upstream.route("http://vllm.test/metrics", response(200, VLLM_FIXTURE.encode()))
|
||||
assert client.get("/api/gufo/metrics").status_code == 404
|
||||
assert client.get("/api/gufo/slots").status_code == 404
|
||||
body = client.get("/api/vllm/metrics").json()
|
||||
assert body["ok"] is True
|
||||
assert upstream.requested_urls == ["http://vllm.test/metrics"]
|
||||
|
||||
|
||||
def test_config_reports_which_engines_exist(client, monkeypatch) -> None:
|
||||
body = client.get("/api/config").json()
|
||||
assert body["gufo_enabled"] is True and body["vllm_enabled"] is True
|
||||
|
||||
monkeypatch.setattr(app_module, "GUFO_BASE_URL", "")
|
||||
body = client.get("/api/config").json()
|
||||
assert body["gufo_enabled"] is False
|
||||
assert body["gufo_base_url"] == ""
|
||||
assert body["vllm_enabled"] is True # the other engine keeps its section
|
||||
|
||||
|
||||
def test_lifespan_owns_one_shared_client(client) -> None:
|
||||
"""Entering lifespan installs exactly one AsyncClient on app.state and closes it
|
||||
on shutdown — the poll loop depends on connection reuse, and a leaked client
|
||||
would keep sockets open after Ctrl-C."""
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
with TestClient(app_module.app) as running:
|
||||
shared = app_module.app.state.client
|
||||
assert isinstance(shared, httpx.AsyncClient)
|
||||
assert not shared.is_closed
|
||||
assert running.get("/api/config").status_code == 200
|
||||
assert app_module.app.state.client is shared
|
||||
assert shared.is_closed
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# page + misc
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_dashboard_is_served(client) -> None:
|
||||
r = client.get("/")
|
||||
assert r.status_code == 200
|
||||
assert "text/html" in r.headers["content-type"]
|
||||
assert "LLM Monitor" in r.text
|
||||
|
||||
|
||||
def test_dashboard_layout_is_driven_by_the_config_flags(client) -> None:
|
||||
"""The page must render from /api/config's *_enabled flags instead of hardcoding two
|
||||
engines: an absent engine loses its panel, its strip card, its polling and its share
|
||||
of the fleet totals, and nothing configured at all shows the setup card. Asserted on
|
||||
the shipped HTML because the layout logic lives in the inline script."""
|
||||
html = client.get("/").text
|
||||
# one generic engine list, revealed from the backend's flags
|
||||
assert "const ENGINE_IDS" in html and "_enabled`" in html
|
||||
assert "isEnabled(job.engine)" in html # an absent engine is never polled
|
||||
# what gets hidden, and what shows instead when there is nothing to show
|
||||
for marker in ("panel-${engine}", "strip-${engine}", 'id="no-engines"'):
|
||||
assert marker in html
|
||||
|
||||
|
||||
def test_dashboard_loads_no_third_party_assets(client) -> None:
|
||||
"""The engines live on an internet-disconnected network: no CDN, no webfonts,
|
||||
no build step. Only same-origin requests are allowed from the page."""
|
||||
import re
|
||||
|
||||
html = client.get("/").text
|
||||
refs = re.findall(r"""(?:src|href)\s*=\s*["']([^"']+)["']""", html)
|
||||
external = [r for r in refs if r.startswith(("http://", "//"))]
|
||||
assert external == [], f"page references external assets: {external}"
|
||||
|
||||
|
||||
def test_unknown_route_is_json(client) -> None:
|
||||
r = client.get("/api/nope")
|
||||
assert r.status_code == 404
|
||||
assert "no such route" in r.json()["detail"]
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Derived throughput: Δcounter/Δt and the lifetime-average fallback.
|
||||
|
||||
The deployed vLLM exports no rate gauge of any kind (verified: zero series with a
|
||||
throughput/rate name), so prompt tok/s can only be *computed*. These tests pin the
|
||||
precedence — native gauge > measured Δcounter > estimate > n/a — and the counter-reset
|
||||
handling, because a naive delta shows up as a giant negative spike after an engine
|
||||
restart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app import RateEstimator, parse_gufo_metrics, parse_vllm_metrics
|
||||
|
||||
# gufo's generation-speed gauge is ONE request's last decode pass; the counter is every
|
||||
# slot's tokens. 30 tok/s on the gauge with 60 tokens landing per second = two slots.
|
||||
GUFO_SCRAPE = """
|
||||
llamacpp:requests_processing 2
|
||||
llamacpp:tokens_predicted_total 1000
|
||||
llamacpp:predicted_tokens_seconds 30
|
||||
"""
|
||||
|
||||
COUNTERS = """
|
||||
vllm:num_requests_running{engine="0"} 2
|
||||
vllm:prompt_tokens_total{engine="0"} 1000
|
||||
vllm:generation_tokens_total{engine="0"} 400
|
||||
vllm:inter_token_latency_seconds_sum 0.04
|
||||
vllm:inter_token_latency_seconds_count 2
|
||||
"""
|
||||
|
||||
|
||||
def test_first_scrape_has_nothing_to_diff_against() -> None:
|
||||
rates = RateEstimator()
|
||||
assert rates.rate("k", 100.0, now=10.0) == (None, None)
|
||||
|
||||
|
||||
def test_second_scrape_measures_the_delta() -> None:
|
||||
rates = RateEstimator()
|
||||
rates.rate("k", 100.0, now=10.0)
|
||||
rate, window = rates.rate("k", 130.0, now=11.0)
|
||||
assert rate == pytest.approx(30.0)
|
||||
assert window == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_counter_reset_yields_none_not_a_negative_spike() -> None:
|
||||
rates = RateEstimator()
|
||||
rates.rate("k", 5000.0, now=10.0)
|
||||
assert rates.rate("k", 12.0, now=11.0) == (None, None) # engine restarted
|
||||
# and it recovers on the following scrape
|
||||
assert rates.rate("k", 22.0, now=12.0)[0] == pytest.approx(10.0)
|
||||
|
||||
|
||||
def test_absent_counter_and_zero_window_are_none() -> None:
|
||||
rates = RateEstimator()
|
||||
assert rates.rate("k", None, now=1.0) == (None, None)
|
||||
rates.rate("k", 1.0, now=5.0)
|
||||
assert rates.rate("k", 2.0, now=5.0) == (None, None) # dt == 0
|
||||
assert rates.rate("other", 1.0, now=6.0) == (None, None) # unknown key
|
||||
|
||||
|
||||
def test_rate_smooths_over_the_span_once_enough_samples_exist() -> None:
|
||||
# a bare 1s delta jitters (prompts arrive in bursts); the span averages it out
|
||||
rates = RateEstimator(span=10.0)
|
||||
rates.rate("k", 0.0, now=0.0) # outside the span by t=12
|
||||
rates.rate("k", 100.0, now=5.0)
|
||||
rate, window = rates.rate("k", 400.0, now=12.0)
|
||||
assert window == pytest.approx(7.0) # measured from the oldest in-span sample
|
||||
assert rate == pytest.approx(300.0 / 7.0)
|
||||
|
||||
|
||||
def test_only_two_in_window_samples_still_produce_a_shorter_rate() -> None:
|
||||
rates = RateEstimator(span=10.0)
|
||||
rates.rate("k", 0.0, now=0.0)
|
||||
rates.rate("k", 100.0, now=20.0) # first sample is long gone
|
||||
rate, window = rates.rate("k", 150.0, now=21.0)
|
||||
assert window == pytest.approx(1.0)
|
||||
assert rate == pytest.approx(50.0)
|
||||
|
||||
|
||||
def test_history_is_bounded() -> None:
|
||||
rates = RateEstimator(span=1.0, max_samples=8)
|
||||
for i in range(50):
|
||||
rates.rate("k", float(i), now=float(i))
|
||||
assert len(rates._samples["k"]) <= 8
|
||||
|
||||
|
||||
def test_keys_are_independent() -> None:
|
||||
rates = RateEstimator()
|
||||
rates.rate("a", 0.0, now=0.0)
|
||||
rates.rate("b", 0.0, now=0.0)
|
||||
assert rates.rate("a", 10.0, now=1.0)[0] == pytest.approx(10.0)
|
||||
assert rates.rate("b", 4.0, now=1.0)[0] == pytest.approx(4.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# gufo aggregate decode rate: the counter adds slots, the gauge cannot
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_gufo_decode_rate_adds_every_slot_generating_at_once() -> None:
|
||||
"""The whole point: two slots at 30 tok/s must read 60, not 30.
|
||||
|
||||
gufo's `predicted_tokens_seconds` is a single unlabeled gauge holding one request's
|
||||
last decode pass, so it can never sum concurrency. `tokens_predicted_total` advances
|
||||
once per token from every slot, so Δ/Δt is the server-aggregate.
|
||||
"""
|
||||
rates = RateEstimator()
|
||||
parse_gufo_metrics(GUFO_SCRAPE, rates, now=100.0)
|
||||
m = parse_gufo_metrics(GUFO_SCRAPE.replace("1000", "1060"), rates, now=101.0)
|
||||
d = m["decode_tokens_per_second"]
|
||||
assert d["value"] == pytest.approx(60.0)
|
||||
assert d["derived"] is True
|
||||
assert d["window"] == pytest.approx(1.0)
|
||||
assert d["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt"
|
||||
# the single-request gauge stays exposed, untouched, as the native reference
|
||||
assert m["decode_tokens_per_second_native"]["value"] == pytest.approx(30.0)
|
||||
|
||||
|
||||
def test_gufo_first_scrape_falls_back_to_the_gauge_not_na() -> None:
|
||||
# nothing to diff against yet, but the scrape can still answer
|
||||
m = parse_gufo_metrics(GUFO_SCRAPE, RateEstimator(), now=10.0)
|
||||
assert m["decode_tokens_per_second"] == {
|
||||
"value": 30.0,
|
||||
"source": "llamacpp:predicted_tokens_seconds",
|
||||
"derived": False,
|
||||
}
|
||||
|
||||
|
||||
def test_gufo_idle_reads_zero_not_the_stale_gauge() -> None:
|
||||
# an engine that generated nothing in the window is at 0 tok/s; the gauge would
|
||||
# happily keep showing the last session's speed forever
|
||||
rates = RateEstimator()
|
||||
parse_gufo_metrics(GUFO_SCRAPE, rates, now=0.0)
|
||||
m = parse_gufo_metrics(GUFO_SCRAPE, rates, now=1.0)
|
||||
assert m["decode_tokens_per_second"]["value"] == 0.0
|
||||
assert m["decode_tokens_per_second_native"]["value"] == pytest.approx(30.0)
|
||||
|
||||
|
||||
def test_gufo_counter_reset_falls_back_instead_of_going_negative() -> None:
|
||||
rates = RateEstimator()
|
||||
parse_gufo_metrics(GUFO_SCRAPE, rates, now=0.0)
|
||||
# engine restarted: the counter dropped. Never -995 tok/s.
|
||||
m = parse_gufo_metrics(GUFO_SCRAPE.replace("1000", "5"), rates, now=1.0)
|
||||
assert m["decode_tokens_per_second"]["value"] == pytest.approx(30.0)
|
||||
assert m["decode_tokens_per_second"]["derived"] is False
|
||||
# and it recovers on the following scrape
|
||||
m = parse_gufo_metrics(GUFO_SCRAPE.replace("1000", "25"), rates, now=2.0)
|
||||
assert m["decode_tokens_per_second"]["value"] == pytest.approx(20.0)
|
||||
|
||||
|
||||
def test_gufo_alias_rename_starts_a_fresh_window_instead_of_a_fake_rate() -> None:
|
||||
# the estimator keys off the RESOLVED series name, so a renamed counter cannot diff
|
||||
# against a different series' history (and a flapping alias cannot invent throughput)
|
||||
rates = RateEstimator()
|
||||
parse_gufo_metrics("llamacpp:tokens_generated_total 1000\n", rates, now=0.0)
|
||||
m = parse_gufo_metrics("llamacpp:tokens_predicted_total 1000\n", rates, now=1.0)
|
||||
assert m["decode_tokens_per_second"]["value"] is None
|
||||
|
||||
|
||||
def test_gufo_prompt_rate_is_measured_and_adds_slots_too() -> None:
|
||||
"""Prefill gets the same treatment as decode, so a cross-engine prompt sum is additive.
|
||||
|
||||
gufo's `prompt_tokens_seconds` is one request's last prefill pass; the counter is every
|
||||
slot's newly-computed prompt tokens.
|
||||
"""
|
||||
text = "llamacpp:prompt_tokens_total 1000\nllamacpp:prompt_tokens_seconds 700\n"
|
||||
rates = RateEstimator()
|
||||
parse_gufo_metrics(text, rates, now=0.0)
|
||||
m = parse_gufo_metrics(text.replace("1000", "3400"), rates, now=2.0)
|
||||
p = m["prompt_tokens_per_second"]
|
||||
assert p["value"] == pytest.approx(1200.0)
|
||||
assert p["source"] == "computed:Δllamacpp:prompt_tokens_total/Δt"
|
||||
assert p["derived"] is True and p["window"] == pytest.approx(2.0)
|
||||
# the one-request gauge stays exposed next to it
|
||||
assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(700.0)
|
||||
|
||||
|
||||
def test_gufo_prompt_first_scrape_falls_back_to_the_gauge() -> None:
|
||||
text = "llamacpp:prompt_tokens_total 1000\nllamacpp:prompt_tokens_seconds 700\n"
|
||||
m = parse_gufo_metrics(text, RateEstimator(), now=5.0)
|
||||
assert m["prompt_tokens_per_second"] == {
|
||||
"value": 700.0,
|
||||
"source": "llamacpp:prompt_tokens_seconds",
|
||||
"derived": False,
|
||||
}
|
||||
|
||||
|
||||
def test_gufo_prompt_and_decode_keep_separate_windows() -> None:
|
||||
# one estimator, but the two counters must not read each other's history
|
||||
rates = RateEstimator()
|
||||
base = "llamacpp:prompt_tokens_total 1000\nllamacpp:tokens_predicted_total 500\n"
|
||||
parse_gufo_metrics(base, rates, now=0.0)
|
||||
m = parse_gufo_metrics(base.replace("1000", "3000").replace("500", "550"), rates, now=2.0)
|
||||
assert m["prompt_tokens_per_second"]["value"] == pytest.approx(1000.0)
|
||||
assert m["decode_tokens_per_second"]["value"] == pytest.approx(25.0)
|
||||
|
||||
|
||||
def test_gufo_prompt_idle_reads_zero_not_the_stale_gauge() -> None:
|
||||
text = "llamacpp:prompt_tokens_total 1000\nllamacpp:prompt_tokens_seconds 700\n"
|
||||
rates = RateEstimator()
|
||||
parse_gufo_metrics(text, rates, now=0.0)
|
||||
m = parse_gufo_metrics(text, rates, now=1.0)
|
||||
assert m["prompt_tokens_per_second"]["value"] == 0.0
|
||||
assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(700.0)
|
||||
|
||||
|
||||
def test_gufo_parse_without_rates_stays_pure() -> None:
|
||||
assert parse_gufo_metrics(GUFO_SCRAPE) == parse_gufo_metrics(GUFO_SCRAPE)
|
||||
assert parse_gufo_metrics(GUFO_SCRAPE)["decode_tokens_per_second"]["derived"] is False
|
||||
|
||||
|
||||
def test_both_engines_share_one_estimator_without_cross_reading() -> None:
|
||||
# RATES is one object for both engines; the series names keep the histories apart
|
||||
rates = RateEstimator()
|
||||
parse_gufo_metrics(GUFO_SCRAPE, rates, now=0.0)
|
||||
parse_vllm_metrics(COUNTERS, rates, now=0.0)
|
||||
g = parse_gufo_metrics(GUFO_SCRAPE.replace("1000", "1040"), rates, now=1.0)
|
||||
v = parse_vllm_metrics(COUNTERS.replace("400", "410"), rates, now=1.0)
|
||||
assert g["decode_tokens_per_second"]["value"] == pytest.approx(40.0)
|
||||
assert v["decode_tokens_per_second"]["source"] == "computed:Δvllm:generation_tokens_total/Δt"
|
||||
assert v["decode_tokens_per_second"]["value"] == pytest.approx(10.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# parser integration
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_prompt_and_decode_use_measured_deltas_when_rates_are_supplied() -> None:
|
||||
rates = RateEstimator()
|
||||
parse_vllm_metrics(COUNTERS, rates, now=100.0) # seeds the previous sample
|
||||
m = parse_vllm_metrics(
|
||||
COUNTERS.replace("1000", "1500").replace("400", "450"), rates, now=101.0
|
||||
)
|
||||
p = m["prompt_tokens_per_second"]
|
||||
d = m["decode_tokens_per_second"]
|
||||
assert p["derived"] is True and p["window"] == pytest.approx(1.0)
|
||||
assert p["value"] == pytest.approx(500.0)
|
||||
assert p["source"] == "computed:Δvllm:prompt_tokens_total/Δt"
|
||||
assert d["value"] == pytest.approx(50.0)
|
||||
assert d["source"] == "computed:Δvllm:generation_tokens_total/Δt"
|
||||
|
||||
|
||||
def test_first_scrape_falls_back_to_the_lifetime_average() -> None:
|
||||
# no previous sample to diff against, but prefill time + prompt length still answer
|
||||
text = (
|
||||
"vllm:prompt_tokens_total 1000\n"
|
||||
"vllm:request_prompt_tokens_sum 8000\nvllm:request_prompt_tokens_count 2\n"
|
||||
"vllm:request_prefill_time_seconds_sum 1.6\nvllm:request_prefill_time_seconds_count 2\n"
|
||||
)
|
||||
m = parse_vllm_metrics(text, RateEstimator(), now=10.0)
|
||||
p = m["prompt_tokens_per_second"]
|
||||
assert p["value"] == pytest.approx(5000.0) # 4000 tok mean / 0.8 s mean
|
||||
assert p["source"] == "computed:mean(request_prompt_tokens)/mean(request_prefill_time_seconds)"
|
||||
assert m["prefill_tokens_per_second_lifetime"]["value"] == pytest.approx(5000.0)
|
||||
|
||||
|
||||
def test_measured_delta_beats_the_lifetime_average() -> None:
|
||||
rates = RateEstimator()
|
||||
text = (
|
||||
"vllm:prompt_tokens_total 1000\n"
|
||||
"vllm:request_prompt_tokens_sum 8000\nvllm:request_prompt_tokens_count 2\n"
|
||||
"vllm:request_prefill_time_seconds_sum 1.6\nvllm:request_prefill_time_seconds_count 2\n"
|
||||
)
|
||||
parse_vllm_metrics(text, rates, now=0.0)
|
||||
m = parse_vllm_metrics(text.replace("1000", "1200"), rates, now=1.0)
|
||||
assert m["prompt_tokens_per_second"]["source"].startswith("computed:Δ")
|
||||
assert m["prompt_tokens_per_second"]["value"] == pytest.approx(200.0)
|
||||
|
||||
|
||||
def test_native_gauge_still_wins_over_any_derivation() -> None:
|
||||
m = parse_vllm_metrics(
|
||||
"vllm:prompt_throughput 1234.5\nvllm:prompt_tokens_total 10\n",
|
||||
RateEstimator(),
|
||||
now=1.0,
|
||||
)
|
||||
assert m["prompt_tokens_per_second"] == {
|
||||
"value": 1234.5,
|
||||
"source": "vllm:prompt_throughput",
|
||||
"derived": False,
|
||||
}
|
||||
|
||||
|
||||
def test_without_rates_the_parse_stays_pure() -> None:
|
||||
# parse(text) alone must not depend on wall-clock or hidden state
|
||||
a = parse_vllm_metrics(COUNTERS)
|
||||
b = parse_vllm_metrics(COUNTERS)
|
||||
assert a == b
|
||||
assert a["prompt_tokens_per_second"]["value"] is None # no prefill series here
|
||||
assert a["prefill_tokens_per_second_lifetime"]["value"] is None
|
||||
# decode still answers via TPOT, which is stateless
|
||||
assert a["decode_tokens_per_second"]["source"] == "computed:running/tpot"
|
||||
|
||||
|
||||
def test_prompt_tok_s_is_never_a_silent_zero() -> None:
|
||||
"""The whole point: an engine that IS working must not render n/a because it lacks
|
||||
a gauge, and an idle engine must show 0 rather than a stale estimate."""
|
||||
rates = RateEstimator()
|
||||
text = "vllm:prompt_tokens_total 500\n"
|
||||
parse_vllm_metrics(text, rates, now=0.0)
|
||||
m = parse_vllm_metrics(text, rates, now=1.0)
|
||||
assert m["prompt_tokens_per_second"]["value"] == 0.0 # genuinely idle, not unknown
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Ground truth: the payloads captured from the DEPLOYED engines.
|
||||
|
||||
Neither engine exports the series names people expect, and gufo's `/slots` is a bare
|
||||
JSON array. These fixtures are verbatim captures (gufo on deskwork-ai:8731, vLLM on
|
||||
driveripper-ai:8000); if a field stops mapping, the engine renamed something and this
|
||||
test tells you which one instead of the dashboard quietly showing `n/a` in production.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import RateEstimator, parse_gufo_metrics, parse_vllm_metrics
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def gufo_text() -> str:
|
||||
return (FIXTURES / "gufo_metrics_real.txt").read_text()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vllm_text() -> str:
|
||||
return (FIXTURES / "vllm_metrics_real.txt").read_text()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# gufo
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_gufo_real_names_are_the_primary_aliases(gufo_text: str) -> None:
|
||||
m = parse_gufo_metrics(gufo_text)
|
||||
assert m["tokens_prompt_total"]["source"] == "llamacpp:prompt_tokens_total"
|
||||
assert m["tokens_generated_total"]["source"] == "llamacpp:tokens_predicted_total"
|
||||
# gufo names its speed gauges *_seconds while the unit is tokens/second
|
||||
assert m["prompt_tokens_per_second_native"]["source"] == "llamacpp:prompt_tokens_seconds"
|
||||
assert m["decode_tokens_per_second_native"]["source"] == "llamacpp:predicted_tokens_seconds"
|
||||
# parsed without an estimator, both throughput cells degrade to those one-request gauges
|
||||
assert m["prompt_tokens_per_second"]["source"] == "llamacpp:prompt_tokens_seconds"
|
||||
# parsed without an estimator, the throughput cell degrades to that one-request gauge
|
||||
assert m["decode_tokens_per_second"]["source"] == "llamacpp:predicted_tokens_seconds"
|
||||
|
||||
|
||||
def test_gufo_aggregate_decode_rate_resolves_on_the_real_counter(gufo_text: str) -> None:
|
||||
"""The Δ-rates key off the RESOLVED counter names, so they must work on gufo's real
|
||||
names (`tokens_predicted_total`, `prompt_tokens_total`) and land in the same ballpark as
|
||||
gufo's own busy-time counter (`tokens_predicted_seconds_total`).
|
||||
"""
|
||||
rates = RateEstimator()
|
||||
parse_gufo_metrics(gufo_text, rates, now=0.0)
|
||||
bumped = gufo_text.replace(
|
||||
"llamacpp:tokens_predicted_total 173091", "llamacpp:tokens_predicted_total 173191"
|
||||
).replace(
|
||||
# +1000 generated and +1000 prompt tokens over 2s -> 500 tok/s of each
|
||||
"llamacpp:prompt_tokens_total 374263", "llamacpp:prompt_tokens_total 375263"
|
||||
)
|
||||
assert bumped != gufo_text # the capture moved the counter; re-check this test
|
||||
m = parse_gufo_metrics(bumped, rates, now=2.0)
|
||||
d = m["decode_tokens_per_second"]
|
||||
assert d["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt"
|
||||
assert d["value"] == pytest.approx(50.0)
|
||||
# sanity: the lifetime mean from the two cumulative counters is the same order of
|
||||
# magnitude, so the windowed aggregate is not quietly summing something else
|
||||
busy = float(re.search(r"tokens_predicted_seconds_total ([0-9.]+)", gufo_text).group(1))
|
||||
assert 5.0 < (173091.0 / busy) < 100.0
|
||||
|
||||
# prompt throughput from the same two scrapes: gufo's prompt counter EXCLUDES cache hits,
|
||||
# so this is newly-computed prefill work across all slots, not prompt tokens accepted
|
||||
p = m["prompt_tokens_per_second"]
|
||||
assert p["source"] == "computed:Δllamacpp:prompt_tokens_total/Δt"
|
||||
assert p["value"] == pytest.approx(500.0)
|
||||
assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(820.2841172907755)
|
||||
|
||||
|
||||
def test_gufo_every_banner_and_strip_field_resolves(gufo_text: str) -> None:
|
||||
m = parse_gufo_metrics(gufo_text)
|
||||
for key in (
|
||||
"requests_processing",
|
||||
"requests_deferred",
|
||||
"kv_cache_usage_ratio",
|
||||
"tokens_prompt_total",
|
||||
"tokens_generated_total",
|
||||
"prompt_tokens_per_second",
|
||||
"decode_tokens_per_second",
|
||||
"tokens_total",
|
||||
"n_tokens_max",
|
||||
"device_lost_total",
|
||||
):
|
||||
assert m[key]["value"] is not None, f"gufo {key} stopped resolving"
|
||||
assert 0.0 <= m["kv_cache_usage_ratio"]["value"] <= 1.0
|
||||
|
||||
|
||||
def test_gufo_totals_include_cache_hits_which_are_a_separate_counter(gufo_text: str) -> None:
|
||||
m = parse_gufo_metrics(gufo_text)
|
||||
prompt = m["tokens_prompt_total"]["value"]
|
||||
cached = m["tokens_prompt_cached_total"]["value"]
|
||||
generated = m["tokens_generated_total"]["value"]
|
||||
assert m["tokens_total"]["value"] == pytest.approx(prompt + cached + generated)
|
||||
# 29.6M cached vs 0.37M computed on the live box — cache hits dominate
|
||||
assert m["prompt_cache_hit_ratio"]["value"] == pytest.approx(cached / (prompt + cached))
|
||||
assert 0.0 < m["prompt_cache_hit_ratio"]["value"] < 1.0
|
||||
|
||||
|
||||
def test_gufo_speculative_decoding_accept_rate(gufo_text: str) -> None:
|
||||
m = parse_gufo_metrics(gufo_text)
|
||||
accept = m["spec_accept_rate"]
|
||||
assert accept["derived"] is True
|
||||
assert 0.0 <= accept["value"] <= 1.0
|
||||
|
||||
|
||||
def test_gufo_device_lost_is_zero_on_a_healthy_box(gufo_text: str) -> None:
|
||||
assert parse_gufo_metrics(gufo_text)["device_lost_total"]["value"] == 0.0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# gufo /slots
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_gufo_slots_is_a_bare_array_not_an_object() -> None:
|
||||
"""The UI must not assume {"slots": [...]} — the deployed engine returns a list."""
|
||||
data = json.loads((FIXTURES / "gufo_slots_real.json").read_text())
|
||||
assert isinstance(data, list)
|
||||
assert len(data) >= 1
|
||||
assert all(isinstance(slot, dict) and "id" in slot for slot in data)
|
||||
|
||||
|
||||
def test_gufo_slot_fields_the_cards_depend_on() -> None:
|
||||
data = json.loads((FIXTURES / "gufo_slots_real.json").read_text())
|
||||
slot = data[0]
|
||||
for key in (
|
||||
"id",
|
||||
"state",
|
||||
"is_processing",
|
||||
"id_task",
|
||||
"n_prompt_tokens",
|
||||
"n_prompt_tokens_cache",
|
||||
"n_ctx",
|
||||
"next_token",
|
||||
"model",
|
||||
):
|
||||
assert key in slot, f"gufo slot no longer reports {key}"
|
||||
assert slot["next_token"], "next_token[] should still be present, with n_decoded/n_remain"
|
||||
assert "n_decoded" in slot["next_token"][0]
|
||||
# an idle slot reports -1 for "nothing pending", which is not a count
|
||||
if not slot["is_processing"]:
|
||||
assert slot["id_task"] == -1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# vLLM
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_vllm_real_names_are_the_primary_aliases(vllm_text: str) -> None:
|
||||
m = parse_vllm_metrics(vllm_text)
|
||||
assert m["gpu_cache_usage"]["source"] == "vllm:kv_cache_usage_perc" # not gpu_cache_usage
|
||||
assert m["generation_tokens_total"]["source"] == "vllm:generation_tokens_total"
|
||||
assert m["time_per_output_token_seconds"]["source"] == "vllm:inter_token_latency_seconds"
|
||||
|
||||
|
||||
def test_vllm_every_banner_and_card_field_resolves(vllm_text: str) -> None:
|
||||
m = parse_vllm_metrics(vllm_text)
|
||||
for key in (
|
||||
"num_requests_running",
|
||||
"num_requests_waiting",
|
||||
"gpu_cache_usage",
|
||||
"prompt_tokens_total",
|
||||
"generation_tokens_total",
|
||||
"tokens_total",
|
||||
"waiting_capacity",
|
||||
"prefix_cache_hit_ratio",
|
||||
"num_preemptions_total",
|
||||
):
|
||||
assert m[key]["value"] is not None, f"vLLM {key} stopped resolving"
|
||||
for key in (
|
||||
"time_to_first_token_seconds__mean",
|
||||
"time_per_output_token_seconds__mean",
|
||||
"request_queue_time_seconds__mean",
|
||||
):
|
||||
assert m[key]["value"] is not None, f"vLLM {key} stopped resolving"
|
||||
assert 0.0 <= m["gpu_cache_usage"]["value"] <= 1.0
|
||||
assert 0.0 <= m["prefix_cache_hit_ratio"]["value"] <= 1.0
|
||||
|
||||
|
||||
def test_vllm_waiting_reasons_from_the_live_labels(vllm_text: str) -> None:
|
||||
# the deployed build tags reasons as capacity/deferred, not defragmentation
|
||||
m = parse_vllm_metrics(vllm_text)
|
||||
assert "capacity" in m["waiting_by_reason"]["value"]
|
||||
|
||||
|
||||
def test_vllm_histograms_survive_engine_and_model_labels(vllm_text: str) -> None:
|
||||
m = parse_vllm_metrics(vllm_text)
|
||||
ttft = m["time_to_first_token_seconds"]
|
||||
assert ttft["count"] > 0 and ttft["mean"] > 0
|
||||
# single engine label set here, so an interpolated p95 is legitimate
|
||||
assert ttft["p95"] is None or ttft["p95"] >= ttft["mean"]
|
||||
|
||||
|
||||
def test_vllm_prompt_tok_s_is_answered_even_without_a_rate_gauge(vllm_text: str) -> None:
|
||||
# this build exports no throughput series at all, so prompt tok/s comes from the
|
||||
# prefill histograms; it must not render n/a on a healthy, busy engine
|
||||
m = parse_vllm_metrics(vllm_text)
|
||||
p = m["prompt_tokens_per_second"]
|
||||
assert p["value"] is not None, "prompt tok/s regressed to n/a"
|
||||
assert p["derived"] is True
|
||||
assert p["source"] == "computed:mean(request_prompt_tokens)/mean(request_prefill_time_seconds)"
|
||||
assert p["value"] > 1000 # live box sits around 4.7k tok/s; single digits mean a bug
|
||||
assert m["prefill_tokens_per_second_lifetime"]["value"] == pytest.approx(p["value"])
|
||||
|
||||
|
||||
def test_vllm_prompt_tok_s_switches_to_measured_rate_once_scraping(vllm_text: str) -> None:
|
||||
from app import RateEstimator
|
||||
|
||||
rates = RateEstimator()
|
||||
parse_vllm_metrics(vllm_text, rates, now=1000.0)
|
||||
m = parse_vllm_metrics(vllm_text, rates, now=1001.0) # same counters -> zero throughput
|
||||
p = m["prompt_tokens_per_second"]
|
||||
assert p["source"] == "computed:Δvllm:prompt_tokens_total/Δt"
|
||||
assert p["window"] == pytest.approx(1.0)
|
||||
# unchanged capture -> genuinely zero throughput in that window, not a missing gauge
|
||||
assert p["value"] == 0.0
|
||||
|
||||
|
||||
def test_vllm_decode_throughput_is_derived_on_this_build(vllm_text: str) -> None:
|
||||
# no vllm:*throughput gauge exists on the deployed version -> computed from TPOT
|
||||
m = parse_vllm_metrics(vllm_text)
|
||||
assert m["decode_tokens_per_second"]["derived"] is True
|
||||
assert m["decode_tokens_per_second"]["source"] == "computed:running/tpot"
|
||||
Reference in New Issue
Block a user