Add homelab dashboards: system, LLM, and UPS on one domain
Three dashboards served by one FastAPI app with a SQLite history store
(raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana,
no CDN, no build step. Nothing is fetched from the internet at runtime.
* system: a stdlib-only agent (agent/host_agent.py) reports /proc and
/sys counters; the dashboard turns them into rates over a labelled
window, so every number says how it was measured.
* llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house
exposition parser. Measured delta/elapsed throughput beats a native
per-request gauge, and each value carries its provenance.
* ups: a NUT wire-protocol client plus health checks that name what
they looked at. Commands are gated by NUT_ALLOW_ACTIONS,
DASH_READ_ONLY and a DANGER confirmation, re-checked server-side.
Missing values render as n/a rather than 0, and an unconfigured source
answers 404 {"configured": false} without dialling anything, so "not
deployed here" never looks like an outage.
253 tests at 94% coverage against captured real fixtures; ruff and
pyright clean.
LLM-Generated: true
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
"""Shared fixtures: real captures for shape, fakes for behaviour.
|
||||
|
||||
Anything that must match an upstream (an agent snapshot, an engine's /metrics text, the NUT
|
||||
wire protocol) is asserted against a verbatim capture in tests/fixtures/. Everything that
|
||||
needs a *failure* — a dead host, a refused command, a counter reset — is a fake here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "src"
|
||||
if str(SRC) not in sys.path:
|
||||
sys.path.insert(0, str(SRC))
|
||||
|
||||
from dashboards.rates import RateEstimator # noqa: E402
|
||||
from dashboards.store import SampleStore # noqa: E402
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path: Path) -> Any:
|
||||
store = SampleStore(tmp_path / "t.sqlite3")
|
||||
try:
|
||||
yield store
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(db: Any) -> Any:
|
||||
"""Alias: some tests read better with `store`, others with `db`. Same object."""
|
||||
return db
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent_snapshot() -> dict[str, Any]:
|
||||
"""A real driveripper snapshot (Threadripper + R9700, 48 logical CPUs)."""
|
||||
return json.loads((FIXTURES / "agent_driveripper.json").read_text())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent_snapshots() -> dict[str, dict[str, Any]]:
|
||||
"""Every captured host, keyed by name — fleet code paths need >1 host to mean anything."""
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for path in sorted(FIXTURES.glob("agent_*.json")):
|
||||
out[path.stem.removeprefix("agent_")] = json.loads(path.read_text())
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def metrics_text() -> Any:
|
||||
"""metrics_text('gufo') / metrics_text('vllm') -> the captured exposition verbatim."""
|
||||
def load(engine: str) -> str:
|
||||
path = FIXTURES / f"metrics_{engine}.txt"
|
||||
if not path.exists():
|
||||
pytest.skip(f"missing fixture {path.name} — run scripts/capture-fixtures.sh --with-llm")
|
||||
return path.read_text()
|
||||
|
||||
return load
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rates() -> RateEstimator:
|
||||
return RateEstimator(span=10.0)
|
||||
Vendored
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
Vendored
+1
@@ -0,0 +1 @@
|
||||
{"ok": true, "schema": 1, "agent_version": "1.0.0", "ts": 1792178443.0656452, "duration_ms": 10.22, "host": {"hostname": "gateway", "os_pretty": "AlmaLinux 10.2 (Lavender Lion)", "os_id": "almalinux", "kernel": "6.12.0-211.65.1.el10_2.x86_64_v2", "arch": "x86_64", "python": "3.12.15", "cpu_model": "AMD Ryzen Threadripper 3960X 24-Core Processor", "cpu_cores_logical": 2, "cpu_cores_physical": 2, "memory_total_bytes": 2051493888.0, "boot_ts": 1791967457.1014743, "uptime_seconds": 210985.95, "container": false, "agent_version": "1.0.0"}, "cpu": {"modes": {"user": 561498.0, "nice": 2756.0, "system": 122750.0, "idle": 41118362.0, "iowait": 290460.0, "irq": 24373.0, "softirq": 46665.0, "steal": 4918.0, "guest": 0.0, "guest_nice": 0.0, "total": 42171782.0}, "cores": [{"core": 0, "modes": {"user": 285993.0, "nice": 1771.0, "system": 60082.0, "idle": 20538975.0, "iowait": 143245.0, "irq": 14642.0, "softirq": 38622.0, "steal": 3640.0, "guest": 0.0, "guest_nice": 0.0, "total": 21086970.0}}, {"core": 1, "modes": {"user": 275504.0, "nice": 985.0, "system": 62667.0, "idle": 20579386.0, "iowait": 147214.0, "irq": 9730.0, "softirq": 8043.0, "steal": 1277.0, "guest": 0.0, "guest_nice": 0.0, "total": 21084806.0}}], "clock_ticks_per_second": 100, "load1": 0.01, "load5": 0.07, "load15": 0.08, "procs_running": 1.0, "procs_blocked": 2.0, "ctxt_total": 64391428.0, "processes_total": 343402.0, "freq_mhz": null, "freq_max_mhz": null}, "memory": {"total_bytes": 2051493888.0, "free_bytes": 172544000.0, "available_bytes": 1400086528.0, "used_bytes": 651407360.0, "buffers_bytes": 106156032.0, "cached_bytes": 1315049472.0, "dirty_bytes": 2158592.0, "swap_total_bytes": 0.0, "swap_used_bytes": 0.0, "committed_bytes": 720392192.0}, "filesystems": [{"device": "/dev/vda4", "mountpoint": "/", "fstype": "ext4", "total_bytes": 9197531136, "used_bytes": 5252460544, "free_bytes": 3455754240, "used_pct": 60.316157493618384, "read_only": false}, {"device": "/dev/vda3", "mountpoint": "/boot", "fstype": "ext4", "total_bytes": 1020702720, "used_bytes": 272543744, "free_bytes": 677695488, "used_pct": 28.68159246870582, "read_only": false}, {"device": "/dev/vda2", "mountpoint": "/boot/efi", "fstype": "vfat", "total_bytes": 209489920, "used_bytes": 9560064, "free_bytes": 199929856, "used_pct": 4.56349594290742, "read_only": false}], "disk_io": {"devices": [{"name": "vda", "model": null, "rotational": true, "size_bytes": 10737418240.0, "reads_total": 14464.0, "writes_total": 1989589.0, "read_bytes_total": 645659136.0, "write_bytes_total": 22551089664.0, "in_flight": 0.0, "io_time_ms_total": 2882080.0, "weighted_io_ms_total": 6688324.0, "discard_bytes_total": 0.0}], "device_names": ["vda"]}, "network": {"interfaces": [{"name": "eth0", "rx_bytes_total": 14580699012.0, "rx_packets_total": 22626423.0, "rx_errors_total": 0.0, "rx_dropped_total": 287.0, "tx_bytes_total": 10807075678.0, "tx_packets_total": 10084825.0, "tx_errors_total": 0.0, "tx_dropped_total": 0.0, "speed_mbps": null}]}, "gpus": [], "temperatures": [], "cpu_temperature_celsius": null, "processes": {"total": 142, "top_rss": [{"pid": 592, "name": "systemd-journal", "rss_bytes": 105857024.0}, {"pid": 1182, "name": "haproxy", "rss_bytes": 99045376.0}, {"pid": 723, "name": "firewalld", "rss_bytes": 45318144.0}, {"pid": 1091, "name": "polkitd", "rss_bytes": 41594880.0}, {"pid": 1, "name": "systemd", "rss_bytes": 37564416.0}], "top_cpu": [{"pid": 342784, "name": "python3", "cpu_pct": 2.680722502695548}, {"pid": 1, "name": "systemd", "cpu_pct": 0.0}, {"pid": 2, "name": "kthreadd", "cpu_pct": 0.0}, {"pid": 3, "name": "pool_workqueue_release", "cpu_pct": 0.0}, {"pid": 4, "name": "kworker/R-rcu_gp", "cpu_pct": 0.0}]}, "errors": []}
|
||||
Vendored
+45
@@ -0,0 +1,45 @@
|
||||
# HELP gufo_device_lost_total Confirmed GPU context losses
|
||||
# TYPE gufo_device_lost_total counter
|
||||
gufo_device_lost_total 0
|
||||
# HELP llamacpp:prompt_tokens_total Total prompt tokens processed, excluding cache hits
|
||||
# TYPE llamacpp:prompt_tokens_total counter
|
||||
llamacpp:prompt_tokens_total 1350666
|
||||
# HELP llamacpp:prompt_tokens_cached_total Total prompt tokens reused from cache
|
||||
# TYPE llamacpp:prompt_tokens_cached_total counter
|
||||
llamacpp:prompt_tokens_cached_total 124529105
|
||||
# HELP llamacpp:prompt_seconds_total Prompt process time
|
||||
# TYPE llamacpp:prompt_seconds_total counter
|
||||
llamacpp:prompt_seconds_total 1478.1443332270005
|
||||
# HELP llamacpp:tokens_predicted_total Total tokens generated
|
||||
# TYPE llamacpp:tokens_predicted_total counter
|
||||
llamacpp:tokens_predicted_total 877336
|
||||
# HELP llamacpp:tokens_predicted_seconds_total Predict process time
|
||||
# TYPE llamacpp:tokens_predicted_seconds_total counter
|
||||
llamacpp:tokens_predicted_seconds_total 26565.247833160014
|
||||
# HELP llamacpp:n_tokens_max Largest observed n_tokens.
|
||||
# TYPE llamacpp:n_tokens_max counter
|
||||
llamacpp:n_tokens_max 252622
|
||||
# HELP llamacpp:spec_decode_num_drafts_total Total speculative verification rounds
|
||||
# TYPE llamacpp:spec_decode_num_drafts_total counter
|
||||
llamacpp:spec_decode_num_drafts_total 291738
|
||||
# HELP llamacpp:spec_decode_num_draft_tokens_total Total draft tokens proposed
|
||||
# TYPE llamacpp:spec_decode_num_draft_tokens_total counter
|
||||
llamacpp:spec_decode_num_draft_tokens_total 789028
|
||||
# HELP llamacpp:spec_decode_num_accepted_tokens_total Total draft tokens accepted
|
||||
# TYPE llamacpp:spec_decode_num_accepted_tokens_total counter
|
||||
llamacpp:spec_decode_num_accepted_tokens_total 578514
|
||||
# HELP llamacpp:prompt_tokens_seconds Prompt processing speed in tokens per second
|
||||
# TYPE llamacpp:prompt_tokens_seconds gauge
|
||||
llamacpp:prompt_tokens_seconds 160.47362399350274
|
||||
# HELP llamacpp:predicted_tokens_seconds Generation speed in tokens per second
|
||||
# TYPE llamacpp:predicted_tokens_seconds gauge
|
||||
llamacpp:predicted_tokens_seconds 42.621031668072597
|
||||
# HELP llamacpp:requests_processing Number of admitted requests, including cache preparation
|
||||
# TYPE llamacpp:requests_processing gauge
|
||||
llamacpp:requests_processing 0
|
||||
# HELP llamacpp:requests_deferred Number of requests waiting for a session
|
||||
# TYPE llamacpp:requests_deferred gauge
|
||||
llamacpp:requests_deferred 0
|
||||
# HELP llamacpp:kv_cache_usage_ratio In-flight prompt and generated tokens over sessions times context; excludes retained cache
|
||||
# TYPE llamacpp:kv_cache_usage_ratio gauge
|
||||
llamacpp:kv_cache_usage_ratio 0
|
||||
Vendored
+662
@@ -0,0 +1,662 @@
|
||||
# HELP python_gc_objects_collected_total Objects collected during gc
|
||||
# TYPE python_gc_objects_collected_total counter
|
||||
python_gc_objects_collected_total{generation="0"} 166913.0
|
||||
python_gc_objects_collected_total{generation="1"} 27377.0
|
||||
python_gc_objects_collected_total{generation="2"} 1409.0
|
||||
# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC
|
||||
# TYPE python_gc_objects_uncollectable_total counter
|
||||
python_gc_objects_uncollectable_total{generation="0"} 0.0
|
||||
python_gc_objects_uncollectable_total{generation="1"} 0.0
|
||||
python_gc_objects_uncollectable_total{generation="2"} 0.0
|
||||
# HELP python_gc_collections_total Number of times this generation was collected
|
||||
# TYPE python_gc_collections_total counter
|
||||
python_gc_collections_total{generation="0"} 4631.0
|
||||
python_gc_collections_total{generation="1"} 420.0
|
||||
python_gc_collections_total{generation="2"} 12.0
|
||||
# HELP python_info Python platform information
|
||||
# TYPE python_info gauge
|
||||
python_info{implementation="CPython",major="3",minor="12",patchlevel="3",version="3.12.3"} 1.0
|
||||
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
|
||||
# TYPE process_virtual_memory_bytes gauge
|
||||
process_virtual_memory_bytes 3.2837738496e+10
|
||||
# HELP process_resident_memory_bytes Resident memory size in bytes.
|
||||
# TYPE process_resident_memory_bytes gauge
|
||||
process_resident_memory_bytes 2.370977792e+09
|
||||
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
|
||||
# TYPE process_start_time_seconds gauge
|
||||
process_start_time_seconds 1.79210782535e+09
|
||||
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
|
||||
# TYPE process_cpu_seconds_total counter
|
||||
process_cpu_seconds_total 457.6
|
||||
# HELP process_open_fds Number of open file descriptors.
|
||||
# TYPE process_open_fds gauge
|
||||
process_open_fds 43.0
|
||||
# HELP process_max_fds Maximum number of open file descriptors.
|
||||
# TYPE process_max_fds gauge
|
||||
process_max_fds 524288.0
|
||||
# HELP vllm:estimated_flops_per_gpu_total Estimated number of floating point operations per GPU (for Model Flops Utilization calculations).
|
||||
# TYPE vllm:estimated_flops_per_gpu_total counter
|
||||
vllm:estimated_flops_per_gpu_total{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:estimated_flops_per_gpu_created Estimated number of floating point operations per GPU (for Model Flops Utilization calculations).
|
||||
# TYPE vllm:estimated_flops_per_gpu_created gauge
|
||||
vllm:estimated_flops_per_gpu_created{engine="0",model_name="juggernaut"} 1.7921081022034154e+09
|
||||
# HELP vllm:estimated_read_bytes_per_gpu_total Estimated number of bytes read from memory per GPU (for Model Flops Utilization calculations).
|
||||
# TYPE vllm:estimated_read_bytes_per_gpu_total counter
|
||||
vllm:estimated_read_bytes_per_gpu_total{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:estimated_read_bytes_per_gpu_created Estimated number of bytes read from memory per GPU (for Model Flops Utilization calculations).
|
||||
# TYPE vllm:estimated_read_bytes_per_gpu_created gauge
|
||||
vllm:estimated_read_bytes_per_gpu_created{engine="0",model_name="juggernaut"} 1.792108102203435e+09
|
||||
# HELP vllm:estimated_write_bytes_per_gpu_total Estimated number of bytes written to memory per GPU (for Model Flops Utilization calculations).
|
||||
# TYPE vllm:estimated_write_bytes_per_gpu_total counter
|
||||
vllm:estimated_write_bytes_per_gpu_total{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:estimated_write_bytes_per_gpu_created Estimated number of bytes written to memory per GPU (for Model Flops Utilization calculations).
|
||||
# TYPE vllm:estimated_write_bytes_per_gpu_created gauge
|
||||
vllm:estimated_write_bytes_per_gpu_created{engine="0",model_name="juggernaut"} 1.7921081022034492e+09
|
||||
# HELP vllm:num_requests_running Number of requests in model execution batches.
|
||||
# TYPE vllm:num_requests_running gauge
|
||||
vllm:num_requests_running{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:num_requests_waiting Number of requests waiting to be processed.
|
||||
# TYPE vllm:num_requests_waiting gauge
|
||||
vllm:num_requests_waiting{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:num_requests_waiting_by_reason Number of waiting requests by reason. Reason labels: 'capacity' = waiting for scheduling capacity; 'deferred' = deferred by transient constraints (LoRA budget, KV transfer, blocked status). Sum of all reasons equals vllm:num_requests_waiting.
|
||||
# TYPE vllm:num_requests_waiting_by_reason gauge
|
||||
vllm:num_requests_waiting_by_reason{engine="0",model_name="juggernaut",reason="capacity"} 0.0
|
||||
vllm:num_requests_waiting_by_reason{engine="0",model_name="juggernaut",reason="deferred"} 0.0
|
||||
# HELP vllm:engine_sleep_state Engine sleep state; awake = 0 means engine is sleeping; awake = 1 means engine is awake; weights_offloaded = 1 means sleep level 1; discard_all = 1 means sleep level 2.
|
||||
# TYPE vllm:engine_sleep_state gauge
|
||||
vllm:engine_sleep_state{engine="0",model_name="juggernaut",sleep_state="awake"} 1.0
|
||||
vllm:engine_sleep_state{engine="0",model_name="juggernaut",sleep_state="weights_offloaded"} 0.0
|
||||
vllm:engine_sleep_state{engine="0",model_name="juggernaut",sleep_state="discard_all"} 0.0
|
||||
# HELP vllm:kv_cache_usage_perc KV-cache usage. 1 means 100 percent usage.
|
||||
# TYPE vllm:kv_cache_usage_perc gauge
|
||||
vllm:kv_cache_usage_perc{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:prefix_cache_queries_total Prefix cache queries, in terms of number of queried tokens.
|
||||
# TYPE vllm:prefix_cache_queries_total counter
|
||||
vllm:prefix_cache_queries_total{engine="0",model_name="juggernaut"} 1.8389909e+07
|
||||
# HELP vllm:prefix_cache_queries_created Prefix cache queries, in terms of number of queried tokens.
|
||||
# TYPE vllm:prefix_cache_queries_created gauge
|
||||
vllm:prefix_cache_queries_created{engine="0",model_name="juggernaut"} 1.7921081022037191e+09
|
||||
# HELP vllm:prefix_cache_hits_total Prefix cache hits, in terms of number of cached tokens.
|
||||
# TYPE vllm:prefix_cache_hits_total counter
|
||||
vllm:prefix_cache_hits_total{engine="0",model_name="juggernaut"} 1.210496e+07
|
||||
# HELP vllm:prefix_cache_hits_created Prefix cache hits, in terms of number of cached tokens.
|
||||
# TYPE vllm:prefix_cache_hits_created gauge
|
||||
vllm:prefix_cache_hits_created{engine="0",model_name="juggernaut"} 1.792108102203731e+09
|
||||
# HELP vllm:external_prefix_cache_queries_total External prefix cache queries from KV connector cross-instance cache sharing, in terms of number of queried tokens.
|
||||
# TYPE vllm:external_prefix_cache_queries_total counter
|
||||
vllm:external_prefix_cache_queries_total{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:external_prefix_cache_queries_created External prefix cache queries from KV connector cross-instance cache sharing, in terms of number of queried tokens.
|
||||
# TYPE vllm:external_prefix_cache_queries_created gauge
|
||||
vllm:external_prefix_cache_queries_created{engine="0",model_name="juggernaut"} 1.7921081022037423e+09
|
||||
# HELP vllm:external_prefix_cache_hits_total External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens.
|
||||
# TYPE vllm:external_prefix_cache_hits_total counter
|
||||
vllm:external_prefix_cache_hits_total{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:external_prefix_cache_hits_created External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens.
|
||||
# TYPE vllm:external_prefix_cache_hits_created gauge
|
||||
vllm:external_prefix_cache_hits_created{engine="0",model_name="juggernaut"} 1.792108102203755e+09
|
||||
# HELP vllm:mm_cache_queries_total Multi-modal cache queries, in terms of number of queried items.
|
||||
# TYPE vllm:mm_cache_queries_total counter
|
||||
vllm:mm_cache_queries_total{engine="0",model_name="juggernaut"} 8.0
|
||||
# HELP vllm:mm_cache_queries_created Multi-modal cache queries, in terms of number of queried items.
|
||||
# TYPE vllm:mm_cache_queries_created gauge
|
||||
vllm:mm_cache_queries_created{engine="0",model_name="juggernaut"} 1.7921081022037659e+09
|
||||
# HELP vllm:mm_cache_hits_total Multi-modal cache hits, in terms of number of cached items.
|
||||
# TYPE vllm:mm_cache_hits_total counter
|
||||
vllm:mm_cache_hits_total{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:mm_cache_hits_created Multi-modal cache hits, in terms of number of cached items.
|
||||
# TYPE vllm:mm_cache_hits_created gauge
|
||||
vllm:mm_cache_hits_created{engine="0",model_name="juggernaut"} 1.7921081022037778e+09
|
||||
# HELP vllm:num_preemptions_total Cumulative number of preemption from the engine.
|
||||
# TYPE vllm:num_preemptions_total counter
|
||||
vllm:num_preemptions_total{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:num_preemptions_created Cumulative number of preemption from the engine.
|
||||
# TYPE vllm:num_preemptions_created gauge
|
||||
vllm:num_preemptions_created{engine="0",model_name="juggernaut"} 1.7921081022037895e+09
|
||||
# HELP vllm:prompt_tokens_total Number of prefill tokens processed.
|
||||
# TYPE vllm:prompt_tokens_total counter
|
||||
vllm:prompt_tokens_total{engine="0",model_name="juggernaut"} 1.8389909e+07
|
||||
# HELP vllm:prompt_tokens_created Number of prefill tokens processed.
|
||||
# TYPE vllm:prompt_tokens_created gauge
|
||||
vllm:prompt_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022038019e+09
|
||||
# HELP vllm:prompt_tokens_by_source_total Number of prompt tokens by source.
|
||||
# TYPE vllm:prompt_tokens_by_source_total counter
|
||||
vllm:prompt_tokens_by_source_total{engine="0",model_name="juggernaut",source="local_compute"} 6.284949e+06
|
||||
vllm:prompt_tokens_by_source_total{engine="0",model_name="juggernaut",source="local_cache_hit"} 1.210496e+07
|
||||
vllm:prompt_tokens_by_source_total{engine="0",model_name="juggernaut",source="external_kv_transfer"} 0.0
|
||||
# HELP vllm:prompt_tokens_by_source_created Number of prompt tokens by source.
|
||||
# TYPE vllm:prompt_tokens_by_source_created gauge
|
||||
vllm:prompt_tokens_by_source_created{engine="0",model_name="juggernaut",source="local_compute"} 1.7921081022038164e+09
|
||||
vllm:prompt_tokens_by_source_created{engine="0",model_name="juggernaut",source="local_cache_hit"} 1.7921081022038229e+09
|
||||
vllm:prompt_tokens_by_source_created{engine="0",model_name="juggernaut",source="external_kv_transfer"} 1.7921081022038274e+09
|
||||
# HELP vllm:prompt_tokens_cached_total Number of cached prompt tokens (local + external).
|
||||
# TYPE vllm:prompt_tokens_cached_total counter
|
||||
vllm:prompt_tokens_cached_total{engine="0",model_name="juggernaut"} 1.210496e+07
|
||||
# HELP vllm:prompt_tokens_cached_created Number of cached prompt tokens (local + external).
|
||||
# TYPE vllm:prompt_tokens_cached_created gauge
|
||||
vllm:prompt_tokens_cached_created{engine="0",model_name="juggernaut"} 1.7921081022038393e+09
|
||||
# HELP vllm:generation_tokens_total Number of generation tokens processed.
|
||||
# TYPE vllm:generation_tokens_total counter
|
||||
vllm:generation_tokens_total{engine="0",model_name="juggernaut"} 611163.0
|
||||
# HELP vllm:generation_tokens_created Number of generation tokens processed.
|
||||
# TYPE vllm:generation_tokens_created gauge
|
||||
vllm:generation_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022038517e+09
|
||||
# HELP vllm:request_success_total Count of successfully processed requests.
|
||||
# TYPE vllm:request_success_total counter
|
||||
vllm:request_success_total{engine="0",finished_reason="stop",model_name="juggernaut"} 3488.0
|
||||
vllm:request_success_total{engine="0",finished_reason="length",model_name="juggernaut"} 0.0
|
||||
vllm:request_success_total{engine="0",finished_reason="abort",model_name="juggernaut"} 0.0
|
||||
vllm:request_success_total{engine="0",finished_reason="error",model_name="juggernaut"} 0.0
|
||||
vllm:request_success_total{engine="0",finished_reason="repetition",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:request_success_created Count of successfully processed requests.
|
||||
# TYPE vllm:request_success_created gauge
|
||||
vllm:request_success_created{engine="0",finished_reason="stop",model_name="juggernaut"} 1.792108102203872e+09
|
||||
vllm:request_success_created{engine="0",finished_reason="length",model_name="juggernaut"} 1.7921081022038789e+09
|
||||
vllm:request_success_created{engine="0",finished_reason="abort",model_name="juggernaut"} 1.7921081022038846e+09
|
||||
vllm:request_success_created{engine="0",finished_reason="error",model_name="juggernaut"} 1.7921081022038896e+09
|
||||
vllm:request_success_created{engine="0",finished_reason="repetition",model_name="juggernaut"} 1.7921081022038949e+09
|
||||
# HELP vllm:request_prompt_tokens Number of prefill tokens processed.
|
||||
# TYPE vllm:request_prompt_tokens histogram
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 5.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 5.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 30.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 133.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 612.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 1135.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 2002.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3126.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3227.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3290.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3383.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3482.0
|
||||
vllm:request_prompt_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prompt_tokens_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prompt_tokens_sum{engine="0",model_name="juggernaut"} 1.8352277e+07
|
||||
# HELP vllm:request_prompt_tokens_created Number of prefill tokens processed.
|
||||
# TYPE vllm:request_prompt_tokens_created gauge
|
||||
vllm:request_prompt_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022039344e+09
|
||||
# HELP vllm:request_generation_tokens Number of generation tokens processed.
|
||||
# TYPE vllm:request_generation_tokens histogram
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 6.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 45.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 819.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 3206.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 3379.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 3423.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 3464.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3486.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_generation_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_generation_tokens_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_generation_tokens_sum{engine="0",model_name="juggernaut"} 610140.0
|
||||
# HELP vllm:request_generation_tokens_created Number of generation tokens processed.
|
||||
# TYPE vllm:request_generation_tokens_created gauge
|
||||
vllm:request_generation_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022039804e+09
|
||||
# HELP vllm:iteration_tokens_total Histogram of number of tokens per engine_step.
|
||||
# TYPE vllm:iteration_tokens_total histogram
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="1.0",model_name="juggernaut"} 317139.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="8.0",model_name="juggernaut"} 461626.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="16.0",model_name="juggernaut"} 461631.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="32.0",model_name="juggernaut"} 461631.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="64.0",model_name="juggernaut"} 461632.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="128.0",model_name="juggernaut"} 461692.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="256.0",model_name="juggernaut"} 461852.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="512.0",model_name="juggernaut"} 462266.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="1024.0",model_name="juggernaut"} 462849.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="2048.0",model_name="juggernaut"} 463840.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="4096.0",model_name="juggernaut"} 464991.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="8192.0",model_name="juggernaut"} 465088.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="16384.0",model_name="juggernaut"} 465097.0
|
||||
vllm:iteration_tokens_total_bucket{engine="0",le="+Inf",model_name="juggernaut"} 465099.0
|
||||
vllm:iteration_tokens_total_count{engine="0",model_name="juggernaut"} 465099.0
|
||||
vllm:iteration_tokens_total_sum{engine="0",model_name="juggernaut"} 6.896112e+06
|
||||
# HELP vllm:iteration_tokens_total_created Histogram of number of tokens per engine_step.
|
||||
# TYPE vllm:iteration_tokens_total_created gauge
|
||||
vllm:iteration_tokens_total_created{engine="0",model_name="juggernaut"} 1.7921081022040136e+09
|
||||
# HELP vllm:request_max_num_generation_tokens Histogram of maximum number of requested generation tokens.
|
||||
# TYPE vllm:request_max_num_generation_tokens histogram
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 6.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 45.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 819.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 3206.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 3379.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 3423.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 3464.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3486.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_max_num_generation_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_max_num_generation_tokens_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_max_num_generation_tokens_sum{engine="0",model_name="juggernaut"} 610140.0
|
||||
# HELP vllm:request_max_num_generation_tokens_created Histogram of maximum number of requested generation tokens.
|
||||
# TYPE vllm:request_max_num_generation_tokens_created gauge
|
||||
vllm:request_max_num_generation_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022040458e+09
|
||||
# HELP vllm:request_params_n Histogram of the n request parameter.
|
||||
# TYPE vllm:request_params_n histogram
|
||||
vllm:request_params_n_bucket{engine="0",le="1.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_n_bucket{engine="0",le="2.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_n_bucket{engine="0",le="5.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_n_bucket{engine="0",le="10.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_n_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_n_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_n_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_n_sum{engine="0",model_name="juggernaut"} 3488.0
|
||||
# HELP vllm:request_params_n_created Histogram of the n request parameter.
|
||||
# TYPE vllm:request_params_n_created gauge
|
||||
vllm:request_params_n_created{engine="0",model_name="juggernaut"} 1.7921081022040787e+09
|
||||
# HELP vllm:request_params_max_tokens Histogram of the max_tokens request parameter.
|
||||
# TYPE vllm:request_params_max_tokens histogram
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 1.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 1.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3139.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3140.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3183.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_max_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_max_tokens_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_params_max_tokens_sum{engine="0",model_name="juggernaut"} 1.7011909e+07
|
||||
# HELP vllm:request_params_max_tokens_created Histogram of the max_tokens request parameter.
|
||||
# TYPE vllm:request_params_max_tokens_created gauge
|
||||
vllm:request_params_max_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022041035e+09
|
||||
# HELP vllm:time_to_first_token_seconds Histogram of time to first token in seconds.
|
||||
# TYPE vllm:time_to_first_token_seconds histogram
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.001",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.005",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.01",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.02",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.04",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.06",model_name="juggernaut"} 0.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.08",model_name="juggernaut"} 4.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.1",model_name="juggernaut"} 4.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.25",model_name="juggernaut"} 592.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 1312.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="0.75",model_name="juggernaut"} 1924.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 2266.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 3433.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 3485.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="7.5",model_name="juggernaut"} 3488.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3489.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3490.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3490.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="80.0",model_name="juggernaut"} 3490.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="160.0",model_name="juggernaut"} 3490.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="640.0",model_name="juggernaut"} 3490.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="2560.0",model_name="juggernaut"} 3490.0
|
||||
vllm:time_to_first_token_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3490.0
|
||||
vllm:time_to_first_token_seconds_count{engine="0",model_name="juggernaut"} 3490.0
|
||||
vllm:time_to_first_token_seconds_sum{engine="0",model_name="juggernaut"} 2904.5298397541046
|
||||
# HELP vllm:time_to_first_token_seconds_created Histogram of time to first token in seconds.
|
||||
# TYPE vllm:time_to_first_token_seconds_created gauge
|
||||
vllm:time_to_first_token_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022041366e+09
|
||||
# HELP vllm:inter_token_latency_seconds Histogram of inter-token latency in seconds.
|
||||
# TYPE vllm:inter_token_latency_seconds histogram
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.01",model_name="juggernaut"} 0.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.025",model_name="juggernaut"} 0.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.05",model_name="juggernaut"} 596060.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.075",model_name="juggernaut"} 605658.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.1",model_name="juggernaut"} 605758.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.15",model_name="juggernaut"} 606021.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.2",model_name="juggernaut"} 606268.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 606488.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.4",model_name="juggernaut"} 606669.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 606814.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="0.75",model_name="juggernaut"} 607341.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 607398.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="7.5",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="80.0",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_count{engine="0",model_name="juggernaut"} 607673.0
|
||||
vllm:inter_token_latency_seconds_sum{engine="0",model_name="juggernaut"} 22579.6616614901
|
||||
# HELP vllm:inter_token_latency_seconds_created Histogram of inter-token latency in seconds.
|
||||
# TYPE vllm:inter_token_latency_seconds_created gauge
|
||||
vllm:inter_token_latency_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022041779e+09
|
||||
# HELP vllm:request_time_per_output_token_seconds Histogram of time_per_output_token_seconds per request.
|
||||
# TYPE vllm:request_time_per_output_token_seconds histogram
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.01",model_name="juggernaut"} 0.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.025",model_name="juggernaut"} 0.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.05",model_name="juggernaut"} 3383.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.075",model_name="juggernaut"} 3485.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.1",model_name="juggernaut"} 3487.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.15",model_name="juggernaut"} 3487.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.2",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.4",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.75",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="7.5",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="80.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_time_per_output_token_seconds_sum{engine="0",model_name="juggernaut"} 125.03691733873717
|
||||
# HELP vllm:request_time_per_output_token_seconds_created Histogram of time_per_output_token_seconds per request.
|
||||
# TYPE vllm:request_time_per_output_token_seconds_created gauge
|
||||
vllm:request_time_per_output_token_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022042146e+09
|
||||
# HELP vllm:e2e_request_latency_seconds Histogram of e2e request latency in seconds.
|
||||
# TYPE vllm:e2e_request_latency_seconds histogram
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 0.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 2.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 5.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 5.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 14.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 57.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 211.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 1389.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3241.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3339.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3369.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3404.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3418.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3436.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3449.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3480.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3486.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0
|
||||
vllm:e2e_request_latency_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:e2e_request_latency_seconds_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:e2e_request_latency_seconds_sum{engine="0",model_name="juggernaut"} 25442.03079867363
|
||||
# HELP vllm:e2e_request_latency_seconds_created Histogram of e2e request latency in seconds.
|
||||
# TYPE vllm:e2e_request_latency_seconds_created gauge
|
||||
vllm:e2e_request_latency_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022042491e+09
|
||||
# HELP vllm:request_queue_time_seconds Histogram of time spent in WAITING phase for request.
|
||||
# TYPE vllm:request_queue_time_seconds histogram
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 3485.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 3485.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 3485.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 3485.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 3487.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 3487.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 3487.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_queue_time_seconds_sum{engine="0",model_name="juggernaut"} 6.650666017842013
|
||||
# HELP vllm:request_queue_time_seconds_created Histogram of time spent in WAITING phase for request.
|
||||
# TYPE vllm:request_queue_time_seconds_created gauge
|
||||
vllm:request_queue_time_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022042854e+09
|
||||
# HELP vllm:request_inference_time_seconds Histogram of time spent in RUNNING phase for request.
|
||||
# TYPE vllm:request_inference_time_seconds histogram
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 0.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 2.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 6.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 6.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 14.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 61.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 214.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 1394.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3248.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3339.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3369.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3404.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3418.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3436.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3449.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3480.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3486.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_inference_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_inference_time_seconds_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_inference_time_seconds_sum{engine="0",model_name="juggernaut"} 25331.654430066672
|
||||
# HELP vllm:request_inference_time_seconds_created Histogram of time spent in RUNNING phase for request.
|
||||
# TYPE vllm:request_inference_time_seconds_created gauge
|
||||
vllm:request_inference_time_seconds_created{engine="0",model_name="juggernaut"} 1.79210810220433e+09
|
||||
# HELP vllm:request_prefill_time_seconds Histogram of time spent in PREFILL phase for request.
|
||||
# TYPE vllm:request_prefill_time_seconds histogram
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 844.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 1341.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 2035.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 2314.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 3230.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 3384.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 3444.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 3484.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3487.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3487.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_time_seconds_sum{engine="0",model_name="juggernaut"} 2786.310553313582
|
||||
# HELP vllm:request_prefill_time_seconds_created Histogram of time spent in PREFILL phase for request.
|
||||
# TYPE vllm:request_prefill_time_seconds_created gauge
|
||||
vllm:request_prefill_time_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022043684e+09
|
||||
# HELP vllm:request_decode_time_seconds Histogram of time spent in DECODE phase for request.
|
||||
# TYPE vllm:request_decode_time_seconds histogram
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 0.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 4.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 6.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 7.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 22.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 165.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 452.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 2092.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3278.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3354.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3376.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3406.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3422.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3438.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3450.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3481.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3486.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_decode_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_decode_time_seconds_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_decode_time_seconds_sum{engine="0",model_name="juggernaut"} 22545.34387675309
|
||||
# HELP vllm:request_decode_time_seconds_created Histogram of time spent in DECODE phase for request.
|
||||
# TYPE vllm:request_decode_time_seconds_created gauge
|
||||
vllm:request_decode_time_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022044446e+09
|
||||
# HELP vllm:request_num_preemptions Histogram of the number of times a request was preempted.
|
||||
# TYPE vllm:request_num_preemptions histogram
|
||||
vllm:request_num_preemptions_bucket{engine="0",le="1.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_bucket{engine="0",le="2.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_bucket{engine="0",le="3.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_bucket{engine="0",le="4.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_bucket{engine="0",le="5.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_bucket{engine="0",le="10.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_num_preemptions_sum{engine="0",model_name="juggernaut"} 0.0
|
||||
# HELP vllm:request_num_preemptions_created Histogram of the number of times a request was preempted.
|
||||
# TYPE vllm:request_num_preemptions_created gauge
|
||||
vllm:request_num_preemptions_created{engine="0",model_name="juggernaut"} 1.792108102204479e+09
|
||||
# HELP vllm:request_prefill_kv_computed_tokens Histogram of new KV tokens computed during prefill (excluding cached tokens).
|
||||
# TYPE vllm:request_prefill_kv_computed_tokens histogram
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 5.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 6.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 32.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 136.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 628.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 1206.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 2210.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3423.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3485.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3486.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_kv_computed_tokens_count{engine="0",model_name="juggernaut"} 3488.0
|
||||
vllm:request_prefill_kv_computed_tokens_sum{engine="0",model_name="juggernaut"} 6.267701e+06
|
||||
# HELP vllm:request_prefill_kv_computed_tokens_created Histogram of new KV tokens computed during prefill (excluding cached tokens).
|
||||
# TYPE vllm:request_prefill_kv_computed_tokens_created gauge
|
||||
vllm:request_prefill_kv_computed_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022045038e+09
|
||||
# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig
|
||||
# TYPE vllm:cache_config_info gauge
|
||||
vllm:cache_config_info{_block_size_resolved="True",block_size="1568",cache_dtype="fp8",enable_mamba_fine_grained_prefix_cache="False",enable_prefix_caching="True",engine="0",gpu_memory_utilization="0.95",is_attention_free="False",kv_cache_dtype_skip_layers="[]",kv_cache_layout="None",kv_cache_max_concurrency="2.1333333333333333",kv_cache_memory_bytes="None",kv_cache_size_tokens="279620",kv_offloading_backend="native",kv_offloading_size="None",kv_sharing_fast_prefill="False",mamba_block_size="1568",mamba_cache_dtype="auto",mamba_cache_mode="align",mamba_page_size_padded="None",mamba_ssm_cache_dtype="float32",num_cpu_blocks="None",num_gpu_blocks="192",num_gpu_blocks_override="None",prefix_cache_retention_interval="0",prefix_caching_hash_algo="sha256",prefix_match_unit="None",replayssm_buffer_len="16",skip_page_size_padded="None",sliding_window="None",use_kda_recoverssm="False",use_replayssm="False",user_specified_block_size="False",user_specified_mamba_block_size="False"} 1.0
|
||||
# HELP http_requests_total Total number of requests by method, status and handler.
|
||||
# TYPE http_requests_total counter
|
||||
http_requests_total{handler="/v1/models",method="GET",status="2xx"} 236.0
|
||||
http_requests_total{handler="/v1/chat/completions",method="POST",status="2xx"} 3490.0
|
||||
http_requests_total{handler="/v1/chat/completions",method="POST",status="4xx"} 1.0
|
||||
# HELP http_requests_created Total number of requests by method, status and handler.
|
||||
# TYPE http_requests_created gauge
|
||||
http_requests_created{handler="/v1/models",method="GET",status="2xx"} 1.7921081971217349e+09
|
||||
http_requests_created{handler="/v1/chat/completions",method="POST",status="2xx"} 1.792108217381334e+09
|
||||
http_requests_created{handler="/v1/chat/completions",method="POST",status="4xx"} 1.7921641238703468e+09
|
||||
# HELP http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
|
||||
# TYPE http_request_size_bytes summary
|
||||
http_request_size_bytes_count{handler="/v1/models"} 236.0
|
||||
http_request_size_bytes_sum{handler="/v1/models"} 0.0
|
||||
http_request_size_bytes_count{handler="/v1/chat/completions"} 3491.0
|
||||
http_request_size_bytes_sum{handler="/v1/chat/completions"} 7.4767849e+07
|
||||
# HELP http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
|
||||
# TYPE http_request_size_bytes_created gauge
|
||||
http_request_size_bytes_created{handler="/v1/models"} 1.7921081971218035e+09
|
||||
http_request_size_bytes_created{handler="/v1/chat/completions"} 1.7921082173813493e+09
|
||||
# HELP http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
|
||||
# TYPE http_response_size_bytes summary
|
||||
http_response_size_bytes_count{handler="/v1/models"} 236.0
|
||||
http_response_size_bytes_sum{handler="/v1/models"} 113516.0
|
||||
http_response_size_bytes_count{handler="/v1/chat/completions"} 3491.0
|
||||
http_response_size_bytes_sum{handler="/v1/chat/completions"} 4.443412e+06
|
||||
# HELP http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated.
|
||||
# TYPE http_response_size_bytes_created gauge
|
||||
http_response_size_bytes_created{handler="/v1/models"} 1.7921081971218395e+09
|
||||
http_response_size_bytes_created{handler="/v1/chat/completions"} 1.7921082173813689e+09
|
||||
# HELP http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations.
|
||||
# TYPE http_request_duration_highr_seconds histogram
|
||||
http_request_duration_highr_seconds_bucket{le="0.01"} 236.0
|
||||
http_request_duration_highr_seconds_bucket{le="0.025"} 236.0
|
||||
http_request_duration_highr_seconds_bucket{le="0.05"} 236.0
|
||||
http_request_duration_highr_seconds_bucket{le="0.075"} 236.0
|
||||
http_request_duration_highr_seconds_bucket{le="0.1"} 236.0
|
||||
http_request_duration_highr_seconds_bucket{le="0.25"} 237.0
|
||||
http_request_duration_highr_seconds_bucket{le="0.5"} 239.0
|
||||
http_request_duration_highr_seconds_bucket{le="0.75"} 241.0
|
||||
http_request_duration_highr_seconds_bucket{le="1.0"} 242.0
|
||||
http_request_duration_highr_seconds_bucket{le="1.5"} 251.0
|
||||
http_request_duration_highr_seconds_bucket{le="2.0"} 294.0
|
||||
http_request_duration_highr_seconds_bucket{le="2.5"} 446.0
|
||||
http_request_duration_highr_seconds_bucket{le="3.0"} 678.0
|
||||
http_request_duration_highr_seconds_bucket{le="3.5"} 917.0
|
||||
http_request_duration_highr_seconds_bucket{le="4.0"} 1126.0
|
||||
http_request_duration_highr_seconds_bucket{le="4.5"} 1351.0
|
||||
http_request_duration_highr_seconds_bucket{le="5.0"} 1625.0
|
||||
http_request_duration_highr_seconds_bucket{le="7.5"} 3191.0
|
||||
http_request_duration_highr_seconds_bucket{le="10.0"} 3478.0
|
||||
http_request_duration_highr_seconds_bucket{le="30.0"} 3643.0
|
||||
http_request_duration_highr_seconds_bucket{le="60.0"} 3688.0
|
||||
http_request_duration_highr_seconds_bucket{le="+Inf"} 3727.0
|
||||
http_request_duration_highr_seconds_count 3727.0
|
||||
http_request_duration_highr_seconds_sum 25494.72712990234
|
||||
# HELP http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations.
|
||||
# TYPE http_request_duration_highr_seconds_created gauge
|
||||
http_request_duration_highr_seconds_created 1.7921081024104605e+09
|
||||
# HELP http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important.
|
||||
# TYPE http_request_duration_seconds histogram
|
||||
http_request_duration_seconds_bucket{handler="/v1/models",le="0.1",method="GET"} 236.0
|
||||
http_request_duration_seconds_bucket{handler="/v1/models",le="0.5",method="GET"} 236.0
|
||||
http_request_duration_seconds_bucket{handler="/v1/models",le="1.0",method="GET"} 236.0
|
||||
http_request_duration_seconds_bucket{handler="/v1/models",le="+Inf",method="GET"} 236.0
|
||||
http_request_duration_seconds_count{handler="/v1/models",method="GET"} 236.0
|
||||
http_request_duration_seconds_sum{handler="/v1/models",method="GET"} 0.09283922897884622
|
||||
http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="0.1",method="POST"} 0.0
|
||||
http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="0.5",method="POST"} 3.0
|
||||
http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="1.0",method="POST"} 6.0
|
||||
http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="+Inf",method="POST"} 3491.0
|
||||
http_request_duration_seconds_count{handler="/v1/chat/completions",method="POST"} 3491.0
|
||||
http_request_duration_seconds_sum{handler="/v1/chat/completions",method="POST"} 25494.634290673363
|
||||
# HELP http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important.
|
||||
# TYPE http_request_duration_seconds_created gauge
|
||||
http_request_duration_seconds_created{handler="/v1/models",method="GET"} 1.7921081971218753e+09
|
||||
http_request_duration_seconds_created{handler="/v1/chat/completions",method="POST"} 1.7921082173813894e+09
|
||||
# HELP vllm:tool_call_parser_invocations_total Total number of ToolParser invocations. Non-streaming increments once per choice; streaming increments once per delta.
|
||||
# TYPE vllm:tool_call_parser_invocations_total counter
|
||||
vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="chat_completions"} 31000.0
|
||||
vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="responses"} 0.0
|
||||
vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="other"} 0.0
|
||||
vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="chat_completions"} 42734.0
|
||||
vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="responses"} 0.0
|
||||
vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="other"} 0.0
|
||||
vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="chat_completions"} 0.0
|
||||
vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="responses"} 0.0
|
||||
vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="other"} 0.0
|
||||
vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="chat_completions"} 3139.0
|
||||
vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="responses"} 0.0
|
||||
vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="other"} 0.0
|
||||
# HELP vllm:tool_call_parser_invocations_created Total number of ToolParser invocations. Non-streaming increments once per choice; streaming increments once per delta.
|
||||
# TYPE vllm:tool_call_parser_invocations_created gauge
|
||||
vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="chat_completions"} 1.7921081024108987e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="responses"} 1.792108102410908e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="other"} 1.7921081024109144e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="chat_completions"} 1.7921081024109201e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="responses"} 1.7921081024109259e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="other"} 1.792108102410931e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="chat_completions"} 1.7921081024109368e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="responses"} 1.7921081024109426e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="other"} 1.7921081024109476e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="chat_completions"} 1.792108102410953e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="responses"} 1.7921081024109583e+09
|
||||
vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="other"} 1.7921081024109638e+09
|
||||
@@ -0,0 +1,284 @@
|
||||
"""The host agent: stdlib-only, read-only by construction, and wrong on no box.
|
||||
|
||||
Three things are asserted here:
|
||||
* parsing of /proc and /sys files exactly as Linux writes them (fake roots, so the
|
||||
weird cases are testable on any machine);
|
||||
* the collector produces a complete, non-crashing snapshot against the *real* /proc of
|
||||
whatever runs the tests, because that is the code that ships;
|
||||
* the HTTP surface answers GETs, needs a token when it is exposed, and has no write path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
AGENT_PATH = ROOT / "agent" / "host_agent.py"
|
||||
|
||||
spec = importlib.util.spec_from_file_location("host_agent", AGENT_PATH)
|
||||
assert spec and spec.loader
|
||||
host_agent = importlib.util.module_from_spec(spec)
|
||||
sys.modules["host_agent"] = host_agent
|
||||
spec.loader.exec_module(host_agent)
|
||||
|
||||
linux_only = pytest.mark.skipif(not sys.platform.startswith("linux"),
|
||||
reason="reads /proc and /sys")
|
||||
|
||||
|
||||
# ---- source-level helpers ------------------------------------------------- #
|
||||
|
||||
@pytest.mark.parametrize("lspci,expected", [
|
||||
("Advanced Micro Devices, Inc. [AMD/ATI] Navi 48 [Radeon AI PRO R9700]",
|
||||
"Radeon AI PRO R9700"),
|
||||
("Advanced Micro Devices, Inc. [AMD/ATI] Strix Halo [Radeon Graphics / Radeon 8050S "
|
||||
"Graphics / Radeon 8060S Graphics]", "Radeon 8060S Graphics"),
|
||||
("Advanced Micro Devices, Inc. [AMD/ATI] Phoenix1", "Phoenix1"),
|
||||
("00:02.0 VGA compatible controller: Intel Corporation UHD Graphics 620",
|
||||
"UHD Graphics 620"),
|
||||
("NVIDIA GB200", "NVIDIA GB200"),
|
||||
("Vendor 1af1 Device 0001", "Vendor 1af1 Device 0001"),
|
||||
(None, None),
|
||||
("", None),
|
||||
])
|
||||
def test_gpu_model_is_shortened_for_legends_without_losing_identity(lspci, expected) -> None:
|
||||
"""`ATI` on a legend is worse than no name: it is true of every AMD GPU on the box."""
|
||||
assert host_agent.short_gpu_model(lspci) == expected
|
||||
|
||||
|
||||
def test_gpu_model_is_capped_so_a_legend_cannot_blow_up() -> None:
|
||||
long_name = "X" * 200
|
||||
assert len(host_agent.short_gpu_model(long_name)) <= 41 # 40 + ellipsis
|
||||
|
||||
|
||||
# ---- /proc and /sys parsing (fake roots) ---------------------------------- #
|
||||
|
||||
def write_file(root: Path, relative: str, text: str) -> None:
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text)
|
||||
|
||||
|
||||
def test_read_number_takes_the_first_field(tmp_path: Path) -> None:
|
||||
"""/proc/uptime is `210036.18 9613750.86`; float() of the whole line is None."""
|
||||
write_file(tmp_path, "proc/uptime", "210036.18 9613750.86\n")
|
||||
collector = host_agent.Collector(str(tmp_path))
|
||||
assert collector.read_number("proc/uptime") == 210036.18
|
||||
|
||||
|
||||
def test_read_number_handles_missing_and_junk_files(tmp_path: Path) -> None:
|
||||
collector = host_agent.Collector(str(tmp_path))
|
||||
assert collector.read_number("proc/nope") is None
|
||||
write_file(tmp_path, "proc/junk", "\n")
|
||||
assert collector.read_number("proc/junk") is None
|
||||
|
||||
|
||||
def test_core_counts_dedupe_socket_and_core(tmp_path: Path) -> None:
|
||||
"""One /proc/cpuinfo block per *thread*: counting 'core id' lines said '1 core'."""
|
||||
blocks = []
|
||||
for thread in range(8): # 2 sockets x 2 cores x 2 threads
|
||||
socket, core = thread % 2, (thread // 2) % 2
|
||||
blocks.append(f"processor\t: {thread}\nphysical id\t: {socket}\ncore id\t\t: {core}\n")
|
||||
write_file(tmp_path, "proc/cpuinfo", "\n".join(blocks))
|
||||
collector = host_agent.Collector(str(tmp_path))
|
||||
assert collector.core_counts() == (8, 4)
|
||||
|
||||
|
||||
def test_core_counts_falls_back_to_smt_width_on_sysfs(tmp_path: Path) -> None:
|
||||
"""Some kernels/vms give no physical id/core id; cpu0's sibling list still says SMT=2."""
|
||||
write_file(tmp_path, "proc/cpuinfo", "processor\t: 0\nprocessor\t: 1\n")
|
||||
write_file(tmp_path, "sys/devices/system/cpu/cpu0/topology/thread_siblings_list", "0,1\n")
|
||||
collector = host_agent.Collector(str(tmp_path))
|
||||
logical, physical = collector.core_counts()
|
||||
assert (logical, physical) == (2, 1)
|
||||
|
||||
|
||||
def test_core_counts_survive_a_kernel_with_no_hints(tmp_path: Path) -> None:
|
||||
collector = host_agent.Collector(str(tmp_path))
|
||||
assert collector.core_counts()[1] is None, "unknown must be None, never 1"
|
||||
|
||||
|
||||
def test_cpu_modes_total_excludes_guest_ticks(tmp_path: Path) -> None:
|
||||
"""/proc/stat's idle/iowait are already inside 'total'; guest is inside user on Linux."""
|
||||
write_file(tmp_path, "proc/stat",
|
||||
"cpu 100 10 20 500 5 3 2 1 0 0\nintr 1\nctxt 2\nprocesses 3\n")
|
||||
collector = host_agent.Collector(str(tmp_path))
|
||||
sample = collector.cpu()["modes"]
|
||||
assert sample["user"] == 100 and sample["idle"] == 500
|
||||
assert sample["total"] == pytest.approx(100 + 10 + 20 + 500 + 5 + 3 + 2 + 1)
|
||||
|
||||
|
||||
# ---- the real collector --------------------------------------------------- #
|
||||
|
||||
@linux_only
|
||||
def test_snapshot_against_the_live_system_is_complete() -> None:
|
||||
collector = host_agent.Collector("/")
|
||||
snap = collector.snapshot()
|
||||
for key in ("schema", "ts", "host", "cpu", "memory", "filesystems", "disk_io",
|
||||
"network", "gpus", "temperatures", "processes", "errors"):
|
||||
assert key in snap, f"{key} missing from a real snapshot"
|
||||
assert snap["host"]["hostname"]
|
||||
assert snap["cpu"]["modes"]["total"] > 0
|
||||
assert len(snap["cpu"]["cores"]) >= 1
|
||||
assert snap["memory"]["total_bytes"] > 0
|
||||
assert snap["filesystems"], "the box it runs on has at least one mounted filesystem"
|
||||
assert snap["errors"] == [], f"degraded sections must be empty on a healthy box: {snap['errors']}"
|
||||
|
||||
|
||||
def test_captured_fixture_has_the_shape_the_dashboard_depends_on(agent_snapshot) -> None:
|
||||
"""Ground truth from driveripper: if the agent's schema drifts, this fails first."""
|
||||
assert agent_snapshot["schema"] == host_agent.SCHEMA_VERSION
|
||||
host = agent_snapshot["host"]
|
||||
assert host["hostname"] == "driveripper"
|
||||
assert (host["cpu_cores_physical"], host["cpu_cores_logical"]) == (24, 48)
|
||||
assert host["uptime_seconds"] > 0
|
||||
gpu = agent_snapshot["gpus"][0]
|
||||
assert gpu["model"] == "Radeon AI PRO R9700"
|
||||
assert "model_full" in gpu and "Navi 48" in gpu["model_full"]
|
||||
assert gpu["vram_total_bytes"] > 0
|
||||
# the dashboard renders n/a for anything a driver omits — it must never invent a 0
|
||||
assert gpu["busy_pct"] is None or 0.0 <= gpu["busy_pct"] <= 100.0
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_uptime_and_boot_ts_are_reported() -> None:
|
||||
"""Both are needed: uptime for the table, boot_ts to detect a counter-reset reboot."""
|
||||
info = host_agent.Collector("/").host_info()
|
||||
assert info["uptime_seconds"] and info["uptime_seconds"] > 0
|
||||
assert info["boot_ts"] and info["boot_ts"] <= time.time()
|
||||
assert abs((time.time() - info["boot_ts"]) - info["uptime_seconds"]) < 60
|
||||
assert info["cpu_cores_logical"] and info["cpu_cores_physical"]
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_snapshot_is_repeatable_and_does_not_deadlock() -> None:
|
||||
"""snapshot() reads processes() internally; a non-reentrant lock used to hang it."""
|
||||
collector = host_agent.Collector("/")
|
||||
first = collector.snapshot()
|
||||
second = collector.snapshot(top=0)
|
||||
assert first["processes"]["total"] > 0
|
||||
assert second["processes"]["top_cpu"] == [], "top=0 must skip the expensive scan"
|
||||
|
||||
|
||||
@linux_only
|
||||
def test_disk_io_comes_from_sysfs_not_proc_diskstats() -> None:
|
||||
"""/proc/diskstats field order has moved between kernels; /sys/block/<dev>/stat has not."""
|
||||
devices = host_agent.Collector("/").disk_io()["devices"]
|
||||
assert devices
|
||||
for device in devices:
|
||||
assert device["reads_total"] is None or device["reads_total"] >= 0
|
||||
assert device["size_bytes"] is None or device["size_bytes"] > 0
|
||||
|
||||
|
||||
# ---- HTTP surface --------------------------------------------------------- #
|
||||
|
||||
@pytest.fixture
|
||||
def agent_server():
|
||||
"""Run the real handler on an ephemeral loopback port; yield (get, port)."""
|
||||
servers: list[ThreadingHTTPServer] = []
|
||||
|
||||
def start(token: str | None = None) -> tuple[str, int]:
|
||||
host_agent.Handler.collector = host_agent.Collector("/")
|
||||
host_agent.Handler.token = token
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), host_agent.Handler)
|
||||
servers.append(httpd)
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
return f"http://127.0.0.1:{httpd.server_address[1]}", httpd.server_address[1]
|
||||
|
||||
yield start
|
||||
for httpd in servers:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
||||
|
||||
def _get(url: str, headers: dict[str, str] | None = None) -> tuple[int, dict]:
|
||||
request = urllib.request.Request(url, headers=headers or {})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status, json.loads(response.read())
|
||||
except urllib.error.HTTPError as error:
|
||||
body = error.read()
|
||||
try:
|
||||
return error.code, json.loads(body)
|
||||
except json.JSONDecodeError: # pragma: no cover - handler always sends JSON
|
||||
return error.code, {"raw": body.decode(errors="replace")}
|
||||
|
||||
|
||||
def test_health_info_schema_snapshot_all_answer(agent_server) -> None:
|
||||
base, _port = agent_server()
|
||||
for path in ("/health", "/info", "/schema", "/snapshot"):
|
||||
status, body = _get(base + path)
|
||||
assert status == 200, path
|
||||
assert body["ok"] is True, path
|
||||
|
||||
|
||||
def test_snapshot_query_param_controls_the_process_scan(agent_server) -> None:
|
||||
base, _port = agent_server()
|
||||
_status, body = _get(base + "/snapshot?top=0")
|
||||
assert body["processes"]["top_cpu"] == []
|
||||
_status, full = _get(base + "/snapshot?top=3")
|
||||
assert len(full["processes"]["top_cpu"]) <= 3
|
||||
|
||||
|
||||
def test_unknown_path_is_404_not_a_snapshot(agent_server) -> None:
|
||||
base, _port = agent_server()
|
||||
status, body = _get(base + "/etc/shadow")
|
||||
assert status == 404
|
||||
assert body["ok"] is False
|
||||
|
||||
|
||||
def test_token_is_required_when_configured_but_health_stays_open(agent_server) -> None:
|
||||
base, _port = agent_server(token="s3cret")
|
||||
assert _get(base + "/snapshot")[0] == 401
|
||||
assert _get(base + "/snapshot", {"Authorization": "Bearer wrong"})[0] == 401
|
||||
assert _get(base + "/snapshot", {"Authorization": "Bearer s3cret"})[0] == 200
|
||||
# /health stays open so a monitoring probe needs no secret
|
||||
assert _get(base + "/health")[0] == 200
|
||||
|
||||
|
||||
def test_agent_has_no_write_path(agent_server) -> None:
|
||||
"""No do_POST/do_PUT exists: read-only is structural, not a policy toggle."""
|
||||
base, _port = agent_server()
|
||||
request = urllib.request.Request(base + "/snapshot", data=b"{}", method="POST")
|
||||
with pytest.raises(urllib.error.HTTPError) as excinfo:
|
||||
urllib.request.urlopen(request, timeout=5)
|
||||
assert excinfo.value.code in (405, 501)
|
||||
|
||||
|
||||
def test_refuses_to_bind_off_loopback_without_a_token() -> None:
|
||||
"""0.0.0.0 with no token would hand host inventory to the whole LAN."""
|
||||
assert host_agent.main(["--host", "0.0.0.0", "--port", "0"]) == 2
|
||||
|
||||
|
||||
def test_once_mode_prints_json_and_exits(capsys) -> None:
|
||||
assert host_agent.main(["--once"]) == 0
|
||||
printed = capsys.readouterr().out
|
||||
assert json.loads(printed)["ok"] is True
|
||||
|
||||
|
||||
def test_agent_is_dependency_free() -> None:
|
||||
"""The agent must run on a stock distro python: stdlib imports, and few of them.
|
||||
|
||||
Two checks, because they fail for different reasons: anything outside the stdlib means a
|
||||
`pip install` on every monitored host (deployment pain), while a growing footprint means
|
||||
the agent can break on an older interpreter than the dashboard's own Python.
|
||||
"""
|
||||
text = AGENT_PATH.read_text()
|
||||
imports = re.findall(r"^(?:import|from)\s+([a-zA-Z0-9_.]+)", text, re.M)
|
||||
# indented (function-local) imports count too — subprocess is loaded lazily on purpose
|
||||
imports += re.findall(r"^\s+(?:import|from)\s+([a-zA-Z0-9_.]+)", text, re.M)
|
||||
top = {name.split(".")[0] for name in imports}
|
||||
third_party = top - set(sys.stdlib_module_names) - {"host_agent"}
|
||||
assert not third_party, f"agent grew a non-stdlib import: {third_party}"
|
||||
assert top <= {"__future__", "argparse", "collections", "dataclasses", "http", "json",
|
||||
"os", "pathlib", "re", "socket", "subprocess", "threading", "time",
|
||||
"typing"}, f"agent import footprint grew: {sorted(top)}"
|
||||
@@ -0,0 +1,346 @@
|
||||
"""HTTP contract: what the pages may assume, and what must never leak.
|
||||
|
||||
The behaviours here are the ones a screenshot cannot show: an unconfigured source answering
|
||||
404 `configured: false` instead of dialling a hostname that does not exist; a bearer token
|
||||
that stops at the backend; the single write endpoint refusing on every gate; no external
|
||||
asset the network can fail to deliver.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from dashboards.app import create_app
|
||||
from dashboards.config import Settings, Target
|
||||
from dashboards.store import Event, Sample, SampleStore
|
||||
|
||||
# refused immediately, and never dials out of the test environment
|
||||
DEAD_URL = "http://127.0.0.1:1"
|
||||
|
||||
|
||||
def build_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, simulate: bool = False,
|
||||
**overrides: Any) -> TestClient:
|
||||
"""A TestClient whose lifespan runs, against a private DB and no real network."""
|
||||
if simulate:
|
||||
monkeypatch.setenv("NUT_SIMULATE", "1")
|
||||
else:
|
||||
monkeypatch.delenv("NUT_SIMULATE", raising=False)
|
||||
db = str(tmp_path / "api.sqlite3")
|
||||
settings = Settings(db_path=db, **overrides)
|
||||
store = SampleStore(db)
|
||||
app = create_app(settings, store)
|
||||
client = TestClient(app)
|
||||
client.__enter__()
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
||||
with build_client(tmp_path, monkeypatch) as test_client:
|
||||
yield test_client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ups_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
||||
"""NUT on the labelled simulator, actions allowed — the dev configuration."""
|
||||
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
||||
nut_allow_actions=True) as c:
|
||||
yield c
|
||||
|
||||
|
||||
def wait_for_payload(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
response = client.get(path)
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
time.sleep(0.1)
|
||||
pytest.fail(f"{path} never became available")
|
||||
|
||||
|
||||
# ---- unconfigured: say so, and do not dial -------------------------------- #
|
||||
|
||||
def test_config_reports_every_source_as_off(client) -> None:
|
||||
body = client.get("/api/config").json()
|
||||
assert body["system"]["enabled"] is False
|
||||
assert body["system"]["targets"] == []
|
||||
assert body["llm"]["gufo"]["enabled"] is False
|
||||
assert body["llm"]["vllm"]["enabled"] is False
|
||||
assert body["ups"]["enabled"] is False
|
||||
assert body["ups"]["actions_enabled"] is False
|
||||
assert body["ranges"], "the page builds its window buttons from this list"
|
||||
|
||||
|
||||
def test_health_with_nothing_configured_is_not_a_failure(client) -> None:
|
||||
"""Nothing configured is 'empty', not 'broken' — and the page must not go red."""
|
||||
body = client.get("/api/health").json()
|
||||
assert body["status"] in ("ok", "empty")
|
||||
assert body["ok"] is True
|
||||
assert body["sources"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/api/system/snapshot", "/api/llm/snapshot",
|
||||
"/api/ups/snapshot", "/api/ups/commands"])
|
||||
def test_unconfigured_source_is_404_configured_false(client, path) -> None:
|
||||
response = client.get(path)
|
||||
assert response.status_code == 404
|
||||
assert response.json()["configured"] is False
|
||||
assert "detail" in response.json()
|
||||
|
||||
|
||||
def test_summary_still_answers_with_no_sources(client) -> None:
|
||||
body = client.get("/api/summary").json()
|
||||
assert body["system"]["enabled"] is False
|
||||
assert body["system"]["fleet"] is None
|
||||
assert body["llm"]["enabled"] is False
|
||||
assert body["ups"]["enabled"] is False
|
||||
|
||||
|
||||
def test_unknown_route_is_json_not_html(client) -> None:
|
||||
response = client.get("/api/nope")
|
||||
assert response.status_code == 404
|
||||
assert response.headers["content-type"].startswith("application/json")
|
||||
assert "no such route" in response.json()["detail"]
|
||||
|
||||
|
||||
# ---- tokens --------------------------------------------------------------- #
|
||||
|
||||
def test_agent_tokens_never_reach_the_browser(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("NUT_SIMULATE", "")
|
||||
targets = [Target(name="driveripper", url="http://driveripper:9120",
|
||||
headers={"Authorization": "Bearer SEKRIT-TOKEN"})]
|
||||
db = str(tmp_path / "tokens.sqlite3")
|
||||
store = SampleStore(db)
|
||||
app = create_app(Settings(db_path=db, host_targets=tuple(targets)), store)
|
||||
with TestClient(app) as client:
|
||||
body = client.get("/api/config")
|
||||
assert "SEKRIT-TOKEN" not in body.text
|
||||
target = body.json()["system"]["targets"][0]
|
||||
assert target["name"] == "driveripper"
|
||||
assert "SEKRIT" not in target["url"] and "@" not in target["url"]
|
||||
# and a host that never answered stays in the list, marked, with the fleet card intact
|
||||
snapshot = wait_for_state(client, "/api/system/snapshot")
|
||||
assert snapshot["hosts"][0]["name"] == "driveripper"
|
||||
assert snapshot["hosts"][0].get("unreachable") is True
|
||||
assert snapshot["fleet"]["hosts_total"] == 1
|
||||
assert snapshot["fleet"]["hosts_live"] == 0
|
||||
assert snapshot["fleet"]["memory_total_bytes"] is None
|
||||
store.close()
|
||||
|
||||
|
||||
def wait_for_state(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
response = client.get(path)
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
time.sleep(0.1)
|
||||
pytest.fail(f"{path} never answered")
|
||||
|
||||
|
||||
def test_target_problems_are_surfaced_not_swallowed(client) -> None:
|
||||
body = client.get("/api/config").json()
|
||||
assert body["system"]["target_problems"] == []
|
||||
|
||||
|
||||
# ---- pages ---------------------------------------------------------------- #
|
||||
|
||||
@pytest.mark.parametrize("path,title", [
|
||||
("/", "overview"), ("/system", "system"), ("/llm", "llm"), ("/ups", "ups"),
|
||||
])
|
||||
def test_pages_serve_and_declare_themselves(client, path, title) -> None:
|
||||
response = client.get(path)
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers["content-type"]
|
||||
assert title in response.text.lower()
|
||||
|
||||
|
||||
def test_no_page_loads_anything_from_the_internet(client) -> None:
|
||||
"""No CDN, no webfonts: this dashboard has to work with no route to the internet."""
|
||||
for path in ("/", "/system", "/llm", "/ups"):
|
||||
html = client.get(path).text
|
||||
for fragment in ("//cdn.", "https://fonts.", "//unpkg.com", "//cdnjs"):
|
||||
assert fragment not in html, f"{path} references {fragment}"
|
||||
|
||||
|
||||
def test_static_assets_are_served(client) -> None:
|
||||
for asset in ("/static/dash.css", "/static/app.js", "/static/charts.js",
|
||||
"/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js",
|
||||
"/static/page-index.js"):
|
||||
assert client.get(asset).status_code == 200, asset
|
||||
|
||||
|
||||
def test_api_responses_are_not_cacheable(client) -> None:
|
||||
for path in ("/api/config", "/api/health", "/api/summary"):
|
||||
assert "no-store" in client.get(path).headers.get("cache-control", "")
|
||||
|
||||
|
||||
# ---- history -------------------------------------------------------------- #
|
||||
|
||||
def test_history_returns_seeded_samples(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
db = str(tmp_path / "hist.sqlite3")
|
||||
store = SampleStore(db)
|
||||
now = time.time()
|
||||
for i in range(30):
|
||||
store.add("host", "driveripper", "cpu.usage_pct", 10.0 + i, ts=now - 300 + i)
|
||||
app = create_app(Settings(db_path=db, host_targets=(Target("driveripper", DEAD_URL),)), store)
|
||||
with TestClient(app) as client:
|
||||
body = client.get("/api/history", params={"source": "host", "host": "driveripper",
|
||||
"metric": "cpu.usage_pct", "range": "15m"})
|
||||
assert response_ok(body)
|
||||
points = body.json()["points"]
|
||||
assert len(points) >= 25
|
||||
assert points[-1]["value"] == pytest.approx(39.0)
|
||||
store.close()
|
||||
|
||||
|
||||
def test_history_rejects_an_unknown_range(client) -> None:
|
||||
response = client.get("/api/history", params={"source": "host", "host": "h",
|
||||
"metric": "m", "range": "banana"})
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_history_requires_the_series_coordinates(client) -> None:
|
||||
assert client.get("/api/history").status_code == 422
|
||||
|
||||
|
||||
def test_metrics_endpoint_lists_what_actually_has_history(client) -> None:
|
||||
body = client.get("/api/metrics").json()
|
||||
assert body["metrics"] == []
|
||||
|
||||
|
||||
# ---- events --------------------------------------------------------------- #
|
||||
|
||||
def test_events_are_returned_in_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
db = str(tmp_path / "events.sqlite3")
|
||||
store = SampleStore(db)
|
||||
store.log_event(Event(ts=time.time(), source="host", host="deskwork", kind="down",
|
||||
severity="error", message="deskwork is not answering"))
|
||||
app = create_app(Settings(db_path=db), store)
|
||||
with TestClient(app) as client:
|
||||
events = client.get("/api/events").json()["events"]
|
||||
assert events[0]["host"] == "deskwork"
|
||||
assert events[0]["severity"] == "error"
|
||||
store.close()
|
||||
|
||||
|
||||
# ---- the one write endpoint ------------------------------------------------ #
|
||||
|
||||
def test_command_endpoint_is_post_only(ups_client) -> None:
|
||||
assert ups_client.post("/api/ups/command", json={}).status_code in (400, 403, 404)
|
||||
assert ups_client.put("/api/ups/command", json={}).status_code == 405
|
||||
assert ups_client.delete("/api/ups/command").status_code == 405
|
||||
|
||||
|
||||
def test_commands_are_offered_when_actions_are_enabled(ups_client) -> None:
|
||||
body = wait_for_payload(ups_client, "/api/ups/snapshot")
|
||||
assert body["simulated"] is True, "the simulator must label itself"
|
||||
assert body["actions_enabled"] is True
|
||||
ups = body["ups"][0]
|
||||
enabled = [item for item in ups["commands"] if item["enabled"]]
|
||||
assert enabled, "with NUT_ALLOW_ACTIONS=1 and a simulator, safe commands are sendable"
|
||||
assert all(item["risk"] in ("safe", "caution", "dangerous") for item in ups["commands"])
|
||||
|
||||
|
||||
def test_a_safe_command_actually_reaches_the_driver(ups_client) -> None:
|
||||
body = wait_for_payload(ups_client, "/api/ups/snapshot")
|
||||
name = body["ups"][0]["name"]
|
||||
response = ups_client.post("/api/ups/command",
|
||||
json={"ups": name, "command": "test.panel.start"})
|
||||
assert response.status_code == 200, response.text
|
||||
payload = response.json()
|
||||
assert payload["ok"] is True and payload["simulated"] is True
|
||||
events = ups_client.get("/api/events").json()["events"]
|
||||
assert any("test.panel.start" in event["message"] or "Panel test" in event["message"]
|
||||
for event in events), "every sent command must leave an audit trail"
|
||||
|
||||
|
||||
def test_a_dangerous_command_needs_the_confirmation_word(ups_client) -> None:
|
||||
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
||||
deep = "test.battery.start.deep"
|
||||
refusal = ups_client.post("/api/ups/command", json={"ups": name, "command": deep})
|
||||
assert refusal.status_code == 422
|
||||
assert "DANGER" in refusal.json()["detail"]
|
||||
assert ups_client.post("/api/ups/command",
|
||||
json={"ups": name, "command": deep, "confirm": "DANGER"}).status_code == 200
|
||||
|
||||
|
||||
def test_unknown_commands_are_refused_even_with_actions_on(ups_client) -> None:
|
||||
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
||||
for command in ("load.format", "rm -rf /", "INSTCMD x y"):
|
||||
response = ups_client.post("/api/ups/command", json={"ups": name, "command": command})
|
||||
assert response.status_code == 400, command
|
||||
|
||||
|
||||
def test_commands_for_an_unknown_ups_are_404(ups_client) -> None:
|
||||
wait_for_payload(ups_client, "/api/ups/snapshot")
|
||||
response = ups_client.post("/api/ups/command",
|
||||
json={"ups": "not-real", "command": "beeper.mute"})
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_missing_fields_are_a_400(ups_client) -> None:
|
||||
assert ups_client.post("/api/ups/command", json={"ups": ""}).status_code == 400
|
||||
|
||||
|
||||
def test_actions_flag_off_blocks_every_command(tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1") as client:
|
||||
body = wait_for_payload(client, "/api/ups/snapshot")
|
||||
name = body["ups"][0]["name"]
|
||||
assert body["actions_enabled"] is False
|
||||
response = client.post("/api/ups/command",
|
||||
json={"ups": name, "command": "beeper.mute"})
|
||||
assert response.status_code == 403
|
||||
assert "NUT_ALLOW_ACTIONS" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_read_only_beats_the_actions_flag(tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
||||
nut_allow_actions=True, read_only=True) as client:
|
||||
body = wait_for_payload(client, "/api/ups/snapshot")
|
||||
name = body["ups"][0]["name"]
|
||||
response = client.post("/api/ups/command",
|
||||
json={"ups": name, "command": "beeper.mute"})
|
||||
assert response.status_code == 403
|
||||
commands = wait_for_payload(client, "/api/ups/commands")
|
||||
assert commands["read_only"] is True
|
||||
assert all(item["enabled"] is False for ups in commands["ups"]
|
||||
for item in ups["commands"])
|
||||
|
||||
|
||||
def test_ups_history_answers_for_simulated_ups(ups_client) -> None:
|
||||
wait_for_payload(ups_client, "/api/ups/snapshot")
|
||||
time.sleep(1.2)
|
||||
body = ups_client.get("/api/ups/history", params={"metrics": "ups.charge_pct",
|
||||
"range": "1h"}).json()
|
||||
assert body["ups"], "the UPS page needs a name to chart before any history exists"
|
||||
|
||||
|
||||
def response_ok(response) -> bool:
|
||||
assert response.status_code == 200, response.text
|
||||
return True
|
||||
|
||||
|
||||
def test_samples_written_by_the_poller_are_queryable(tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The write path (poller -> store) and the read path (/api/history) must agree on names."""
|
||||
db = str(tmp_path / "roundtrip.sqlite3")
|
||||
store = SampleStore(db)
|
||||
store.add_many([Sample("llm", "gufo", "llm.decode_tokens_per_second", 40.0, time.time() - 60),
|
||||
Sample("llm", "gufo", "llm.decode_tokens_per_second", 45.0, time.time() - 30)])
|
||||
app = create_app(Settings(db_path=db), store)
|
||||
with TestClient(app) as client:
|
||||
metrics = client.get("/api/metrics", params={"source": "llm"}).json()["metrics"]
|
||||
assert "llm.decode_tokens_per_second" in metrics
|
||||
points = client.get("/api/history", params={"source": "llm", "host": "gufo",
|
||||
"metric": "llm.decode_tokens_per_second",
|
||||
"range": "15m"}).json()["points"]
|
||||
assert [p["value"] for p in points] == [40.0, 45.0]
|
||||
store.close()
|
||||
@@ -0,0 +1,443 @@
|
||||
"""Collectors: the fetch half of every poll, exercised with a fake transport.
|
||||
|
||||
`httpx.MockTransport` means these run at the speed of the parser and still cover what the
|
||||
live hosts do badly: a 401 from a rotated agent token, a half-degraded agent snapshot, a
|
||||
gufo server that only answers `/v1/slots`, an engine that counts nothing yet.
|
||||
|
||||
NUT is covered here for the *monitor* bookkeeping (payload shape, command gating, status
|
||||
transitions); the socket protocol itself is in test_nut.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from dashboards.collectors import (
|
||||
NutMonitor,
|
||||
UpstreamError,
|
||||
fetch_engine,
|
||||
fetch_host_snapshot,
|
||||
)
|
||||
from dashboards.config import Target
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.sources import nut
|
||||
from dashboards.sources.host import HostCollector
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
def client_for(handler: Any) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(transport=httpx.MockTransport(handler),
|
||||
base_url="http://agent.test")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# host agent
|
||||
# --------------------------------------------------------------------------- #
|
||||
def agent_body() -> dict[str, Any]:
|
||||
return json.loads((FIXTURES / "agent_driveripper.json").read_text())
|
||||
|
||||
|
||||
def advance_cpu(body: dict[str, Any], *, total: float, idle: float) -> None:
|
||||
"""Move the jiffy counters the way a real /proc/stat does between polls."""
|
||||
modes = body["cpu"]["modes"]
|
||||
for mode, value in modes.items():
|
||||
modes[mode] = value + (total if mode == "total" else idle if mode == "idle" else 0)
|
||||
|
||||
|
||||
async def test_a_real_agent_snapshot_parses() -> None:
|
||||
body = agent_body()
|
||||
calls: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/snapshot"
|
||||
calls.append(request.url.path)
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
target = Target("driveripper", "http://agent.test")
|
||||
collector = HostCollector(RateEstimator())
|
||||
async with client_for(handler) as client:
|
||||
first = (await fetch_host_snapshot(client, target, collector)).payload
|
||||
# CPU usage is Δjiffies/Δt: one scrape has no window, so it is n/a and not 0 %
|
||||
assert first["cpu"]["usage_pct"] is None
|
||||
# the counters have to actually move — total 0 delta means "cannot divide", not idle
|
||||
same = (await fetch_host_snapshot(client, target, collector)).payload
|
||||
assert same["cpu"]["usage_pct"] is None, "no counter movement must read as n/a"
|
||||
advance_cpu(body, total=1000, idle=700)
|
||||
result = await fetch_host_snapshot(client, target, collector)
|
||||
payload = result.payload
|
||||
assert payload["cpu"]["usage_pct"] == pytest.approx(30.0, abs=1.0)
|
||||
assert payload["name"] == "driveripper"
|
||||
assert payload["cpu"]["usage_pct"] is not None
|
||||
assert payload["memory"]["total_bytes"] > 0
|
||||
# real captures have real GPUs; the legend wants the short name, the tooltip the full one
|
||||
assert payload["gpus"], "driveripper's capture must yield at least one GPU"
|
||||
assert payload["gpus"][0]["model"] and "model_full" in payload["gpus"][0]
|
||||
assert any(s.metric == "cpu.usage_pct" for s in result.samples)
|
||||
|
||||
|
||||
async def test_target_headers_are_sent_to_the_agent() -> None:
|
||||
seen: dict[str, str] = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen["authorization"] = request.headers.get("authorization", "")
|
||||
return httpx.Response(200, json=agent_body())
|
||||
|
||||
target = Target("gateway", "http://agent.test", headers={"Authorization": "Bearer tok"})
|
||||
async with client_for(handler) as client:
|
||||
await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
|
||||
assert seen["authorization"] == "Bearer tok"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,fragment", [
|
||||
(401, "token"),
|
||||
(403, "HTTP 403"),
|
||||
(500, "HTTP 500"),
|
||||
])
|
||||
async def test_agent_rejections_say_which_one_it_was(status, fragment) -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(status, text="nope")
|
||||
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError) as excinfo:
|
||||
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
assert fragment in excinfo.value.message
|
||||
assert excinfo.value.status == 502
|
||||
|
||||
|
||||
async def test_a_refused_connection_is_an_upstream_error_not_a_traceback() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("connection refused")
|
||||
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError, match="unreachable"):
|
||||
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
|
||||
|
||||
async def test_a_slow_agent_is_a_gateway_timeout() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ReadTimeout("too slow")
|
||||
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError) as excinfo:
|
||||
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
assert excinfo.value.status == 504
|
||||
|
||||
|
||||
async def test_html_instead_of_json_is_reported_as_such() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, text="<html><body>502 Bad Gateway</body></html>")
|
||||
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError, match="non-JSON"):
|
||||
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
|
||||
|
||||
async def test_an_agent_answering_not_ok_is_not_silently_empty() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"})
|
||||
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError, match="procfs unreadable"):
|
||||
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
|
||||
|
||||
async def test_a_degraded_agent_still_renders_and_says_what_broke() -> None:
|
||||
body = agent_body()
|
||||
body["errors"] = {"temperatures": "no hwmon sensors"}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
async with client_for(handler) as client:
|
||||
result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
assert result.payload["degraded"] == {"temperatures": "no hwmon sensors"}
|
||||
assert result.payload["memory"]["total_bytes"] > 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# LLM engines
|
||||
# --------------------------------------------------------------------------- #
|
||||
GUFO = (FIXTURES / "metrics_gufo.txt").read_text()
|
||||
VLLM = (FIXTURES / "metrics_vllm.txt").read_text()
|
||||
|
||||
|
||||
def bump(text: str, series: str, amount: float) -> str:
|
||||
lines = []
|
||||
for line in text.splitlines():
|
||||
if line.startswith(series + " "):
|
||||
head, _, value = line.rpartition(" ")
|
||||
line = f"{head} {float(value) + amount}"
|
||||
lines.append(line)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
async def test_first_gufo_poll_has_no_measured_rate_yet() -> None:
|
||||
"""One scrape cannot produce Δ/Δt. The cell is n/a, not a stale single-request gauge."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/metrics":
|
||||
return httpx.Response(200, text=GUFO)
|
||||
return httpx.Response(404, text="nope")
|
||||
|
||||
async with client_for(handler) as client:
|
||||
result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(),
|
||||
with_slots=True)
|
||||
decode = result.payload["metrics"]["decode_tokens_per_second"]
|
||||
assert decode["value"] is None
|
||||
assert decode["source"] is None
|
||||
assert result.payload["sessions"]["active"] == 0.0
|
||||
assert "no /slots answer" in result.payload["sessions"]["detail"]
|
||||
|
||||
|
||||
async def test_second_gufo_poll_measures_throughput_over_a_window() -> None:
|
||||
state = {"text": GUFO}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/metrics":
|
||||
return httpx.Response(200, text=state["text"])
|
||||
if request.url.path == "/slots":
|
||||
return httpx.Response(200, json=[{"index": 0, "n_ctx": 8192, "predicted": 12}])
|
||||
return httpx.Response(404)
|
||||
|
||||
rates = RateEstimator(span=10.0)
|
||||
async with client_for(handler) as client:
|
||||
await fetch_engine(client, "gufo", "http://engine.test", "", rates)
|
||||
state["text"] = bump(GUFO, "llamacpp:tokens_predicted_total", 40.0)
|
||||
result = await fetch_engine(client, "gufo", "http://engine.test", "", rates,
|
||||
with_slots=True)
|
||||
decode = result.payload["metrics"]["decode_tokens_per_second"]
|
||||
assert decode["source"].startswith("computed:Δllamacpp:")
|
||||
assert decode["derived"] is True
|
||||
assert decode["window"] > 0
|
||||
assert decode["value"] == pytest.approx(40.0 / decode["window"], rel=0.05)
|
||||
assert result.payload["sessions"]["source"] == "gufo /slots"
|
||||
assert result.payload["sessions"]["list"][0]["index"] == 0
|
||||
|
||||
|
||||
async def test_gufo_slots_alias_is_used_when_plain_slots_is_missing() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/metrics":
|
||||
return httpx.Response(200, text=GUFO)
|
||||
if request.url.path == "/v1/slots":
|
||||
return httpx.Response(200, json={"slots": [{"index": 1}]})
|
||||
return httpx.Response(404)
|
||||
|
||||
async with client_for(handler) as client:
|
||||
result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(),
|
||||
with_slots=True)
|
||||
assert result.payload["sessions"]["list"] == [{"index": 1}]
|
||||
|
||||
|
||||
async def test_vllm_metrics_parse_and_carry_the_api_key() -> None:
|
||||
seen: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request.headers.get("authorization", ""))
|
||||
return httpx.Response(200, text=VLLM)
|
||||
|
||||
async with client_for(handler) as client:
|
||||
result = await fetch_engine(client, "vllm", "http://engine.test", "SECRET",
|
||||
RateEstimator())
|
||||
assert seen == ["Bearer SECRET"]
|
||||
metrics = result.payload["metrics"]
|
||||
assert metrics["num_requests_running"]["value"] is not None
|
||||
# vLLM publishes no per-session list: the page is told, rather than shown an empty table
|
||||
assert "counts, not sessions" in result.payload["sessions"]["detail"]
|
||||
assert any(s.metric.startswith("llm.") for s in result.samples)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [404, 500, 503])
|
||||
async def test_engine_http_errors_name_the_status(status) -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(status, text="")
|
||||
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError, match=f"HTTP {status}"):
|
||||
await fetch_engine(client, "vllm", "http://engine.test", "", RateEstimator())
|
||||
|
||||
|
||||
async def test_engine_timeout_is_504() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ReadTimeout("engine busy")
|
||||
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError) as excinfo:
|
||||
await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator())
|
||||
assert excinfo.value.status == 504
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# NUT monitor
|
||||
# --------------------------------------------------------------------------- #
|
||||
class FakeSimulator:
|
||||
"""Stands in for NutSimulator: hands back readings we control."""
|
||||
|
||||
def __init__(self, readings: list[nut.UpsReading]) -> None:
|
||||
self.readings = readings
|
||||
|
||||
async def read(self) -> list[nut.UpsReading]:
|
||||
return list(self.readings)
|
||||
|
||||
|
||||
def reading(status: str, **vars: str) -> nut.UpsReading:
|
||||
item = nut.UpsReading(name="cyberpower", description="CP1350PFCLCD", ts=1_700_000_000.0)
|
||||
item.vars = {"ups.status": status, "battery.charge": "92", "ups.load": "35",
|
||||
"battery.runtime": "1800", **vars}
|
||||
item.commands = ["beeper.mute", "test.panel.start", "test.battery.start.deep"]
|
||||
return item
|
||||
|
||||
|
||||
class SettingsStub:
|
||||
nut_host, nut_port, nut_timeout = "ups.test", 3493, 5.0
|
||||
nut_user = nut_password = ""
|
||||
nut_interval = 5.0
|
||||
nut_enabled = True
|
||||
actions_allowed = True
|
||||
read_only = False
|
||||
|
||||
|
||||
async def test_monitor_payload_labels_the_simulator() -> None:
|
||||
monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
|
||||
result = await monitor.collect()
|
||||
payload = result.payload
|
||||
assert payload["simulated"] is True
|
||||
assert payload["server"] is None, "a simulator must not claim a server address"
|
||||
assert payload["actions_enabled"] is True
|
||||
card = payload["ups"][0]
|
||||
assert card["name"] == "cyberpower"
|
||||
assert card["status"] == "OL"
|
||||
assert card["numbers"]["charge_pct"] == 92.0
|
||||
# a driver that reports no battery date, temperature, or self-test result is `unknown`,
|
||||
# never a silent "healthy" — and never a failure either
|
||||
assert card["health"]["overall"] == "unknown", card["health"]
|
||||
assert card["health"]["counts"]["fail"] == 0
|
||||
assert card["health"]["counts"]["warn"] == 0
|
||||
# commands are offered with their risk, and the dangerous one is present but marked
|
||||
risks = {item["command"]: item for item in card["commands"]}
|
||||
assert risks["beeper.mute"]["enabled"] is True
|
||||
assert risks["test.battery.start.deep"]["risk"] == "dangerous"
|
||||
|
||||
|
||||
async def test_a_fully_report_ups_is_called_healthy() -> None:
|
||||
full = reading("OL", **{"battery.temperature": "27.4", "battery.mfr.date": "2024-06-01",
|
||||
"battery.test.result": "OK", "input.voltage": "230"})
|
||||
full.vars["battery.charge"] = "100"
|
||||
card = (await NutMonitor(SettingsStub(), simulator=FakeSimulator([full])).collect()).payload["ups"][0]
|
||||
assert card["health"]["overall"] == "healthy", card["health"]
|
||||
assert card["health"]["counts"]["unknown"] == 0
|
||||
|
||||
|
||||
async def test_monitor_sends_history_samples_for_every_ups_metric() -> None:
|
||||
monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OB")]))
|
||||
result = await monitor.collect()
|
||||
names = {sample.metric for sample in result.samples}
|
||||
assert "ups.charge_pct" in names
|
||||
assert "ups.load_pct" in names
|
||||
assert all(sample.source == "ups" for sample in result.samples)
|
||||
|
||||
|
||||
async def test_status_changes_become_one_event_each() -> None:
|
||||
monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
|
||||
assert (await monitor.collect()).events == []
|
||||
monitor.simulator = FakeSimulator([reading("OL")])
|
||||
assert (await monitor.collect()).events == [], "an unchanged status is not news"
|
||||
monitor.simulator = FakeSimulator([reading("OB")])
|
||||
events = (await monitor.collect()).events
|
||||
assert len(events) == 1
|
||||
assert events[0].kind == "status"
|
||||
assert "OL → OB" in events[0].message
|
||||
assert events[0].severity == "warn", "on battery is a warning, not a failure"
|
||||
monitor.simulator = FakeSimulator([reading("OL FUB")])
|
||||
events = (await monitor.collect()).events
|
||||
assert events[0].severity == "error", "FUB (the UPS decided to power off) is a failure"
|
||||
|
||||
|
||||
async def test_a_self_test_result_is_reported_when_it_changes() -> None:
|
||||
monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
|
||||
await monitor.collect()
|
||||
# the variable is `ups.test.result` (drivers also use battery.test.result) — see VAR_MAP
|
||||
monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "INPROGRESS"})])
|
||||
assert (await monitor.collect()).events == [], "first sighting is not news"
|
||||
monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "FAIL"})])
|
||||
events = (await monitor.collect()).events
|
||||
assert [event.kind for event in events] == ["test"]
|
||||
assert events[0].severity == "warn"
|
||||
monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "OK"})])
|
||||
assert (await monitor.collect()).events[0].severity == "info"
|
||||
|
||||
|
||||
async def test_gating_is_applied_where_the_commands_are_built() -> None:
|
||||
class ReadOnly(SettingsStub):
|
||||
actions_allowed = False
|
||||
|
||||
monitor = NutMonitor(ReadOnly(), simulator=FakeSimulator([reading("OL")]))
|
||||
card = (await monitor.collect()).payload["ups"][0]
|
||||
assert all(item["enabled"] is False for item in card["commands"])
|
||||
assert all(item["reason"] for item in card["commands"]), "every refusal must explain itself"
|
||||
assert set(card["advertised_commands"]) == {"beeper.mute", "test.panel.start",
|
||||
"test.battery.start.deep"}, (
|
||||
"the driver still advertises them — the refusal is ours, and must say so")
|
||||
|
||||
|
||||
class FakeNutClient:
|
||||
"""The real upsd branch without a socket: vars work, the optional lists do not."""
|
||||
|
||||
instances: ClassVar[list[FakeNutClient]] = []
|
||||
|
||||
def __init__(self, host: str, port: int, **kwargs: Any) -> None:
|
||||
self.host, self.port, self.kwargs = host, port, kwargs
|
||||
self.entered = False
|
||||
FakeNutClient.instances.append(self)
|
||||
|
||||
async def __aenter__(self) -> FakeNutClient:
|
||||
self.entered = True
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: Any) -> None:
|
||||
self.entered = False
|
||||
|
||||
async def list_ups(self) -> list[dict[str, str]]:
|
||||
return [{"name": "cyberpower", "description": "CP1350PFCLCD"}]
|
||||
|
||||
async def list_vars(self, ups: str) -> dict[str, str]:
|
||||
return {"ups.status": "OL", "battery.charge": "100", "battery.temperature": "301.1"}
|
||||
|
||||
async def list_cmds(self, ups: str) -> list[str]:
|
||||
raise nut.NutError("this driver exports no commands")
|
||||
|
||||
async def list_rw_vars(self, ups: str) -> list[str]:
|
||||
raise nut.NutError("LIST RW denied")
|
||||
|
||||
|
||||
async def test_real_upsd_branch_survives_drivers_that_refuse_the_optional_lists(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(nut, "NutClient", FakeNutClient)
|
||||
monitor = NutMonitor(SettingsStub())
|
||||
result = await monitor.collect()
|
||||
client = FakeNutClient.instances[-1]
|
||||
assert client.entered is False, "the connection must not outlive one poll"
|
||||
assert client.kwargs["timeout"] == 5.0
|
||||
card = result.payload["ups"][0]
|
||||
assert card["commands"] == [] and card["advertised_commands"] == []
|
||||
assert card["numbers"]["charge_pct"] == 100.0
|
||||
# a driver reporting Kelvin is normalised, not charted as 301 °C
|
||||
assert card["numbers"]["battery_temperature"] == pytest.approx(301.1 - 273.15, abs=0.01)
|
||||
samples = {sample.metric: sample.value for sample in result.samples}
|
||||
assert samples["ups.battery_temperature_c"] == pytest.approx(27.95, abs=0.01)
|
||||
|
||||
|
||||
async def test_a_real_server_names_itself_in_the_payload(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(nut, "NutClient", FakeNutClient)
|
||||
payload = (await NutMonitor(SettingsStub()).collect()).payload
|
||||
assert payload["simulated"] is False
|
||||
assert payload["server"] == "ups.test:3493"
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Configuration: env-only, no defaults that point at somebody else's machine."""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import os
|
||||
from dataclasses import FrozenInstanceError
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards import config
|
||||
from dashboards.config import Settings, parse_targets
|
||||
|
||||
|
||||
def test_parses_whitespace_and_comma_separated_targets() -> None:
|
||||
targets, problems = parse_targets("a=http://h1:9120 b=http://h2:9120,c=http://h3:9120")
|
||||
assert [t.name for t in targets] == ["a", "b", "c"]
|
||||
assert [t.url for t in targets] == ["http://h1:9120", "http://h2:9120", "http://h3:9120"]
|
||||
assert problems == []
|
||||
|
||||
|
||||
def test_trailing_slash_is_stripped_so_urls_do_not_double_up() -> None:
|
||||
targets, _ = parse_targets("a=http://h1:9120/")
|
||||
assert targets[0].url == "http://h1:9120"
|
||||
|
||||
|
||||
def test_one_bad_entry_does_not_sink_the_others() -> None:
|
||||
targets, problems = parse_targets("good=http://h:1 broken-no-equals another=ftp://h:1")
|
||||
assert [t.name for t in targets] == ["good"]
|
||||
assert len(problems) == 2
|
||||
assert any("not name=url" in p for p in problems)
|
||||
assert any("http://" in p for p in problems)
|
||||
|
||||
|
||||
def test_empty_input_yields_no_targets_and_no_noise() -> None:
|
||||
assert parse_targets("") == ([], [])
|
||||
assert parse_targets(" \n ") == ([], [])
|
||||
|
||||
|
||||
def test_token_in_url_becomes_a_bearer_header_and_stays_out_of_the_display_url() -> None:
|
||||
targets, problems = parse_targets("driveripper=http://SEKRIT@driveripper:9120")
|
||||
assert problems == []
|
||||
target = targets[0]
|
||||
assert target.headers == {"Authorization": "Bearer SEKRIT"}
|
||||
assert target.url == "http://driveripper:9120", "the token must not stay in the URL"
|
||||
assert target.display_url == "http://driveripper:9120"
|
||||
assert "SEKRIT" not in target.display_url
|
||||
|
||||
|
||||
def test_target_names_are_restricted_because_they_are_db_keys() -> None:
|
||||
"""Names key the history DB and appear in URLs, so they are validated, not guessed.
|
||||
|
||||
Whitespace separates entries, which means "Bad Name=http://h" is two entries — both
|
||||
reported, because the operator needs to see exactly what was rejected.
|
||||
"""
|
||||
targets, problems = parse_targets("Bad=http://h:1 /=http://h:1 ok-1_2=http://h:1")
|
||||
assert [t.name for t in targets] == ["ok-1_2"]
|
||||
assert len(problems) == 2
|
||||
assert any("Bad" in p for p in problems)
|
||||
|
||||
_targets, problems = parse_targets("Bad Name=http://h:1")
|
||||
assert len(problems) == 2, "a space splits the entry, so both halves are rejected"
|
||||
|
||||
|
||||
def test_settings_default_to_no_sources_configured() -> None:
|
||||
settings = Settings()
|
||||
assert settings.host_targets == ()
|
||||
assert settings.gufo_url == "" and settings.vllm_url == ""
|
||||
assert settings.nut_host == ""
|
||||
assert settings.nut_enabled is False
|
||||
assert settings.any_source_enabled is False
|
||||
|
||||
|
||||
def test_actions_need_the_flag_and_are_dead_under_read_only() -> None:
|
||||
assert Settings(nut_allow_actions=True, read_only=False).actions_allowed is True
|
||||
assert Settings(nut_allow_actions=True, read_only=True).actions_allowed is False
|
||||
assert Settings(nut_allow_actions=False, read_only=False).actions_allowed is False
|
||||
|
||||
|
||||
def test_nut_host_accepts_a_nut_scheme_prefix() -> None:
|
||||
assert Settings(nut_host="nut://ups-box:3493").nut_enabled is True
|
||||
|
||||
|
||||
def test_load_reads_host_targets_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HOST_TARGETS", "deskwork=http://127.0.0.1:9122")
|
||||
settings = config.load()
|
||||
assert [t.name for t in settings.host_targets] == ["deskwork"]
|
||||
|
||||
|
||||
def test_env_driven_scalars_are_read_at_import(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The dataclass defaults read os.environ, so a reload must pick up a changed env."""
|
||||
monkeypatch.setenv("DASH_PORT", "8123")
|
||||
monkeypatch.setenv("RETAIN_RAW_HOURS", "6")
|
||||
monkeypatch.setenv("NUT_ALLOW_ACTIONS", "1")
|
||||
reloaded = importlib.reload(config)
|
||||
try:
|
||||
assert reloaded.SETTINGS.port == 8123
|
||||
assert reloaded.SETTINGS.raw_keep_hours == 6
|
||||
assert reloaded.SETTINGS.actions_allowed is True
|
||||
finally:
|
||||
monkeypatch.delenv("DASH_PORT")
|
||||
monkeypatch.delenv("RETAIN_RAW_HOURS")
|
||||
monkeypatch.delenv("NUT_ALLOW_ACTIONS")
|
||||
importlib.reload(config)
|
||||
|
||||
|
||||
def test_flags_accept_the_usual_spellings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
for truthy in ("1", "true", "TRUE", "yes", "on"):
|
||||
monkeypatch.setenv("DASH_READ_ONLY", truthy)
|
||||
assert config._flag("DASH_READ_ONLY") is True, truthy
|
||||
for falsey in ("0", "false", "", "no"):
|
||||
monkeypatch.setenv("DASH_READ_ONLY", falsey)
|
||||
assert config._flag("DASH_READ_ONLY") is False, falsey
|
||||
|
||||
|
||||
def test_numeric_env_must_be_a_number_or_the_default_wins(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HOST_INTERVAL_SECONDS", "not-a-number")
|
||||
assert config._num("HOST_INTERVAL_SECONDS", 10.0) == 10.0
|
||||
monkeypatch.setenv("HOST_INTERVAL_SECONDS", "2.5")
|
||||
assert config._num("HOST_INTERVAL_SECONDS", 10.0) == 2.5
|
||||
|
||||
|
||||
def test_settings_is_immutable_so_nobody_mutates_shared_state() -> None:
|
||||
settings: Any = Settings()
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
settings.port = 1 # type: ignore[misc]
|
||||
|
||||
|
||||
# ---- the .env convenience -------------------------------------------------- #
|
||||
|
||||
def test_no_env_file_is_the_normal_case(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
|
||||
monkeypatch.setenv("DASH_ENV_FILE", str(tmp_path / "absent.env"))
|
||||
assert config.load_env_file() is None
|
||||
|
||||
|
||||
def test_env_file_seeds_values_a_deployed_env_does_not_set(monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Any) -> None:
|
||||
env = tmp_path / ".env"
|
||||
env.write_text("DASH_PORT=9999\nNUT_HOST=ups.internal\n")
|
||||
monkeypatch.setenv("DASH_ENV_FILE", str(env))
|
||||
monkeypatch.delenv("DASH_PORT", raising=False)
|
||||
monkeypatch.delenv("NUT_HOST", raising=False)
|
||||
assert config.load_env_file() == env
|
||||
# Settings reads its defaults when the module is imported, so the reload is what a real
|
||||
# process gets for free and what the test has to ask for explicitly
|
||||
reloaded = importlib.reload(config)
|
||||
assert reloaded.SETTINGS.port == 9999
|
||||
assert reloaded.SETTINGS.nut_host == "ups.internal"
|
||||
|
||||
|
||||
def test_a_real_environment_variable_beats_the_file(monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Any) -> None:
|
||||
"""The container/quadlet env is the source of truth; the file is for laptops."""
|
||||
env = tmp_path / ".env"
|
||||
env.write_text("DASH_PORT=9999\n")
|
||||
monkeypatch.setenv("DASH_ENV_FILE", str(env))
|
||||
monkeypatch.setenv("DASH_PORT", "7777")
|
||||
reloaded = importlib.reload(config)
|
||||
assert os.environ["DASH_PORT"] == "7777"
|
||||
assert reloaded.SETTINGS.port == 7777
|
||||
|
||||
|
||||
def test_env_file_expands_other_variables(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
|
||||
"""`.env.example` shows `DASH_DB=${HOME}/...` — the file has to mean what it shows."""
|
||||
env = tmp_path / ".env"
|
||||
env.write_text("MY_BASE=/srv/dash\nDASH_DB=${MY_BASE}/dashboards.sqlite3\n")
|
||||
monkeypatch.setenv("DASH_ENV_FILE", str(env))
|
||||
monkeypatch.delenv("MY_BASE", raising=False)
|
||||
monkeypatch.delenv("DASH_DB", raising=False)
|
||||
reloaded = importlib.reload(config)
|
||||
assert reloaded.SETTINGS.db_path == "/srv/dash/dashboards.sqlite3"
|
||||
|
||||
|
||||
def test_the_example_file_is_never_loaded(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None:
|
||||
example = tmp_path / ".env.example"
|
||||
example.write_text("NUT_HOST=should-not-load\n")
|
||||
monkeypatch.setenv("DASH_ENV_FILE", str(example))
|
||||
monkeypatch.delenv("NUT_HOST", raising=False)
|
||||
reloaded = importlib.reload(config)
|
||||
assert reloaded.load_env_file() is None
|
||||
assert reloaded.SETTINGS.nut_host == ""
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Agent snapshot -> dashboard payload: rates, null-safety, and honest fleet totals.
|
||||
|
||||
The rule this file defends hardest is *null in, null out*. A missing sensor rendered as 0
|
||||
is a dashboard that says "the GPU is idle" when it means "the driver said nothing", and on
|
||||
a machine you are debugging at 2am that difference is the whole point.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.sources.host import HostCollector, fleet_summary, parse_host_snapshot
|
||||
|
||||
TICKS_PER_SECOND = 100.0
|
||||
|
||||
|
||||
def make_collector() -> HostCollector:
|
||||
return HostCollector(RateEstimator(span=60.0))
|
||||
|
||||
|
||||
def advance(snapshot: dict[str, Any], dt: float = 10.0, *, total_ticks: float = 1000.0,
|
||||
idle_ticks: float = 900.0, read_bytes: float = 0.0, write_bytes: float = 0.0,
|
||||
rx_bytes: float = 0.0, tx_bytes: float = 0.0) -> dict[str, Any]:
|
||||
"""A second scrape `dt` later with counters moved by known amounts."""
|
||||
snap = copy.deepcopy(snapshot)
|
||||
snap["ts"] = float(snapshot["ts"]) + dt
|
||||
modes = snap["cpu"]["modes"]
|
||||
idle_parts = {"idle": idle_ticks * 0.9, "iowait": idle_ticks * 0.1}
|
||||
for key, value in modes.items():
|
||||
if key == "total":
|
||||
modes["total"] = value + total_ticks
|
||||
elif key in idle_parts:
|
||||
modes[key] = value + idle_parts[key]
|
||||
elif key != "guest" and key != "guest_nice":
|
||||
modes[key] = value + (total_ticks - idle_ticks) / 5
|
||||
for device in snap["disk_io"]["devices"]:
|
||||
device["read_bytes_total"] += read_bytes
|
||||
device["write_bytes_total"] += write_bytes
|
||||
device["reads_total"] += read_bytes / 4096
|
||||
device["writes_total"] += write_bytes / 4096
|
||||
for iface in snap["network"]["interfaces"]:
|
||||
iface["rx_bytes_total"] += rx_bytes
|
||||
iface["tx_bytes_total"] += tx_bytes
|
||||
return snap
|
||||
|
||||
|
||||
def parse_twice(agent_snapshot) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
collector = make_collector()
|
||||
first = parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0)
|
||||
second_snap = advance(agent_snapshot, 10.0, read_bytes=4_096_000, write_bytes=2_048_000,
|
||||
rx_bytes=100_000, tx_bytes=50_000)
|
||||
second = parse_host_snapshot("driveripper", second_snap, collector, now=1_010.0)
|
||||
return first.payload, second.payload
|
||||
|
||||
|
||||
def test_first_scrape_has_no_rates_but_reports_instantaneous_values(agent_snapshot) -> None:
|
||||
collector = make_collector()
|
||||
payload = parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0).payload
|
||||
assert payload["cpu"]["usage_pct"] is None, "one sample cannot make a rate"
|
||||
assert payload["cpu"]["load1"] is not None, "load is a gauge, available immediately"
|
||||
assert payload["memory"]["used_pct"] is not None
|
||||
assert payload["disks"]["read_bytes_per_s"] is None
|
||||
assert payload["network"]["rx_bytes_per_s"] is None
|
||||
assert payload["window_seconds"] is None
|
||||
|
||||
|
||||
def test_second_scrape_measures_the_window_it_claims(agent_snapshot) -> None:
|
||||
"""Two scrapes are enough — a dashboard that needs three looks to say anything about
|
||||
CPU is a dashboard whose first minute of history is a hole."""
|
||||
_first, payload = parse_twice(agent_snapshot)
|
||||
assert payload["window_seconds"] == 10.0
|
||||
usage = payload["cpu"]["usage_pct"]
|
||||
assert usage is not None
|
||||
# 1000 ticks total, 900 idle over 10s => exactly 10% busy, whatever the host was doing
|
||||
assert usage == 10.0
|
||||
|
||||
|
||||
def test_iowait_counts_as_idle_but_is_still_reported(agent_snapshot) -> None:
|
||||
_first, payload = parse_twice(agent_snapshot)
|
||||
modes = payload["cpu"]["modes_pct"]
|
||||
assert modes["iowait"] is not None and modes["iowait"] > 0
|
||||
assert sum(v for v in modes.values() if v is not None) <= 100.0001
|
||||
|
||||
|
||||
def test_per_core_series_is_one_point_per_logical_cpu(agent_snapshot) -> None:
|
||||
collector = make_collector()
|
||||
parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0)
|
||||
payload = parse_host_snapshot("driveripper", advance(agent_snapshot, 10.0), collector,
|
||||
now=1_010.0).payload
|
||||
cores = payload["cpu"]["core_usage_pct"]
|
||||
assert len(cores) == payload["info"]["cpu_cores_logical"] == 48
|
||||
assert all(value is None or 0.0 <= value <= 100.0 for value in cores)
|
||||
|
||||
|
||||
def test_disk_and_network_rates_are_bytes_per_second_over_the_window(agent_snapshot) -> None:
|
||||
_first, payload = parse_twice(agent_snapshot)
|
||||
devices = payload["disks"]["devices"]
|
||||
assert devices, "captured host has block devices"
|
||||
# the fixture scrape added the same amount to every device/interface, so the per-device
|
||||
# figure is delta/dt and the headline total is the sum of what the payload shows
|
||||
assert devices[0]["read_bytes_per_s"] == 4_096_000 / 10.0
|
||||
assert payload["disks"]["read_bytes_per_s"] == sum(
|
||||
d["read_bytes_per_s"] or 0.0 for d in devices)
|
||||
assert payload["disks"]["write_bytes_per_s"] == sum(
|
||||
d["write_bytes_per_s"] or 0.0 for d in devices)
|
||||
interfaces = payload["network"]["interfaces"]
|
||||
assert payload["network"]["rx_bytes_per_s"] == 100_000 / 10.0 * len(interfaces)
|
||||
assert payload["network"]["tx_bytes_per_s"] == 50_000 / 10.0 * len(interfaces)
|
||||
|
||||
|
||||
def test_idle_machine_reports_zero_not_none_once_measured(agent_snapshot) -> None:
|
||||
"""0 and n/a are different claims: 'nothing happened' vs 'we could not tell'."""
|
||||
collector = make_collector()
|
||||
parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0)
|
||||
quiet = advance(agent_snapshot, 10.0, total_ticks=1_000.0, idle_ticks=1_000.0)
|
||||
payload = parse_host_snapshot("driveripper", quiet, collector, now=1_010.0).payload
|
||||
assert payload["cpu"]["usage_pct"] == 0.0
|
||||
assert payload["disks"]["read_bytes_per_s"] == 0.0
|
||||
assert payload["network"]["rx_bytes_per_s"] == 0.0
|
||||
|
||||
|
||||
def test_missing_sensors_stay_missing(agent_snapshot) -> None:
|
||||
snap = copy.deepcopy(agent_snapshot)
|
||||
snap["temperatures"] = [] # the agent reports a flat sensor list
|
||||
snap["cpu_temperature_celsius"] = None # ...plus its own pick for "the CPU temp"
|
||||
snap["cpu"]["freq_mhz"] = None
|
||||
for gpu in snap["gpus"]:
|
||||
gpu["power_watts"] = None
|
||||
gpu["temps"] = {}
|
||||
collector = make_collector()
|
||||
payload = parse_host_snapshot("driveripper", snap, collector, now=1_000.0).payload
|
||||
assert payload["cpu"]["temp_c"] is None
|
||||
assert payload["cpu"]["freq_mhz"] is None
|
||||
assert payload["temperatures"]["max_c"] is None
|
||||
assert payload["temperatures"]["sensors"] == []
|
||||
for gpu in payload["gpus"]:
|
||||
assert gpu["power_watts"] is None
|
||||
assert gpu["temps"] == {}
|
||||
assert gpu["temp_c"] is None
|
||||
|
||||
|
||||
def test_no_sample_is_ever_written_as_zero_for_a_missing_value(agent_snapshot) -> None:
|
||||
snap = copy.deepcopy(agent_snapshot)
|
||||
snap["cpu"]["freq_mhz"] = None
|
||||
snap["gpus"] = []
|
||||
collector = make_collector()
|
||||
result = parse_host_snapshot("driveripper", snap, collector, now=1_000.0)
|
||||
for sample in result.samples:
|
||||
assert sample.value is not None
|
||||
assert math.isfinite(sample.value), f"{sample.metric} is {sample.value}"
|
||||
assert not any(s.metric == "cpu.freq_mhz" for s in result.samples)
|
||||
|
||||
|
||||
def test_gpu_pool_percent_accounts_for_vram_and_gtt(agent_snapshot) -> None:
|
||||
"""On an APU the model lives in GTT; reporting only vram says '32 MB used' on a 128 GB box."""
|
||||
collector = make_collector()
|
||||
payload = parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0).payload
|
||||
gpu = payload["gpus"][0]
|
||||
assert gpu["model_full"] and gpu["model"]
|
||||
assert gpu["vram_total_bytes"] > 0
|
||||
assert gpu["mem_used_pct"] is not None and 0.0 <= gpu["mem_used_pct"] <= 100.0
|
||||
assert "gtt_total_bytes" in gpu
|
||||
|
||||
|
||||
def test_reboot_is_reported_as_an_event(agent_snapshot) -> None:
|
||||
collector = make_collector()
|
||||
parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0)
|
||||
snap = advance(agent_snapshot, 10.0)
|
||||
snap["host"]["boot_ts"] = float(agent_snapshot["host"]["boot_ts"]) + 5_000
|
||||
result = parse_host_snapshot("driveripper", snap, collector, now=1_010.0)
|
||||
reboots = [e for e in result.events if e.kind == "reboot"]
|
||||
assert len(reboots) == 1
|
||||
assert reboots[0].severity == "warn"
|
||||
assert "rebooted" in reboots[0].message
|
||||
|
||||
|
||||
def test_agent_errors_travel_to_the_ui(agent_snapshot) -> None:
|
||||
snap = copy.deepcopy(agent_snapshot)
|
||||
snap["errors"] = ["disk_io: permission denied reading /sys/block/sdb"]
|
||||
payload = parse_host_snapshot("driveripper", snap, make_collector(), now=1_000.0).payload
|
||||
assert payload["agent"]["errors"], "a partial snapshot must be labelled partial"
|
||||
|
||||
|
||||
def test_counter_reset_from_a_reboot_does_not_chart_a_spike(agent_snapshot) -> None:
|
||||
collector = make_collector()
|
||||
parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0)
|
||||
snap = copy.deepcopy(agent_snapshot)
|
||||
snap["cpu"]["modes"]["total"] = 1.0 # counter restarted from zero
|
||||
payload = parse_host_snapshot("driveripper", snap, collector, now=1_010.0).payload
|
||||
assert payload["cpu"]["usage_pct"] is None
|
||||
|
||||
|
||||
def test_fleet_summary_ignores_hosts_that_are_not_answering(agent_snapshot) -> None:
|
||||
payload = parse_host_snapshot("driveripper", agent_snapshot, make_collector(),
|
||||
now=1_000.0).payload
|
||||
live = copy.deepcopy(payload)
|
||||
live["cpu"]["usage_pct"] = 40.0
|
||||
live["info"]["cpu_cores_logical"] = 48
|
||||
dead = copy.deepcopy(payload)
|
||||
dead["name"] = "deskwork"
|
||||
dead["cpu"]["usage_pct"] = None
|
||||
fleet = fleet_summary([live, dead])
|
||||
assert fleet["hosts_total"] == 2
|
||||
assert fleet["hosts_live"] == 1
|
||||
assert fleet["cpu_usage_pct_avg"] == 40.0, "a dead host must not drag the average to 0"
|
||||
|
||||
|
||||
def test_fleet_summary_weights_cpu_by_cores(agent_snapshot) -> None:
|
||||
base = parse_host_snapshot("driveripper", agent_snapshot, make_collector(),
|
||||
now=1_000.0).payload
|
||||
big = copy.deepcopy(base)
|
||||
big["cpu"]["usage_pct"] = 10.0
|
||||
big["info"]["cpu_cores_logical"] = 100
|
||||
small = copy.deepcopy(base)
|
||||
small["name"] = "gateway"
|
||||
small["cpu"]["usage_pct"] = 90.0
|
||||
small["info"]["cpu_cores_logical"] = 2
|
||||
fleet = fleet_summary([big, small])
|
||||
assert fleet["cpu_usage_pct_avg"] == pytest.approx((10 * 100 + 90 * 2) / 102)
|
||||
assert fleet["cpu_cores"] == 102
|
||||
|
||||
|
||||
def test_fleet_summary_worst_disk_and_memory_totals(agent_snapshot) -> None:
|
||||
base = parse_host_snapshot("driveripper", agent_snapshot, make_collector(),
|
||||
now=1_000.0).payload
|
||||
fleet = fleet_summary([base, base])
|
||||
assert fleet["disk_worst_used_pct"] == pytest.approx(base["disks"]["worst_used_pct"])
|
||||
assert fleet["memory_total_bytes"] == pytest.approx(2 * base["memory"]["total_bytes"])
|
||||
assert fleet["memory_used_pct"] == pytest.approx(
|
||||
base["memory"]["used_bytes"] / base["memory"]["total_bytes"] * 100)
|
||||
|
||||
|
||||
def test_fleet_summary_of_nothing_is_empty_not_zero() -> None:
|
||||
fleet = fleet_summary([])
|
||||
assert fleet["hosts_total"] == 0
|
||||
assert fleet["cpu_usage_pct_avg"] is None
|
||||
assert fleet["memory_used_bytes"] is None
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
"""LLM engine parsing: real captures, provenance on every number, absent never 0.
|
||||
|
||||
The engines do not export the names people expect, so these tests are pinned to captures of
|
||||
the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.sources.llm import (
|
||||
engine_result,
|
||||
fleet_totals,
|
||||
parse_gufo_metrics,
|
||||
parse_vllm_metrics,
|
||||
series_names,
|
||||
)
|
||||
|
||||
|
||||
def bump_metric(text: str, name: str, delta: float) -> str:
|
||||
"""Return the exposition with one counter increased — a scrape `dt` later."""
|
||||
pattern = re.compile(rf"^({re.escape(name)}(?:\{{[^}}]*\}})?) (\S+)$", re.M)
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
return f"{match.group(1)} {float(match.group(2)) + delta}"
|
||||
|
||||
bumped, count = pattern.subn(replace, text)
|
||||
assert count, f"{name} not present in the capture"
|
||||
return bumped
|
||||
|
||||
|
||||
# ---- gufo / llama.cpp ----------------------------------------------------- #
|
||||
|
||||
def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
|
||||
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
||||
assert metrics["tokens_generated_total"]["value"] == 877_336.0
|
||||
assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0
|
||||
assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0
|
||||
assert metrics["requests_processing"]["value"] == 0.0
|
||||
assert metrics["device_lost_total"]["value"] == 0.0
|
||||
assert metrics["kv_cache_usage_ratio"]["value"] == 0.0
|
||||
|
||||
|
||||
def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None:
|
||||
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
||||
accept = metrics["spec_accept_rate"]["value"]
|
||||
assert accept == pytest.approx(578_514 / 789_028)
|
||||
assert str(metrics["spec_accept_rate"]["source"]).startswith("computed")
|
||||
|
||||
|
||||
def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None:
|
||||
"""gufo's prompt counter excludes cache hits; the lifetime total must not."""
|
||||
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
||||
total = metrics["tokens_total"]["value"]
|
||||
assert total == 1_350_666 + 124_529_105 + 877_336
|
||||
|
||||
|
||||
def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None:
|
||||
"""llamacpp:predicted_tokens_seconds is ONE request's last pass — never the headline."""
|
||||
text = metrics_text("gufo")
|
||||
rates = RateEstimator(span=10)
|
||||
now = 1_000.0
|
||||
first = parse_gufo_metrics(text, rates, now)
|
||||
assert "decode_tokens_per_second" not in first or first["decode_tokens_per_second"]["value"] is None
|
||||
|
||||
text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0)
|
||||
metrics = parse_gufo_metrics(text, rates, now + 10.0)
|
||||
decode = metrics["decode_tokens_per_second"]
|
||||
assert decode["value"] == pytest.approx(40.0)
|
||||
assert decode["window"] == pytest.approx(10.0)
|
||||
assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"]
|
||||
# the native gauge is still there, labelled, because it answers a different question
|
||||
assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)
|
||||
|
||||
|
||||
def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None:
|
||||
text = metrics_text("gufo")
|
||||
rates = RateEstimator(span=10)
|
||||
parse_gufo_metrics(text, rates, 1_000.0)
|
||||
text = bump_metric(text, "llamacpp:prompt_tokens_total", 1_000.0)
|
||||
metrics = parse_gufo_metrics(text, rates, 1_010.0)
|
||||
assert metrics["prompt_tokens_per_second"]["value"] == pytest.approx(100.0)
|
||||
|
||||
|
||||
def test_gufo_counter_reset_produces_n_a_not_a_spike(metrics_text) -> None:
|
||||
text = metrics_text("gufo")
|
||||
rates = RateEstimator(span=60)
|
||||
parse_gufo_metrics(text, rates, 1_000.0)
|
||||
restarted = bump_metric(text, "llamacpp:tokens_predicted_total", -800_000.0)
|
||||
metrics = parse_gufo_metrics(restarted, rates, 1_010.0)
|
||||
assert metrics.get("decode_tokens_per_second", {}).get("value") is None
|
||||
|
||||
|
||||
def test_absent_gufo_metrics_are_none_with_no_source() -> None:
|
||||
metrics = parse_gufo_metrics("# nothing here\n")
|
||||
assert metrics["requests_processing"]["value"] is None
|
||||
assert metrics["requests_processing"]["source"] is None
|
||||
assert metrics.get("spec_accept_rate", {}).get("value") is None
|
||||
|
||||
|
||||
# ---- vLLM ----------------------------------------------------------------- #
|
||||
|
||||
def test_vllm_counters_and_gauges(metrics_text) -> None:
|
||||
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
||||
assert metrics["prompt_tokens_total"]["value"] == pytest.approx(1.8389909e7)
|
||||
assert metrics["generation_tokens_total"]["value"] == pytest.approx(611_163.0)
|
||||
usage = metrics["gpu_cache_usage"]["value"]
|
||||
assert usage is None or 0.0 <= usage <= 1.0, "a ratio summed over label sets would exceed 1"
|
||||
assert metrics["num_requests_running"]["value"] is not None
|
||||
assert metrics["num_preemptions_total"]["value"] is not None
|
||||
|
||||
|
||||
def test_vllm_latency_histograms(metrics_text) -> None:
|
||||
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
||||
ttft = metrics["time_to_first_token_seconds"]
|
||||
assert ttft and ttft["count"] > 0 and ttft["mean"] > 0
|
||||
assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"]
|
||||
assert metrics["time_per_output_token_seconds"]["mean"] > 0
|
||||
|
||||
|
||||
def test_vllm_per_request_speed_is_the_reciprocal_of_tpot(metrics_text) -> None:
|
||||
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
||||
tpot = metrics["time_per_output_token_seconds"]["mean"]
|
||||
per_request = metrics["per_request_decode_tokens_per_second"]
|
||||
assert per_request["value"] == pytest.approx(1.0 / tpot)
|
||||
assert per_request["derived"] is True
|
||||
|
||||
|
||||
def test_vllm_prefix_cache_hit_ratio(metrics_text) -> None:
|
||||
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
||||
ratio = metrics["prefix_cache_hit_ratio"]["value"]
|
||||
assert ratio is not None and 0.0 <= ratio <= 1.0
|
||||
assert "hits" in metrics["prefix_cache_hit_ratio"]["source"]
|
||||
|
||||
|
||||
def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) -> None:
|
||||
text = metrics_text("vllm")
|
||||
rates = RateEstimator(span=10)
|
||||
parse_vllm_metrics(text, rates, 2_000.0)
|
||||
text = bump_metric(text, "vllm:generation_tokens_total", 250.0)
|
||||
metrics = parse_vllm_metrics(text, rates, 2_010.0)
|
||||
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0)
|
||||
|
||||
|
||||
def test_vllm_waiting_requests_are_broken_down_by_reason(metrics_text) -> None:
|
||||
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
||||
reasons = metrics["waiting_by_reason"]["value"]
|
||||
assert isinstance(reasons, dict)
|
||||
|
||||
|
||||
def test_vllm_empty_exposition_is_all_none() -> None:
|
||||
metrics = parse_vllm_metrics("")
|
||||
assert metrics["num_requests_running"]["value"] is None
|
||||
assert metrics["time_to_first_token_seconds"] is None
|
||||
assert metrics["per_request_decode_tokens_per_second"]["value"] is None
|
||||
|
||||
|
||||
# ---- Result assembly ------------------------------------------------------ #
|
||||
|
||||
def test_engine_result_payload_and_samples(metrics_text) -> None:
|
||||
rates = RateEstimator(span=10)
|
||||
result = engine_result("gufo", "http://gufo.internal:8731", metrics_text("gufo"), rates,
|
||||
now=3_000.0)
|
||||
payload = result.payload
|
||||
assert payload["engine"] == "gufo"
|
||||
assert payload["upstream"] == "http://gufo.internal:8731"
|
||||
assert payload["series_count"] > 10
|
||||
assert payload["unparsed"] == 0
|
||||
assert payload["sessions"]["detail"], "a missing /slots must be explained, not blank"
|
||||
for sample in result.samples:
|
||||
assert sample.source == "llm" and sample.host == "gufo"
|
||||
assert sample.metric.startswith("llm.")
|
||||
assert sample.value == sample.value and sample.value is not None # no NaN
|
||||
|
||||
|
||||
def test_engine_result_charts_a_measured_rate_after_the_second_poll(metrics_text) -> None:
|
||||
rates = RateEstimator(span=10)
|
||||
engine_result("gufo", "http://g:1", metrics_text("gufo"), rates, now=3_000.0)
|
||||
bumped = bump_metric(metrics_text("gufo"), "llamacpp:tokens_predicted_total", 100.0)
|
||||
result = engine_result("gufo", "http://g:1", bumped, rates, now=3_010.0)
|
||||
metrics = result.payload["metrics"]
|
||||
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(10.0)
|
||||
assert any(s.metric == "llm.decode_tokens_per_second" for s in result.samples)
|
||||
|
||||
|
||||
def test_sessions_pass_through_slots_for_gufo() -> None:
|
||||
slots = [{"id": 0, "idle": False, "n_prompt": 12, "n_predict": 40}]
|
||||
result = engine_result("gufo", "http://g:1", "", RateEstimator(), slots=slots, now=1.0)
|
||||
sessions = result.payload["sessions"]
|
||||
assert sessions["list"] == slots
|
||||
assert sessions["source"] == "gufo /slots"
|
||||
|
||||
|
||||
def test_sessions_say_vllm_has_no_per_session_view() -> None:
|
||||
result = engine_result("vllm", "http://v:1", "", RateEstimator(), now=1.0)
|
||||
sessions = result.payload["sessions"]
|
||||
assert sessions["list"] == []
|
||||
assert "vLLM" in sessions["detail"]
|
||||
|
||||
|
||||
# ---- fleet totals --------------------------------------------------------- #
|
||||
|
||||
def _engine(name: str, **values) -> dict:
|
||||
metrics = {key: {"value": value} for key, value in values.items()}
|
||||
return {"engine": name, "metrics": metrics}
|
||||
|
||||
|
||||
def test_fleet_totals_sum_only_engines_that_answered() -> None:
|
||||
engines = [
|
||||
_engine("gufo", decode_tokens_per_second=40.0, prompt_tokens_per_second=10.0,
|
||||
kv_cache_usage_ratio=0.25, requests_processing=1.0, requests_deferred=0.0,
|
||||
tokens_total=100.0),
|
||||
_engine("vllm", decode_tokens_per_second=10.0, prompt_tokens_per_second=5.0,
|
||||
gpu_cache_usage=0.75, num_requests_running=2.0, num_requests_waiting=3.0,
|
||||
tokens_total=200.0),
|
||||
{}, # an engine whose poll failed
|
||||
]
|
||||
totals = fleet_totals(engines)
|
||||
assert totals["engines_total"] == 3
|
||||
assert totals["engines_live"] == 2
|
||||
assert totals["decode_tokens_per_second"] == 50.0
|
||||
assert totals["prompt_tokens_per_second"] == 15.0
|
||||
assert totals["requests_active"] == 3.0
|
||||
assert totals["requests_waiting"] == 3.0
|
||||
assert totals["kv_cache_usage_max"] == 0.75, "cache pressure is a max, never a sum"
|
||||
assert totals["tokens_total"] == 300.0
|
||||
|
||||
|
||||
def test_fleet_totals_report_the_gufo_vllm_seam() -> None:
|
||||
totals = fleet_totals([_engine("gufo"), _engine("vllm")])
|
||||
assert totals["seams"], "the two engines count prompt tokens differently; say so"
|
||||
|
||||
|
||||
def test_fleet_totals_of_nothing_is_none_not_zero() -> None:
|
||||
totals = fleet_totals([])
|
||||
assert totals["decode_tokens_per_second"] is None
|
||||
assert totals["tokens_total"] is None
|
||||
assert totals["engines_live"] == 0
|
||||
|
||||
|
||||
def test_series_names_are_the_parse_catalog() -> None:
|
||||
assert "tokens_generated_total" in series_names("gufo")
|
||||
assert "generation_tokens_total" in series_names("vllm")
|
||||
assert "generation_tokens_total" not in series_names("gufo")
|
||||
|
||||
|
||||
def test_gufo_a_measured_zero_is_not_replaced_by_the_gauge(metrics_text) -> None:
|
||||
"""`measured or gauge` made an idle engine advertise its last request's speed."""
|
||||
text = metrics_text("gufo")
|
||||
rates = RateEstimator(span=10)
|
||||
parse_gufo_metrics(text, rates, 1_000.0)
|
||||
metrics = parse_gufo_metrics(text, rates, 1_010.0) # counters unchanged: truly idle
|
||||
decode = metrics["decode_tokens_per_second"]
|
||||
assert decode["value"] == 0.0
|
||||
assert "predicted_tokens_seconds" not in str(decode["source"]), "native gauge must not leak in"
|
||||
|
||||
|
||||
def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None:
|
||||
text = metrics_text("vllm")
|
||||
rates = RateEstimator(span=10)
|
||||
parse_vllm_metrics(text, rates, 2_000.0)
|
||||
metrics = parse_vllm_metrics(text, rates, 2_010.0)
|
||||
decode = metrics["decode_tokens_per_second"]
|
||||
assert decode["value"] == 0.0
|
||||
assert "running/tpot" not in str(decode["source"])
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Startup wiring: `python -m dashboards` / the `dashboards` console script.
|
||||
|
||||
Two things are worth pinning: the reload path must hand uvicorn an import string (an
|
||||
app object cannot be reloaded), and debugpy must stay completely inert unless asked.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from dashboards import __main__ as entry
|
||||
from dashboards import debug as debug_module
|
||||
from dashboards.config import Settings
|
||||
|
||||
|
||||
class FakeUvicorn(ModuleType):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("uvicorn")
|
||||
self.calls: list[tuple[Any, dict[str, Any]]] = []
|
||||
|
||||
def run(self, target: Any, **kwargs: Any) -> None:
|
||||
self.calls.append((target, kwargs))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_uvicorn(monkeypatch: pytest.MonkeyPatch) -> FakeUvicorn:
|
||||
module = FakeUvicorn()
|
||||
monkeypatch.setitem(sys.modules, "uvicorn", module)
|
||||
return module
|
||||
|
||||
|
||||
def test_main_serves_the_assembled_app(fake_uvicorn: FakeUvicorn, tmp_path: Any,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Settings is frozen on purpose, so the module attribute is replaced, not poked. The DB
|
||||
# path is private: a test must not open the developer's running database.
|
||||
monkeypatch.setattr(entry, "SETTINGS",
|
||||
Settings(host="0.0.0.0", port=8090, db_path=str(tmp_path / "main.sqlite3")))
|
||||
monkeypatch.delenv("DASH_RELOAD", raising=False)
|
||||
entry.main()
|
||||
(target, kwargs), = fake_uvicorn.calls
|
||||
assert kwargs == {"host": "0.0.0.0", "port": 8090}
|
||||
# FastAPI keeps included routers in a private wrapper, so the honest check is a request
|
||||
assert isinstance(target, FastAPI)
|
||||
with TestClient(target) as client:
|
||||
assert client.get("/api/health").status_code == 200
|
||||
assert client.get("/ups").status_code == 200
|
||||
|
||||
|
||||
def test_reload_mode_passes_an_import_string(fake_uvicorn: FakeUvicorn, tmp_path: Any,
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(entry, "SETTINGS", Settings(db_path=str(tmp_path / "reload.sqlite3")))
|
||||
# uvicorn cannot reload an object it was handed, only a module:app string
|
||||
monkeypatch.setenv("DASH_RELOAD", "1")
|
||||
entry.main()
|
||||
(target, kwargs), = fake_uvicorn.calls
|
||||
assert target == "dashboards.app:app"
|
||||
assert kwargs["reload"] is True
|
||||
|
||||
|
||||
def test_debugpy_is_inert_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("DEBUGPY", raising=False)
|
||||
assert debug_module.maybe_listen() is False
|
||||
|
||||
|
||||
def test_debugpy_listens_when_asked(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls: list[tuple[str, int]] = []
|
||||
fake = ModuleType("debugpy")
|
||||
fake.listen = lambda addr: calls.append(addr) # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, "debugpy", fake)
|
||||
monkeypatch.setenv("DEBUGPY", "yes")
|
||||
monkeypatch.setenv("DEBUGPY_PORT", "5699")
|
||||
assert debug_module.maybe_listen() is True
|
||||
assert calls == [("0.0.0.0", 5699)]
|
||||
|
||||
|
||||
def test_debugpy_missing_is_a_message_not_a_crash(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("DEBUGPY", "1")
|
||||
# a None entry makes `import debugpy` raise ImportError, as it does in a prod venv
|
||||
monkeypatch.setitem(sys.modules, "debugpy", None) # type: ignore[arg-type]
|
||||
assert debug_module.maybe_listen() is False
|
||||
@@ -0,0 +1,486 @@
|
||||
"""NUT: the wire protocol, the health rules, and the gates around the only write path.
|
||||
|
||||
The protocol tests run against a fake upsd over a real socket — this client speaks a
|
||||
line protocol with quoting rules, and a mock that agrees with my memory of it proves
|
||||
nothing. The health rules and the command gates are pure functions and are asserted
|
||||
directly, because those are what decide whether a battery test gets sent.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from dashboards.sources import nut
|
||||
from dashboards.sources.nut import NutClient, NutError, UpsReading, normalise_celsius
|
||||
from dashboards.sources.nut_sim import NutSimulator
|
||||
|
||||
VAR_ROWS = {
|
||||
"battery.charge": "100",
|
||||
"battery.runtime": "3720",
|
||||
"battery.voltage": "27.3",
|
||||
"battery.temperature": "300.4", # kelvin, as several drivers report it
|
||||
"battery.date": "2021/04/01",
|
||||
"battery.type": "Pb",
|
||||
"input.voltage": "121.8",
|
||||
"input.voltage.nominal": "120",
|
||||
"input.transfer.low": "92",
|
||||
"input.transfer.high": "139",
|
||||
"output.voltage": "120.9",
|
||||
"ups.load": "34",
|
||||
"ups.realpower.nominal": "900",
|
||||
"ups.load.high": "1500",
|
||||
"ups.status": "OL CHRG",
|
||||
"ups.mfr": "Cyber Power Systems",
|
||||
"ups.model": "PR1500LCDRT2U",
|
||||
"battery.test.result": "PASSED",
|
||||
"battery.test.date": "2026-09-01",
|
||||
}
|
||||
|
||||
CMDS = ["beeper.mute", "test.panel.start", "test.battery.start.quick",
|
||||
"test.battery.start.deep", "load.off"]
|
||||
|
||||
|
||||
class FakeUpsd:
|
||||
"""Just enough of upsd's protocol to exercise the client, over a real socket."""
|
||||
|
||||
def __init__(self, *, require_auth: bool = False, close_early: bool = False,
|
||||
hang: bool = False) -> None:
|
||||
self.require_auth = require_auth
|
||||
self.close_early = close_early
|
||||
self.hang = hang
|
||||
self.received: list[str] = []
|
||||
self.authenticated = False
|
||||
self._server: asyncio.AbstractServer | None = None
|
||||
|
||||
async def start(self) -> int:
|
||||
self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0)
|
||||
await self._server.start_serving()
|
||||
return int(self._server.sockets[0].getsockname()[1])
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._server:
|
||||
self._server.close()
|
||||
await self._server.wait_closed()
|
||||
|
||||
async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
if self.close_early:
|
||||
writer.close()
|
||||
return
|
||||
while True:
|
||||
raw = await reader.readline()
|
||||
if not raw:
|
||||
break
|
||||
line = raw.decode().strip()
|
||||
self.received.append(line)
|
||||
verb, _, rest = line.partition(" ")
|
||||
await self._respond(writer, verb, rest)
|
||||
|
||||
async def _respond(self, writer: asyncio.StreamWriter, verb: str, rest: str) -> None:
|
||||
def send(text: str) -> None:
|
||||
writer.write((text + "\n").encode())
|
||||
|
||||
if verb == "LIST":
|
||||
kind, _, target = rest.partition(" ")
|
||||
if kind == "UPS":
|
||||
send("BEGIN LIST UPS")
|
||||
send('UPS "cyberpower" "Cyber Power Systems PR1500LCDRT2U"')
|
||||
send("END LIST UPS")
|
||||
elif kind == "VAR":
|
||||
send("BEGIN LIST VAR")
|
||||
for key, value in VAR_ROWS.items():
|
||||
send(f'VAR "{_strip(target)}" "{key}" "{value}"')
|
||||
send("END LIST VAR")
|
||||
elif kind == "CMD":
|
||||
send("BEGIN LIST CMD")
|
||||
for command in CMDS:
|
||||
send(f'CMD "{_strip(target)}" "{command}"')
|
||||
send("END LIST CMD")
|
||||
elif kind == "RW":
|
||||
send("BEGIN LIST RW")
|
||||
send(f'RW "{_strip(target)}" "battery.energysave" "0"')
|
||||
send("END LIST RW")
|
||||
else:
|
||||
send("ERR UNKNOWN-COMMAND")
|
||||
elif verb == "USER":
|
||||
self.user = rest
|
||||
send("OK")
|
||||
elif verb == "PASS":
|
||||
self.authenticated = not self.require_auth or getattr(self, "user", "") == "monitor"
|
||||
send("OK" if self.authenticated else "ERR ACCESS-DENIED")
|
||||
elif verb == "INSTCMD":
|
||||
if self.require_auth and not self.authenticated:
|
||||
send("ERR ACCESS-DENIED")
|
||||
elif self.hang:
|
||||
await asyncio.sleep(30)
|
||||
else:
|
||||
send("OK Send shorthand command successful")
|
||||
elif verb == "LOGOUT":
|
||||
send("OK Goodbye")
|
||||
else:
|
||||
send("ERR UNKNOWN-COMMAND")
|
||||
|
||||
|
||||
def _strip(value: str) -> str:
|
||||
return value.strip().strip('"')
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def upsd():
|
||||
server = FakeUpsd()
|
||||
port = await server.start()
|
||||
yield server, port
|
||||
await server.stop()
|
||||
|
||||
|
||||
async def test_list_ups_parses_names_and_descriptions(upsd) -> None:
|
||||
_server, port = upsd
|
||||
async with NutClient("127.0.0.1", port) as client:
|
||||
ups_list = await client.list_ups()
|
||||
assert ups_list[0]["name"] == "cyberpower"
|
||||
assert ups_list[0]["description"] == "Cyber Power Systems PR1500LCDRT2U"
|
||||
|
||||
|
||||
async def test_list_vars_keeps_spaces_and_quotes(upsd) -> None:
|
||||
_server, port = upsd
|
||||
async with NutClient("127.0.0.1", port) as client:
|
||||
variables = await client.list_vars("cyberpower")
|
||||
assert variables["ups.mfr"] == "Cyber Power Systems"
|
||||
assert variables["battery.charge"] == "100"
|
||||
assert variables["ups.status"] == "OL CHRG"
|
||||
|
||||
|
||||
async def test_list_cmds_returns_what_the_driver_offers(upsd) -> None:
|
||||
_server, port = upsd
|
||||
async with NutClient("127.0.0.1", port) as client:
|
||||
commands = await client.list_cmds("cyberpower")
|
||||
assert commands == CMDS
|
||||
|
||||
|
||||
async def test_list_rw_vars_names_the_variables_not_the_ups(upsd) -> None:
|
||||
"""`RW <ups> <variable>`: reading field 1 returns the UPS name as a writable variable."""
|
||||
_server, port = upsd
|
||||
async with NutClient("127.0.0.1", port) as client:
|
||||
rw = await client.list_rw_vars("cyberpower")
|
||||
assert rw == ["battery.energysave"]
|
||||
|
||||
|
||||
async def test_instcmd_without_credentials_refuses_locally(upsd) -> None:
|
||||
"""No socket write: a dashboard must not even try when it knows it cannot authorise."""
|
||||
_server, port = upsd
|
||||
async with NutClient("127.0.0.1", port) as client:
|
||||
with pytest.raises(NutError) as excinfo:
|
||||
await client.instcmd("cyberpower", "test.panel.start")
|
||||
assert excinfo.value.code == "ACCESS-DENIED"
|
||||
assert "INSTCMD" not in _server.received or not any("INSTCMD" in line for line in _server.received)
|
||||
|
||||
|
||||
async def test_instcmd_with_credentials_succeeds() -> None:
|
||||
server = FakeUpsd(require_auth=True)
|
||||
port = await server.start()
|
||||
try:
|
||||
async with NutClient("127.0.0.1", port, username="monitor", password="pw") as client:
|
||||
assert await client.login() is True
|
||||
reply = await client.instcmd("cyberpower", "test.panel.start")
|
||||
assert "OK" in reply
|
||||
assert any(line.startswith("INSTCMD") for line in server.received)
|
||||
finally:
|
||||
await server.stop()
|
||||
|
||||
|
||||
async def test_upsd_rejection_carries_the_error_code() -> None:
|
||||
server = FakeUpsd(require_auth=True)
|
||||
port = await server.start()
|
||||
try:
|
||||
async with NutClient("127.0.0.1", port, username="nobody", password="pw") as client:
|
||||
with pytest.raises(NutError) as excinfo:
|
||||
await client.login()
|
||||
assert excinfo.value.code == "ACCESS-DENIED"
|
||||
assert "upsd.users" in str(excinfo.value) or "refused" in str(excinfo.value) or str(excinfo.value)
|
||||
finally:
|
||||
await server.stop()
|
||||
|
||||
|
||||
async def test_connection_refused_is_reported_as_unreachable() -> None:
|
||||
"""A closed port must read as 'upsd is not answering', never as an empty UPS list."""
|
||||
client = NutClient("127.0.0.1", 1, timeout=1.0)
|
||||
with pytest.raises(NutError) as excinfo:
|
||||
await client.connect()
|
||||
assert excinfo.value.code == "UNREACHABLE"
|
||||
|
||||
|
||||
async def test_server_closing_early_is_reported() -> None:
|
||||
server = FakeUpsd(close_early=True)
|
||||
port = await server.start()
|
||||
try:
|
||||
async with NutClient("127.0.0.1", port, timeout=2.0) as client:
|
||||
with pytest.raises(NutError) as excinfo:
|
||||
await client.list_ups()
|
||||
assert excinfo.value.code == "CLOSED"
|
||||
finally:
|
||||
await server.stop()
|
||||
|
||||
|
||||
async def test_a_silent_upsd_times_out_instead_of_hanging() -> None:
|
||||
server = FakeUpsd(hang=True)
|
||||
port = await server.start()
|
||||
try:
|
||||
async with NutClient("127.0.0.1", port, timeout=0.3) as client:
|
||||
with pytest.raises(NutError) as excinfo:
|
||||
await client._send("INSTCMD cyberpower beeper.mute")
|
||||
await client._read_line()
|
||||
assert excinfo.value.code == "TIMEOUT"
|
||||
finally:
|
||||
await server.stop()
|
||||
|
||||
|
||||
# ---- reading semantics ---------------------------------------------------- #
|
||||
|
||||
def reading(**overrides: str) -> UpsReading:
|
||||
values = dict(VAR_ROWS)
|
||||
values.update(overrides)
|
||||
return UpsReading(name="cyberpower", description="Cyber Power PR1500LCDRT2U",
|
||||
vars=values, commands=list(CMDS))
|
||||
|
||||
|
||||
def test_status_flags_and_state_chips() -> None:
|
||||
ups = reading()
|
||||
assert ups.status_flags == ["OL", "CHRG"]
|
||||
labels = {state["label"] for state in ups.states()}
|
||||
assert "online" in labels and "charging" in labels
|
||||
assert all(state["severity"] in ("ok", "warn", "fail", "info") for state in ups.states())
|
||||
|
||||
|
||||
def test_unknown_status_flag_is_shown_not_dropped() -> None:
|
||||
ups = reading(**{"ups.status": "OL OFF"})
|
||||
ids = {state["id"] for state in ups.states()}
|
||||
assert "OFF" in ids, "an unrecognised-in-combination flag must still be shown, not swallowed"
|
||||
|
||||
|
||||
def test_on_battery_is_derived_from_the_flag_not_from_charge() -> None:
|
||||
assert reading().on_battery is False
|
||||
assert reading(**{"ups.status": "OB"}).on_battery is True
|
||||
|
||||
|
||||
def test_alarms_are_split_and_cleaned() -> None:
|
||||
ups = reading(**{"ups.alarm": "BATT.VERIFYFAIL, OVERLOAD "})
|
||||
assert ups.alarms == ["BATT.VERIFYFAIL", "OVERLOAD"]
|
||||
|
||||
|
||||
def test_number_returns_none_for_absent_and_non_numeric() -> None:
|
||||
ups = reading()
|
||||
assert ups.number("battery.charge") == 100.0
|
||||
assert ups.number("does.not.exist") is None
|
||||
assert ups.number("ups.status") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw,expected", [
|
||||
(None, None), (27.4, 27.4), (300.4, 27.25), (150.0, 150.0), (151.0, -122.15),
|
||||
])
|
||||
def test_kelvin_is_detected_because_the_driver_cannot_be_trusted(raw, expected) -> None:
|
||||
result = normalise_celsius(raw)
|
||||
assert result == (None if expected is None else pytest.approx(expected, abs=0.01))
|
||||
|
||||
|
||||
# ---- health assessment ---------------------------------------------------- #
|
||||
|
||||
def test_a_healthy_ups_reports_every_check_as_ok() -> None:
|
||||
health = nut.assess(reading(**{"battery.temperature": "27.4", "battery.mfr.date": "2026-01-01",
|
||||
"ups.status": "OL"}), now=1_767_225_600.0)
|
||||
assert health["overall"] == "healthy"
|
||||
assert health["counts"]["fail"] == 0 and health["counts"]["warn"] == 0
|
||||
assert any(check["name"] == "Power path" for check in health["checks"])
|
||||
|
||||
|
||||
def test_on_battery_with_little_runtime_is_a_failure() -> None:
|
||||
health = nut.assess(reading(**{"ups.status": "OB", "battery.charge": "18",
|
||||
"battery.runtime": "90",
|
||||
"battery.temperature": "27.4"}), now=1_767_225_600.0)
|
||||
assert health["overall"] == "at risk"
|
||||
runtime = next(c for c in health["checks"] if c["name"] == "Runtime estimate")
|
||||
assert runtime["status"] in ("warn", "fail")
|
||||
|
||||
|
||||
def test_old_battery_is_warned_about_with_the_actual_age() -> None:
|
||||
health = nut.assess(reading(**{"battery.temperature": "27.4", "battery.mfr.date": "2019-01-01"}),
|
||||
now=1_767_225_600.0)
|
||||
age = next(c for c in health["checks"] if c["name"] == "Battery age")
|
||||
assert age["status"] == "warn"
|
||||
assert "2019" in age["detail"] or "7" in age["detail"]
|
||||
|
||||
|
||||
def test_a_kelvin_temperature_is_not_reported_as_a_melting_battery() -> None:
|
||||
health = nut.assess(reading(), now=1_767_225_600.0) # 300.4 in the fixture
|
||||
temp = next(c for c in health["checks"] if c["name"] == "Battery temperature")
|
||||
assert temp["status"] == "ok"
|
||||
assert "kelvin" in temp["detail"]
|
||||
assert 20 < temp["value"] < 35
|
||||
|
||||
|
||||
def test_a_hot_battery_fails() -> None:
|
||||
health = nut.assess(reading(**{"battery.temperature": "52"}), now=1_767_225_600.0)
|
||||
temp = next(c for c in health["checks"] if c["name"] == "Battery temperature")
|
||||
assert temp["status"] == "fail"
|
||||
|
||||
|
||||
def test_a_failed_self_test_is_surfaced() -> None:
|
||||
health = nut.assess(reading(**{"battery.test.result": "FAILED", "battery.temperature": "27.4"}),
|
||||
now=1_767_225_600.0)
|
||||
check = next(c for c in health["checks"] if c["name"] == "Last self-test")
|
||||
assert check["status"] in ("warn", "fail")
|
||||
|
||||
|
||||
def test_missing_test_history_is_unknown_not_ok() -> None:
|
||||
values = {key: value for key, value in VAR_ROWS.items()
|
||||
if not key.startswith("battery.test")}
|
||||
values["battery.temperature"] = "27.4"
|
||||
health = nut.assess(nut.UpsReading(name="cyberpower", vars=values, commands=[]),
|
||||
now=1_767_225_600.0)
|
||||
check = next(c for c in health["checks"] if c["name"] == "Last self-test")
|
||||
assert check["status"] == "unknown"
|
||||
|
||||
|
||||
def test_alarms_drive_the_overall_verdict() -> None:
|
||||
health = nut.assess(reading(**{"ups.alarm": "BATT.FAIL", "battery.temperature": "27.4"}),
|
||||
now=1_767_225_600.0)
|
||||
assert health["overall"] in ("degraded", "at risk")
|
||||
|
||||
|
||||
# ---- command gating ------------------------------------------------------- #
|
||||
|
||||
def base_commands() -> list[str]:
|
||||
return list(CMDS)
|
||||
|
||||
|
||||
def by_name(items: list[dict], command: str) -> dict:
|
||||
return next(item for item in items if item["command"] == command)
|
||||
|
||||
|
||||
def test_commands_are_disabled_when_actions_are_off() -> None:
|
||||
items = nut.available_commands(base_commands(), allow_actions=False, read_only=False, has_credentials=True,
|
||||
on_battery=False)
|
||||
deep = by_name(items, "test.battery.start.deep")
|
||||
assert deep["enabled"] is False
|
||||
assert "NUT_ALLOW_ACTIONS" in deep["reason"]
|
||||
|
||||
|
||||
def test_commands_are_disabled_without_credentials_that_have_actions() -> None:
|
||||
items = nut.available_commands(base_commands(), allow_actions=True, read_only=False, has_credentials=False,
|
||||
on_battery=False)
|
||||
assert by_name(items, "beeper.mute")["enabled"] is False
|
||||
assert "upsd.users" in by_name(items, "beeper.mute")["reason"]
|
||||
|
||||
|
||||
def test_dangerous_commands_are_blocked_while_on_battery() -> None:
|
||||
"""A deep test while already on battery can end the runtime early — that is the whole
|
||||
thing the UPS is protecting against, so the UI must not offer it."""
|
||||
items = nut.available_commands(base_commands(), allow_actions=True, read_only=False, has_credentials=True,
|
||||
on_battery=True)
|
||||
deep = by_name(items, "test.battery.start.deep")
|
||||
assert deep["enabled"] is False
|
||||
assert "battery" in deep["reason"].lower()
|
||||
safe = by_name(items, "beeper.mute")
|
||||
assert safe["enabled"] is True
|
||||
|
||||
|
||||
def test_read_only_overrides_everything_else() -> None:
|
||||
items = nut.available_commands(base_commands(), allow_actions=True, read_only=True,
|
||||
has_credentials=True, on_battery=False)
|
||||
assert items and all(item["enabled"] is False for item in items)
|
||||
assert any("DASH_READ_ONLY" in (item["reason"] or "") for item in items)
|
||||
|
||||
|
||||
def test_risk_levels_are_labelled_per_command() -> None:
|
||||
items = nut.available_commands(base_commands(), allow_actions=True, read_only=False, has_credentials=True,
|
||||
on_battery=False)
|
||||
assert by_name(items, "beeper.mute")["risk"] == nut.RISK_SAFE
|
||||
assert by_name(items, "test.battery.start.deep")["risk"] == nut.RISK_DANGEROUS
|
||||
assert by_name(items, "load.off")["risk"] == nut.RISK_DANGEROUS
|
||||
|
||||
|
||||
def test_commands_the_driver_does_not_advertise_are_never_offered() -> None:
|
||||
items = nut.available_commands(["beeper.mute"], allow_actions=True, read_only=False, has_credentials=True,
|
||||
on_battery=False)
|
||||
assert [item["command"] for item in items] == ["beeper.mute"]
|
||||
|
||||
|
||||
def test_commands_outside_the_risk_catalog_are_never_offered() -> None:
|
||||
"""A driver can advertise anything; the dashboard only sends what it has a risk label for."""
|
||||
items = nut.available_commands(["beeper.mute", "something.new"], allow_actions=True,
|
||||
read_only=False, has_credentials=True, on_battery=False)
|
||||
assert [item["command"] for item in items] == ["beeper.mute"]
|
||||
|
||||
|
||||
def test_command_advice_catalog_is_complete_for_the_ui() -> None:
|
||||
advice = nut.command_advice("test.battery.start.deep")
|
||||
assert advice and advice["risk"] == nut.RISK_DANGEROUS
|
||||
assert advice["label"] and advice["help"]
|
||||
assert nut.command_advice("not.a.command") is None
|
||||
|
||||
|
||||
# ---- payload & simulator -------------------------------------------------- #
|
||||
|
||||
def test_payload_carries_numbers_health_and_what_the_driver_offers() -> None:
|
||||
payload = nut.payload(reading(), now=1_767_225_600.0)
|
||||
assert payload["name"] == "cyberpower"
|
||||
assert payload["identity"]["model"] == "PR1500LCDRT2U"
|
||||
assert payload["numbers"]["charge_pct"] == 100.0
|
||||
assert payload["numbers"]["battery_temperature"] == pytest.approx(27.25, abs=0.01)
|
||||
assert payload["health"]["overall"] in ("healthy", "degraded", "at risk")
|
||||
assert payload["advertised_commands"] == CMDS
|
||||
assert payload["on_battery"] is False
|
||||
# the raw dump keeps the original kelvin string: normalising for charts must not
|
||||
# destroy the value the operator would compare against `upsc`
|
||||
assert payload["extra"] or payload["status"] == "OL CHRG"
|
||||
|
||||
|
||||
def test_history_samples_only_include_real_numbers() -> None:
|
||||
values = {key: value for key, value in VAR_ROWS.items() if key != "battery.runtime"}
|
||||
samples = nut.history_samples(nut.UpsReading(name="cyberpower", vars=values, commands=[]))
|
||||
assert "ups.charge_pct" in samples
|
||||
assert "ups.battery_runtime_seconds" not in samples
|
||||
assert all(isinstance(value, float) and value == value for value in samples.values())
|
||||
|
||||
|
||||
def test_history_samples_normalise_the_temperature() -> None:
|
||||
samples = nut.history_samples(reading())
|
||||
assert samples["ups.battery_temperature_c"] == pytest.approx(27.25, abs=0.01)
|
||||
|
||||
|
||||
async def test_simulator_reads_like_upsd_and_is_labelled() -> None:
|
||||
readings = await NutSimulator().read()
|
||||
assert readings, "the simulator must present at least one UPS"
|
||||
for ups in readings:
|
||||
assert "(simulated)" in ups.description, "simulated data must be identifiable"
|
||||
assert ups.vars["battery.charge"]
|
||||
assert "test.battery.start.deep" in ups.commands
|
||||
|
||||
|
||||
async def test_simulator_commands_change_the_model_like_a_real_driver() -> None:
|
||||
sim = NutSimulator()
|
||||
name = (await sim.read())[0].name
|
||||
assert "OK" in sim.command(name, "test.battery.start.quick")
|
||||
after = (await sim.read())[0]
|
||||
assert after.on_battery is True, "a quick test puts the load on battery"
|
||||
assert "OK" in sim.command(name, "test.battery.stop")
|
||||
assert (await sim.read())[0].on_battery is False
|
||||
|
||||
|
||||
async def test_simulator_discharges_while_on_battery() -> None:
|
||||
"""A simulated UPS that never drains cannot exercise a runtime or health path."""
|
||||
sim = NutSimulator()
|
||||
name = (await sim.read())[0].name
|
||||
before = float((await sim.read())[0].vars["battery.charge"])
|
||||
sim.command(name, "test.battery.start.deep")
|
||||
ups = next(item for item in sim.ups if item.name == name)
|
||||
ups.on_battery = True
|
||||
for _ in range(30):
|
||||
ups.step(60.0) # half an hour of model time on battery
|
||||
assert ups.charge < before
|
||||
|
||||
|
||||
async def test_simulator_command_for_unknown_ups_raises() -> None:
|
||||
sim = NutSimulator()
|
||||
with pytest.raises(KeyError):
|
||||
sim.command("nope", "beeper.mute")
|
||||
@@ -0,0 +1,217 @@
|
||||
"""The poll loop: isolation, last-good-value, and events that mark transitions only.
|
||||
|
||||
These three behaviours are why the dashboard can be trusted when something is broken, so
|
||||
they are asserted directly rather than through HTTP:
|
||||
|
||||
* a source that throws must not take the loop, the hub, or its neighbours down;
|
||||
* a failed poll keeps the previous payload and is marked stale — never zeroed;
|
||||
* ok→fail and fail→ok each write exactly one event; 500 more failures write nothing;
|
||||
* maintenance failing is logged, because a silent rollup failure means every long-window
|
||||
chart is quietly empty (that bug is what the logging was added for).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards.poller import Hub, Job, Result
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.store import Event, Sample, SampleStore
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Any) -> Iterator[SampleStore]:
|
||||
instance = SampleStore(tmp_path / "poller.sqlite3")
|
||||
try:
|
||||
yield instance
|
||||
finally:
|
||||
instance.close()
|
||||
|
||||
|
||||
def ok_result(value: float = 1.0) -> Result:
|
||||
return Result(payload={"value": value},
|
||||
samples=[Sample("host", "gw", "cpu.usage_pct", value, time.time())])
|
||||
|
||||
|
||||
async def failing() -> Result:
|
||||
raise ConnectionError("agent is down")
|
||||
|
||||
|
||||
async def test_a_successful_poll_stores_payload_samples_and_health(store) -> None:
|
||||
job = Job("host:gw", "system", "gw", 0.5, lambda: asyncio.sleep(0, ok_result(7.0)))
|
||||
outcome = await job.run_once(store)
|
||||
assert outcome is True, job.state.error
|
||||
assert job.state.ok is True and job.state.error is None
|
||||
assert job.state.payload == {"value": 7.0}
|
||||
assert job.state.successes == 1 and job.state.samples_written == 1
|
||||
assert job.state.latency_ms is not None
|
||||
assert store.recent_events(limit=5) == [], "a first successful poll is not a recovery"
|
||||
|
||||
|
||||
async def test_a_failing_poll_keeps_the_last_good_payload(store) -> None:
|
||||
"""The UI shows `stale 12s` against real numbers; it must not show zeros or n/a."""
|
||||
job = Job("host:gw", "system", "gw", 0.5, lambda: asyncio.sleep(0, ok_result(7.0)))
|
||||
await job.run_once(store)
|
||||
job.fn = failing
|
||||
assert await job.run_once(store) is False
|
||||
assert job.state.ok is False
|
||||
assert job.state.payload == {"value": 7.0}, "a dead host must not blank its last values"
|
||||
assert job.state.error == "ConnectionError: agent is down"
|
||||
assert job.state.as_json()["age_seconds"] is not None
|
||||
|
||||
|
||||
async def test_transitions_write_one_event_each_and_spam_writes_none(store) -> None:
|
||||
job = Job("host:gw", "system", "gw", 0.5, lambda: asyncio.sleep(0, ok_result()),
|
||||
grace_failures=1)
|
||||
await job.run_once(store)
|
||||
job.fn = failing
|
||||
for _ in range(5):
|
||||
await job.run_once(store)
|
||||
events = store.recent_events(limit=50)
|
||||
down = [event for event in events if event.kind == "unreachable"]
|
||||
assert len(down) == 1, "five consecutive failures must not log five events"
|
||||
assert down[0].host == "gw", "events carry the display label, not the internal job key"
|
||||
assert down[0].severity == "error"
|
||||
|
||||
job.fn = lambda: asyncio.sleep(0, ok_result())
|
||||
await job.run_once(store)
|
||||
events = store.recent_events(limit=50)
|
||||
up = [event for event in events if event.kind == "reachable"]
|
||||
assert len(up) == 1
|
||||
assert up[0].message == "gw is answering again after 5 failures"
|
||||
|
||||
|
||||
async def test_one_failure_reads_as_a_failure_not_failures(store) -> None:
|
||||
job = Job("host:gw", "system", "gw", 0.5, lambda: asyncio.sleep(0, ok_result()))
|
||||
await job.run_once(store)
|
||||
job.fn = failing
|
||||
await job.run_once(store)
|
||||
job.fn = lambda: asyncio.sleep(0, ok_result())
|
||||
await job.run_once(store)
|
||||
up = [event for event in store.recent_events(limit=50) if event.kind == "reachable"]
|
||||
assert up[0].message.endswith("after 1 failure")
|
||||
|
||||
|
||||
async def test_grace_allows_a_blip_without_an_event(store) -> None:
|
||||
"""One failure inside the grace window must not page anybody."""
|
||||
job = Job("host:gw", "system", "gw", 0.5, failing, grace_failures=2)
|
||||
await job.run_once(store)
|
||||
assert store.recent_events(limit=10) == []
|
||||
await job.run_once(store)
|
||||
events = store.recent_events(limit=10)
|
||||
assert [event.kind for event in events] == ["unreachable"]
|
||||
assert events[0].severity == "warn", "never answered is a config problem, not an outage"
|
||||
assert "never answered" in events[0].message
|
||||
for _ in range(4):
|
||||
await job.run_once(store)
|
||||
assert len(store.recent_events(limit=10)) == 1, "one event per outage, not per poll"
|
||||
|
||||
|
||||
async def test_a_slow_poll_never_overlaps_itself(store) -> None:
|
||||
calls = 0
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def slow() -> Result:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
await gate.wait()
|
||||
return ok_result()
|
||||
|
||||
job = Job("host:gw", "system", "gw", 0.5, slow)
|
||||
first = asyncio.create_task(job.run_once(store))
|
||||
await asyncio.sleep(0.01)
|
||||
assert await job.run_once(store) is False, "an overlapping poll double-counts rates"
|
||||
gate.set()
|
||||
assert await first is True
|
||||
assert calls == 1
|
||||
|
||||
|
||||
async def test_a_counter_that_resets_is_reported_not_silently_negative() -> None:
|
||||
"""The reset path is what ships: an agent restart must not read as negative throughput."""
|
||||
rates = RateEstimator(span=10.0)
|
||||
first, _ = rates.rate("cpu.total", 1000.0, now=100.0)
|
||||
assert first is None, "one sample cannot produce a rate"
|
||||
second, window = rates.rate("cpu.total", 2000.0, now=110.0)
|
||||
assert second == pytest.approx(100.0) and window == pytest.approx(10.0)
|
||||
after, _ = rates.rate("cpu.total", 5.0, now=120.0) # agent restarted
|
||||
assert after is None, "a reset counter must read n/a, never as negative throughput"
|
||||
|
||||
|
||||
async def test_the_hub_runs_every_job_and_stops_cleanly(store) -> None:
|
||||
counts = {"a": 0, "b": 0}
|
||||
|
||||
async def good(key: str) -> Result:
|
||||
counts[key] += 1
|
||||
return ok_result()
|
||||
|
||||
hub = Hub(store)
|
||||
hub.add(Job("host:a", "system", "a", 0.5, lambda: good("a")))
|
||||
hub.add(Job("host:b", "system", "b", 0.5, failing))
|
||||
hub.add(Job("llm:off", "llm", "off", 0.5, failing, enabled=False))
|
||||
hub.start()
|
||||
try:
|
||||
await asyncio.sleep(1.2)
|
||||
finally:
|
||||
await hub.stop()
|
||||
assert counts["a"] >= 2, "a neighbour that throws must not slow the healthy one"
|
||||
states = {state["key"]: state for state in hub.states()}
|
||||
assert states["host:a"]["ok"] is True
|
||||
assert states["host:b"]["ok"] is False
|
||||
assert states["llm:off"]["configured"] is False, "not configured is not the same as broken"
|
||||
assert hub.jobs["host:b"].state.failures >= 1
|
||||
|
||||
|
||||
async def test_states_can_be_filtered_to_one_dashboard(store) -> None:
|
||||
hub = Hub(store)
|
||||
hub.add(Job("host:a", "system", "a", 0.5, lambda: asyncio.sleep(0, ok_result())))
|
||||
hub.add(Job("nut", "ups", "NUT server", 0.5, lambda: asyncio.sleep(0, ok_result())))
|
||||
assert [s["key"] for s in hub.states("ups")] == ["nut"]
|
||||
|
||||
|
||||
async def test_maintenance_failure_is_logged_not_swallowed(store, monkeypatch,
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""A broken rollup means every chart past the raw window is empty. Retrying quietly
|
||||
turned a data-loss bug into nothing at all — this is the regression that added the log."""
|
||||
def boom() -> dict[str, int]:
|
||||
raise RuntimeError("no such table: sample_rollup")
|
||||
|
||||
monkeypatch.setattr(store, "maintain", boom)
|
||||
hub = Hub(store)
|
||||
task = asyncio.create_task(hub._maintain_loop(interval=0.02))
|
||||
try:
|
||||
with caplog.at_level(logging.ERROR, logger="dashboards.poller"):
|
||||
await asyncio.sleep(0.15)
|
||||
finally:
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert "store maintenance failed" in caplog.text
|
||||
assert "no such table" in caplog.text, "the log line must carry the real reason"
|
||||
|
||||
|
||||
async def test_maintenance_runs_from_the_hub_and_records_its_pass(store) -> None:
|
||||
hub = Hub(store)
|
||||
calls = []
|
||||
store.maintain = lambda: calls.append(1) or {"to_minute": 0, "to_hour": 0, # type: ignore[method-assign]
|
||||
"pruned": 0}
|
||||
task = asyncio.create_task(hub._maintain_loop(interval=0.02))
|
||||
try:
|
||||
await asyncio.sleep(0.1)
|
||||
finally:
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert calls, "the maintenance loop must actually call maintain"
|
||||
|
||||
|
||||
async def test_events_written_by_a_job_land_in_the_store(store) -> None:
|
||||
event = Event(1_700_000_000.0, "ups", "cyberpower", "status", "warn", "OL → OB")
|
||||
job = Job("nut", "ups", "NUT server", 0.5,
|
||||
lambda: asyncio.sleep(0, Result(payload={}, events=[event])))
|
||||
await job.run_once(store)
|
||||
assert [e.message for e in store.recent_events(limit=5)] == ["OL → OB"]
|
||||
@@ -0,0 +1,112 @@
|
||||
"""The hand-rolled Prometheus parser, tested against real engine expositions.
|
||||
|
||||
This parser exists so the app has no Prometheus dependency for the two upstreams that
|
||||
already speak the format. That makes it load-bearing on text we do not control: llama.cpp
|
||||
emits names with colons, vLLM emits label sets and histograms, and both are free to change
|
||||
them. Fixtures are captured verbatim (scripts/capture-fixtures.sh --with-llm).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
from dashboards.prom import Prom
|
||||
|
||||
|
||||
def test_parses_counters_and_gauges_from_a_real_capture(metrics_text) -> None:
|
||||
prom = Prom(metrics_text("gufo"))
|
||||
assert prom.scalar(["llamacpp:tokens_predicted_total"]) == 877_336.0
|
||||
assert prom.scalar(["llamacpp:kv_cache_usage_ratio"]) == 0.0
|
||||
assert prom.parse_errors == 0
|
||||
|
||||
|
||||
def test_help_and_type_comments_are_not_samples(metrics_text) -> None:
|
||||
prom = Prom(metrics_text("gufo"))
|
||||
assert not any(name.startswith("#") for name in prom.series)
|
||||
assert "# HELP llamacpp:prompt_tokens_total" not in prom.series
|
||||
|
||||
|
||||
def test_labels_are_parsed_and_filterable(metrics_text) -> None:
|
||||
prom = Prom(metrics_text("vllm"))
|
||||
running = prom.values(["vllm:num_requests_running"])
|
||||
assert running, "vLLM exposes num_requests_running"
|
||||
by_model = prom.by_label(["vllm:generation_tokens_total"], "model_name")
|
||||
assert by_model and all(model for model in by_model)
|
||||
|
||||
|
||||
def test_aggregations(metrics_text) -> None:
|
||||
prom = Prom(metrics_text("vllm"))
|
||||
total = prom.scalar(["vllm:prompt_tokens_total"], agg="sum")
|
||||
assert total is not None and total > 1e6
|
||||
# ratios must never be summed across label sets — a 3-way sum would read as 300% used
|
||||
usage = prom.scalar(["vllm:kv_cache_usage_perc"], agg="max")
|
||||
assert usage is not None and 0.0 <= usage <= 1.0
|
||||
|
||||
|
||||
def test_missing_series_is_none_not_zero(metrics_text) -> None:
|
||||
prom = Prom(metrics_text("gufo"))
|
||||
assert prom.scalar(["vllm:does_not_exist"]) is None
|
||||
assert prom.values(["vllm:does_not_exist"]) == []
|
||||
assert prom.by_label(["nope"], "model_name") == {}
|
||||
|
||||
|
||||
def test_histogram_mean_and_p95(metrics_text) -> None:
|
||||
hist = Prom(metrics_text("vllm")).histogram(["vllm:time_to_first_token_seconds"])
|
||||
assert hist is not None
|
||||
assert hist["count"] and hist["count"] > 0
|
||||
assert hist["mean"] and hist["mean"] > 0
|
||||
assert hist["p95"] is not None
|
||||
assert hist["p95"] >= hist["mean"], "p95 below the mean means the interpolation is wrong"
|
||||
assert math.isfinite(hist["p95"])
|
||||
|
||||
|
||||
def test_histogram_resolves_through_suffixes(metrics_text) -> None:
|
||||
prom = Prom(metrics_text("vllm"))
|
||||
# the bare name never appears in the exposition; only _bucket/_sum/_count do
|
||||
assert "vllm:time_to_first_token_seconds" not in prom.series
|
||||
assert prom.resolve_hist(["vllm:time_to_first_token_seconds"]) == "vllm:time_to_first_token_seconds"
|
||||
assert prom.resolve(["vllm:time_to_first_token_seconds"]) is None
|
||||
|
||||
|
||||
def test_resolve_returns_the_first_present_name() -> None:
|
||||
prom = Prom('a_total 5\nb_total 7\n')
|
||||
assert prom.resolve(["missing_total", "b_total", "a_total"]) == "b_total"
|
||||
assert prom.resolve(["nope"]) is None
|
||||
|
||||
|
||||
def test_label_match_narrows_the_series() -> None:
|
||||
prom = Prom('x{engine="0"} 1\nx{engine="1"} 2\n')
|
||||
assert prom.scalar(["x"], agg="sum") == 3
|
||||
assert prom.scalar(["x"], label_match={"engine": "1"}) == 2
|
||||
assert prom.scalar(["x"], label_match={"engine": "9"}) is None
|
||||
|
||||
|
||||
def test_malformed_lines_are_counted_not_fatal() -> None:
|
||||
prom = Prom("good_metric 1\nthis is not prometheus\nalso_bad{unclosed 2\n")
|
||||
assert prom.scalar(["good_metric"]) == 1.0
|
||||
assert prom.parse_errors >= 1
|
||||
|
||||
|
||||
def test_empty_exposition_is_harmless() -> None:
|
||||
prom = Prom("")
|
||||
assert prom.series == {}
|
||||
assert prom.scalar(["anything"]) is None
|
||||
assert prom.histogram(["anything"]) is None
|
||||
|
||||
|
||||
def test_quantile_ignores_non_finite_values() -> None:
|
||||
prom = Prom('m_sum 10\ncount 2\nm{le="1"} NaN\n')
|
||||
assert prom.scalar(["m"], label_match={"le": "1"}) is None
|
||||
|
||||
|
||||
def test_multiple_histogram_label_sets_do_not_invent_a_p95(metrics_text) -> None:
|
||||
"""Mixing per-DP-rank buckets into one cumulative curve would fabricate a percentile."""
|
||||
two = (
|
||||
'h_sum{dp="0"} 10\nh_sum{dp="1"} 10\n'
|
||||
'h_count{dp="0"} 2\nh_count{dp="1"} 2\n'
|
||||
'h_bucket{dp="0",le="1"} 1\nh_bucket{dp="0",le="+Inf"} 2\n'
|
||||
'h_bucket{dp="1",le="1"} 1\nh_bucket{dp="1",le="+Inf"} 2\n'
|
||||
)
|
||||
hist = Prom(two).histogram(["h"])
|
||||
assert hist is not None
|
||||
assert hist["mean"] == 5.0
|
||||
assert hist["p95"] is None, "p95 across mixed label sets is a made-up number"
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Rate estimation over a labelled window.
|
||||
|
||||
Every throughput number in this dashboard is Δcounter/Δt. If the estimator returns a
|
||||
plausible-looking number for a counter that restarted, or a silent 0 for a scrape that
|
||||
arrived late, the UI cannot tell the difference — so the None cases matter more here than
|
||||
the happy path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards.rates import RateEstimator
|
||||
|
||||
|
||||
def test_first_sample_has_no_rate() -> None:
|
||||
r = RateEstimator(span=10)
|
||||
assert r.rate("cpu", 100.0, now=1_000.0) == (None, None)
|
||||
|
||||
|
||||
def test_second_sample_gives_rate_and_window() -> None:
|
||||
r = RateEstimator(span=10)
|
||||
r.rate("cpu", 100.0, now=1_000.0)
|
||||
rate, window = r.rate("cpu", 300.0, now=1_010.0)
|
||||
assert rate == 20.0
|
||||
assert window == 10.0
|
||||
|
||||
|
||||
def test_window_is_labelled_with_the_span_actually_used() -> None:
|
||||
r = RateEstimator(span=10)
|
||||
r.rate("x", 0.0, now=0.0)
|
||||
r.rate("x", 10.0, now=2.0) # inside the span
|
||||
rate, window = r.rate("x", 30.0, now=11.0)
|
||||
# base is the sample 11s ago (outside the span) -> falls back to the previous sample
|
||||
assert window == 9.0
|
||||
assert rate == pytest.approx(20.0 / 9.0)
|
||||
|
||||
|
||||
def test_none_value_does_not_poison_the_window() -> None:
|
||||
r = RateEstimator(span=10)
|
||||
r.rate("x", 10.0, now=0.0)
|
||||
assert r.rate("x", None, now=1.0) == (None, None)
|
||||
rate, _window = r.rate("x", 30.0, now=2.0)
|
||||
assert rate == 10.0, "the None scrape must not become the baseline"
|
||||
|
||||
|
||||
def test_nan_and_inf_are_treated_as_missing() -> None:
|
||||
r = RateEstimator(span=10)
|
||||
r.rate("x", 1.0, now=0.0)
|
||||
for bad in (float("nan"), float("inf")):
|
||||
assert r.rate("x", bad, now=1.0) == (None, None)
|
||||
|
||||
|
||||
def test_counter_reset_yields_none_once_then_recovers() -> None:
|
||||
"""A restart must not report a huge negative — or, worse, a huge positive — rate."""
|
||||
r = RateEstimator(span=60)
|
||||
r.rate("tokens", 5_000.0, now=0.0)
|
||||
rate, _ = r.rate("tokens", 10.0, now=10.0) # process restarted
|
||||
assert rate is None
|
||||
rate, _ = r.rate("tokens", 60.0, now=20.0)
|
||||
assert rate == 5.0, "the window re-seeds from the post-restart counter"
|
||||
|
||||
|
||||
def test_flat_counter_reports_zero_not_none() -> None:
|
||||
r = RateEstimator(span=10)
|
||||
r.rate("idle", 5.0, now=0.0)
|
||||
rate, window = r.rate("idle", 5.0, now=5.0)
|
||||
assert rate == 0.0 and window == 5.0
|
||||
|
||||
|
||||
def test_keys_are_independent() -> None:
|
||||
r = RateEstimator(span=10)
|
||||
r.rate("a", 0.0, now=0.0)
|
||||
r.rate("b", 100.0, now=0.0)
|
||||
assert r.rate("a", 10.0, now=1.0)[0] == 10.0
|
||||
assert r.rate("b", 100.0, now=1.0)[0] == 0.0
|
||||
|
||||
|
||||
def test_forget_drops_a_removed_target() -> None:
|
||||
r = RateEstimator(span=10)
|
||||
for key in ("driveripper:cpu.total", "driveripper:disk.read", "deskwork:cpu.total"):
|
||||
r.rate(key, 1.0, now=0.0)
|
||||
assert r.forget("driveripper:") == 2
|
||||
assert r.keys() == ["deskwork:cpu.total"]
|
||||
# and the surviving target is untouched
|
||||
assert r.rate("deskwork:cpu.total", 11.0, now=1.0)[0] == 10.0
|
||||
|
||||
|
||||
def test_history_is_trimmed_to_max_samples() -> None:
|
||||
r = RateEstimator(span=1e6, max_samples=8)
|
||||
for i in range(50):
|
||||
r.rate("x", float(i), now=float(i))
|
||||
assert len(r._samples["x"]) <= 8 # bounded memory is the contract
|
||||
assert all(math.isfinite(v) for _, v in r._samples["x"])
|
||||
@@ -0,0 +1,254 @@
|
||||
"""History store: rollups, retention, and the honesty of an empty chart.
|
||||
|
||||
The rollup queries are plain SQL with positional bindings, and a mismatch there used to
|
||||
raise on every maintenance tick while the poller swallowed it — so every window past the
|
||||
raw retention was silently empty. These tests fail loudly if that ever happens again.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards.store import (
|
||||
HOUR,
|
||||
MINUTE,
|
||||
RES_HOUR,
|
||||
RES_MIN,
|
||||
RES_RAW,
|
||||
Event,
|
||||
Sample,
|
||||
SampleStore,
|
||||
samples_from_flat,
|
||||
select_window,
|
||||
)
|
||||
|
||||
|
||||
def add_series(db, source: str, host: str, metric: str, values, *, start: float, step: float,
|
||||
res: str = RES_RAW) -> None:
|
||||
for i, value in enumerate(values):
|
||||
# the private handle is the point: writing at an explicit resolution is what
|
||||
# the resolution-choice tests need
|
||||
db._db.execute(
|
||||
"INSERT INTO sample (source, host, metric, res, ts, value) VALUES (?,?,?,?,?,?)",
|
||||
(source, host, metric, res, start + i * step, float(value)),
|
||||
)
|
||||
db._db.commit()
|
||||
|
||||
|
||||
def test_store_starts_in_wal_mode(db) -> None:
|
||||
"""Readers poll while the writer commits; without WAL the API would hit locked errors."""
|
||||
assert db._db.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
|
||||
|
||||
|
||||
def test_add_and_query_roundtrip(db) -> None:
|
||||
now = time.time()
|
||||
db.add("host", "driveripper", "cpu.usage_pct", 42.5, ts=now)
|
||||
points = db.query("host", "driveripper", "cpu.usage_pct", start=now - 60)
|
||||
assert points == [(now, 42.5)]
|
||||
|
||||
|
||||
def test_query_is_bounded_and_ascending(db) -> None:
|
||||
base = 1_700_000_000.0
|
||||
add_series(db, "host", "h", "m", [3, 1, 2], start=base, step=10)
|
||||
assert db.query("host", "h", "m", start=base - 1) == [(base, 3.0), (base + 10, 1.0), (base + 20, 2.0)]
|
||||
assert len(db.query("host", "h", "m", start=base - 1, limit=2)) == 2
|
||||
|
||||
|
||||
def test_query_tail_keeps_the_newest_rows(db) -> None:
|
||||
base = 1_700_000_000.0
|
||||
add_series(db, "host", "h", "m", range(100), start=base, step=10)
|
||||
head = db.query("host", "h", "m", start=base - 1, limit=3)
|
||||
tail = db.query("host", "h", "m", start=base - 1, limit=3, tail=True)
|
||||
assert [v for _, v in head] == [0.0, 1.0, 2.0]
|
||||
assert [v for _, v in tail] == [97.0, 98.0, 99.0]
|
||||
assert tail == sorted(tail), "tail must still come back oldest-first"
|
||||
|
||||
|
||||
def test_samples_from_flat_drops_unknown_and_none() -> None:
|
||||
payload: dict[str, Any] = {"a": 1.0, "b": None, "c": "nope"}
|
||||
samples = samples_from_flat("host", "gw", payload, ts=5.0)
|
||||
assert [(s.metric, s.value) for s in samples] == [("a", 1.0)]
|
||||
|
||||
|
||||
def test_maintain_rolls_raw_into_minutes(db) -> None:
|
||||
"""Regression: the rollup statement had 7 placeholders and 6 bindings."""
|
||||
now = time.time()
|
||||
# two full, aligned minute buckets — see the hour test for why alignment matters
|
||||
minute = int(now // MINUTE) * MINUTE
|
||||
add_series(db, "host", "h", "cpu.usage_pct", [10, 20, 30, 40], start=minute - 4 * MINUTE,
|
||||
step=1)
|
||||
add_series(db, "host", "h", "cpu.usage_pct", [1, 1, 1, 1], start=minute - 3 * MINUTE, step=1)
|
||||
counts = db.maintain(now=now)
|
||||
assert counts["to_minute"] > 0
|
||||
minutes = db.query("host", "h", "cpu.usage_pct", res=RES_MIN, start=now - 3600)
|
||||
assert minutes, "no 1m rollup was written"
|
||||
assert [round(value) for _ts, value in minutes] == [25, 1]
|
||||
|
||||
|
||||
def test_maintain_is_idempotent(db) -> None:
|
||||
"""Rollups run on a timer: running twice must not double-count or duplicate rows."""
|
||||
now = time.time()
|
||||
minute = int(now // MINUTE) * MINUTE
|
||||
add_series(db, "host", "h", "m", [10, 20], start=minute - 4 * MINUTE, step=1)
|
||||
first = db.maintain(now=now)
|
||||
rows = db.query("host", "h", "m", res=RES_MIN, start=now - 3600)
|
||||
count = db.count()
|
||||
second = db.maintain(now=now)
|
||||
assert rows == db.query("host", "h", "m", res=RES_MIN, start=now - 3600)
|
||||
assert first["to_minute"] == second["to_minute"]
|
||||
assert db.count() == count, "re-running maintenance duplicated samples"
|
||||
|
||||
|
||||
def test_maintain_rolls_minutes_into_hours(db) -> None:
|
||||
now = time.time()
|
||||
# buckets are aligned to the epoch (`ts/3600*3600`), not to "hours from now", so the
|
||||
# fixtures start on a boundary or the two hours mix into a third, averaged bucket
|
||||
hour = int(now // HOUR) * HOUR
|
||||
# two *complete* hours of minute buckets: the current and previous hour are still
|
||||
# filling in, and maintenance must not touch them (see the partial-bucket test)
|
||||
add_series(db, "host", "h", "m", [1] * 60, start=hour - 5 * HOUR, step=60, res=RES_MIN)
|
||||
add_series(db, "host", "h", "m", [3] * 60, start=hour - 4 * HOUR, step=60, res=RES_MIN)
|
||||
counts = db.maintain(now=now)
|
||||
assert counts["to_hour"] > 0
|
||||
hours = db.query("host", "h", "m", res=RES_HOUR, start=now - 86_400)
|
||||
assert sorted(round(v) for _, v in hours) == [1.0, 3.0]
|
||||
|
||||
|
||||
|
||||
def test_maintain_prunes_expired_raw(db) -> None:
|
||||
now = time.time()
|
||||
old = now - (db.raw_keep_seconds + 7200)
|
||||
add_series(db, "host", "h", "m", [1, 2], start=old, step=60)
|
||||
add_series(db, "host", "h", "fresh", [1], start=now - 60, step=1)
|
||||
counts = db.maintain(now=now)
|
||||
assert counts["pruned"] >= 2
|
||||
assert db.query("host", "h", "m", start=0) == []
|
||||
|
||||
|
||||
def test_maintain_records_timestamp_in_stats(db) -> None:
|
||||
db.maintain()
|
||||
stats = db.stats()
|
||||
assert stats["last_maintain"] is not None
|
||||
assert "samples_by_res" in stats and "path" in stats
|
||||
|
||||
|
||||
def test_pick_res_prefers_the_finest_resolution_that_fits(db) -> None:
|
||||
now = time.time()
|
||||
add_series(db, "host", "h", "m", list(range(500)), start=now - 3600, step=1)
|
||||
assert db.pick_res("host", "h", "m", 3600, 600) == RES_RAW
|
||||
add_series(db, "host", "h", "m", list(range(4000)), start=now - 7200, step=1)
|
||||
assert db.pick_res("host", "h", "m", 7200, 600) == RES_RAW, "raw still fits"
|
||||
add_series(db, "host", "h", "m", list(range(120)), start=now - 7200, step=60,
|
||||
res=RES_MIN)
|
||||
# 4500 raw points blow the 600 budget, so the minute rollup takes over
|
||||
assert db.pick_res("host", "h", "m", 7200, 600) == RES_MIN
|
||||
|
||||
|
||||
def test_pick_res_never_picks_an_empty_resolution(db) -> None:
|
||||
"""A fresh install has raw only: choosing 1h because it 'fits' would chart nothing."""
|
||||
now = time.time()
|
||||
add_series(db, "host", "h", "m", [1] * 2000, start=now - 7200, step=1)
|
||||
assert db.pick_res("host", "h", "m", 86_400, 600) == RES_RAW
|
||||
|
||||
|
||||
def test_history_falls_back_to_raw_when_rollups_lag(db) -> None:
|
||||
now = time.time()
|
||||
add_series(db, "host", "h", "m", list(range(1500)), start=now - 1500, step=1)
|
||||
points = db.history("host", "h", "m", span=3600, max_points=600)
|
||||
assert len(points) > 0, "empty chart while raw data exists"
|
||||
assert len(points) <= 600
|
||||
|
||||
|
||||
def test_history_always_keeps_the_newest_point(db) -> None:
|
||||
now = time.time()
|
||||
values = list(range(2000))
|
||||
add_series(db, "host", "h", "m", values, start=now - 2000, step=1)
|
||||
points = db.history("host", "h", "m", span=3600, max_points=50)
|
||||
assert points[-1][1] == pytest.approx(values[-1])
|
||||
|
||||
|
||||
def test_series_names_and_hosts(db) -> None:
|
||||
db.add("host", "a", "cpu.usage_pct", 1.0)
|
||||
db.add("host", "a", "mem.used_pct", 2.0)
|
||||
db.add("host", "b", "cpu.usage_pct", 3.0)
|
||||
db.add("llm", "gufo", "llm.decode_tokens_per_second", 4.0)
|
||||
assert db.series_names("host", "a") == ["cpu.usage_pct", "mem.used_pct"]
|
||||
assert db.hosts("host") == ["a", "b"]
|
||||
assert db.series_names("llm") == ["llm.decode_tokens_per_second"]
|
||||
|
||||
|
||||
def test_delete_series_cleans_up(db) -> None:
|
||||
db.add("host", "a", "gone.metric", 1.0)
|
||||
db.add("host", "a", "keep.metric", 2.0)
|
||||
assert db.delete_series("host", "a", "gone.metric") == 1
|
||||
assert db.series_names("host", "a") == ["keep.metric"]
|
||||
|
||||
|
||||
def test_events_roundtrip_and_filter(db) -> None:
|
||||
now = time.time()
|
||||
db.log_event(Event(ts=now, source="host", host="a", kind="down", severity="error",
|
||||
message="a is not answering", data={"fails": 3}))
|
||||
db.log_event(Event(ts=now + 1, source="nut", host="cyberpower", kind="state",
|
||||
severity="warn", message="OL → OB"))
|
||||
everything = db.recent_events(10)
|
||||
assert [e.host for e in everything] == ["cyberpower", "a"]
|
||||
assert everything[-1].data == {"fails": 3}
|
||||
assert [e.source for e in db.recent_events(10, source="nut")] == ["nut"]
|
||||
assert len(db.recent_events(1)) == 1
|
||||
# the API hands out plain JSON, so the boundary has to be serialisable
|
||||
assert everything[0].as_json()["message"] == "OL → OB"
|
||||
|
||||
|
||||
def test_event_as_json_is_serialisable(db) -> None:
|
||||
event = Event(ts=1.0, source="host", host="a", kind="up", severity="info", message="ok")
|
||||
payload = event.as_json()
|
||||
assert payload["severity"] == "info" and payload["data"] is None
|
||||
|
||||
|
||||
def test_count_filters_by_resolution(db) -> None:
|
||||
now = time.time()
|
||||
db.add("host", "h", "m", 1.0, ts=now)
|
||||
add_series(db, "host", "h", "m", [2.0], start=now, step=1, res=RES_MIN)
|
||||
assert db.count() == 2
|
||||
assert db.count(res=RES_RAW) == 1
|
||||
assert db.count(res=RES_HOUR) == 0
|
||||
|
||||
|
||||
def test_add_many_reports_inserted_rows(db) -> None:
|
||||
now = time.time()
|
||||
rows = [Sample("host", "h", f"m{i}", float(i), now) for i in range(5)]
|
||||
assert db.add_many(rows) == 5
|
||||
assert db.count() == 5
|
||||
|
||||
|
||||
def test_store_reopens_without_losing_history(tmp_path) -> None:
|
||||
path = tmp_path / "reopen.sqlite3"
|
||||
first = SampleStore(path)
|
||||
now = time.time()
|
||||
first.add("host", "h", "m", 7.0, ts=now)
|
||||
first.close()
|
||||
second = SampleStore(path)
|
||||
try:
|
||||
assert second.query("host", "h", "m", start=now - 5) == [(now, 7.0)]
|
||||
finally:
|
||||
second.close()
|
||||
|
||||
|
||||
def test_select_window_maps_range_onto_presets() -> None:
|
||||
assert select_window(60, [300, 3600, 86_400]) == 300
|
||||
assert select_window(3600, [300, 3600, 86_400]) == 3600
|
||||
assert select_window(86_401, [300, 3600, 86_400]) == 86_400
|
||||
|
||||
|
||||
def test_rollup_does_not_average_a_partial_minute(db) -> None:
|
||||
"""A bucket is only rolled once its whole window is in the past, or it gets rewritten."""
|
||||
now = time.time()
|
||||
add_series(db, "host", "h", "m", [1, 1, 1], start=now - 5, step=1) # current minute
|
||||
db.maintain(now=now)
|
||||
assert db.query("host", "h", "m", res=RES_MIN, start=now - 600) == []
|
||||
|
||||
|
||||
def test_hour_constant_is_used_in_retention(db) -> None:
|
||||
assert db.raw_keep_seconds == pytest.approx(48 * HOUR)
|
||||
Reference in New Issue
Block a user