commit c4bb374ccc1e3be9a28311ebd2cda024f64e8f36 Author: ducoterra Date: Fri Oct 16 17:20:45 2026 -0400 Add homelab dashboards: system, LLM, and UPS on one domain Three dashboards served by one FastAPI app with a SQLite history store (raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana, no CDN, no build step. Nothing is fetched from the internet at runtime. * system: a stdlib-only agent (agent/host_agent.py) reports /proc and /sys counters; the dashboard turns them into rates over a labelled window, so every number says how it was measured. * llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house exposition parser. Measured delta/elapsed throughput beats a native per-request gauge, and each value carries its provenance. * ups: a NUT wire-protocol client plus health checks that name what they looked at. Commands are gated by NUT_ALLOW_ACTIONS, DASH_READ_ONLY and a DANGER confirmation, re-checked server-side. Missing values render as n/a rather than 0, and an unconfigured source answers 404 {"configured": false} without dialling anything, so "not deployed here" never looks like an outage. 253 tests at 94% coverage against captured real fixtures; ruff and pyright clean. LLM-Generated: true diff --git a/.env.dev.example b/.env.dev.example new file mode 100644 index 0000000..fa3b6c7 --- /dev/null +++ b/.env.dev.example @@ -0,0 +1,17 @@ +# Local development overrides. Copy to .env.dev and edit the host names to yours. +# cp .env.dev.example .env.dev +# ./scripts/dev-tunnels.sh start # ssh tunnels to each host's agent +# set -a; . .env.dev; set +a # then run the dashboard +# +# DEV_TUNNELS entries are ::. +# scripts/dev-tunnels.sh forwards each one to 127.0.0.1: so the dashboard on +# the laptop reads live fleet data with nothing listening on the hosts. +export DEV_TUNNELS="9121:driveripper-ops:9120 9122:deskwork-ops:9120 9123:gateway-ops:9120" + +# Point the dashboard at those tunnels. +export HOST_TARGETS="driveripper=http://127.0.0.1:9121 deskwork=http://127.0.0.1:9122 gateway=http://127.0.0.1:9123" +export GUFO_BASE_URL=http://127.0.0.1:8731 +export VLLM_BASE_URL=http://127.0.0.1:8000 +export NUT_HOST=127.0.0.1 +export NUT_PORT=3493 +export DASH_PORT=8090 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..474835e --- /dev/null +++ b/.env.example @@ -0,0 +1,71 @@ +# dashboards — configuration. Copy to .env (or set these in the container env file). +# Every value is read once at start; restart after changing any of them. + +# --------------------------------------------------------------------------- # +# Server +# --------------------------------------------------------------------------- # +DASH_HOST=127.0.0.1 +DASH_PORT=8090 +# SQLite file (WAL). Put it on persistent storage in deployment. +DASH_DB=data/dashboards.sqlite3 +# DASH_READ_ONLY=1 kills every mutating endpoint regardless of other flags +DASH_READ_ONLY=0 + +# --------------------------------------------------------------------------- # +# System dashboard: hosts running agent/host_agent.py +# Space or comma separated name=url. A bearer token goes in the URL userinfo +# (http://TOKEN@host:9120) and is never sent to the browser. +# --------------------------------------------------------------------------- # +# HOST_TARGETS=driveripper=http://SECRET@driveripper.example.net:9120 deskwork=http://SECRET@deskwork.example.net:9120 +HOST_TARGETS= +HOST_INTERVAL_SECONDS=10 +HOST_TIMEOUT_SECONDS=4 + +# --------------------------------------------------------------------------- # +# LLM dashboard: either engine is optional — unset means "not deployed here" +# and its half of the page disappears instead of showing a dead panel. +# --------------------------------------------------------------------------- # +GUFO_BASE_URL= +GUFO_API_KEY= +VLLM_BASE_URL= +VLLM_API_KEY= +LLM_INTERVAL_SECONDS=2 +LLM_TIMEOUT_SECONDS=3 +# Window the token rates are measured over. A bare 1s delta jitters because +# prompts arrive in bursts. +RATE_WINDOW_SECONDS=10 + +# --------------------------------------------------------------------------- # +# UPS dashboard: the NUT data server (upsd) +# Needs upsd reachable: `LISTEN 0.0.0.0` + PORT 3493 in upsd.conf, and in upsd.users +# [dashboards] +# password = "..." +# actions = none # add `actions = -` (or specific cmds) for test buttons +# --------------------------------------------------------------------------- # +NUT_HOST= +NUT_PORT=3493 +NUT_USER= +NUT_PASSWORD= +NUT_INTERVAL_SECONDS=5 +NUT_TIMEOUT_SECONDS=4 +# Instant commands (battery test, beeper, calibration) are the only write path in the +# app. Requires NUT_USER plus this flag; the UI still confirms every command. +NUT_ALLOW_ACTIONS=0 + +# --------------------------------------------------------------------------- # +# History retention (raw -> 1m rollup -> 1h rollup) +# --------------------------------------------------------------------------- # +RETAIN_RAW_HOURS=48 +RETAIN_MINUTE_DAYS=30 +RETAIN_HOUR_DAYS=400 + +# --------------------------------------------------------------------------- # +# Browser poll rates +# --------------------------------------------------------------------------- # +POLL_MS_SYSTEM=10000 +POLL_MS_LLM=2000 +POLL_MS_UPS=5000 + +# Debug: DEBUGPY=1 makes the app listen for a debugger on 5678 +DEBUGPY=0 +DEBUGPY_PORT=5678 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6bb79e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +# Python +__pycache__/ +*.py[cod] +.venv/ +.pytest_cache/ +.ruff_cache/ +.coverage +coverage.xml +htmlcov/ +*.egg-info/ + +# Local state: the SQLite history DB and anything else runtime-written +data/ +*.sqlite3 +*.sqlite3-wal +*.sqlite3-shm + +# Secrets / local config — never commit. The `.example` files are documentation, so they +# are un-ignored explicitly: `.env.*` would otherwise swallow them and a fresh clone would +# have no annotated list of what to set. +.env +.env.* +!.env.example +!.env.dev.example + +# Agent binaries staged for deployment +agent/*.tar.gz + +# Editor / OS +.vscode/ +.idea/ +.DS_Store + +# Agent/pipeline scratch (the .agents/ roadmap itself IS tracked) +.agents/phase-sessions/ +.agents/pipeline.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..63d51cc --- /dev/null +++ b/README.md @@ -0,0 +1,142 @@ +# Homelab Dashboards + +Three dashboards, one domain, one process, one SQLite file: + +| Page | What it answers | Source | +| ----------- | ------------------------------------------------------------------------------- | -------------------------------------------------- | +| `/` | Is anything wrong, and which dashboard do I need? | all of the below, one request | +| `/system` | What are my machines doing — CPU, memory, disk, GPU, network, and the history of all of it | `agent/host_agent.py`, our own stdlib agent per host | +| `/llm` | Tokens/sec, active sessions, queue depth, KV-cache pressure, latency percentiles | llama.cpp-family (gufo) + vLLM `/metrics`, read-only | +| `/ups` | Battery health, runtime left, power events, and battery-test buttons | NUT `upsd` on 3493 (its own wire protocol client) | + +No Prometheus, no Grafana, no CDN, no build step. Nothing is fetched from the internet at +runtime; every asset is served by this app. + +## Quickstart + +```bash +uv sync --python 3.14 # deps + the package, editable +cp .env.example .env # then edit +uv run dashboards # http://127.0.0.1:8090 +uv run uvicorn dashboards.app:app --port 8090 # equivalent +uv run pytest # unit tests, no network needed +``` + +Developing against the real fleet from a laptop (agents stay unexposed — ssh forwards +them to localhost): + +```bash +cp .env.dev.example .env.dev # your ssh aliases + which ports +python3 - <<'PY' # push the agent to each host (once) +import shlex, subprocess, pathlib +for host in ("driveripper-ops", "deskwork-ops", "gateway-ops"): + subprocess.run(["ssh", host, "mkdir -p ~/lab-agent"], check=True) + subprocess.run(["scp", "agent/host_agent.py", f"{host}:~/lab-agent/"], check=True) +PY +./scripts/dev-tunnels.sh start # 9121/9122/9123 -> each host's agent +set -a; . ./.env.dev; set +a +uv run dashboards +``` + +## The host agent + +`agent/host_agent.py` is one file, **stdlib only**, Python 3.10+. It reads `/proc`, +`/sys/block`, `/sys/class/drm` and `statvfs` and answers `GET /snapshot` as JSON. + +```bash +python3 host_agent.py --once # print one snapshot, exit +python3 host_agent.py --host 0.0.0.0 --port 9120 --token … # serve, token required +deploy/install-agent.sh # systemd unit (run with sudo) +``` + +It is read-only by construction: there is no code path that writes, and the HTTP server +only answers `GET`. Counters are reported as counters — the dashboard computes rates over +a labelled window, so the agent never has to remember who polled it last. + +## Configuration + +Everything comes from the environment (see `.env.example` for the annotated list). At +startup a `.env` next to the checkout is loaded first if it exists — as a *seed*: a +variable the container, systemd unit, or your shell already set always wins. Point +`DASH_ENV_FILE` somewhere else to keep several installs on one box. `.env.example` is +documentation and is never loaded, and `scripts/dev-server.sh` exports `.env.dev` itself +(see the quickstart) because dev settings must not leak into a production process. + +| Variable | Meaning | +| ------------------------------------- | -------------------------------------------------------------- | +| `HOST_TARGETS` | `name=url` per host running the agent; a bearer token may ride in the URL userinfo (`http://TOKEN@host:9120`) | +| `HOST_INTERVAL_SECONDS` | poll interval for hosts (default 10) | +| `GUFO_BASE_URL` / `VLLM_BASE_URL` | LLM engines; unset means *not deployed here*, and its half of the page disappears | +| `RATE_WINDOW_SECONDS` | the Δ window behind every measured token rate (default 10) | +| `NUT_HOST` / `NUT_PORT` | upsd address. Reads work unauthenticated in stock NUT | +| `NUT_USER` / `NUT_PASSWORD` | only needed for instant commands (battery tests) | +| `NUT_ALLOW_ACTIONS` | `1` enables the one write endpoint in the app | +| `NUT_SIMULATE` | `1` replaces upsd with a **labelled** simulator (development only) | +| `DASH_READ_ONLY` | `1` disables every mutating endpoint regardless of other flags | +| `DASH_DB` | SQLite path (default `data/dashboards.sqlite3`) | +| `RETAIN_RAW_HOURS` / `RETAIN_MINUTE_DAYS` / `RETAIN_HOUR_DAYS` | history rollup and pruning | + +A source that is not configured answers `404 {"configured": false}` **without dialling**, +and the page says "not configured" instead of showing a red panel for something you never +installed. + +## Architecture + +``` +agent/host_agent.py (each host) gufo / vLLM upsd (NUT) + │ GET /snapshot │ GET /metrics │ LIST UPS / LIST VAR + ▼ ▼ ▼ +src/dashboards/collectors.py ────────────────────────────────────────────┐ + │ parse + Δcounter/Δt (rates.py) │ + ▼ ▼ +sources/host.py · sources/llm.py · sources/nut.py poller.py (one task per source) + │ samples events │ + ▼ ▼ +store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── api.py → web/ +``` + +* `poller.py` — every source gets its own asyncio task, its own health record, and its own + error handling. A dead host keeps its last good values and is labelled stale; it never + blanks or blocks another source. +* `store.py` — raw samples roll up to 1-minute and 1-hour averages and the raw rows are + pruned, so a year of history is a few hundred MB rather than a disk emergency. +* `web/` — four static pages, one stylesheet, hand-rolled SVG charts (`charts.js`). No + framework, no CDN, no bundler. + +## Testing + +```bash +uv run pytest # unit + API tests, fully mocked +uv run pytest --cov # coverage gate +uv run pytest -m live --live # optional: hit the real fleet/engines +uv run ruff check src tests agent && uv run pyright # lint + types +./scripts/capture-fixtures.sh # re-record ground-truth captures +``` + +Test philosophy: **real captures as fixtures, mocks for behaviour.** Anything that must +match an upstream (`/proc` layouts, vLLM metric names, NUT protocol framing) is asserted +against a verbatim capture in `tests/fixtures/`; behaviour that needs a failure — upsd +refusing a command, an engine returning 500, a host rebooting — is a fake. `-m live` +opt-in tests are the bridge: they run only when you point them at real hardware. + +## Battery-test buttons + +Instant commands are the only mutating endpoint in the application (`POST /api/ups/command`) +and they are gated four times: `NUT_ALLOW_ACTIONS=1`, not `DASH_READ_ONLY`, an `upsd.users` +account that actually has `actions`, and a confirmation in the UI — destructive ones +(deep test, calibration, `load.off`) require typing `DANGER`. The server re-checks every +gate; hiding a button is not a safety control. Commands that would cut power are refused +outright while the UPS is already on battery. + +## Deployment + +`Containerfile` builds a slim image (the agent is copied in too, so one image can serve a +host if you want). In the deployment repo the pattern is the same as every other service: +quadlet container + env template + nginx/SNI on the single domain. + +```bash +podman build -t dashboards . +podman run --rm -p 8090:8090 --env-file dashboards.env dashboards +``` + +Keep the SQLite file on persistent storage (`-v dashboards-data:/var/lib/dashboards`). diff --git a/agent/host_agent.py b/agent/host_agent.py new file mode 100644 index 0000000..5a52b92 --- /dev/null +++ b/agent/host_agent.py @@ -0,0 +1,971 @@ +#!/usr/bin/env python3 +"""host_agent.py — read-only host metrics agent for the homelab dashboards. + +**Stdlib only, one file, Python 3.10+.** Copy it to a host, run it, and the dashboard can +read that host. It never writes anything anywhere: every number comes from reading +/proc, /sys and statvfs, and the HTTP server only answers GET. + + GET /health -> {"ok": true, "hostname": ..., "version": ...} + GET /info -> slow-changing identity (os, kernel, cpu model, cores, gpu list) + GET /snapshot -> everything, one JSON document (see SCHEMA below) + GET /schema -> this docstring's SCHEMA section, for self-documenting deploys + +Design rules +------------ +* **Counters stay counters.** The agent reports cumulative values (jiffies, bytes, ops) + and the *dashboard* computes rates with a labelled window. A stateless agent cannot + lie about an interval, survive a restart silently, or be polled twice by two dashboards + and halve its own rate. +* **Absent is null, never 0.** A host without a GPU, or a sysfs file only root can read, + yields `null`. The UI renders `n/a`. Turning "unknown" into 0 is the one mistake a + monitoring agent is not allowed to make. +* **Every read is guarded.** A vanished /proc entry (process exited mid-scan) skips that + item; the snapshot still succeeds. The poll is degraded, never failed. +* **The only state** is a small pid->cpu-time cache so `processes.top_cpu` can answer a + question /proc cannot (CPU needs two samples). It is bounded and cleared when stale. + +SCHEMA (GET /snapshot) +---------------------- + schema int -> 1 + ts float unix seconds (agent clock) + duration_ms float collection cost + agent_version str + host {hostname, os_pretty, kernel, arch, python, boot_ts, uptime_seconds, + cpu_model, cpu_cores_logical, cpu_cores_physical, container} + cpu {modes: {user,nice,system,idle,iowait,irq,softirq,steal,guest, + guest_nice,total} [clock ticks, cumulative], + cores: [{core, modes: {...}}], + load1, load5, load15, procs_running, procs_blocked, + ctxt_total, processes_total, + freq_mhz, freq_max_mhz} + memory {total_bytes, available_bytes, used_bytes, free_bytes, buffers_bytes, + cached_bytes, dirty_bytes, swap_total_bytes, swap_used_bytes, + committed_bytes} + filesystems [{device, mountpoint, fstype, total_bytes, used_bytes, free_bytes, + used_pct, read_only}] + disk_io {devices: [{name, model, rotational, size_bytes, reads_total, + writes_total, read_bytes_total, write_bytes_total, + in_flight, io_time_ms_total, weighted_io_ms_total, + discard_bytes_total}], + device_names: [str]} # whole disks from /sys/block, no partitions + network {interfaces: [{name, rx_bytes_total, tx_bytes_total, rx_packets_total, + tx_packets_total, rx_errors_total, tx_errors_total, + rx_dropped_total, tx_dropped_total, speed_mbps}]} + gpus [{index, card, vendor, model, pci_slot, pci_address, driver, busy_pct, + vram_total_bytes, vram_used_bytes, gtt_total_bytes, gtt_used_bytes, + power_watts, temps: {label: celsius}, clocks: {label: mhz}}] + temperatures [{kind: hwmon|thermal, chip, label, celsius}] + processes {total, top_rss: [{pid, name, rss_bytes}], + top_cpu: [{pid, name, cpu_pct}]} + errors [str] reads that failed; non-empty means the snapshot is partial + +Deployment: see deploy/install-agent.sh (systemd unit, binds 127.0.0.1 unless you pass +--host, requires a bearer token via --token or AGENT_TOKEN when bound off-loopback). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import socket +import threading +import time +from collections.abc import Callable +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +AGENT_VERSION = "1.0.0" +SCHEMA_VERSION = 1 +PAGE_SIZE = 0 # resolved from sysconf at runtime + +# Filesystems that are not storage. Mirrors the spirit of node_exporter's default +# --collector.filesystem.fs-types-exclude, trimmed to what actually appears here. +PSEUDO_FSTYPES = { + "autofs", "binfmt_misc", "bpf", "cgroup", "cgroup2", "configfs", "debugfs", "devpts", + "devtmpfs", "efivarfs", "fuse.gvfsd-fuse", "fusectl", "hugetlbfs", "iso9660", "mqueue", + "nsfs", "overlay", "proc", "procfs", "pstore", "ramfs", "rpc_pipefs", "securityfs", + "selinuxfs", "squashfs", "sysfs", "tmpfs", "tracefs", "fuse.lxcfs", "binfmt", +} +PSEUDO_MOUNT_PREFIXES = ("/proc", "/sys", "/dev", "/run", "/var/lib/containers/storage", + "/var/lib/docker", "/snap", "/boot/grub2", "/mnt/iso") + +# Interface name prefixes that are containers/bridges talking to themselves. +VIRTUAL_NET_PREFIXES = ("lo", "veth", "docker", "br-", "virbr", "br", "tap", "vnet", + "cni", "kube", "flannel", "dummy", "ifb", "sit", "teql") + +# /sys/block entries that are not physical media. dm-* is deliberately here too: an LVM +# or LUKS volume is carved out of a disk we already count, so charting both double-counts. +VIRTUAL_DISK_PREFIXES = ("loop", "ram", "zram", "sr", "dm-", "md127") + +# pci slot -> human model, filled on first look (lspci output cannot change at runtime). +_GPU_MODEL_CACHE: dict[str, str | None] = {} + +VENDOR_NAMES = {"0x1002": "AMD", "0x10de": "NVIDIA", "0x8086": "Intel", "0x15ad": "VMware", + "0x1af4": "Virtio", "0x1a03": "ASPEED", "0x10ec": "Realtek"} + +# sysfs names worth showing as "the CPU temperature", in preference order. +CPU_TEMP_CHIPS = ("k10temp", "zenpower", "coretemp", "cpu_thermal", "soc_thermal", "acpitz") + +# Bracket groups in lspci output that name the vendor rather than the product. +_GPU_VENDOR_TAG = re.compile(r"(?:amd|ati|nvidia|intel|apple|matrox|s3|vmware|red hat|xinxiang" + r"|cirrus logic)(?:\s*/\s*(?:amd|ati|nvidia|intel))?", re.I) +# "Advanced Micro Devices, Inc.", "Intel Corporation", "NVIDIA Corporation" … +# The trailing `\b` matters: without it, "VGA compatible controller" splits on the "co" of +# "compatible" and the legend reads "mpatible controller: …". +_GPU_CORP_SUFFIX = re.compile(r"\b(?:Incorporated|Incorporation|Corporation|Company|Limited" + r"|Inc|Corp|Ltd|Co)\b\.?,?\s*", re.I) +# "00:02.0 VGA compatible controller: Intel …" — slot and class are not part of the name. +_GPU_PCI_PREFIX = re.compile(r"^[0-9a-fA-F]{2,4}:[0-9a-fA-F]{2}\.[0-9a-fA-F]\s*[^:]{0,60}:\s*") + + +def short_gpu_model(name: str | None) -> str | None: + """Turn an lspci description into something a chart legend can hold. + + lspci says `Advanced Micro Devices, Inc. [AMD/ATI] Navi 48 [Radeon AI PRO R9700]`; + the useful product name is the last bracket group, and AMD lists APU aliases separated + by slashes where the last one is the strongest part. The full string is still reported + as `model_full` so nothing is lost — this is display, not identity. + """ + if not name: + return None + groups = re.findall(r"\[([^\]]+)\]", name) + # [AMD/ATI] and [NVIDIA] are vendor tags, not products — keeping them turned an iGPU + # named Phoenix1 into a legend that said "ATI". + products = [g for g in groups if not _GPU_VENDOR_TAG.fullmatch(g.strip())] + if products: + candidate = products[-1].split("/")[-1].strip() + else: + rest = _GPU_PCI_PREFIX.sub("", name) + rest = re.sub(r"\[[^\]]+\]", " ", rest) # drop vendor tags + candidate = _GPU_CORP_SUFFIX.split(rest, maxsplit=1)[-1].strip(" ,;-") + candidate = candidate.strip() or name.strip() + return candidate[:40] + "\u2026" if len(candidate) > 40 else candidate + + +def _to_float(text: str) -> float | None: + try: + value = float(text) + except (TypeError, ValueError): + return None + return value if value == value and abs(value) != float("inf") else None + + +def _at(values: list[float | None], index: int) -> float | None: + """Element `index`, or None. /proc and /sys files are truncated more often than you'd think. + + This was a closure over the enclosing loop variable, which is the bug pattern B023 warns + about: escape the loop and it reads the *last* iteration's counters. It takes its data now. + """ + return values[index] if index < len(values) else None + + +class Collector: + """Reads one host's /proc and /sys. `root` is injectable so tests can hand it a tree + of fixture files instead of a live machine.""" + + def __init__(self, root: str | os.PathLike[str] = "/", *, top_processes: int = 5, + net_all: bool = False) -> None: + self.root = Path(root) + self.top_processes = top_processes + self.net_all = net_all + self._lock = threading.Lock() # one HTTP handler at a time may touch the pid cache + self._pid_cpu: dict[int, tuple[float, int, str]] = {} # pid -> (ts, jiffies, name) + self._boot_ts: float | None = None + self._page_size = os.sysconf("SC_PAGE_SIZE") or 4096 + self._ticks = os.sysconf("SC_CLK_TCK") or 100 + self._cpu_model_cache: str | None = None + + # -- tiny fs helpers ----------------------------------------------------- # + def _path(self, *parts: str) -> Path: + return self.root.joinpath(*parts) + + def read_text(self, *parts: str) -> str | None: + try: + return self._path(*parts).read_text(errors="replace") + except (OSError, ValueError): + return None + + def read_lines(self, *parts: str) -> list[str]: + text = self.read_text(*parts) + return text.splitlines() if text else [] + + def read_number(self, *parts: str) -> float | None: + """First number in the file. + + `/proc/uptime` is `210036.18 9613750.86`, while `/sys/.../scaling_cur_freq` is bare. + Parsing the whole file as a float returns None for exactly the files that matter, + which reads as "this kernel has no uptime" rather than "you forgot to split". + """ + text = self.read_text(*parts) + if not text: + return None + fields = text.split() + return _to_float(fields[0]) if fields else None + + # -- cpu ----------------------------------------------------------------- # + def cpu(self) -> dict[str, Any]: + """Jiffies per mode, cumulatively since boot. Divide by SC_CLK_TCK for seconds.""" + modes: dict[str, float] = {} + cores: list[dict[str, Any]] = [] + for line in self.read_lines("proc/stat"): + if not line.startswith("cpu"): + break + fields = line.split() + name = fields[0] + values = [float(v) for v in fields[1:11]] if len(fields) > 1 else [] + if not values: + continue + while len(values) < 10: + values.append(0.0) + sample = dict(zip(MODE_NAMES, values[:10], strict=True)) + # guest and guest_nice are already counted inside user/nice on Linux + sample["total"] = sum(values[:8]) + if name == "cpu": + modes = sample + elif name[3:].isdigit(): + cores.append({"core": int(name[3:]), "modes": sample}) + load = self.read_text("proc/loadavg") + load_parts = load.split() if load else [] + procs_running = procs_blocked = ctxt_total = processes_total = None + for line in self.read_lines("proc/stat"): + if line.startswith("procs_running"): + procs_running = _to_float(line.split()[-1]) + elif line.startswith("procs_blocked"): + procs_blocked = _to_float(line.split()[-1]) + elif line.startswith("ctxt"): + ctxt_total = _to_float(line.split()[-1]) + elif line.startswith("processes"): + processes_total = _to_float(line.split()[-1]) + freq = self.cpu_frequency() + return { + "modes": modes or None, + "cores": cores, + "clock_ticks_per_second": self._ticks, + "load1": _to_float(load_parts[0]) if len(load_parts) > 0 else None, + "load5": _to_float(load_parts[1]) if len(load_parts) > 1 else None, + "load15": _to_float(load_parts[2]) if len(load_parts) > 2 else None, + "procs_running": procs_running, + "procs_blocked": procs_blocked, + "ctxt_total": ctxt_total, + "processes_total": processes_total, + "freq_mhz": freq["current"], + "freq_max_mhz": freq["max"], + } + + def cpu_frequency(self) -> dict[str, float | None]: + """Average current / best-case maximum, in MHz.""" + current: list[float] = [] + maximum: list[float] = [] + base = self._path("sys/devices/system/cpu") + if base.is_dir(): + for cpu in sorted(base.glob("cpu[0-9]*")): + cur = self.read_number(str(cpu.relative_to(self.root)), "cpufreq/scaling_cur_freq") + if cur is None: + cur = self.read_number(str(cpu.relative_to(self.root)), "cpufreq/cpuinfo_cur_freq") + if cur is not None: + current.append(cur / 1000.0) + mx = self.read_number(str(cpu.relative_to(self.root)), "cpufreq/cpuinfo_max_freq") + if mx is not None: + maximum.append(mx / 1000.0) + return { + "current": (sum(current) / len(current)) if current else None, + "max": max(maximum) if maximum else None, + } + + def cpu_model(self) -> str | None: + if self._cpu_model_cache: + return self._cpu_model_cache + for line in self.read_lines("proc/cpuinfo"): + if line.lower().startswith(("model name", "cpu model", "hardware")): + _, _, value = line.partition(":") + if value.strip(): + self._cpu_model_cache = value.strip() + return self._cpu_model_cache + # arm64 fallback: compatible = "arm,cortex-a76" + text = self.read_text("proc/device-tree/compatible") + if text: + self._cpu_model_cache = text.strip("\x00").split(",")[-1].strip() + return self._cpu_model_cache + return None + + def core_counts(self) -> tuple[int | None, int | None]: + text = self.read_text("proc/cpuinfo") or "" + logical = len(re.findall(r"^processor\s*:", text, re.M)) or (os.cpu_count() or None) + # one block per *thread*, so dedupe (socket, core) — counting how many "core id:" + # lines exist yields 1 for the set of colons, which is how a 24-core TR reports + # "1 physical core" and makes a fleet table quietly meaningless. + seen: set[tuple[str, str]] = set() + for block in text.split("\n\n"): + socket = re.search(r"^physical id\s*:\s*(\d+)", block, re.M) + core = re.search(r"^core id\s*:\s*(\d+)", block, re.M) + if socket and core: + seen.add((socket.group(1), core.group(1))) + physical: int | None = len(seen) or None + if physical is None and logical: + threads = self._threads_per_core() + if threads: + physical = max(1, logical // threads) + return (logical or None), physical + + def _threads_per_core(self) -> int | None: + """SMT width from cpu0's sibling list: `0,24` or `0-1` → 2. VMs often omit it.""" + listing = self.read_text("sys/devices/system/cpu/cpu0/topology/thread_siblings_list") + if not listing: + return None + count = 0 + for part in listing.strip().split(","): + start, dash, end = part.partition("-") + first = _to_float(start) + if first is None: + continue + last = _to_float(end) if dash else None + count += int((last if last is not None else first) - first) + 1 + return count or None + + # -- memory -------------------------------------------------------------- # + def memory(self) -> dict[str, float | None]: + raw: dict[str, float] = {} + for line in self.read_lines("proc/meminfo"): + name, _, rest = line.partition(":") + value = _to_float(rest.split()[0]) if rest.split() else None + if value is not None: + raw[name.strip()] = value * 1024.0 # meminfo is in kB + if not raw: + return dict.fromkeys(MEMORY_KEYS) + total = raw.get("MemTotal") + available = raw.get("MemAvailable", raw.get("MemFree")) + swap_total = raw.get("SwapTotal", 0.0) + swap_free = raw.get("SwapFree", 0.0) + out: dict[str, float | None] = { + "total_bytes": total, + "free_bytes": raw.get("MemFree"), + "available_bytes": available, + "used_bytes": (total - available) if (total is not None and available is not None) else None, + "buffers_bytes": raw.get("Buffers"), + "cached_bytes": (raw.get("Cached", 0.0) + raw.get("SReclaimable", 0.0)) or None, + "dirty_bytes": raw.get("Dirty"), + "swap_total_bytes": swap_total, + "swap_used_bytes": swap_total - swap_free, + "committed_bytes": raw.get("Committed_AS"), + } + for key in MEMORY_KEYS: + out.setdefault(key, None) + return out + + # -- filesystems --------------------------------------------------------- # + def filesystems(self) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + for line in self.read_lines("proc/mounts"): + parts = line.split() + if len(parts) < 3: + continue + device, mountpoint, fstype = parts[0], parts[1], parts[2] + mountpoint = mountpoint.replace("\\040", " ").replace("\\011", "\t") + if fstype in PSEUDO_FSTYPES: + continue + if any(mountpoint == p or mountpoint.startswith(p + "/") for p in PSEUDO_MOUNT_PREFIXES): + continue + key = (device, mountpoint) + if key in seen: + continue # bind mounts / btrfs subvolume aliases + seen.add(key) + try: + st = os.statvfs(mountpoint) + except OSError: + continue + block = st.f_frsize or st.f_bsize + total = st.f_blocks * block + free = st.f_bavail * block # what an unprivileged writer can use + used = (st.f_blocks - st.f_bfree) * block + denominator = used + free # df's "Use%" basis, not raw total + out.append({ + "device": device, + "mountpoint": mountpoint, + "fstype": fstype, + "total_bytes": total or None, + "used_bytes": used, + "free_bytes": free, + "used_pct": (used / denominator * 100.0) if denominator > 0 else None, + "read_only": bool(st.f_flag & os.ST_RDONLY), + }) + out.sort(key=lambda item: item["mountpoint"]) + return out + + # -- disk io ------------------------------------------------------------- # + def disk_io(self) -> dict[str, Any]: + """Whole disks via /sys/block//stat, which is documented ABI-stable. + + /proc/diskstats is NOT: the discard/flush counters were inserted at different + offsets in different kernels, so "which index is in-flight" depends on the build. + Measured on driveripper (kernel 7.2): its diskstats line and its /sys/block/stat + line are the same 17 numbers — classic 11 first (in-flight at index 8, matching + /sys/block//inflight), then the discard/flush tail. So the classic 11 are + read from fixed indexes and the tail only when the 17-number shape is present. + + /sys/block lists whole disks only (nvme0n1, not nvme0n1p1), which is exactly the + set we want: partitions and dm-* layers would double-count the same bytes. + """ + devices: list[dict[str, Any]] = [] + names: list[str] = [] + base = self._path("sys/block") + if not base.is_dir(): + return {"devices": [], "device_names": []} + for entry in sorted(base.iterdir(), key=lambda p: p.name): + name = entry.name + if name.startswith(VIRTUAL_DISK_PREFIXES): + continue + numbers = (self.read_text("sys/block", name, "stat") or "").split() + if len(numbers) < 11: + continue + values = [_to_float(v) for v in numbers] + rotational = self.read_number("sys/block", name, "queue/rotational") + model = (self.read_text("sys/block", name, "device/model") or "").strip() or None + names.append(name) + devices.append({ + "name": name, + "model": model, + "rotational": None if rotational is None else rotational == 1.0, + "size_bytes": (self.read_number("sys/block", name, "size") or 0) * 512.0 or None, + "reads_total": _at(values, 0), + "writes_total": _at(values, 4), + "read_bytes_total": (_at(values, 2) or 0.0) * 512.0, + "write_bytes_total": (_at(values, 6) or 0.0) * 512.0, + "in_flight": _at(values, 8), + "io_time_ms_total": _at(values, 9), + "weighted_io_ms_total": _at(values, 10), + # discard/flush tail: only this shape is verified (see docstring) + "discard_bytes_total": ((_at(values, 13) or 0.0) * 512.0 + if len(values) >= 17 else None), + }) + return {"devices": devices, "device_names": names} + + # -- network ------------------------------------------------------------- # + def network(self) -> dict[str, Any]: + interfaces: list[dict[str, Any]] = [] + for line in self.read_lines("proc/net/dev"): + name, sep, rest = line.partition(":") + if not sep: + continue + name = name.strip() + if not self.net_all and name.startswith(VIRTUAL_NET_PREFIXES): + continue + fields = [_to_float(v) for v in rest.split()] + if len(fields) < 9: + continue + + interfaces.append({ + "name": name, + "rx_bytes_total": _at(fields, 0), + "rx_packets_total": _at(fields, 1), + "rx_errors_total": _at(fields, 2), + "rx_dropped_total": _at(fields, 3), + "tx_bytes_total": _at(fields, 8), + "tx_packets_total": _at(fields, 9), + "tx_errors_total": _at(fields, 10), + "tx_dropped_total": _at(fields, 11), + "speed_mbps": self._net_speed(name), + }) + interfaces.sort(key=lambda item: item["name"]) + return {"interfaces": interfaces} + + def _net_speed(self, name: str) -> float | None: + value = self.read_number("sys/class/net", name, "speed") + return value if value and value > 0 else None + + # -- gpus ---------------------------------------------------------------- # + def gpus(self) -> list[dict[str, Any]]: + """AMD GPUs via DRM sysfs (gpu_busy_percent, mem_info_vram_*, power, hwmon temps). + + That set exists on amdgpu since kernel ~4.x and is world-readable, which is why + this agent needs no root. NVIDIA/Intel are not wired up: the fleet has no such + card, and guessing at sysfs paths we cannot test would be worse than `null`. + """ + out: list[dict[str, Any]] = [] + base = self._path("sys/class/drm") + if not base.is_dir(): + return out + for card in sorted(base.glob("card[0-9]*")): + name = card.name + if "-" in name or name == "card": # only cardN, not cardN-DP-1 connectors + continue + device = card / "device" + if not device.is_dir(): + continue + busy = self.read_number(str(device.relative_to(self.root)), "gpu_busy_percent") + vram_total = self.read_number(str(device.relative_to(self.root)), "mem_info_vram_total") + vram_used = self.read_number(str(device.relative_to(self.root)), "mem_info_vram_used") + if busy is None and vram_total is None: + continue # not a compute-capable GPU (connector-only / virtual) + pci_id = self._read_uevent(device, "PCI_ID") # "1002:7551" + slot = self._read_uevent(device, "PCI_SLOT_NAME") # "0000:4d:00.0" + vendor_id, _, device_id = (pci_id or "").partition(":") + driver = self._driver_name(device) + full_model = self._gpu_model(slot, vendor_id, device_id) + out.append({ + "index": len(out), + "card": name, + "vendor": VENDOR_NAMES.get(f"0x{vendor_id}" if vendor_id else "", vendor_id or None), + "model": short_gpu_model(full_model), + "model_full": full_model, + "pci_slot": slot, + "pci_address": pci_id or None, + "driver": driver, + "busy_pct": busy, + "vram_total_bytes": vram_total, + "vram_used_bytes": vram_used, + # On an APU (Strix Halo) the dedicated carve-out is tiny and the model + # actually lives in GTT (shared system RAM): report both, never silently + # add them into one number that hides which pool was used. + "gtt_total_bytes": self.read_number(str(device.relative_to(self.root)), + "mem_info_gtt_total"), + "gtt_used_bytes": self.read_number(str(device.relative_to(self.root)), + "mem_info_gtt_used"), + "power_watts": self._gpu_power(device), + "temps": self._gpu_temps(device), + "clocks": self._gpu_clocks(device), + }) + return out + + def _gpu_power(self, device: Path) -> float | None: + """Board power in watts. hwmon `power*_input`/`*_average`, microWatts, PPT preferred. + + Not always present: the amdgpu build on driveripper ships power1_label=PPT with no + matching input file, so this honestly returns None instead of inventing a number. + """ + hwmon_dir = device / "hwmon" + if not hwmon_dir.is_dir(): + return None + best: float | None = None + for hwmon in hwmon_dir.glob("hwmon*"): + for suffix in ("input", "average", "max"): + for candidate in sorted(hwmon.glob(f"power*_{suffix}")): + key = candidate.stem.split("_")[0] + label_file = hwmon / f"{key}_label" + label = (self.read_text(str(label_file.relative_to(self.root))) or "").strip().lower() + value = self.read_number(str(candidate.relative_to(self.root))) + if value is None: + continue + watts = value / 1_000_000.0 + if best is None: + best = watts + if "ppt" in label or "package" in label: + return watts + return best + + def _gpu_clocks(self, device: Path) -> dict[str, float]: + """Clocks in MHz, keyed by their sysfs label (sclk / mclk).""" + clocks: dict[str, float] = {} + hwmon_dir = device / "hwmon" + if not hwmon_dir.is_dir(): + return clocks + for hwmon in hwmon_dir.glob("hwmon*"): + for input_file in sorted(hwmon.glob("freq*_input")): + key = input_file.stem.replace("_input", "") + label_file = hwmon / f"{key}_label" + label = (self.read_text(str(label_file.relative_to(self.root))) or key).strip() \ + if label_file.exists() else key + value = self.read_number(str(input_file.relative_to(self.root))) + if value is not None: + clocks[label] = value / 1_000_000.0 + return clocks + + def _read_uevent(self, device: Path, key: str) -> str | None: + for line in self.read_lines(str(device.relative_to(self.root)), "uevent"): + name, _, value = line.partition("=") + if name == key: + return value.strip().lower() + return None + + def _driver_name(self, device: Path) -> str | None: + driver = device / "driver" + try: + return os.path.basename(os.readlink(driver)) + except OSError: + return None + + def _gpu_model(self, slot: str | None, vendor_id: str | None, device_id: str | None) -> str | None: + """Human GPU name. lspci when it exists (cached: the answer never changes at + runtime), otherwise vendor + device id — never a bare "gpu".""" + cache_key = slot or f"{vendor_id}:{device_id}" + if cache_key in _GPU_MODEL_CACHE: + return _GPU_MODEL_CACHE[cache_key] + model: str | None = None + if slot: + found = self._lspci_name(slot.split(":", 1)[-1] if slot.count(":") == 2 else slot) + if found: + model = found + if model is None: + vendor = VENDOR_NAMES.get(f"0x{vendor_id}" if vendor_id else "") + if vendor and device_id: + model = f"{vendor} GPU ({vendor_id}:{device_id})" + elif vendor: + model = vendor + _GPU_MODEL_CACHE[cache_key] = model + return model + + def _lspci_name(self, slot: str) -> str | None: + """`lspci -nns ` -> the text after the class, e.g. 'Navi 48 [Radeon ...]'. + + Optional: lspci is not guaranteed to be installed, so the caller must cope with + None. The subprocess is bounded and never raises into the snapshot. + """ + try: + import subprocess # noqa: PLC0415 - only needed for one lspci call + + raw = subprocess.run(["lspci", "-nns", slot], capture_output=True, text=True, + timeout=3, check=False) + except Exception: # noqa: BLE001 - missing lspci / no permissions is normal + return None + if raw.returncode != 0 or not raw.stdout.strip(): + return None + line = raw.stdout.strip().splitlines()[0] + _, _, rest = line.partition(":") + _, _, rest = rest.partition(":") + rest = re.sub(r"\s*\[[0-9a-fA-F]{4}:[0-9a-fA-F]{4}\]", "", rest) + rest = re.sub(r"\s*\(rev [0-9a-fA-F]+\)\s*$", "", rest) + return rest.strip() or None + + def _gpu_temps(self, device: Path) -> dict[str, float]: + temps: dict[str, float] = {} + hwmon_dir = device / "hwmon" + if not hwmon_dir.is_dir(): + return temps + for hwmon in hwmon_dir.glob("hwmon*"): + for input_file in sorted(hwmon.glob("temp*_input")): + label = input_file.stem.replace("_input", "") + named = hwmon / f"{label}_label" + if named.exists(): + label = (self.read_text(str(named.relative_to(self.root))) or label).strip() + value = self.read_number(str(input_file.relative_to(self.root))) + if value is not None: + temps[label] = value / 1000.0 + return temps + + # -- temperatures -------------------------------------------------------- # + def temperatures(self) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + hwmon = self._path("sys/class/hwmon") + if hwmon.is_dir(): + for unit in sorted(hwmon.glob("hwmon*")): + chip = (self.read_text(str(unit.relative_to(self.root)), "name") or unit.name).strip() + for input_file in sorted(unit.glob("temp*_input")): + key = input_file.stem.replace("_input", "") + label_file = unit / f"{key}_label" + label = chip + if label_file.exists(): + label = (self.read_text(str(label_file.relative_to(self.root))) or chip).strip() + value = self.read_number(str(input_file.relative_to(self.root))) + if value is not None: + out.append({"kind": "hwmon", "chip": chip, "label": label, + "celsius": value / 1000.0}) + thermal = self._path("sys/class/thermal") + if thermal.is_dir(): + for zone in sorted(thermal.glob("thermal_zone*")): + ztype = (self.read_text(str(zone.relative_to(self.root)), "type") or zone.name).strip() + value = self.read_number(str(zone.relative_to(self.root)), "temp") + if value is not None: + out.append({"kind": "thermal", "chip": ztype, "label": ztype, + "celsius": value / 1000.0}) + return out + + def cpu_temperature(self, temps: list[dict[str, Any]]) -> float | None: + for chip in CPU_TEMP_CHIPS: + picks = [t["celsius"] for t in temps if t["chip"] == chip and t["celsius"] is not None] + if picks: + return max(picks) # Tctl/Tdie style: the hottest sensor on that chip + return None + + # -- processes ----------------------------------------------------------- # + def processes(self) -> dict[str, Any]: + """Top-N by RSS (instant) and by CPU (needs the pid cache — see module docstring).""" + now = time.time() + seen: set[int] = set() + rss_rows: list[dict[str, Any]] = [] + cpu_rows: list[dict[str, Any]] = [] + total = 0 + proc = self._path("proc") + try: + entries = list(proc.iterdir()) + except OSError: + return {"total": None, "top_rss": [], "top_cpu": []} + for entry in entries: + if not entry.name.isdigit(): + continue + pid = int(entry.name) + stat = self.read_text("proc", entry.name, "stat") + if not stat: + continue + fields = _split_stat(stat) + if not fields: + continue + total += 1 + seen.add(pid) + name = fields[0] + utime = _to_float(fields[11]) if len(fields) > 11 else None + stime = _to_float(fields[12]) if len(fields) > 12 else None + jiffies = (utime + stime) if (utime is not None and stime is not None) else None + rss_pages = self._rss_pages(entry.name) + if rss_pages is not None: + rss_rows.append({"pid": pid, "name": name, "rss_bytes": rss_pages * self._page_size}) + previous = self._pid_cpu.get(pid) + if jiffies is not None: + if previous is not None and previous[0] < now: + elapsed = now - previous[0] + used = (jiffies - previous[1]) / self._ticks + if elapsed > 0 and used >= 0: + cpu_rows.append({"pid": pid, "name": name, + "cpu_pct": used / elapsed * 100.0}) + self._pid_cpu[pid] = (now, jiffies, name) + with self._lock: + for pid in list(self._pid_cpu): + if pid not in seen: + del self._pid_cpu[pid] + rss_rows.sort(key=lambda row: row["rss_bytes"], reverse=True) + cpu_rows.sort(key=lambda row: row["cpu_pct"], reverse=True) + return {"total": total, "top_rss": rss_rows[: self.top_processes], + "top_cpu": cpu_rows[: self.top_processes]} + + def _rss_pages(self, pid_dir: str) -> float | None: + """statm field 2 is resident set size in pages.""" + parts = (self.read_text("proc", pid_dir, "statm") or "").split() + return _to_float(parts[1]) if len(parts) > 1 else None + + # -- host identity ------------------------------------------------------- # + def host_info(self) -> dict[str, Any]: + uptime = self.read_number("proc/uptime") + if uptime is not None and self._boot_ts is None: + self._boot_ts = time.time() - uptime + release: dict[str, str] = {} + for line in self.read_lines("etc/os-release"): + key, sep, value = line.partition("=") + if sep: + release[key] = value.strip('"') + logical, physical = self.core_counts() + meminfo = self.memory() + try: + uname = os.uname() + except AttributeError: # pragma: no cover - non-POSIX + uname = None + return { + "hostname": socket.gethostname(), + "os_pretty": release.get("PRETTY_NAME") or " ".join( + v for v in (release.get("NAME"), release.get("VERSION")) if v) or None, + "os_id": release.get("ID"), + "kernel": uname.release if uname else None, + "arch": uname.machine if uname else None, + "python": f"{os.sys.version_info.major}.{os.sys.version_info.minor}.{os.sys.version_info.micro}", + "cpu_model": self.cpu_model(), + "cpu_cores_logical": logical, + "cpu_cores_physical": physical, + "memory_total_bytes": meminfo.get("total_bytes"), + "boot_ts": self._boot_ts, + "uptime_seconds": uptime, + "container": self._in_container(), + "agent_version": AGENT_VERSION, + } + + def _in_container(self) -> bool: + for marker in ("run/.containerenv", ".dockerenv"): + if self._path(marker).exists(): + return True + cgroup = self.read_text("proc/1/cgroup") or "" + return bool(re.search(r"(docker|containerd|kubepods|libpod)", cgroup)) + + # -- everything ---------------------------------------------------------- # + def snapshot(self, *, top: int | None = None) -> dict[str, Any]: + started = time.perf_counter() + errors: list[str] = [] + info = self._safe(self.host_info, "host", errors) + temps = self._safe(self.temperatures, "temperatures", errors) or [] + if top is not None: + self.top_processes = top + # no lock here: processes() takes it itself, and threading.Lock is not reentrant + processes = self._safe(self.processes, "processes", errors) or {"total": None, + "top_rss": [], + "top_cpu": []} + return { + "schema": SCHEMA_VERSION, + "agent_version": AGENT_VERSION, + "ts": time.time(), + "duration_ms": round((time.perf_counter() - started) * 1000.0, 2), + "host": info or {}, + "cpu": self._safe(self.cpu, "cpu", errors) or {}, + "memory": self._safe(self.memory, "memory", errors) or {}, + "filesystems": self._safe(self.filesystems, "filesystems", errors) or [], + "disk_io": self._safe(self.disk_io, "disk_io", errors) or {"devices": [], "device_names": []}, + "network": self._safe(self.network, "network", errors) or {"interfaces": []}, + "gpus": self._safe(self.gpus, "gpus", errors) or [], + "temperatures": temps, + "cpu_temperature_celsius": self.cpu_temperature(temps), + "processes": processes, + "errors": errors, + } + + def _safe(self, fn: Callable[[], Any], name: str, errors: list[str]) -> Any: + """One failing collector degrades its own section to empty; never the snapshot.""" + try: + return fn() + except Exception as exc: # noqa: BLE001 - a bad /proc read must not fail the poll + errors.append(f"{name}: {type(exc).__name__}: {exc}") + return None + + +MODE_NAMES = ("user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", + "guest", "guest_nice") +MEMORY_KEYS = ("total_bytes", "free_bytes", "available_bytes", "used_bytes", "buffers_bytes", + "cached_bytes", "dirty_bytes", "swap_total_bytes", "swap_used_bytes", + "committed_bytes") + +# /proc//stat comm is in parens and may itself contain spaces and parens. +_STAT_COMM_RE = re.compile(r"^\d+ \((?P.*)\) (?P.*)$", re.S) + + +def _split_stat(stat: str) -> list[str] | None: + match = _STAT_COMM_RE.match(stat.strip()) + if not match: + return None + return [match.group("comm"), *match.group("rest").split()] + + +# --------------------------------------------------------------------------- # +# HTTP server +# --------------------------------------------------------------------------- # +STATUS_PAGE = """ +{hostname} — host agent + +

{hostname}

+ + + + + + +
agentv{version}
os{os_pretty}
kernel{kernel}
cpu{cpu_model}
uptime{uptime}
+

/snapshot · /info · + /health · /schema

+""" + + +class Handler(BaseHTTPRequestHandler): + server_version = f"host_agent/{AGENT_VERSION}" + collector: Collector + token: str | None = None + + def do_GET(self) -> None: # the BaseHTTPRequestHandler API name — keep it + path = self.path.split("?", 1)[0].rstrip("/") or "/" + if self.token and path != "/health": + given = self.headers.get("Authorization", "") + if given != f"Bearer {self.token}": + self._send(401, {"ok": False, "detail": "missing or bad bearer token"}) + return + if path == "/": + self._status_page() + elif path == "/health": + self._send(200, {"ok": True, "hostname": socket.gethostname(), + "version": AGENT_VERSION, "ts": time.time()}) + elif path == "/info": + self._send(200, {"ok": True, "info": self.collector.host_info(), + "gpus": self.collector.gpus()}) + elif path == "/schema": + self._send(200, {"ok": True, "schema": SCHEMA_VERSION, "doc": __doc__}) + elif path == "/snapshot": + query = self.path.split("?", 1)[1] if "?" in self.path else "" + top = None + for part in query.split("&"): + if part.startswith("top="): + try: + top = max(0, int(part[4:])) + except ValueError: + top = None + self._send(200, {"ok": True, **self.collector.snapshot(top=top)}) + else: + self._send(404, {"ok": False, "detail": f"no such path: {path}"}) + + def _status_page(self) -> None: + info = self.collector.host_info() + uptime = info.get("uptime_seconds") + body = STATUS_PAGE.format( + hostname=info.get("hostname") or "unknown", + version=AGENT_VERSION, + os_pretty=info.get("os_pretty") or "unknown", + kernel=info.get("kernel") or "unknown", + cpu_model=info.get("cpu_model") or "unknown", + uptime=f"{uptime / 86400:.1f} days" if uptime else "unknown", + ) + data = body.encode() + self.send_response(200) + self.send_header("content-type", "text/html; charset=utf-8") + self.send_header("content-length", str(len(data))) + self.send_header("cache-control", "no-store") + self.end_headers() + self.wfile.write(data) + + def _send(self, status: int, body: dict[str, Any]) -> None: + data = json.dumps(body, default=str).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + self.send_header("cache-control", "no-store") + self.end_headers() + self.wfile.write(data) + + def log_message(self, fmt: str, *args: Any) -> None: + # One line per poll, journald-friendly, and never with query strings (a token + # pasted by mistake in a URL should not live in the journal). + if "%s" in fmt: + fmt = fmt % args # the log_message idiom from BaseHTTPRequestHandler + print(f"[agent] {self.address_string()} {fmt}", flush=True) + + +def serve(host: str, port: int, collector: Collector, token: str | None) -> None: + Handler.collector = collector + Handler.token = token + httpd = ThreadingHTTPServer((host, port), Handler) + print(f"[agent] listening on http://{host}:{port} " + f"({'token required' if token else 'no token'}); pid {os.getpid()}", flush=True) + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("[agent] shutting down", flush=True) + finally: + httpd.server_close() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0], + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("--host", default=os.environ.get("AGENT_HOST", "127.0.0.1"), + help="bind address; 0.0.0.0 exposes host metrics to the network") + parser.add_argument("--port", type=int, default=int(os.environ.get("AGENT_PORT", "9120"))) + parser.add_argument("--token", default=os.environ.get("AGENT_TOKEN", ""), + help="bearer token required on every request except /health") + parser.add_argument("--root", default="/", help="filesystem root to read (for tests)") + parser.add_argument("--top", type=int, default=5, help="processes per top-N list") + parser.add_argument("--net-all", action="store_true", help="include veth/bridge interfaces") + parser.add_argument("--once", action="store_true", help="print one snapshot as JSON and exit") + args = parser.parse_args(argv) + + collector = Collector(args.root, top_processes=args.top, net_all=args.net_all) + if args.once: + print(json.dumps({"ok": True, **collector.snapshot()}, indent=2, default=str)) + return 0 + if args.host not in ("127.0.0.1", "localhost", "::1") and not args.token: + print("[agent] refusing to bind off-loopback without --token (host metrics are " + "sensitive); pass --token or set AGENT_TOKEN", flush=True) + return 2 + serve(args.host, args.port, collector, args.token or None) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9135b32 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[project] +name = "dashboards" +version = "0.1.0" +description = "Homelab dashboards on one domain: system resources, LLM engines, and UPS battery health" +readme = "README.md" +requires-python = ">=3.13" +# Deliberately small. Everything we collect is either our own agent's JSON (stdlib only, +# see agent/host_agent.py) or an upstream Prometheus exposition parsed by our own parser +# (src/dashboards/prom.py). No prometheus_client, no Grafana, no CDN, no bundler. +dependencies = [ + "fastapi>=0.143.3", + "httpx>=0.28.1", + "uvicorn[standard]>=0.54.0", + "pydantic>=2.14.1", + "python-dotenv>=1.2.4", +] + +[dependency-groups] +dev = [ + "pytest>=9.1.1", + "pytest-cov>=7.1.0", + "pytest-asyncio>=1.4.0", + "ruff>=0.17.1", + "pyright>=1.1.414", + "debugpy>=1.8.22", +] + +# Installed editable by `uv sync` (src layout), so `uv run dashboards` and +# `uv run uvicorn dashboards.app:app` both work without PYTHONPATH gymnastics. +[build-system] +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" + +[project.scripts] +dashboards = "dashboards.__main__:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/dashboards"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q --strict-markers" +# auto: async tests run without a decorator on every single one +asyncio_mode = "auto" +filterwarnings = ["error::DeprecationWarning"] + +[tool.coverage.run] +source = ["src/dashboards"] +omit = ["tests/*"] + +[tool.coverage.report] +show_missing = true +skip_covered = false + +[tool.ruff] +line-length = 110 +target-version = "py313" +src = ["src", "tests", "agent"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "C4", "SIM", "RUF", "BLE", "PLC"] +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +# FastAPI's dependency injection is a call in a default argument; that is the API, not a bug. +"src/dashboards/api.py" = ["B008"] + +[tool.pyright] +include = ["src", "tests"] +pythonVersion = "3.13" +typeCheckingMode = "standard" diff --git a/scripts/capture-fixtures.sh b/scripts/capture-fixtures.sh new file mode 100755 index 0000000..624bb6e --- /dev/null +++ b/scripts/capture-fixtures.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Re-record ground-truth captures into tests/fixtures/. +# +# ./scripts/capture-fixtures.sh # needs the tunnels up: scripts/dev-tunnels.sh start +# ./scripts/capture-fixtures.sh --with-llm # also scrape the live engines' /metrics +# +# Fixtures are verbatim, not hand-written: a fixture you typed is a test of your memory. +# When a host's kernel or an engine's metric names change, re-run this and read the diff. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +FIX="$ROOT/tests/fixtures" +mkdir -p "$FIX" + +HOSTS=(driveripper:9121 deskwork:9122 gateway:9123) +# Engine URLs come from the same env the app uses, so this script carries no deployment +# detail of its own — and never records a tokenised URL into a fixture by accident. +if [ -f "$ROOT/.env.dev" ]; then + set -a; . "$ROOT/.env.dev"; set +a +fi +ENGINES=() +[ -n "${GUFO_BASE_URL:-}" ] && ENGINES+=("gufo:${GUFO_BASE_URL%/}/metrics") +[ -n "${VLLM_BASE_URL:-}" ] && ENGINES+=("vllm:${VLLM_BASE_URL%/}/metrics") + +failed=0 + +for entry in "${HOSTS[@]}"; do + name=${entry%%:*}; port=${entry##*:} + url="http://127.0.0.1:${port}/snapshot" + if curl -fsS --max-time 8 "$url" -o "$FIX/agent_${name}.json"; then + echo " ✓ agent_${name}.json ($(wc -c <"$FIX/agent_${name}.json") bytes)" + else + echo " ✗ agent_${name}.json — no agent on $url (scripts/dev-tunnels.sh start?)" + failed=1 + fi +done + +if [ "${1:-}" = "--with-llm" ]; then + if [ ${#ENGINES[@]} -eq 0 ]; then + echo " ! no GUFO_BASE_URL / VLLM_BASE_URL set (see .env.dev) — nothing to capture" + failed=1 + fi + for entry in "${ENGINES[@]}"; do + name=${entry%%:*}; url=${entry#*:} + if curl -fsS --max-time 20 "$url" -o "$FIX/metrics_${name}.txt"; then + echo " ✓ metrics_${name}.txt ($(wc -l <"$FIX/metrics_${name}.txt") lines)" + else + echo " ✗ metrics_${name}.txt — $url unreachable (VPN? engine down?)" + failed=1 + fi + done +else + echo " (skipped engine /metrics captures — pass --with-llm when the engines are reachable)" +fi + +# Secrets never belong in a fixture: engines can expose /metrics behind a token in the URL, +# and a hostname you use at home is a hostname in a public repo. +if grep -rlE "Authorization|Bearer |://[^/]*:[^/@]*@" "$FIX" >/dev/null 2>&1; then + echo " ! fixture contains what looks like a credential — scrub it before committing" + failed=1 +fi + +# Two things leak out of a real capture that nobody notices: LUKS/partition UUIDs (which +# identify *your* disks, and appear in `device` strings as /dev/mapper/luks-) and the +# FQDN a host thinks it has. They are replaced deterministically so re-capturing does not +# churn the fixture, and the tests keep the same shapes to assert against. +python3 - "$FIX" <<'PY' +import pathlib +import re +import sys + +uuid_re = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +SYNTHETIC = "00000000-0000-4000-8000-" # already-scrubbed markers: re-running must not churn +fix = pathlib.Path(sys.argv[1]) +changed = [] +for path in sorted(fix.iterdir()): + if path.suffix not in (".json", ".txt"): + continue + text = path.read_text() + seen: dict[str, str] = {} + + def swap(match: re.Match[str]) -> str: + value = match.group(0) + if value.startswith(SYNTHETIC): + return value + if value not in seen: + seen[value] = f"00000000-0000-4000-8000-{len(seen) + 1:012d}" + return seen[value] + + text = uuid_re.sub(swap, text) + # "hostname": "gw.lab.example.com" -> "gw": the dashboard's host key is the target name + # from HOST_TARGETS, never whatever the box calls itself + text = re.sub(r'("hostname": "[^"]*?)\.[^"]*(")', r"\1\2", text) + if text != path.read_text(): + path.write_text(text) + changed.append(path.name) +print(f" scrubbed identifiers in: {', '.join(changed)}" if changed else " fixtures carry no identifiers") +PY + +exit "$failed" diff --git a/scripts/capture-node-exporter.sh b/scripts/capture-node-exporter.sh new file mode 100755 index 0000000..4f46a7a --- /dev/null +++ b/scripts/capture-node-exporter.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Capture a REAL node_exporter exposition from each monitored host into tests/fixtures/. +# +# Runs node_exporter in the foreground on the remote host for a few seconds, scrapes it +# over loopback there, and saves the text. Nothing is installed as a service and no port +# is opened: the only network hop is ssh. +# +# ./scripts/capture-node-exporter.sh # all hosts +# ./scripts/capture-node-exporter.sh driveripper-ops # one host +# +# Why: metric names must be verified against the deployed exporter (see AGENTS.md), never +# guessed from the upstream README. +set -uo pipefail + +VERSION="${NODE_EXPORTER_VERSION:-1.12.1}" +OUT="$(cd "$(dirname "$0")/.." && pwd)/tests/fixtures" +HOSTS=("${@:-driveripper-ops deskwork-ops gateway-ops}") +PORT=9199 +ARGS="--collector.drm --collector.thermal_zone --collector.cpufreq --collector.hwmon --web.listen-address=127.0.0.1:${PORT}" + +mkdir -p "$OUT" +command -v ssh >/dev/null || { echo "ssh not found"; exit 1; } + +for host in $HOSTS; do + echo "=== $host" + ssh -n -o ConnectTimeout=10 "$host" "set -e + mkdir -p \$HOME/exporters && cd \$HOME/exporters + if [ ! -x node_exporter ]; then + curl -sSL -o ne.tgz https://github.com/prometheus/node_exporter/releases/download/v${VERSION}/node_exporter-${VERSION}.linux-amd64.tar.gz + tar xzf ne.tgz && mv node_exporter-${VERSION}.linux-amd64/node_exporter . + rm -rf ne.tgz node_exporter-${VERSION}.linux-amd64 + fi + timeout 8 ./node_exporter ${ARGS} >/dev/null 2>&1 & + sleep 3 + curl -s --max-time 5 127.0.0.1:${PORT}/metrics + " > "$OUT/node_exporter_${host}.txt" 2>/dev/null + lines=$(wc -l < "$OUT/node_exporter_${host}.txt") + if [ "$lines" -lt 20 ]; then + echo " FAILED (only $lines lines) — check ssh access to $host" + rm -f "$OUT/node_exporter_${host}.txt" + continue + fi + echo " saved $lines lines -> tests/fixtures/node_exporter_${host}.txt" + echo " gpu: $(grep -cE '^node_gpu' "$OUT/node_exporter_${host}.txt") node_gpu series" + grep -oE '^node_gpu[a-z_]*' "$OUT/node_exporter_${host}.txt" | sort -u | sed 's/^/ /' | head -20 +done diff --git a/scripts/dev-server.sh b/scripts/dev-server.sh new file mode 100755 index 0000000..aee0cc4 --- /dev/null +++ b/scripts/dev-server.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Local development server: start|stop|status|restart|logs|tunnel-check +# +# ./scripts/dev-server.sh restart # reads .env.dev if present, runs on DASH_PORT +# +# The server keeps running after this shell exits (nohup + pidfile) so you can browse it +# while an agent works. Logs: data/logs/dev.log +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +PIDFILE="$ROOT/data/dev-server.pid" +LOG="$ROOT/data/logs/dev.log" +mkdir -p "$ROOT/data/logs" + +running() { [ -f "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; } + +start() { + if running; then + echo "already running (pid $(cat "$PIDFILE")) — http://127.0.0.1:${DASH_PORT:-8090}" + return 0 + fi + if [ -f "$ROOT/.env.dev" ]; then + set -a; . "$ROOT/.env.dev"; set +a + echo "loaded .env.dev" + fi + PORT="${DASH_PORT:-8090}" + nohup .venv/bin/dashboards >"$LOG" 2>&1 & + echo $! >"$PIDFILE" + sleep 3 + if running && curl -sf -o /dev/null "http://127.0.0.1:${PORT}/api/health"; then + echo "started pid $(cat "$PIDFILE") — http://127.0.0.1:${PORT}" + else + echo "FAILED to start; last lines of $LOG:" + tail -20 "$LOG" + return 1 + fi +} + +stop() { + if running; then + kill "$(cat "$PIDFILE")" 2>/dev/null + sleep 1 + echo "stopped" + else + echo "not running" + fi + rm -f "$PIDFILE" +} + +case "${1:-start}" in + start) start ;; + stop) stop ;; + restart) stop; start ;; + status) + if running; then + echo "running pid $(cat "$PIDFILE")" + curl -s "http://127.0.0.1:${DASH_PORT:-8090}/api/health" \ + | .venv/bin/python -c "import json,sys; d=json.load(sys.stdin); print('status:', d['status']); [print(' ', s['key'], 'ok' if s['ok'] else 'DOWN', (s['error'] or '')[:90]) for s in d['sources']]" + else + echo "not running" + fi ;; + logs) tail -n 40 "$LOG" ;; + *) echo "usage: $0 start|stop|restart|status|logs"; exit 2 ;; +esac diff --git a/scripts/dev-tunnels.sh b/scripts/dev-tunnels.sh new file mode 100755 index 0000000..97286ea --- /dev/null +++ b/scripts/dev-tunnels.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Dev-only: run agent/host_agent.py on each monitored host and forward it to localhost, +# so `make dev` on the laptop sees live fleet data without opening a single port. +# +# ./scripts/dev-tunnels.sh start|stop|status|restart +# +# Each tunnel is ssh -L :127.0.0.1: 'python3 agent/host_agent.py' +# and restarts itself if the link drops. Production is the opposite: deploy/install-agent.sh +# installs a systemd unit that binds the LAN address with a bearer token, and the dashboard +# dials it directly. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUN="$ROOT/data/tunnels" +LOGS="$ROOT/data/logs" +# Where the agent lives on each host. `~` is written into the generated script literally so +# the REMOTE shell expands it (a local $HOME would be the wrong path over there). +AGENT_REMOTE_DIR="${AGENT_REMOTE_DIR:-~/lab-agent}" +AGENT_COMMAND="${AGENT_COMMAND:-python3 host_agent.py}" + +# Targets come from .env.dev (see .env.dev.example) — never hardcoded here, so this file +# stays portable and holds no hostnames of ours. +[ -f "$ROOT/.env.dev" ] && . "$ROOT/.env.dev" +DEV_TUNNELS="${DEV_TUNNELS:-}" +if [ -z "$DEV_TUNNELS" ]; then + echo "no DEV_TUNNELS set — copy .env.dev.example to .env.dev first"; exit 2 +fi +TUNNELS=() +for entry in $DEV_TUNNELS; do + # 9121:driveripper-ops:9120 -> "9121 driveripper-ops 9120" + TUNNELS+=("$(echo "$entry" | tr ':' ' ')") +done + +mkdir -p "$RUN" "$LOGS" + +start() { + for spec in "${TUNNELS[@]}"; do + read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" + pidfile="$RUN/$host.pid" + if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then + echo " $host already tunnelling on :$local (pid $(cat "$pidfile"))" + continue + fi + # A previous agent survives the ssh session that started it, and the next agent then + # dies on bind() with "address already in use" — which looks like a broken agent rather + # than a leaked process. Clear it in a SEPARATE ssh session: putting pkill in the same + # command line as the agent makes pkill match that very command line and shoot itself. + cat > "$RUN/$host.sh" </dev/null 2>&1 + sleep 1 + ssh -nT -o ServerAliveInterval=30 -o ServerAliveCountMax=4 -o ExitOnForwardFailure=yes \ + -L $local:127.0.0.1:$remote "$host" \ + "cd $AGENT_REMOTE_DIR && exec $AGENT_COMMAND --host 127.0.0.1 --port $remote" + echo "[tunnel] $host link dropped at \$(date -Is) (exit \$?); retrying in 5s" + sleep 5 +done +EOF + chmod +x "$RUN/$host.sh" + nohup "$RUN/$host.sh" >>"$LOGS/tunnel-$host.log" 2>&1 & + echo $! >"$pidfile" + echo " $host -> http://127.0.0.1:$local (pid $!)" + done +} + +stop() { + for spec in "${TUNNELS[@]}"; do + read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" + pidfile="$RUN/$host.pid" + [ -f "$pidfile" ] || continue + pid=$(cat "$pidfile") + pkill -P "$pid" 2>/dev/null + kill "$pid" 2>/dev/null + # -N would mean "no remote command", so the forward is kept by a plain ssh running the + # agent; kill any leftover forward for this host by pattern, not just by parent. + pkill -f "ssh -nT.*-L $local:127.0.0.1:$remote $host" 2>/dev/null + # The remote agent outlives its ssh session, and the next one then dies on bind() with + # "address already in use" — which looks like a broken agent, not a leaked process. + timeout 10 ssh -n "$host" "pkill -f '[h]ost_agent.py --host 127.0.0.1 --port $remote'" \ + >/dev/null 2>&1 + rm -f "$pidfile" + echo " stopped $host" + done +} + +status() { + for spec in "${TUNNELS[@]}"; do + read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" + body=$(curl -s --max-time 4 "http://127.0.0.1:$local/health" || true) + if [ -n "$body" ]; then + echo " OK $host http://127.0.0.1:$local $body" + else + echo " DOWN $host http://127.0.0.1:$local" + fi + done +} + +case "${1:-start}" in + start) start ;; + stop) stop ;; + restart) stop; sleep 1; start ;; + status) status ;; + *) echo "usage: $0 start|stop|status|restart"; exit 2 ;; +esac diff --git a/src/dashboards/__init__.py b/src/dashboards/__init__.py new file mode 100644 index 0000000..10e6868 --- /dev/null +++ b/src/dashboards/__init__.py @@ -0,0 +1,10 @@ +"""Homelab dashboards: system resources, LLM engines, and UPS battery health. + +One FastAPI app, one domain, one SQLite file. See AGENTS.md for the rules and +.agent(s)/PLAN.md for the locked decisions. +""" + +from .config import SETTINGS + +__version__ = "0.1.0" +__all__ = ["SETTINGS", "__version__"] diff --git a/src/dashboards/__main__.py b/src/dashboards/__main__.py new file mode 100644 index 0000000..f3d263f --- /dev/null +++ b/src/dashboards/__main__.py @@ -0,0 +1,28 @@ +"""`python -m dashboards` — the same server as `uv run uvicorn dashboards.app:app`. + + DASH_PORT=8090 DEBUGPY=1 python -m dashboards +""" + +from __future__ import annotations + +import os + +from .app import create_app +from .config import SETTINGS +from .debug import maybe_listen + + +def main() -> None: + import uvicorn # noqa: PLC0415 - only the entry point needs uvicorn + + maybe_listen() + reload = os.environ.get("DASH_RELOAD", "").lower() in ("1", "true", "yes", "on") + if reload: + uvicorn.run("dashboards.app:app", host=SETTINGS.host, port=SETTINGS.port, reload=True) + else: + # pass the object: create_app() is cheap and avoids a second import of config + uvicorn.run(create_app(), host=SETTINGS.host, port=SETTINGS.port) + + +if __name__ == "__main__": + main() diff --git a/src/dashboards/api.py b/src/dashboards/api.py new file mode 100644 index 0000000..3566d97 --- /dev/null +++ b/src/dashboards/api.py @@ -0,0 +1,399 @@ +"""HTTP API. GET everywhere except one endpoint: `POST /api/ups/command`. + +Everything a browser can see is derived from the in-memory payloads (fast, no DB on the +hot path) plus SQLite for anything older than a few minutes. Two rules from llm_monitor that +are worth restating here because they are easy to break by accident: + +* `/api/config` never leaks a token or password — target URLs are passed through + `Target.display_url`, which strips userinfo. +* an unconfigured source answers 404 `{"configured": false}` without dialling anything, so + the page can tell "not deployed here" apart from "deployed and broken". +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any + +from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request + +from .collectors import NutMonitor +from .config import Settings +from .poller import Hub +from .sources import nut +from .sources.host import fleet_summary +from .sources.llm import fleet_totals +from .store import Event, SampleStore + +NO_STORE = {"cache-control": "no-store, max-age=0"} +RANGE_SECONDS = {"5m": 300, "15m": 900, "30m": 1800, "1h": 3600, "3h": 10_800, + "6h": 21_600, "12h": 43_200, "24h": 86_400, "7d": 604_800, + "30d": 2_592_000, "90d": 7_776_000} + + +@dataclass +class AppContext: + """Everything the routes need, assembled once at startup and passed around.""" + + settings: Settings + store: SampleStore + hub: Hub + nut_monitor: NutMonitor | None = None + started_at: float = 0.0 + + +def get_context(request: Request) -> AppContext: + """The context built during lifespan. FastAPI hands it to every route.""" + return request.app.state.ctx # type: ignore[no-any-return] + + +def parse_range(value: str) -> float: + """`1h` / `7d` / `45m` / raw seconds. Unknown values are a 400, not a silent default: + a chart that quietly shows the wrong window is worse than an error.""" + raw = (value or "1h").strip().lower() + if raw in RANGE_SECONDS: + return float(RANGE_SECONDS[raw]) + if raw[-1] in ("s", "m", "h", "d") and raw[:-1].isdigit(): + multiplier = {"s": 1, "m": 60, "h": 3600, "d": 86_400}[raw[-1]] + return float(int(raw[:-1]) * multiplier) + if raw.isdigit(): + return float(raw) + raise HTTPException(400, f"unknown range {value!r}; try 5m 1h 24h 7d 30d") + + +def _history(store: SampleStore, source: str, host: str, metric: str, span: float, + points: int) -> dict[str, Any]: + series = store.history(source, host, metric, span=span, max_points=points) + return { + "source": source, + "host": host, + "metric": metric, + "range_seconds": span, + "points": [{"ts": ts, "value": value} for ts, value in series], + } + + +# --------------------------------------------------------------------------- # +# Common routes +# --------------------------------------------------------------------------- # +def common_router() -> APIRouter: + router = APIRouter(prefix="/api", tags=["meta"]) + + @router.get("/config") + async def config(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + settings = ctx.settings + return { + "app": {"name": "dashboards", "version": "0.1.0", + "started_at": ctx.started_at, "read_only": settings.read_only}, + "system": { + "enabled": bool(settings.host_targets), + "interval_seconds": settings.host_interval, + "poll_ms": settings.poll_ms_system, + "targets": [{"name": t.name, "url": t.display_url} for t in settings.host_targets], + "target_problems": list(settings.host_target_problems), + }, + "llm": { + "gufo": {"enabled": bool(settings.gufo_url), "base_url": settings.gufo_url, + "auth": bool(settings.gufo_api_key)}, + "vllm": {"enabled": bool(settings.vllm_url), "base_url": settings.vllm_url, + "auth": bool(settings.vllm_api_key)}, + "interval_seconds": settings.llm_interval, + "poll_ms": settings.poll_ms_llm, + "rate_window_seconds": settings.rate_window, + }, + "ups": { + "enabled": settings.nut_enabled, + "server": f"{settings.nut_host}:{settings.nut_port}" if settings.nut_enabled else None, + "auth": bool(settings.nut_user), + "interval_seconds": settings.nut_interval, + "poll_ms": settings.poll_ms_ups, + "actions_enabled": settings.actions_allowed, + "simulated": bool(ctx.nut_monitor and ctx.nut_monitor.simulated), + }, + "ranges": sorted(RANGE_SECONDS, key=lambda key: RANGE_SECONDS[key]), + } + + @router.get("/health") + async def health(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + states = ctx.hub.states() + enabled = [s for s in states if s["configured"]] + healthy = [s for s in enabled if s["ok"]] + status = "ok" if enabled and len(healthy) == len(enabled) else ( + "degraded" if healthy else ("empty" if not enabled else "down")) + # a fresh install with nothing configured is empty, not broken — `ok` answers "is the + # app working", the banner on the page answers "is there anything to look at" + return {"ok": status in ("ok", "empty"), "status": status, "sources": states, + "db": ctx.store.stats(), "uptime_seconds": time.time() - ctx.started_at} + + @router.get("/events") + def events(limit: int = Query(60, ge=1, le=500), source: str | None = None, + since: float | None = None, + ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + """Event log (transitions, UPS state changes, actions taken). + + Served by a sync handler on purpose: SQLite reads belong in Starlette's + threadpool, not on the event loop the collectors share. + """ + rows = ctx.store.recent_events(limit, source=source, since=since) + return {"events": [row.as_json() for row in rows]} + + @router.get("/history") + def history(source: str = Query(...), host: str = Query(...), metric: str = Query(...), + range: str = Query("1h"), points: int = Query(600, ge=10, le=4000), + ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + return _history(ctx.store, source, host, metric, parse_range(range), points) + + @router.get("/metrics") + def metrics(source: str | None = None, host: str | None = None, + ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + """Which series actually have history — the honest answer to "why is this chart empty".""" + return {"metrics": ctx.store.series_names(source, host)} + + @router.get("/summary") + async def summary(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + """One call for the landing page: is each dashboard alive, and what stands out.""" + hosts = [job.state.payload for job in ctx.hub.jobs.values() + if job.dashboard == "system" and job.state.payload] + engines = [job.state.payload for job in ctx.hub.jobs.values() + if job.dashboard == "llm" and job.state.payload] + ups_payload = None + nut_job = ctx.hub.jobs.get("nut") + if nut_job and nut_job.state.payload: + ups_payload = nut_job.state.payload + return { + "ts": time.time(), + "system": {"enabled": bool(ctx.settings.host_targets), + "fleet": fleet_summary(hosts) if hosts else None, + "hosts": [{"name": h["name"], "os": (h.get("info") or {}).get("os_pretty"), + "cpu_usage_pct": h["cpu"]["usage_pct"], + "load1": h["cpu"].get("load1"), + "cores": (h.get("info") or {}).get("cpu_cores_logical"), + "mem_used_pct": h["memory"]["used_pct"], + "disk_worst_used_pct": h["disks"]["worst_used_pct"], + "gpu_busy_pct": [g.get("busy_pct") for g in h.get("gpus") or []], + "temp_c": h["cpu"].get("temp_c"), + "uptime_seconds": (h.get("info") or {}).get("uptime_seconds")} + for h in hosts], + "states": ctx.hub.states("system")}, + "llm": {"enabled": bool(ctx.settings.gufo_url or ctx.settings.vllm_url), + "fleet": fleet_totals(engines) if engines else None, + "engines": [{"engine": e["engine"], + "decode_tokens_per_second": e["metrics"].get( + "decode_tokens_per_second", {}).get("value"), + "active": e["sessions"].get("active")} + for e in engines], + "states": ctx.hub.states("llm")}, + "ups": {"enabled": ctx.settings.nut_enabled, + "simulated": bool(ups_payload and ups_payload.get("simulated")), + "ups": [{"name": card["name"], "overall": card["health"]["overall"], + "status": card["status"], + "charge_pct": card["numbers"].get("charge_pct"), + "load_pct": card["numbers"].get("load_pct"), + "runtime_seconds": card["numbers"].get("battery_runtime")} + for card in (ups_payload or {}).get("ups", [])], + "states": ctx.hub.states("ups")}, + "recent_events": [event.as_json() for event in ctx.store.recent_events(12)], + } + + return router + + +# --------------------------------------------------------------------------- # +# System dashboard +# --------------------------------------------------------------------------- # +def system_router() -> APIRouter: + router = APIRouter(prefix="/api/system", tags=["system"]) + + @router.get("/snapshot") + async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + if not ctx.settings.host_targets: + raise HTTPException(404, detail={"configured": False, + "detail": "no hosts configured — set HOST_TARGETS"}) + hosts = [] + for job in ctx.hub.jobs.values(): + if job.dashboard != "system": + continue + payload = job.state.payload + if payload is None: + # keep an unreachable host in the list: absent from the list and present-but- + # red are different statements, and only one of them is a config problem + payload = {"name": job.label, "ts": job.state.last_attempt, "unreachable": True} + hosts.append(payload) + hosts.sort(key=lambda item: item.get("name") or "") + return {"ts": time.time(), "hosts": hosts, "fleet": fleet_summary(hosts), + "states": ctx.hub.states("system"), + "window_seconds": ctx.settings.host_interval} + + @router.get("/history") + def history(metrics: str = Query("cpu.usage_pct"), hosts: str = Query(""), + range: str = Query("1h"), points: int = Query(600, ge=10, le=4000), + ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + """One request for a whole chart: N hosts x M metrics. + + `hosts` is comma-separated, or empty for every configured target. Charting three + hosts should cost one round trip, not three, because the browser polls this. + """ + span = parse_range(range) + wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()] + wanted_hosts = [item.strip() for item in hosts.split(",") if item.strip()] or [ + target.name for target in ctx.settings.host_targets] + series = [_history(ctx.store, "host", host, metric, span, points) + for host in wanted_hosts for metric in wanted_metrics] + return {"hosts": wanted_hosts, "range_seconds": span, "series": series} + + return router + + +# --------------------------------------------------------------------------- # +# LLM dashboard +# --------------------------------------------------------------------------- # +def llm_router() -> APIRouter: + router = APIRouter(prefix="/api/llm", tags=["llm"]) + + @router.get("/snapshot") + async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + if not (ctx.settings.gufo_url or ctx.settings.vllm_url): + raise HTTPException(404, detail={"configured": False, + "detail": "no engines configured — set " + "GUFO_BASE_URL / VLLM_BASE_URL"}) + engines = [] + for job in ctx.hub.jobs.values(): + if job.dashboard != "llm": + continue + payload = job.state.payload or { + "engine": job.label, "metrics": {}, "sessions": {"active": None, "list": []}, + "unreachable": True} + engines.append(payload) + return {"ts": time.time(), "engines": engines, "fleet": fleet_totals(engines), + "states": ctx.hub.states("llm")} + + @router.get("/history") + def history(metrics: str = Query("decode_tokens_per_second"), engines: str = Query(""), + range: str = Query("1h"), points: int = Query(600, ge=10, le=4000), + ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + span = parse_range(range) + wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()] + wanted_engines = [item.strip() for item in engines.split(",") if item.strip()] or [ + name for name, url in (("gufo", ctx.settings.gufo_url), ("vllm", ctx.settings.vllm_url)) if url] + return {"engines": wanted_engines, "range_seconds": span, + "series": [_history(ctx.store, "llm", engine, f"llm.{metric}", span, points) + for engine in wanted_engines for metric in wanted_metrics]} + + return router + + +# --------------------------------------------------------------------------- # +# UPS dashboard +# --------------------------------------------------------------------------- # +def ups_router() -> APIRouter: + router = APIRouter(prefix="/api/ups", tags=["ups"]) + + def require_monitor(ctx: AppContext) -> NutMonitor: + if ctx.nut_monitor is None or not ctx.settings.nut_enabled: + raise HTTPException(404, detail={"configured": False, + "detail": "no NUT server configured — set NUT_HOST"}) + return ctx.nut_monitor + + @router.get("/snapshot") + async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + monitor = require_monitor(ctx) + job = ctx.hub.jobs.get("nut") + payload = (job.state.payload if job else None) or { + "ts": time.time(), "ups": [], "simulated": monitor.simulated, + "actions_enabled": ctx.settings.actions_allowed} + events = ctx.store.recent_events(40, source="ups") + return {**payload, "states": ctx.hub.states("ups"), + "events": [event.as_json() for event in events]} + + @router.get("/history") + def history(metrics: str = Query("ups.charge_pct"), ups: str = Query(""), + range: str = Query("24h"), points: int = Query(600, ge=10, le=4000), + ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + span = parse_range(range) + wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()] + job = ctx.hub.jobs.get("nut") + wanted_ups = [item.strip() for item in ups.split(",") if item.strip()] or [ + card["name"] for card in ((job.state.payload if job else None) or {}).get("ups", [])] + return {"ups": wanted_ups, "range_seconds": span, + "series": [_history(ctx.store, "ups", name, metric, span, points) + for name in wanted_ups for metric in wanted_metrics]} + + @router.get("/commands") + async def commands(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + """What may be sent right now, and why not when it may not.""" + require_monitor(ctx) + job = ctx.hub.jobs.get("nut") + payload = (job.state.payload if job else None) or {"ups": []} + return {"actions_enabled": ctx.settings.actions_allowed, + "read_only": ctx.settings.read_only, + "risk_levels": [nut.RISK_SAFE, nut.RISK_CAUTION, nut.RISK_DANGEROUS], + "catalog": [{"command": key, **value} for key, value in nut.COMMANDS.items()], + "ups": [{"name": card["name"], "commands": card["commands"]} + for card in payload.get("ups", [])]} + + @router.post("/command") + async def send_command(ctx: AppContext = Depends(get_context), + payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + """The one write endpoint in the application. Deliberately loud about it. + + Gated on: NUT_ALLOW_ACTIONS, not DASH_READ_ONLY, credentials that carry `actions` + in upsd.users, the command being advertised by the driver for that UPS, and — for + anything that can cut power — the browser echoing the confirmation word back. + """ + monitor = require_monitor(ctx) + ups_name = str(payload.get("ups") or "").strip() + command = str(payload.get("command") or "").strip() + confirm = str(payload.get("confirm") or "").strip() + if not ups_name or not command: + raise HTTPException(400, "both `ups` and `command` are required") + if not ctx.settings.actions_allowed: + raise HTTPException(403, "UPS actions are disabled — set NUT_ALLOW_ACTIONS=1 " + "(and don't run with DASH_READ_ONLY=1)") + + job = ctx.hub.jobs.get("nut") + card = next((item for item in ((job.state.payload or {}).get("ups", []) if job else []) + if item["name"] == ups_name), None) + if card is None: + raise HTTPException(404, f"unknown UPS {ups_name!r} — not in the last poll") + advice = nut.command_advice(command) + if advice is None: + raise HTTPException(400, f"{command} is not a command this dashboard will send") + if command not in card.get("advertised_commands", []): + raise HTTPException(400, f"the driver for {ups_name} does not advertise {command}") + if advice["risk"] == nut.RISK_DANGEROUS and confirm != "DANGER": + raise HTTPException(422, f"{advice['label']} can cut power: send " + f"confirm=\"DANGER\" to prove you read the warning") + if any(item["command"] == command and not item["enabled"] for item in card["commands"]): + blocked = next(item for item in card["commands"] if item["command"] == command) + raise HTTPException(409, blocked["reason"] or f"{command} is not available") + + try: + if monitor.simulated: + detail = monitor.simulator.command(ups_name, command) # type: ignore[union-attr] + else: + async with nut.NutClient( + ctx.settings.nut_host, ctx.settings.nut_port, + timeout=ctx.settings.nut_timeout, + username=ctx.settings.nut_user, password=ctx.settings.nut_password, + ) as client: + await client.login() + detail = await client.instcmd(ups_name, command) + except nut.NutError as exc: + ctx.store.log_event(Event(time.time(), "ups", ups_name, "action", "error", + f"{advice['label']} on {ups_name} failed: {exc}", + {"command": command, "code": exc.code})) + raise HTTPException(502, f"upsd refused {command}: {exc}") from exc + + ctx.store.log_event(_action_event(ups_name, command, advice, detail)) + return {"ok": True, "ups": ups_name, "command": command, "detail": detail, + "risk": advice["risk"], "simulated": monitor.simulated, + "hint": "watch the state card — the driver answers over the next few polls"} + + return router + + +def _action_event(ups_name: str, command: str, advice: dict[str, str], detail: str) -> Event: + severity = "warn" if advice["risk"] != nut.RISK_SAFE else "info" + return Event(time.time(), "ups", ups_name, "action", severity, + f"{advice['label']} sent to {ups_name}: {detail}", {"command": command}) diff --git a/src/dashboards/app.py b/src/dashboards/app.py new file mode 100644 index 0000000..9244398 --- /dev/null +++ b/src/dashboards/app.py @@ -0,0 +1,178 @@ +"""Application assembly: one FastAPI app, four pages, one SQLite file, N poll loops. + +`create_app()` takes injectable settings and store so tests can build an app with an +in-memory DB and no targets — the same code path production uses, without the network. +""" + +from __future__ import annotations + +import os +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import httpx +from fastapi import FastAPI, Request +from fastapi.responses import FileResponse, JSONResponse +from fastapi.staticfiles import StaticFiles +from starlette.exceptions import HTTPException as StarletteHTTPException + +from .api import NO_STORE, AppContext, common_router, llm_router, system_router, ups_router +from .collectors import NutMonitor, UpstreamError, host_jobs, llm_jobs, nut_job +from .config import SETTINGS as DEFAULT_SETTINGS +from .config import Settings +from .poller import Hub +from .rates import RateEstimator +from .sources.host import HostCollector +from .sources.nut_sim import NutSimulator +from .store import SampleStore + +WEB = Path(__file__).resolve().parent / "web" +PAGES = {"/": "index.html", "/system": "system.html", "/llm": "llm.html", "/ups": "ups.html"} +VERSION = "0.1.0" + + +def build_context(settings: Settings, client: httpx.AsyncClient) -> tuple[ + AppContext, SampleStore, Callable[[], None]]: + """Create the store, the collectors, and the hub's jobs. Returns (ctx, store, start).""" + store = SampleStore( + settings.db_path, + raw_keep_hours=settings.raw_keep_hours, + minute_keep_days=settings.minute_keep_days, + hour_keep_days=settings.hour_keep_days, + ) + rates = RateEstimator(span=settings.rate_window) + host_collector = HostCollector(rates) + nut_monitor: NutMonitor | None = None + if settings.nut_enabled: + simulator = NutSimulator() if _truthy(os.environ.get("NUT_SIMULATE")) else None + nut_monitor = NutMonitor(settings, simulator=simulator) + + hub = Hub(store) + for job in host_jobs(list(settings.host_targets), client, host_collector, + settings.host_interval): + hub.add(job) + for job in llm_jobs(settings, client, rates): + hub.add(job) + if nut_monitor is not None: + hub.add(nut_job(settings, nut_monitor, client)) + + ctx = AppContext(settings=settings, store=store, hub=hub, nut_monitor=nut_monitor, + started_at=time.time()) + return ctx, store, hub.start + + +def _truthy(value: str | None) -> bool: + return (value or "").strip().lower() in ("1", "true", "yes", "on") + + +def create_app(settings: Settings | None = None, store: SampleStore | None = None) -> FastAPI: + """`store` is for tests: when supplied it is not created and not closed here.""" + settings = settings or DEFAULT_SETTINGS + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + owned_store: SampleStore | None = store + ctx: AppContext | None = None + client = httpx.AsyncClient( + timeout=httpx.Timeout(max(settings.host_timeout, settings.llm_timeout)), + follow_redirects=False, + limits=httpx.Limits(max_connections=32, max_keepalive_connections=16), + ) + try: + if store is not None: + # injected: rebuild everything except the DB + rates = RateEstimator(span=settings.rate_window) + hub = Hub(store) + nut_monitor = None + if settings.nut_enabled: + simulator = NutSimulator() if _truthy(os.environ.get("NUT_SIMULATE")) else None + nut_monitor = NutMonitor(settings, simulator=simulator) + for job in host_jobs(list(settings.host_targets), client, + HostCollector(rates), settings.host_interval): + hub.add(job) + for job in llm_jobs(settings, client, rates): + hub.add(job) + if nut_monitor is not None: + hub.add(nut_job(settings, nut_monitor, client)) + ctx = AppContext(settings=settings, store=store, hub=hub, + nut_monitor=nut_monitor, started_at=time.time()) + start = hub.start + else: + ctx, owned_store, start = build_context(settings, client) + app.state.ctx = ctx + app.state.hub = ctx.hub + start() + yield + finally: + # `ctx` is None when startup itself failed; the client and DB still need closing + if ctx is not None: + await ctx.hub.stop() + await client.aclose() + if owned_store is not None and store is None: + owned_store.close() + + app = FastAPI(title="Homelab Dashboards", version=VERSION, lifespan=lifespan, + docs_url=None, redoc_url=None, openapi_url=None) + app.mount("/static", StaticFiles(directory=str(WEB / "static")), name="static") + + for path, filename in PAGES.items(): + app.add_api_route(path, _page_handler(filename), methods=["GET"], name=f"page:{path}") + + app.include_router(common_router()) + app.include_router(system_router()) + app.include_router(llm_router()) + app.include_router(ups_router()) + + @app.exception_handler(UpstreamError) + async def upstream_error_handler(request: Request, exc: UpstreamError) -> JSONResponse: + return JSONResponse({"ok": False, "detail": exc.message}, status_code=exc.status, + headers=NO_STORE) + + @app.middleware("http") + async def no_store_for_api(request: Request, call_next): + """Every /api response is `no-store`, success or failure. + + A dashboard is a polling app: a cached snapshot in a proxy or the browser makes a + dead host look alive for as long as the TTL. Static assets keep their caching. + """ + response = await call_next(request) + if request.url.path.startswith("/api"): + response.headers["cache-control"] = "no-store, max-age=0" + return response + + @app.exception_handler(StarletteHTTPException) + async def http_error_handler(request: Request, + exc: StarletteHTTPException) -> JSONResponse: + """One JSON error shape for every failure, deliberate or not. + + A dict detail is a structured answer (`{"configured": false, ...}`) and is spread + into the body so clients can branch on it. A status-specific 404 handler must NOT be + registered: Starlette resolves handlers by status code first, so it would also swallow + the deliberate `404 {"configured": false}` from an unconfigured source — the difference + between "not deployed here" and "deployed and broken" on the page. Unmatched routes + arrive here as Starlette's plain "Not Found" and get relabelled. + """ + body: dict[str, Any] = {"ok": False} + detail: Any = exc.detail # Starlette types this as str | dict | list | None + if isinstance(detail, dict): + body.update(detail) + elif exc.status_code == 404 and str(detail) in ("Not Found", "Not Found."): + body["detail"] = f"no such route: {request.url.path}" + else: + body["detail"] = detail + return JSONResponse(body, status_code=exc.status_code, headers=NO_STORE) + + return app + + +def _page_handler(filename: str) -> Callable[..., Awaitable[FileResponse]]: + async def handler() -> FileResponse: + return FileResponse(WEB / "pages" / filename, media_type="text/html", headers=NO_STORE) + + return handler + + +app = create_app() diff --git a/src/dashboards/collectors.py b/src/dashboards/collectors.py new file mode 100644 index 0000000..ae091b5 --- /dev/null +++ b/src/dashboards/collectors.py @@ -0,0 +1,244 @@ +"""Collectors: the fetch half of each poll. Parse + store live in the Job wrapper. + +Every collector returns a `Result`; every failure is an exception the Job turns into a +red dot plus an event. Nothing here writes to any upstream — with one deliberate exception, +`run_ups_command`, which is the app's only mutating path and is gated in three places. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +import httpx + +from .config import Target +from .poller import Job, Result +from .rates import RateEstimator +from .sources import llm as llm_source +from .sources import nut +from .sources.host import HostCollector, parse_host_snapshot +from .store import Event, Sample + + +class UpstreamError(Exception): + """An upstream answered badly or not at all; 502/504 in the API, red dot in the UI.""" + + def __init__(self, message: str, *, status: int = 502) -> None: + super().__init__(message) + self.message = message + self.status = status + + +# --------------------------------------------------------------------------- # +# System hosts +# --------------------------------------------------------------------------- # +async def fetch_host_snapshot(client: httpx.AsyncClient, target: Target, + host_collector: HostCollector) -> Result: + url = f"{target.url}/snapshot" + try: + response = await client.get(url, headers=target.headers) + except httpx.TimeoutException as exc: + raise UpstreamError(f"{target.name}: {url} timed out", status=504) from exc + except httpx.HTTPError as exc: + raise UpstreamError(f"{target.name}: {url} unreachable ({type(exc).__name__})") from exc + if response.status_code == 401: + raise UpstreamError(f"{target.name}: agent rejected the token (401) — check the " + f"userinfo token in HOST_TARGETS") + if response.status_code >= 400: + raise UpstreamError(f"{target.name}: {url} -> HTTP {response.status_code}") + try: + body = response.json() + except ValueError as exc: + raise UpstreamError(f"{target.name}: agent returned non-JSON " + f"({response.text[:80]!r})") from exc + if not body.get("ok", False): + raise UpstreamError(f"{target.name}: agent said not ok: {body.get('detail')}") + result = parse_host_snapshot(target.name, body, host_collector) + if body.get("errors"): + # The agent degraded a section (a /proc read failed) but still answered. Say so + # in the payload rather than presenting a partial snapshot as a complete one. + result.payload["degraded"] = body["errors"] + return result + + +# --------------------------------------------------------------------------- # +# LLM engines +# --------------------------------------------------------------------------- # +async def fetch_engine(client: httpx.AsyncClient, engine: str, base_url: str, + api_key: str, rates: RateEstimator, *, + with_slots: bool = False) -> Result: + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + url = f"{base_url}/metrics" + try: + response = await client.get(url, headers=headers) + except httpx.TimeoutException as exc: + raise UpstreamError(f"{engine}: {url} timed out", status=504) from exc + except httpx.HTTPError as exc: + raise UpstreamError(f"{engine}: {url} unreachable ({type(exc).__name__})") from exc + if response.status_code >= 400: + raise UpstreamError(f"{engine}: {url} -> HTTP {response.status_code} " + f"{response.reason_phrase}") + text = response.text + + slots: list[dict[str, Any]] | None = None + if with_slots: + # /slots is gufo's; /v1/slots is the documented alias, so a server that only + # exposes the alias still gives us the per-session list. + for path in ("/slots", "/v1/slots"): + try: + slots_response = await client.get(f"{base_url}{path}", headers=headers) + except httpx.HTTPError: + continue + if slots_response.status_code == 200: + try: + data = slots_response.json() + except ValueError: + continue + slots = data if isinstance(data, list) else data.get("slots", []) + break + return llm_source.engine_result(engine, base_url, text, rates, slots=slots) + + +# --------------------------------------------------------------------------- # +# UPS / NUT +# --------------------------------------------------------------------------- # +@dataclass +class NutState: + """Previous poll's per-UPS status, so transitions are events and not spam.""" + + status: dict[str, str] = field(default_factory=dict) + test_results: dict[str, str] = field(default_factory=dict) + + async def read(self, client: httpx.AsyncClient | None, settings: Any) -> Result: + raise NotImplementedError + + +class NutMonitor: + """Polls upsd once and returns every UPS. Also owns the transition bookkeeping.""" + + def __init__(self, settings: Any, simulator: Any | None = None) -> None: + self.settings = settings + self.simulator = simulator + self._status: dict[str, str] = {} + self._test_results: dict[str, str] = {} + self.simulated = simulator is not None + + async def collect(self) -> Result: + now = time.time() + readings: list[nut.UpsReading] = [] + if self.simulator is not None: + readings = await self.simulator.read() + else: + async with nut.NutClient( + self.settings.nut_host, self.settings.nut_port, + timeout=self.settings.nut_timeout, + username=self.settings.nut_user or None, + password=self.settings.nut_password or None, + ) as client: + for entry in await client.list_ups(): + ups = entry["name"] + reading = nut.UpsReading(name=ups, description=entry.get("description", ""), + ts=now) + reading.vars = await client.list_vars(ups) + try: + reading.commands = await client.list_cmds(ups) + except nut.NutError: + reading.commands = [] + try: + reading.writable = await client.list_rw_vars(ups) + except nut.NutError: + reading.writable = [] + readings.append(reading) + + samples = [] + events = [] + payloads = [] + for reading in readings: + card = nut.payload(reading, now=now) + card["commands"] = nut.available_commands( + reading.commands, + allow_actions=self.settings.actions_allowed, + read_only=self.settings.read_only, + # the simulator needs no credentials; refusing its commands would make the + # dev buttons dead while claiming a permissions problem that does not exist + has_credentials=bool(self.settings.nut_user and self.settings.nut_password) or self.simulated, + on_battery=reading.on_battery, + ) + payloads.append(card) + samples.extend( + Sample("ups", reading.name, metric, value, reading.ts) + for metric, value in nut.history_samples(reading).items() + ) + events.extend(self._transitions(reading, now)) + + payload = { + "ts": now, + "server": None if self.simulated else f"{self.settings.nut_host}:{self.settings.nut_port}", + "simulated": self.simulated, + "ups": payloads, + "actions_enabled": self.settings.actions_allowed, + } + return Result(payload=payload, samples=samples, events=events) + + def _transitions(self, reading: nut.UpsReading, now: float) -> list[Event]: + """Log a state change once, at the severity of the state we landed in.""" + out: list[Event] = [] + status = reading.get("status") or "" + previous = self._status.get(reading.name) + if previous is None: + self._status[reading.name] = status + return out + if status != previous: + severities = {s["severity"] for s in reading.states()} + severity = ("error" if "fail" in severities else "warn" if "warn" in severities + else "info") + out.append(Event(now, "ups", reading.name, "status", severity, + f"{reading.name}: {previous or '(no state)'} → {status or '(no state)'}", + {"from": previous, "to": status, + "states": reading.states(), + "charge_pct": reading.number("charge_pct"), + "runtime_seconds": reading.number("battery_runtime")})) + self._status[reading.name] = status + + result = reading.get("test_result") + if result and self._test_results.get(reading.name) != result: + first_seen = reading.name not in self._test_results + self._test_results[reading.name] = result + if not first_seen: # the first sighting is not news, a change is + passed = result.strip().lower() in ("ok", "passed", "pass", "completed") + out.append(Event(now, "ups", reading.name, "test", + "info" if passed else "warn", + f"{reading.name}: self-test reported {result}")) + return out + + +# --------------------------------------------------------------------------- # +# Job factories (what app.py wires into the Hub) +# --------------------------------------------------------------------------- # +def host_jobs(targets: list[Target], client: httpx.AsyncClient, host_collector: HostCollector, + interval: float) -> list[Job]: + return [ + Job(f"host:{target.name}", "system", target.name, interval, + lambda t=target: fetch_host_snapshot(client, t, host_collector)) + for target in targets + ] + + +def llm_jobs(settings: Any, client: httpx.AsyncClient, rates: RateEstimator) -> list[Job]: + jobs: list[Job] = [] + if settings.gufo_url: + jobs.append(Job("llm:gufo", "llm", "gufo", settings.llm_interval, + lambda: fetch_engine(client, "gufo", settings.gufo_url, + settings.gufo_api_key, rates, with_slots=True))) + if settings.vllm_url: + jobs.append(Job("llm:vllm", "llm", "vLLM", settings.llm_interval, + lambda: fetch_engine(client, "vllm", settings.vllm_url, + settings.vllm_api_key, rates))) + return jobs + + +def nut_job(settings: Any, monitor: NutMonitor, client: httpx.AsyncClient) -> Job: + return Job("nut", "ups", "NUT server", settings.nut_interval, monitor.collect, + enabled=settings.nut_enabled) diff --git a/src/dashboards/config.py b/src/dashboards/config.py new file mode 100644 index 0000000..26525f4 --- /dev/null +++ b/src/dashboards/config.py @@ -0,0 +1,199 @@ +"""Environment-driven configuration. Read once at import; restart to pick up changes. + +There is no config file format and no CLI flags on purpose: deployment is a container with +env vars (see Containerfile + the deployment repo's quadlet env template). A `.env` file is +only a convenience for the dev loop — it seeds the environment and never overrides a real +variable. Every value has a safe default EXCEPT the sources, which default to *unset* — an +unset source means "this deployment does not run it", which must render as "not configured", +never as a red panel for a server the operator never installed. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def load_env_file() -> Path | None: + """Seed the environment from `DASH_ENV_FILE` (default: `.env` at the repo root). + + `override=False`, so a variable the container, systemd unit, or shell already set always + beats the file. `.env.example` is documentation and is never loaded, and a missing file is + the normal case in production — not a warning. + """ + raw = os.environ.get("DASH_ENV_FILE", "").strip() + path = Path(raw).expanduser() if raw else ROOT / ".env" + if not path.is_file() or path.name == ".env.example": + return None + from dotenv import load_dotenv # noqa: PLC0415 - only needed when a file exists + + load_dotenv(path, override=False) + return path + + +# This runs here, at the top of the module, because `Settings` below reads the environment +# in its field defaults — which are evaluated once, when the class body executes. Loading the +# file from `load()` instead would seed the environment after those defaults had already been +# taken, and the file would silently do nothing. +load_env_file() + + +def _env(name: str, default: str = "") -> str: + return os.environ.get(name, default).strip() + + +def _num(name: str, default: float) -> float: + raw = _env(name) + try: + return float(raw) if raw else default + except ValueError: + return default + + +def _int(name: str, default: int) -> int: + return int(_num(name, float(default))) + + +def _flag(name: str, default: bool = False) -> bool: + raw = _env(name).lower() + if not raw: + return default + return raw in ("1", "true", "yes", "on") + + +def _base_url(name: str) -> str: + """Normalised base URL, or `""` when unset/blank. No default host, ever.""" + return _env(name).rstrip("/") + + +# A target is `name=url` (comma or whitespace separated). The URL may carry a bearer +# token as userinfo (`http://SECRET@host:9120`) so per-host agent tokens need no extra +# variable; the token never leaves the backend, it is stripped from every /api payload. +_TARGET_SPLIT = re.compile(r"[,\s]+") +_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]*$") + + +@dataclass(frozen=True) +class Target: + """One scrape endpoint. `name` is stable UI text and the history DB's host key.""" + + name: str + url: str + headers: dict[str, str] = field(default_factory=dict) + + @property + def display_url(self) -> str: + """URL with any userinfo removed — safe to show in the browser.""" + return re.sub(r"^(https?://)[^/@]+@", r"\1", self.url) + + +def parse_targets(raw: str) -> tuple[list[Target], list[str]]: + """Parse `name=url,name=url` into targets, plus a list of problems found. + + Bad entries are reported, not raised: one typo in a list of six hosts must not + take the whole dashboard down. The problems surface in /api/config so they are + diagnosable without reading container logs. + """ + targets: list[Target] = [] + problems: list[str] = [] + for chunk in _TARGET_SPLIT.split(raw.strip()): + if not chunk: + continue + name, sep, url = chunk.partition("=") + name, url = name.strip(), url.strip().rstrip("/") + if not sep or not url: + problems.append(f"{chunk!r} is not name=url") + continue + if not _NAME_RE.match(name): + problems.append(f"{name!r} is not a valid target name") + continue + if not url.startswith(("http://", "https://")): + problems.append(f"{name}: url must start with http:// or https://") + continue + headers: dict[str, str] = {} + m = re.match(r"^(https?://)([^/@]+)@(.*)$", url) + if m: + scheme, token, rest = m.groups() + url = f"{scheme}{rest}" + headers["Authorization"] = f"Bearer {token}" + targets.append(Target(name=name, url=url, headers=headers)) + return targets, problems + + +@dataclass(frozen=True) +class Settings: + # --- server ------------------------------------------------------------------- + host: str = _env("DASH_HOST", "127.0.0.1") + port: int = _int("DASH_PORT", 8090) + db_path: str = _env("DASH_DB", str(ROOT / "data" / "dashboards.sqlite3")) + read_only: bool = _flag("DASH_READ_ONLY") # kills every mutating endpoint + + # --- system dashboard --------------------------------------------------------- + host_targets: tuple[Target, ...] = () + host_target_problems: tuple[str, ...] = () + host_interval: float = _num("HOST_INTERVAL_SECONDS", 10) + host_timeout: float = _num("HOST_TIMEOUT_SECONDS", 4) + + # --- llm dashboard ------------------------------------------------------------ + gufo_url: str = _base_url("GUFO_BASE_URL") + gufo_api_key: str = _env("GUFO_API_KEY") + vllm_url: str = _base_url("VLLM_BASE_URL") + vllm_api_key: str = _env("VLLM_API_KEY") + llm_interval: float = _num("LLM_INTERVAL_SECONDS", 2) + llm_timeout: float = _num("LLM_TIMEOUT_SECONDS", 3) + rate_window: float = _num("RATE_WINDOW_SECONDS", 10) + + # --- ups dashboard ------------------------------------------------------------ + nut_host: str = _env("NUT_HOST").removeprefix("nut://") + nut_port: int = _int("NUT_PORT", 3493) + nut_user: str = _env("NUT_USER") + nut_password: str = _env("NUT_PASSWORD") + nut_interval: float = _num("NUT_INTERVAL_SECONDS", 5) + nut_timeout: float = _num("NUT_TIMEOUT_SECONDS", 4) + # Instant commands (battery tests, beeper, calibration) are the only write path in + # the app. Off unless BOTH the env flag is set and DASH_READ_ONLY is not. + nut_allow_actions: bool = _flag("NUT_ALLOW_ACTIONS") + + # --- history ------------------------------------------------------------------ + raw_keep_hours: float = _num("RETAIN_RAW_HOURS", 48) + minute_keep_days: float = _num("RETAIN_MINUTE_DAYS", 30) + hour_keep_days: float = _num("RETAIN_HOUR_DAYS", 400) + + # --- browser polling defaults ------------------------------------------------- + poll_ms_system: int = _int("POLL_MS_SYSTEM", 10_000) + poll_ms_llm: int = _int("POLL_MS_LLM", 2_000) + poll_ms_ups: int = _int("POLL_MS_UPS", 5_000) + + @property + def nut_enabled(self) -> bool: + return bool(self.nut_host) + + @property + def actions_allowed(self) -> bool: + """Coarse switch. Whether a *specific* command can run also depends on the user + having `actions` in upsd.users, which is checked per command (and reported to the + UI as a reason) — requiring credentials here would make the label wrong for the + simulator and would hide the actual fix from the operator.""" + return self.nut_allow_actions and not self.read_only + + @property + def any_source_enabled(self) -> bool: + return bool(self.host_targets) or bool(self.gufo_url or self.vllm_url) or self.nut_enabled + + +def load() -> Settings: + """Settings from the environment as it stands now. + + Every scalar was captured when this module was imported (see `load_env_file`); only the + target list is re-parsed per call, which is what makes a target fixable without a reload + in tests and tooling. + """ + targets, problems = parse_targets(_env("HOST_TARGETS")) + return Settings(host_targets=tuple(targets), host_target_problems=tuple(problems)) + + +SETTINGS = load() diff --git a/src/dashboards/debug.py b/src/dashboards/debug.py new file mode 100644 index 0000000..39cc7e9 --- /dev/null +++ b/src/dashboards/debug.py @@ -0,0 +1,24 @@ +"""Optional debugpy attach, gated by the DEBUGPY env var. + +Default (unset / DEBUGPY=0): this module does nothing and imports nothing heavy. +DEBUGPY=1: listens on DEBUGPY_PORT (default 5678) without blocking startup. +""" + +from __future__ import annotations + +import os + + +def maybe_listen() -> bool: + """Start a debugpy listener when DEBUGPY is truthy. Returns True if it did.""" + if os.environ.get("DEBUGPY", "").lower() not in ("1", "true", "yes", "on"): + return False + port = int(os.environ.get("DEBUGPY_PORT", "5678")) + try: + import debugpy # noqa: PLC0415 - deliberately imported only when requested + except ImportError: + print(f"[debug] DEBUGPY is set but debugpy is not installed (uv sync --dev); port {port} unused", flush=True) + return False + debugpy.listen(("0.0.0.0", port)) + print(f"[debug] debugpy listening on 0.0.0.0:{port}", flush=True) + return True diff --git a/src/dashboards/poller.py b/src/dashboards/poller.py new file mode 100644 index 0000000..d901eba --- /dev/null +++ b/src/dashboards/poller.py @@ -0,0 +1,230 @@ +"""Polling framework: one asyncio loop per source, independent health for each. + +Rules this enforces, because violating them is what makes dashboards untrustworthy: + +* **Isolation.** A source that throws, hangs, or 500s must never blank, block, or delay + another one. Each job has its own task, its own timeout (the collector's problem) and + its own health record. +* **Last good value wins.** A failed poll keeps the previous payload and marks it stale + (`stale 12s` in the UI) instead of showing zeros or `n/a` everywhere. +* **Transitions are events, spam is not.** ok→fail and fail→ok write to the event log; + 500 consecutive failures write one line. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Any + +from .store import Event, Sample, SampleStore + +logger = logging.getLogger("dashboards.poller") + + +@dataclass +class Result: + """What one successful collection produced.""" + + payload: dict[str, Any] + samples: list[Sample] = field(default_factory=list) + events: list[Event] = field(default_factory=list) + + +@dataclass +class SourceState: + key: str # "host:driveripper", "llm:gufo", "nut" + dashboard: str # "system" | "llm" | "ups" + label: str + configured: bool = True + ok: bool = False + error: str | None = None + attempts: int = 0 + successes: int = 0 + failures: int = 0 + consecutive_failures: int = 0 + last_attempt: float | None = None + last_ok: float | None = None + latency_ms: float | None = None + samples_written: int = 0 + payload: dict[str, Any] | None = None + + def as_json(self, now: float | None = None) -> dict[str, Any]: + """Health block for the browser. `age_seconds` only counts successes, so a dead + source reads as "stale 4m" rather than pretending to be current.""" + now = time.time() if now is None else now + return { + "key": self.key, + "dashboard": self.dashboard, + "label": self.label, + "configured": self.configured, + "ok": self.ok, + "error": self.error, + "attempts": self.attempts, + "successes": self.successes, + "failures": self.failures, + "consecutive_failures": self.consecutive_failures, + "last_attempt": self.last_attempt, + "last_ok": self.last_ok, + "age_seconds": (now - self.last_ok) if self.last_ok else None, + "latency_ms": self.latency_ms, + "samples_written": self.samples_written, + } + + +class Job: + """A named poll loop. `fn` does the work; everything else lives here so every + dashboard gets identical health, staleness, and event semantics for free.""" + + def __init__(self, key: str, dashboard: str, label: str, interval: float, + fn: Callable[[], Awaitable[Result]], *, enabled: bool = True, + grace_failures: int = 1) -> None: + self.key = key + self.dashboard = dashboard + self.label = label + self.interval = max(0.5, interval) + self.fn = fn + self.enabled = enabled + self.grace_failures = grace_failures + self.state = SourceState(key=key, dashboard=dashboard, label=label, configured=enabled) + self._down_logged = False + self._stop = asyncio.Event() + self._task: asyncio.Task[None] | None = None + self._inflight = False + + async def run_once(self, store: SampleStore) -> bool: + """One poll, with the health bookkeeping. Returns True on success.""" + if self._inflight: # a slow poll must never overlap itself + return False + self._inflight = True + now = time.time() + self.state.attempts += 1 + self.state.last_attempt = now + was_ok = self.state.ok + try: + started = time.perf_counter() + result = await self.fn() + self.state.latency_ms = round((time.perf_counter() - started) * 1000.0, 1) + except Exception as exc: # noqa: BLE001 - a collector must never kill the loop + self.state.ok = False + self.state.failures += 1 + self.state.consecutive_failures += 1 + self.state.error = f"{type(exc).__name__}: {exc}" + # One event per outage, not per failed poll. A source that has never answered + # gets a warn rather than silence — otherwise a host misconfigured from the first + # boot leaves no trace in the log at all. + if (self.state.consecutive_failures >= self.grace_failures and not self._down_logged): + self._down_logged = True + store.log_event(Event(now, self.dashboard, self.label, "unreachable", + "error" if self.state.successes else "warn", + self.state.error + + ("" if self.state.successes else " (never answered)"))) + return False + finally: + self._inflight = False + + self.state.ok = True + self.state.successes += 1 + self.state.payload = result.payload + # Recovery is news only if we ever said it was down: on a fresh start every source + # would otherwise log "answering again", which reads like an outage happened. + if self._down_logged or (not was_ok and self.state.failures): + count = self.state.consecutive_failures + store.log_event(Event(now, self.dashboard, self.label, "reachable", "info", + f"{self.label} is answering again" + + (f" after {count} failure{'s' if count != 1 else ''}" + if count else ""))) + self._down_logged = False + self.state.consecutive_failures = 0 + self.state.last_ok = now + self.state.error = None + self.state.samples_written += store.add_many(result.samples) + for event in result.events: + store.log_event(event) + return True + + async def _loop(self, store: SampleStore) -> None: + while not self._stop.is_set(): + started = time.perf_counter() + await self.run_once(store) + remaining = self.interval - (time.perf_counter() - started) + try: + await asyncio.wait_for(self._stop.wait(), timeout=max(0.2, remaining)) + except TimeoutError: + continue + + def start(self, store: SampleStore) -> None: + if self._task is None and self.enabled: + self._task = asyncio.create_task(self._loop(store), name=f"poll:{self.key}") + + async def stop(self) -> None: + self._stop.set() + if self._task is not None: + self._task.cancel() + # the cancellation is the point; the task's own outcome is not our problem here. + # CancelledError is a BaseException in 3.8+, so it has to be named explicitly. + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._task + self._task = None + + +class Hub: + """Owns the store and every poll loop; the only thing the API layer talks to.""" + + def __init__(self, store: SampleStore) -> None: + self.store = store + self.jobs: dict[str, Job] = {} + self.started_at = time.time() + self._maintenance: asyncio.Task[None] | None = None + + def add(self, job: Job) -> Job: + self.jobs[job.key] = job + return job + + def start(self) -> None: + for job in self.jobs.values(): + job.start(self.store) + if self._maintenance is None: + self._maintenance = asyncio.create_task(self._maintain_loop(), name="maintain") + + async def stop(self) -> None: + if self._maintenance is not None: + self._maintenance.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._maintenance + self._maintenance = None + for job in self.jobs.values(): + await job.stop() + + async def _maintain_loop(self, interval: float = 60.0) -> None: + """Rollup + prune. Runs in a thread: SQLite writes would otherwise stall the loop.""" + # imported here because only this task needs the thread bridge + from anyio.to_thread import run_sync # noqa: PLC0415 + + while True: + try: + await run_sync(self.store.maintain) + except Exception: + # Logged, never swallowed: a broken rollup means every chart past the raw + # window is quietly empty, and "it retried" is not a record of that. + logger.exception("store maintenance failed; long-window charts may be stale") + await asyncio.sleep(interval) + + def states(self, dashboard: str | None = None) -> list[dict[str, Any]]: + now = time.time() + return [ + job.state.as_json(now) + for job in self.jobs.values() + if dashboard is None or job.dashboard == dashboard + ] + + def payloads(self, dashboard: str | None = None) -> dict[str, dict[str, Any]]: + return { + job.key: job.state.payload + for job in self.jobs.values() + if (dashboard is None or job.dashboard == dashboard) and job.state.payload is not None + } diff --git a/src/dashboards/prom.py b/src/dashboards/prom.py new file mode 100644 index 0000000..1b610e4 --- /dev/null +++ b/src/dashboards/prom.py @@ -0,0 +1,177 @@ +"""Prometheus text-exposition parser. In-house, zero dependencies. + +Only used for the upstreams we do not control: llama.cpp/gufo (`llamacpp:*`) and vLLM +(`vllm:*`). Our own host agent serves JSON, so nothing else here needs this parser and we +never pull in `prometheus_client`. + +Ported from llm_monitor, where it survived a year of both engines renaming series under +it. Keep the aggregation rules (see `Prom.scalar`): they are what stops a summed ratio +from exceeding 1.0 and inventing a percentile. +""" + +from __future__ import annotations + +import math +import re +from collections.abc import Sequence +from typing import Any + +_NAME_RE = r"[a-zA-Z_:][a-zA-Z0-9_:]*" +# name{label="v",...} value [timestamp] +# - the label block is greedy so a '}' inside a quoted value cannot truncate the match; +# the anchored value/end-of-line does that instead. +# - whitespace before '{' and a trailing '# ...' comment are not part of the official +# format, but tolerating them means a quirky exporter shows a number instead of 'n/a'. +_LINE_RE = re.compile( + rf"^(?P{_NAME_RE})(?:[ \t]*\{{(?P.*)\}})?[ \t]+" + r"(?P[+-]?(\d+\.?\d*(e[+-]?\d+)?|inf|nan))" + r"(?:[ \t]+(?P-?\d+))?[ \t]*(?:#.*)?$", + re.IGNORECASE, +) +_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_.:]*)="((?:[^"\\]|\\.)*)"') + + +def _unquote(value: str) -> str: + return re.sub(r"\\(.)", r"\1", value) + + +class Prom: + """Parsed exposition: name -> [(labels, value), ...].""" + + def __init__(self, text: str) -> None: + self.series: dict[str, list[tuple[dict[str, str], float]]] = {} + self.parse_errors = 0 + for raw in text.splitlines(): + line = raw.strip() + if not line or line[0] == "#": + continue + m = _LINE_RE.match(line) + if not m: + self.parse_errors += 1 + continue + try: + value = float(m.group("value")) + except ValueError: # pragma: no cover - regex keeps this unreachable + self.parse_errors += 1 + continue + labels = {_unquote(k): _unquote(v) for k, v in _LABEL_RE.findall(m.group("labels") or "")} + self.series.setdefault(m.group("name"), []).append((labels, value)) + + def resolve(self, names: Sequence[str]) -> str | None: + """First name from `names` that is actually present.""" + return next((n for n in names if n in self.series), None) + + def resolve_hist(self, names: Sequence[str]) -> str | None: + """Like resolve() for histograms: the bare name never appears, only *_bucket/_sum/_count.""" + for n in names: + if any(f"{n}{suffix}" in self.series for suffix in ("_bucket", "_sum", "_count")): + return n + return None + + def values(self, names: Sequence[str], label_match: dict[str, str] | None = None) -> list[float]: + name = self.resolve(names) + if name is None: + return [] + if not label_match: + return [v for _labels, v in self.series[name]] + return [ + v for labels, v in self.series[name] + if all(labels.get(k) == x for k, x in label_match.items()) + ] + + def scalar(self, names: Sequence[str], agg: str = "sum", + label_match: dict[str, str] | None = None) -> float | None: + """Aggregate every matching series, or None when absent. + + agg="sum" counters and per-engine gauges — summing label sets is the fleet total. + agg="max" usage ratios — report the most pressured engine, never a sum over 1.0. + agg="avg" rate gauges — an average across label sets. + """ + found = self.values(names, label_match) + finite = [v for v in found if math.isfinite(v)] + if not finite: + return None + if agg == "max": + return max(finite) + if agg == "avg": + return sum(finite) / len(finite) + return sum(finite) + + def by_label(self, names: Sequence[str], label: str) -> dict[str, float]: + name = self.resolve(names) + if name is None: + return {} + out: dict[str, float] = {} + for labels, value in self.series[name]: + if not math.isfinite(value): + continue + key = labels.get(label, "") + out[key] = out.get(key, 0.0) + value + return out + + def histogram(self, names: Sequence[str]) -> dict[str, Any] | None: + """Mean (+ p95 when interpolable) of a `_sum`/`_count`/`_bucket`.""" + base = self.resolve_hist(names) + if base is None: + return None + total = self.scalar([f"{base}_sum"]) + count = self.scalar([f"{base}_count"]) + buckets = self._buckets(base) + out: dict[str, Any] = { + "count": count, + "sum": total, + "mean": (total / count) if (total is not None and count) else None, + "p95": None, + "source": base, + } + # Interpolate only for a single label set: mixing per-DP-rank histograms into one + # cumulative curve invents a bogus p95. + if buckets and count and self._bucket_label_groups(base) <= 1: + out["p95"] = _quantile(buckets, count, 0.95) + return out + + def _bucket_label_groups(self, base: str) -> int: + groups = set() + for labels, _v in self.series.get(f"{base}_bucket", []): + groups.add(tuple(sorted((k, v) for k, v in labels.items() if k != "le"))) + return len(groups) + + def _buckets(self, base: str) -> list[tuple[float, float]]: + out: list[tuple[float, float]] = [] + for labels, value in self.series.get(f"{base}_bucket", []): + le = labels.get("le") + if le is None or not math.isfinite(value): + continue + bound = math.inf if le.lower() in ("+inf", "inf") else float(le) + out.append((bound, value)) + out.sort(key=lambda item: item[0]) + return out + + +def _quantile(buckets: list[tuple[float, float]], count: float, q: float) -> float | None: + """Linear interpolation inside the bucket containing the `q` quantile.""" + target = q * count + prev_bound, prev_cum = 0.0, 0.0 + for bound, cum in buckets: + if cum >= target: + if math.isinf(bound): + return prev_bound if math.isfinite(prev_bound) else None + span = cum - prev_cum + if span <= 0: + return bound + return prev_bound + (bound - prev_bound) * ((target - prev_cum) / span) + prev_bound, prev_cum = bound, cum + return None + + +def field(value: float | None, source: str | None, derived: bool = False, + window: float | None = None) -> dict[str, Any]: + """Uniform metric cell: the UI renders `n/a` whenever value is None. + + `window` is the Δt a rate was measured over, so the UI can say `Δ10.0s` instead of + implying a scraped gauge it never got. + """ + cell: dict[str, Any] = {"value": value, "source": source, "derived": derived} + if window is not None: + cell["window"] = window + return cell diff --git a/src/dashboards/rates.py b/src/dashboards/rates.py new file mode 100644 index 0000000..b9569b7 --- /dev/null +++ b/src/dashboards/rates.py @@ -0,0 +1,64 @@ +"""Δcounter / Δt over a rolling window — the one piece of mutable state in the app. + +Both upstreams export cumulative counters (tokens generated, jiffies spent, bytes read) +and the dashboards need per-second rates. A bare 1s delta jitters between 0 and spikes +because work arrives in bursts, so the rate is measured over a window (default 10s) and +the window that was actually used travels with the value — the UI prints `Δ10.0s` rather +than implying a scraped gauge it never got. + +Keyed by the *resolved* upstream series name (e.g. `llamacpp:tokens_predicted_total`, or +`driveripper:cpu.total`), so a renamed counter starts a fresh window instead of diffing +two different series into a fabricated rate, and two hosts can never read each other's +samples. +""" + +from __future__ import annotations + +import math +import time +from collections import deque + + +class RateEstimator: + def __init__(self, span: float = 10.0, max_samples: int = 512) -> None: + self.span = span + self.max_samples = max_samples + self._samples: dict[str, deque[tuple[float, float]]] = {} + + def rate(self, key: str, value: float | None, now: float | None = None) -> tuple[float | None, float | None]: + """(per-second, window actually used). (None, window|None) when not computable. + + Never returns a negative rate: a counter going backwards means a restart (or a + renamed series), so the window is dropped and one scrape answers `n/a`. + """ + if value is None or not math.isfinite(value): + return None, None + now = time.time() if now is None else now + samples = self._samples.setdefault(key, deque()) + if samples and value < samples[-1][1]: + samples.clear() + samples.append((now, value)) + while len(samples) > self.max_samples: + samples.popleft() + if len(samples) < 2: + return None, None + + inside = [point for point in samples if now - point[0] <= self.span] + base = inside[0] if len(inside) > 1 else samples[-2] + dt, dv = now - base[0], value - base[1] + cutoff = now - 2 * self.span + while samples and samples[0][0] < cutoff and len(samples) > 2: + samples.popleft() + if dt <= 0 or dv < 0: + return None, dt if dt > 0 else None + return dv / dt, dt + + def forget(self, prefix: str) -> int: + """Drop every window whose key starts with `prefix` (a target that went away).""" + keys = [k for k in self._samples if k.startswith(prefix)] + for key in keys: + del self._samples[key] + return len(keys) + + def keys(self) -> list[str]: + return list(self._samples) diff --git a/src/dashboards/sources/__init__.py b/src/dashboards/sources/__init__.py new file mode 100644 index 0000000..8c7c1c8 --- /dev/null +++ b/src/dashboards/sources/__init__.py @@ -0,0 +1,7 @@ +"""Data sources. Each module turns one upstream's raw payload into our cell shape. + +host -> our own stdlib agent (JSON) +llm -> gufo / vLLM Prometheus expositions (parsed by our own Prom class) +nut -> the upsd wire protocol (reads + instant commands) +nut_sim-> a labelled simulator for development, never a substitute for real hardware +""" diff --git a/src/dashboards/sources/host.py b/src/dashboards/sources/host.py new file mode 100644 index 0000000..44a7939 --- /dev/null +++ b/src/dashboards/sources/host.py @@ -0,0 +1,416 @@ +"""Host source: reads `agent/host_agent.py` JSON and turns counters into rates. + +Division of labour, and why: + +* the agent reads /proc and /sys and reports **cumulative counters** (it cannot know how + long the interval was, and it must not have to remember who polled it last); +* this module owns the **window**: every per-second number is Δcounter/Δt over + `RateEstimator`, and carries the window that produced it, so the UI can print `Δ10.0s`. +* absent values stay None. A host with no GPU has no gpu series; it does not have a GPU at + 0% busy (that would say "idle", which is a different claim from "there is nothing here"). + +Metric names in the history DB use `@stream` for per-device series: +`cpu.usage_pct@core7`, `disk.used_pct@/srv`, `gpu.busy_pct@0`, `net.rx_bytes_per_s@eth0`. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +from ..poller import Result +from ..rates import RateEstimator +from ..store import Sample + +# /proc/stat jiffies we treat as "not doing useful work". +CPU_IDLE_MODES = ("idle", "iowait") + + +@dataclass +class HostTargetState: + """Per-target memory across polls: only what a *comparison* needs.""" + + boot_ts: float | None = None + hostname: str | None = None + + +@dataclass +class HostCollector: + """Holds the rate windows and the reboot detector for every host.""" + + rates: RateEstimator + targets: dict[str, HostTargetState] = field(default_factory=dict) + + def target_state(self, name: str) -> HostTargetState: + return self.targets.setdefault(name, HostTargetState()) + + +def _rate(collector: HostCollector, host: str, key: str, value: float | None, + now: float) -> tuple[float | None, float | None]: + return collector.rates.rate(f"{host}:{key}", value, now) + + +def _ratio_percent(numerator: float | None, denominator: float | None) -> float | None: + if numerator is None or denominator is None or denominator <= 0: + return None + return max(0.0, min(100.0, numerator / denominator * 100.0)) + + +def parse_host_snapshot(name: str, snapshot: dict[str, Any], collector: HostCollector, + *, now: float | None = None) -> Result: + """One agent snapshot -> (UI payload, history samples, transition events).""" + now = time.time() if now is None else now + samples: list[Sample] = [] + events: list[Any] = [] + ts = float(snapshot.get("ts") or now) + + info = snapshot.get("host") or {} + state = collector.target_state(name) + boot_ts = info.get("boot_ts") + if boot_ts is not None: + if state.boot_ts is not None and abs(boot_ts - state.boot_ts) > 1.0: + events.append(_event(now, name, "reboot", "warn", + f"{name} rebooted (agent boot time moved from " + f"{state.boot_ts:.0f} to {boot_ts:.0f})")) + state.boot_ts = boot_ts + if info.get("hostname"): + state.hostname = str(info["hostname"]) + + # ---------------------------------------------------------------- CPU ---- # + cpu_raw = snapshot.get("cpu") or {} + modes = cpu_raw.get("modes") or {} + total_rate, window = _rate(collector, name, "cpu.total", modes.get("total"), now) + # Every counter is sampled on every poll, including the first. Gating the idle modes on + # `total_rate is not None` (as this used to) meant idle was never seeded on poll 1, so + # usage_pct stayed n/a until poll 3 — a dashboard that boots two intervals late, and a + # fleet average that counts a live host as missing for its first minute. + idle_rate, idle_ok = 0.0, True + for mode in CPU_IDLE_MODES: + value, _ = _rate(collector, name, f"cpu.{mode}", modes.get(mode), now) + if value is None: + idle_ok = False + else: + idle_rate += value + busy_rate = max(0.0, total_rate - idle_rate) if (idle_ok and total_rate is not None) else None + usage_pct = (busy_rate / total_rate * 100.0) if (busy_rate is not None and total_rate) else None + + mode_pct: dict[str, float | None] = {} + for mode in ("user", "system", "iowait", "irq", "softirq", "steal", "nice"): + value, _ = _rate(collector, name, f"cpu.{mode}", modes.get(mode), now) + mode_pct[mode] = _ratio_percent(value, total_rate) if total_rate else None + + core_usage: list[dict[str, Any]] = [] + for core in cpu_raw.get("cores") or []: + core_modes = core.get("modes") or {} + core_total, _ = _rate(collector, name, f"cpu.core{core['core']}.total", + core_modes.get("total"), now) + core_idle = 0.0 + core_ok = True + for mode in CPU_IDLE_MODES: + value, _ = _rate(collector, name, f"cpu.core{core['core']}.{mode}", + core_modes.get(mode), now) + if value is None: + core_ok = False + else: + core_idle += value + core_busy = (core_total - core_idle) if (core_ok and core_total is not None) else None + value = _ratio_percent(core_busy, core_total) if core_total else None + core_usage.append({"core": core["core"], "usage_pct": value}) + if value is not None: + samples.append(Sample("host", name, f"cpu.usage_pct@core{core['core']}", value, ts)) + if usage_pct is not None: + samples.append(Sample("host", name, "cpu.usage_pct", usage_pct, ts)) + for mode, value in mode_pct.items(): + if value is not None: + samples.append(Sample("host", name, f"cpu.{mode}_pct", value, ts)) + + ctxt_rate, _ = _rate(collector, name, "cpu.ctxt", cpu_raw.get("ctxt_total"), now) + forks_rate, _ = _rate(collector, name, "cpu.processes", cpu_raw.get("processes_total"), now) + + # -------------------------------------------------------------- Memory --- # + mem = snapshot.get("memory") or {} + mem_total = mem.get("total_bytes") + mem_used = mem.get("used_bytes") + mem_used_pct = _ratio_percent(mem_used, mem_total) + swap_total = mem.get("swap_total_bytes") or 0.0 + swap_used = mem.get("swap_used_bytes") or 0.0 + swap_used_pct = (swap_used / swap_total * 100.0) if swap_total else None + for key in ("total_bytes", "available_bytes", "used_bytes", "cached_bytes", "buffers_bytes", + "swap_total_bytes", "swap_used_bytes"): + if mem.get(key) is not None: + samples.append(Sample("host", name, f"mem.{key}", float(mem[key]), ts)) + if mem_used_pct is not None: + samples.append(Sample("host", name, "mem.used_pct", mem_used_pct, ts)) + if swap_used_pct is not None: + samples.append(Sample("host", name, "mem.swap_used_pct", swap_used_pct, ts)) + + # ----------------------------------------------------------- Filesystems - # + filesystems = [] + for fs in snapshot.get("filesystems") or []: + used_pct = fs.get("used_pct") + filesystems.append(fs) + if used_pct is not None: + samples.append(Sample("host", name, f"disk.used_pct@{fs['mountpoint']}", + float(used_pct), ts)) + if fs.get("used_bytes") is not None: + samples.append(Sample("host", name, f"disk.used_bytes@{fs['mountpoint']}", + float(fs["used_bytes"]), ts)) + scored = [fs for fs in filesystems if fs.get("used_pct") is not None] + worst = max(scored, key=lambda fs: fs["used_pct"]) if scored else None + + # -------------------------------------------------------------- Disk IO -- # + io_devices = [] + read_total = write_total = reads_total = writes_total = 0.0 + any_io = False + util_max: float | None = None + for dev in (snapshot.get("disk_io") or {}).get("devices") or []: + dev_name = dev.get("name") or "?" + read_rate, _ = _rate(collector, name, f"disk.{dev_name}.read_bytes", + dev.get("read_bytes_total"), now) + write_rate, _ = _rate(collector, name, f"disk.{dev_name}.write_bytes", + dev.get("write_bytes_total"), now) + r_ops, _ = _rate(collector, name, f"disk.{dev_name}.reads", dev.get("reads_total"), now) + w_ops, _ = _rate(collector, name, f"disk.{dev_name}.writes", dev.get("writes_total"), now) + io_ms_rate, _ = _rate(collector, name, f"disk.{dev_name}.io_time_ms", + dev.get("io_time_ms_total"), now) + # io_time is milliseconds spent with at least one request outstanding, so + # ms-per-second / 10 == percent of the window the device was busy. + util = None if io_ms_rate is None else max(0.0, min(100.0, io_ms_rate / 10.0)) + row = {"name": dev_name, "model": dev.get("model"), "rotational": dev.get("rotational"), + "size_bytes": dev.get("size_bytes"), + "read_bytes_per_s": read_rate, "write_bytes_per_s": write_rate, + "reads_per_s": r_ops, "writes_per_s": w_ops, "util_pct": util, + "in_flight": dev.get("in_flight")} + io_devices.append(row) + if read_rate is not None: + read_total += read_rate + any_io = True + if write_rate is not None: + write_total += write_rate + any_io = True + if r_ops is not None: + reads_total += r_ops + if w_ops is not None: + writes_total += w_ops + if util is not None: + util_max = util if util_max is None else max(util_max, util) + + # ------------------------------------------------------------- Network --- # + net_rows = [] + rx_total = tx_total = 0.0 + any_net = False + for iface in (snapshot.get("network") or {}).get("interfaces") or []: + iname = iface.get("name") or "?" + rx, _ = _rate(collector, name, f"net.{iname}.rx", iface.get("rx_bytes_total"), now) + tx, _ = _rate(collector, name, f"net.{iname}.tx", iface.get("tx_bytes_total"), now) + net_rows.append({"name": iname, "rx_bytes_per_s": rx, "tx_bytes_per_s": tx, + "speed_mbps": iface.get("speed_mbps"), + "rx_errors_total": iface.get("rx_errors_total"), + "tx_errors_total": iface.get("tx_errors_total"), + "rx_dropped_total": iface.get("rx_dropped_total"), + "tx_dropped_total": iface.get("tx_dropped_total")}) + if rx is not None: + rx_total += rx + any_net = True + samples.append(Sample("host", name, f"net.rx_bytes_per_s@{iname}", rx, ts)) + if tx is not None: + tx_total += tx + samples.append(Sample("host", name, f"net.tx_bytes_per_s@{iname}", tx, ts)) + + # ---------------------------------------------------------------- GPU ---- # + gpus = [] + for gpu in snapshot.get("gpus") or []: + index = gpu.get("index", len(gpus)) + stream = f"@{index}" + vram_used = gpu.get("vram_used_bytes") + vram_total = gpu.get("vram_total_bytes") + gtt_used = gpu.get("gtt_used_bytes") + gtt_total = gpu.get("gtt_total_bytes") + temps = gpu.get("temps") or {} + temp_c = max(temps.values()) if temps else None + pools_used = None if vram_used is None else vram_used + (gtt_used or 0.0) + pools_total = None if vram_total is None else vram_total + (gtt_total or 0.0) + row = { + "index": index, + "card": gpu.get("card"), + "model": gpu.get("model") or gpu.get("card") or f"GPU {index}", + # the short name is what fits a legend; the lspci string belongs in the tooltip + "model_full": gpu.get("model_full") or gpu.get("model"), + "vendor": gpu.get("vendor"), + "driver": gpu.get("driver"), + "pci_address": gpu.get("pci_address"), + "busy_pct": gpu.get("busy_pct"), + "vram_used_bytes": vram_used, + "vram_total_bytes": vram_total, + "vram_used_pct": _ratio_percent(vram_used, vram_total), + "gtt_used_bytes": gtt_used, + "gtt_total_bytes": gtt_total, + # an APU's model lives in GTT, so "VRAM 3%" and "memory 78%" can both be true; + # this is the honest whole-addressable-pool view, kept beside the two pools + "mem_used_bytes": pools_used, + "mem_total_bytes": pools_total, + "mem_used_pct": _ratio_percent(pools_used, pools_total), + "power_watts": gpu.get("power_watts"), + "temps": temps, + "temp_c": temp_c, + "clocks": gpu.get("clocks") or {}, + } + gpus.append(row) + for metric, value in (("busy_pct", row["busy_pct"]), ("vram_used_bytes", vram_used), + ("vram_used_pct", row["vram_used_pct"]), + ("mem_used_pct", row["mem_used_pct"]), + ("power_watts", row["power_watts"]), ("temp_c", temp_c)): + if value is not None: + samples.append(Sample("host", name, f"gpu.{metric}{stream}", float(value), ts)) + + # -------------------------------------------------------- Temperatures --- # + sensors = snapshot.get("temperatures") or [] + cpu_temp = snapshot.get("cpu_temperature_celsius") + values = [s["celsius"] for s in sensors if s.get("celsius") is not None] + max_temp = max(values) if values else None + nvme_values = [s["celsius"] for s in sensors + if s.get("chip") == "nvme" and s.get("celsius") is not None] + nvme_max = max(nvme_values) if nvme_values else None + if cpu_temp is not None: + samples.append(Sample("host", name, "temp.cpu_c", float(cpu_temp), ts)) + if max_temp is not None: + samples.append(Sample("host", name, "temp.max_c", float(max_temp), ts)) + if nvme_max is not None: + samples.append(Sample("host", name, "temp.nvme_max_c", float(nvme_max), ts)) + + # ------------------------------------------------------------ Processes -- # + processes = snapshot.get("processes") or {} + + # ------------------------------------------------------------ Payload ---- # + for metric, value in (("cpu.load1", cpu_raw.get("load1")), + ("cpu.load5", cpu_raw.get("load5")), + ("cpu.load15", cpu_raw.get("load15")), + ("cpu.freq_mhz", cpu_raw.get("freq_mhz")), + ("cpu.context_switches_per_s", ctxt_rate), + ("host.uptime_seconds", info.get("uptime_seconds")), + ("host.up", 1.0), + ("disk.read_bytes_per_s", read_total if any_io else None), + ("disk.write_bytes_per_s", write_total if any_io else None), + ("disk.reads_per_s", reads_total if any_io else None), + ("disk.writes_per_s", writes_total if any_io else None), + ("disk.util_pct", util_max), + ("net.rx_bytes_per_s", rx_total if any_net else None), + ("net.tx_bytes_per_s", tx_total if any_net else None)): + if value is not None: + samples.append(Sample("host", name, metric, float(value), ts)) + if processes.get("total") is not None: + samples.append(Sample("host", name, "host.processes", float(processes["total"]), ts)) + if worst is not None: + samples.append(Sample("host", name, "disk.used_pct", float(worst["used_pct"]), ts)) + + payload = { + "name": name, + "ts": ts, + "window_seconds": window, + "info": info, + "cpu": { + "usage_pct": usage_pct, + "modes_pct": mode_pct, + "load1": cpu_raw.get("load1"), + "load5": cpu_raw.get("load5"), + "load15": cpu_raw.get("load15"), + "cores": len(core_usage) or info.get("cpu_cores_logical"), + "core_usage_pct": [c["usage_pct"] for c in core_usage], + "freq_mhz": cpu_raw.get("freq_mhz"), + "freq_max_mhz": cpu_raw.get("freq_max_mhz"), + "context_switches_per_s": ctxt_rate, + "forks_per_s": forks_rate, + "procs_running": cpu_raw.get("procs_running"), + "procs_blocked": cpu_raw.get("procs_blocked"), + "temp_c": cpu_temp, + }, + "memory": { + "total_bytes": mem_total, + "used_bytes": mem_used, + "available_bytes": mem.get("available_bytes"), + "used_pct": mem_used_pct, + "cached_bytes": mem.get("cached_bytes"), + "buffers_bytes": mem.get("buffers_bytes"), + "swap_total_bytes": swap_total, + "swap_used_bytes": swap_used, + "swap_used_pct": swap_used_pct, + "committed_bytes": mem.get("committed_bytes"), + }, + "disks": { + "filesystems": filesystems, + "worst_used_pct": worst.get("used_pct") if worst else None, + "worst_mount": worst.get("mountpoint") if worst else None, + "read_bytes_per_s": read_total if any_io else None, + "write_bytes_per_s": write_total if any_io else None, + "reads_per_s": reads_total if any_io else None, + "writes_per_s": writes_total if any_io else None, + "util_pct": util_max, + "devices": io_devices, + }, + "network": { + "rx_bytes_per_s": rx_total if any_net else None, + "tx_bytes_per_s": tx_total if any_net else None, + "interfaces": net_rows, + }, + "gpus": gpus, + "temperatures": {"cpu_c": cpu_temp, "max_c": max_temp, "nvme_max_c": nvme_max, + "sensors": sensors}, + "processes": { + "total": processes.get("total"), + "top_cpu": processes.get("top_cpu") or [], + "top_rss": processes.get("top_rss") or [], + }, + "agent": {"duration_ms": snapshot.get("duration_ms"), "errors": snapshot.get("errors") or [], + "version": snapshot.get("agent_version")}, + } + return Result(payload=payload, samples=samples, events=events) + + +def fleet_summary(hosts: list[dict[str, Any]]) -> dict[str, Any]: + """Cross-host totals, computed from live payloads only. + + A host whose poll failed is not "0% busy" and is not in the average — it is absent, + and `hosts_live` vs `hosts_total` says so. Never silently include a dead host. + """ + # A host that has never answered is represented by a stub ({"name", "unreachable"}), so + # every field here is read with .get() — one unreachable agent must not 500 the fleet card. + hosts = [h if isinstance(h, dict) else {} for h in hosts] + live = [h for h in hosts if (h.get("cpu") or {}).get("usage_pct") is not None] + weighted = [(h["cpu"]["usage_pct"], (h.get("info") or {}).get("cpu_cores_logical") or 1) + for h in live] + weight_sum = sum(weight for _value, weight in weighted) + cpu_avg = (sum(value * weight for value, weight in weighted) / weight_sum) if weight_sum else None + mem_total = _sum((h.get("memory") or {}).get("total_bytes") for h in hosts) + mem_used = _sum((h.get("memory") or {}).get("used_bytes") for h in hosts) + disk_pcts = [p for h in hosts if (p := (h.get("disks") or {}).get("worst_used_pct"))] + gpus = [g for h in hosts for g in h.get("gpus") or []] + return { + "hosts_total": len(hosts), + "hosts_live": len(live), + "cpu_cores": _sum((h.get("info") or {}).get("cpu_cores_logical") for h in hosts), + "cpu_usage_pct_avg": cpu_avg, + "memory_total_bytes": mem_total, + "memory_used_bytes": mem_used, + "memory_used_pct": _ratio_percent(mem_used, mem_total), + "disk_worst_used_pct": max(disk_pcts) if disk_pcts else None, + "load1_total": _sum((h["cpu"] or {}).get("load1") for h in live), + "net_rx_bytes_per_s": _sum((h.get("network") or {}).get("rx_bytes_per_s") for h in hosts), + "net_tx_bytes_per_s": _sum((h.get("network") or {}).get("tx_bytes_per_s") for h in hosts), + "gpu_count": len(gpus), + "gpu_busy_pct_avg": (sum(g["busy_pct"] for g in gpus if g.get("busy_pct") is not None) + / max(1, sum(1 for g in gpus if g.get("busy_pct") is not None)) + if any(g.get("busy_pct") is not None for g in gpus) else None), + "power_estimate_w": None, # no whole-host power source yet; deliberately n/a, not 0 + } + + +def _sum(values: Any) -> float | None: + numbers = [float(v) for v in values if isinstance(v, int | float)] + return sum(numbers) if numbers else None + + +def _event(ts: float, host: str, kind: str, severity: str, message: str) -> Any: + from ..store import Event # noqa: PLC0415 - keeps this module import-light + + return Event(ts, "host", host, kind, severity, message) diff --git a/src/dashboards/sources/llm.py b/src/dashboards/sources/llm.py new file mode 100644 index 0000000..bffa4c5 --- /dev/null +++ b/src/dashboards/sources/llm.py @@ -0,0 +1,352 @@ +"""LLM engine source: gufo (llama.cpp-family) and vLLM, read-only. + +Ported from llm_monitor, where these field maps were verified against the deployed +servers rather than against documentation — the engines do not export the names people +expect. Do not "fix" a name here without scraping the real thing: + + curl -s $GUFO_BASE_URL/metrics | grep -v '^#' | sed -E 's/[{ ].*//' | sort -u + curl -s $VLLM_BASE_URL/metrics | grep -v '^#' | sed -E 's/[{ ].*//' | sort -u + +and drop the capture into tests/fixtures/. + +Two rules carried over verbatim, because they are the difference between a truthful and a +plausible dashboard: + +1. **Absent = n/a, never 0.** A renamed metric yields `{"value": null}` and the UI shows + `n/a` — 0 would claim "no traffic", which is a different statement. +2. **Throughput precedence: native aggregate → measured Δcounter/Δt → estimate → n/a.** + vLLM's native throughput gauge really is an aggregate, so it wins. gufo's are ONE + unlabeled gauge holding the last pass of ONE request, so the summed token counter wins + and the gauge survives only as `*_native`. That asymmetry is deliberate; both cases are + labelled in the UI with their provenance. +""" + +from __future__ import annotations + +import math +import time +from collections.abc import Sequence +from typing import Any + +from ..poller import Result +from ..prom import Prom, field +from ..rates import RateEstimator +from ..store import Sample + +GUFO_SERIES: dict[str, tuple[tuple[str, ...], str]] = { + "requests_processing": (("llamacpp:requests_processing", "llama:requests_processing"), "sum"), + "requests_deferred": (("llamacpp:requests_deferred", "llama:requests_deferred"), "sum"), + "kv_cache_usage_ratio": (("llamacpp:kv_cache_usage_ratio", "llama:kv_cache_usage_ratio"), "max"), + # gufo counts prompt cache hits separately: prompt_tokens_total EXCLUDES cached tokens. + "tokens_prompt_total": (("llamacpp:prompt_tokens_total", "llama:prompt_tokens_total", + "llamacpp:tokens_prompt_total"), "sum"), + "tokens_prompt_cached_total": (("llamacpp:prompt_tokens_cached_total", + "llama:prompt_tokens_cached_total"), "sum"), + "tokens_generated_total": (("llamacpp:tokens_predicted_total", "llamacpp:tokens_generated_total", + "llama:tokens_predicted_total", "llama:tokens_generated_total"), "sum"), + # NOT the throughput cell: prefill speed of one request's last pass. + "prompt_tokens_per_second_native": (("llamacpp:prompt_tokens_seconds", + "llamacpp:prompt_tokens_per_second", + "llama:prompt_tokens_seconds"), "avg"), + # NOT the throughput cell: one gauge holding one request's last decode pass. + "decode_tokens_per_second_native": (("llamacpp:predicted_tokens_seconds", + "llamacpp:decode_tokens_per_second", + "llama:predicted_tokens_seconds"), "avg"), + "n_tokens_max": (("llamacpp:n_tokens_max", "llama:n_tokens_max"), "max"), + # GPU context losses (gufo-specific): any non-zero value here is an incident. + "device_lost_total": (("gufo_device_lost_total",), "sum"), + # speculative decoding (MTP on this fleet): accept rate is the useful number. + "spec_drafts_total": (("llamacpp:spec_decode_num_drafts_total",), "sum"), + "spec_draft_tokens_total": (("llamacpp:spec_decode_num_draft_tokens_total",), "sum"), + "spec_accepted_tokens_total": (("llamacpp:spec_decode_num_accepted_tokens_total",), "sum"), +} + +VLLM_SERIES: dict[str, tuple[tuple[str, ...], str]] = { + "num_requests_running": (("vllm:num_requests_running",), "sum"), + "num_requests_waiting": (("vllm:num_requests_waiting",), "sum"), + "gpu_cache_usage": (("vllm:kv_cache_usage_perc", "vllm:gpu_cache_usage"), "max"), + "prompt_tokens_total": (("vllm:prompt_tokens_total",), "sum"), + "generation_tokens_total": (("vllm:generation_tokens_total",), "sum"), + # throughput gauges exist only on some builds; absent -> derived from counters/TPOT + "prompt_tokens_per_second": (("vllm:prompt_throughput", "vllm:avg_prompt_throughput_toks_per_s"), "avg"), + "decode_tokens_per_second_native": (("vllm:generation_throughput", + "vllm:avg_generation_throughput_toks_per_s"), "avg"), + "prefix_cache_queries_total": (("vllm:prefix_cache_queries_total",), "sum"), + "prefix_cache_hits_total": (("vllm:prefix_cache_hits_total",), "sum"), + # preemption = a running request kicked back to the queue: the real KV-pressure signal + "num_preemptions_total": (("vllm:num_preemptions_total",), "sum"), + "num_requests_swapped": (("vllm:num_requests_swapped",), "sum"), +} + +VLLM_HISTOGRAMS: dict[str, tuple[str, ...]] = { + "time_to_first_token_seconds": ("vllm:time_to_first_token_seconds",), + "time_per_output_token_seconds": ("vllm:time_per_output_token_seconds", + "vllm:inter_token_latency_seconds", + "vllm:request_time_per_output_token_seconds"), + "request_queue_time_seconds": ("vllm:request_queue_time_seconds",), + # not shown directly: mean prompt length / mean prefill time = lifetime prefill speed, + # which is what keeps prompt tok/s off n/a on a build with no rate gauge + "request_prefill_time_seconds": ("vllm:request_prefill_time_seconds",), + "request_prompt_tokens": ("vllm:request_prompt_tokens",), + "e2e_request_latency_seconds": ("vllm:e2e_request_latency_seconds",), +} + +#: history series per engine, in the order the LLM page charts them +HISTORY_METRICS: dict[str, tuple[str, ...]] = { + "decode_tokens_per_second": ("decode_tokens_per_second",), + "prompt_tokens_per_second": ("prompt_tokens_per_second",), + "kv_cache_usage": ("kv_cache_usage_ratio", "gpu_cache_usage"), + "requests_running": ("requests_processing", "num_requests_running"), + "requests_waiting": ("requests_deferred", "num_requests_waiting"), + "ttft_seconds": ("time_to_first_token_seconds__mean",), + "ttft_p95_seconds": ("time_to_first_token_seconds__p95",), + "queue_seconds": ("request_queue_time_seconds__mean",), + "prefix_cache_hit_ratio": ("prefix_cache_hit_ratio", "prompt_cache_hit_ratio"), +} + + +def _ratio(numerator: dict[str, Any], denominator: dict[str, Any], source: str) -> dict[str, Any]: + num, den = numerator["value"], denominator["value"] + if num is None or den is None or den == 0: + return field(None, None) + return field(num / den, source, derived=True) + + +def _histogram_mean(out: dict[str, Any], key: str) -> float | None: + cell = out.get(key) + mean = cell.get("mean") if isinstance(cell, dict) else None + return mean if isinstance(mean, float) and math.isfinite(mean) else None + + +def parse_gufo_metrics(text: str, rates: RateEstimator | None = None, + now: float | None = None) -> dict[str, Any]: + """`rates` enables the Δ-based aggregate rates; omitted, the parse stays pure.""" + parsed = Prom(text) + out: dict[str, Any] = {} + for key, (names, agg) in GUFO_SERIES.items(): + out[key] = field(parsed.scalar(names, agg), parsed.resolve(names)) + + def measured(total_key: str) -> dict[str, Any] | None: + """Δcounter/Δt for one cumulative counter, keyed by the RESOLVED series name so an + alias rename starts a fresh window instead of diffing two different series.""" + if rates is None: + return None + source = out[total_key]["source"] + if source is None: + return None + rate, window = rates.rate(source, out[total_key]["value"], now) + if rate is None: + return None + return field(rate, f"computed:Δ{source}/Δt", derived=True, window=window) + + # The counters add every slot; gufo's single gauge cannot. Measured therefore wins for + # both speeds here (the inverse of vLLM, where the native gauge is the aggregate) — and + # when no window exists yet the cell stays n/a instead of borrowing one request's speed. + for ui_key, total_key in ( + ("decode_tokens_per_second", "tokens_generated_total"), + ("prompt_tokens_per_second", "tokens_prompt_total"), + ): + rate = measured(total_key) + out[ui_key] = rate if rate is not None else field(None, None) + + prompt = out["tokens_prompt_total"]["value"] + cached = out["tokens_prompt_cached_total"]["value"] or 0.0 + generated = out["tokens_generated_total"]["value"] + parts = [v for v in (prompt, generated) if v is not None] + out["tokens_total"] = (field(sum(parts) + cached, "computed:prompt+cached+generated") + if parts else field(None, None)) + out["prompt_cache_hit_ratio"] = ( + field(cached / (prompt + cached) if (prompt + cached) > 0 else None, + "computed:cached/(prompt+cached)", derived=True) + if prompt is not None else field(None, None)) + out["spec_accept_rate"] = _ratio(out["spec_accepted_tokens_total"], + out["spec_draft_tokens_total"], + "computed:accepted/draft_tokens") + out["spec_drafts_per_request"] = _ratio(out["spec_drafts_total"], + out["spec_draft_tokens_total"], + "computed:drafts/draft_tokens") + return out + + +def parse_vllm_metrics(text: str, rates: RateEstimator | None = None, + now: float | None = None) -> dict[str, Any]: + parsed = Prom(text) + out: dict[str, Any] = {} + for key, (names, agg) in VLLM_SERIES.items(): + out[key] = field(parsed.scalar(names, agg), parsed.resolve(names)) + + reason_names = ("vllm:num_requests_waiting_by_reason", + "vllm:num_requests_waiting_by_reason_total") + reasons = parsed.by_label(reason_names, "reason") + out["waiting_by_reason"] = {"value": reasons, "source": parsed.resolve(reason_names), + "derived": False} + out["waiting_capacity"] = field(reasons.get("capacity"), parsed.resolve(reason_names)) + + for ui_key, names in VLLM_HISTOGRAMS.items(): + out[ui_key] = parsed.histogram(names) + out[f"{ui_key}__mean"] = field(out[ui_key]["mean"] if out[ui_key] else None, + out[ui_key]["source"] if out[ui_key] else None) + out[f"{ui_key}__p95"] = field(out[ui_key]["p95"] if out[ui_key] else None, + out[ui_key]["source"] if out[ui_key] else None) + + tpot = out["time_per_output_token_seconds"]["mean"] if out["time_per_output_token_seconds"] else None + per_seq = (1.0 / tpot) if tpot else None + out["per_request_decode_tokens_per_second"] = field(per_seq, "computed:1/tpot", derived=True) + + def measured(key: str, total_key: str) -> dict[str, Any] | None: + if rates is None: + return None + rate, window = rates.rate(key, out[total_key]["value"], now) + if rate is None: + return None + return field(rate, f"computed:Δ{VLLM_SERIES[total_key][0][0]}/Δt", derived=True, window=window) + + decode = out["decode_tokens_per_second_native"]["value"] + running = out["num_requests_running"]["value"] + if decode is not None: + out["decode_tokens_per_second"] = field(decode, + out["decode_tokens_per_second_native"]["source"]) + else: + # measured beats the running/tpot estimate: it counts real output tokens, the + # estimate assumes every running request is decoding this instant + measured_decode = measured("vllm:generation_tokens_total", "generation_tokens_total") + if measured_decode is not None: + # `or` here would treat a genuine 0 tok/s as missing and print the lifetime + # estimate for an engine that is provably idle + out["decode_tokens_per_second"] = measured_decode + elif per_seq is not None: + out["decode_tokens_per_second"] = field((running or 0) * per_seq, + "computed:running/tpot", derived=True) + else: + out["decode_tokens_per_second"] = field(None, None) + + prefill_avg = None + prefill_time = _histogram_mean(out, "request_prefill_time_seconds") + prompt_len = _histogram_mean(out, "request_prompt_tokens") + if prefill_time and prompt_len: + prefill_avg = field(prompt_len / prefill_time, + "computed:mean(request_prompt_tokens)/mean(request_prefill_time)", + derived=True) + if out["prompt_tokens_per_second"]["value"] is None: + measured_prefill = measured("vllm:prompt_tokens_total", "prompt_tokens_total") + if measured_prefill is not None: + out["prompt_tokens_per_second"] = measured_prefill + elif prefill_avg is not None: + out["prompt_tokens_per_second"] = prefill_avg + else: + out["prompt_tokens_per_second"] = field(None, None) + # a lifetime average is a different claim from current throughput — labelled apart + out["prefill_tokens_per_second_lifetime"] = prefill_avg or field(None, None) + + parts = [v for v in (out["prompt_tokens_total"]["value"], + out["generation_tokens_total"]["value"]) if v is not None] + out["tokens_total"] = field(sum(parts), "computed:prompt+generated") if parts else field(None, None) + out["prefix_cache_hit_ratio"] = _ratio(out["prefix_cache_hits_total"], + out["prefix_cache_queries_total"], + "computed:hits/queries") + return out + + +def engine_result(engine: str, base_url: str, text: str, rates: RateEstimator, + *, slots: list[dict[str, Any]] | None = None, now: float | None = None) -> Result: + """Parse one engine's /metrics into a poll Result (payload + history samples).""" + now = time.time() if now is None else now + if engine == "gufo": + metrics = parse_gufo_metrics(text, rates, now) + active = metrics["requests_processing"]["value"] + else: + metrics = parse_vllm_metrics(text, rates, now) + active = metrics["num_requests_running"]["value"] + + parsed = Prom(text) + sessions = _sessions(engine, slots, active, metrics) + payload = { + "engine": engine, + "upstream": base_url, + "scraped_at": now, + "metrics": metrics, + "sessions": sessions, + "series_count": len(parsed.series), + "unparsed": parsed.parse_errors, + } + samples: list[Sample] = [] + for metric, candidates in HISTORY_METRICS.items(): + value = next((metrics[key]["value"] for key in candidates + if key in metrics and metrics[key]["value"] is not None), None) + if value is not None: + samples.append(Sample("llm", engine, f"llm.{metric}", float(value), now)) + return Result(payload=payload, samples=samples) + + +def _sessions(engine: str, slots: list[dict[str, Any]] | None, active: float | None, + metrics: dict[str, Any]) -> dict[str, Any]: + """Active sessions, however this engine expresses them. + + gufo exposes /slots (one row per parallel slot, with per-slot tokens and speed), so it + gets a real list. vLLM publishes no per-session endpoint in its /metrics, so it gets + counts and says the list is not available rather than inventing rows. + """ + out: dict[str, Any] = {"active": active, "list": [], "source": None, "detail": None} + if engine == "gufo": + if slots: + out["list"] = slots + out["source"] = "gufo /slots" + else: + out["detail"] = ("no /slots answer — active count still comes from " + "llamacpp:requests_processing") + else: + waiting = metrics.get("num_requests_waiting", {}).get("value") + queued = f", {waiting:g} queued" if waiting else "" + out["detail"] = (f"vLLM reports counts, not sessions: {active or 0:g} running{queued}. " + f"Per-request detail is in the latency histograms.") + return out + + +def fleet_totals(engines: list[dict[str, Any]]) -> dict[str, Any]: + """Cross-engine sums, computed in the API over engines that answered. + + A dead engine keeps its last good values in its own panel but contributes nothing to + a total presented as current — and `engines_live` vs `engines_total` makes that visible. + """ + live = [e for e in engines if e.get("metrics")] + + def sum_metric(key: str) -> float | None: + values = [e["metrics"].get(key, {}).get("value") for e in live] + numbers = [v for v in values if isinstance(v, int | float)] + return sum(numbers) if numbers else None + + def max_metric(key: str) -> float | None: + values = [e["metrics"].get(key, {}).get("value") for e in live + if isinstance(e["metrics"].get(key, {}).get("value"), int | float)] + return max(values) if values else None + + def sum_pairs(keys: Sequence[str]) -> float | None: + """Sum across engines AND across the per-engine key aliases (gufo calls a running + request `requests_processing`, vLLM `num_requests_running`).""" + values = [sum_metric(key) for key in keys] + numbers = [v for v in values if v is not None] + return sum(numbers) if numbers else None + + return { + "engines_total": len(engines), + "engines_live": len(live), + "decode_tokens_per_second": sum_metric("decode_tokens_per_second"), + "prompt_tokens_per_second": sum_metric("prompt_tokens_per_second"), + "requests_active": sum_pairs(("requests_processing", "num_requests_running")), + "requests_waiting": sum_pairs(("requests_deferred", "num_requests_waiting")), + "tokens_total": sum_metric("tokens_total"), + # a real max across both engines' aliases: `a or b` returned gufo's 25% while vLLM + # was sitting at 75%, which is the opposite of a pressure signal + "kv_cache_usage_max": max( + (v for v in (max_metric("kv_cache_usage_ratio"), max_metric("gpu_cache_usage")) + if v is not None), + default=None, + ), + "seams": ("gufo's prompt counter excludes cache-hit tokens while vLLM's includes " + "cached prefix tokens, so the summed prompt rate leans slightly low for " + "gufo. Both are counted honestly per engine."), + } + + +def series_names(engine: str) -> Sequence[str]: + return tuple(GUFO_SERIES if engine == "gufo" else VLLM_SERIES) diff --git a/src/dashboards/sources/nut.py b/src/dashboards/sources/nut.py new file mode 100644 index 0000000..b1505d2 --- /dev/null +++ b/src/dashboards/sources/nut.py @@ -0,0 +1,779 @@ +"""NUT (Network UPS Tools) client — the upsd wire protocol, asyncio, no dependencies. + +`upsc`/`upspy`/`pynut` are either a subprocess dependency or a stalled third-party lib, +and the protocol is a line protocol with ~10 commands. So: talk to upsd on 3493 directly. + +Reading is unauthenticated in stock NUT (`LIST UPS` / `LIST VAR`), so the dashboard works +with a read-only upsd. Instant commands (the battery-test buttons) require a user with +`actions` in upsd.users — that is the ONLY write path in this whole application, and it is +gated three times: `NUT_ALLOW_ACTIONS`, not `DASH_READ_ONLY`, and a confirmation in the UI +(destructive ones require typing the UPS name). + +Verified against NUT 2.8.5 (`/usr/bin/upsc` on driveripper). Protocol shape: + + -> LIST UPS + <- BEGIN LIST UPS + <- UPS "apc" : APC Smart-UPS + <- END LIST UPS + -> LIST VAR apc + <- BEGIN LIST VAR apc + <- VAR "ups.status" "OL" + <- END LIST VAR apc + -> LIST CMD apc ; <- BEGIN LIST CMD apc / CMD test.battery.start.quick / END LIST CMD apc + bad -> ERR ACCESS-DENIED (auth or permission problem) + ok -> OK ... +""" + +from __future__ import annotations + +import asyncio +import contextlib +import re +import time +from dataclasses import dataclass, field +from typing import Any + +DEFAULT_PORT = 3493 + + +class NutError(Exception): + """A NUT protocol error. `code` is the upsd error token (ACCESS-DENIED, ...).""" + + def __init__(self, message: str, *, code: str | None = None) -> None: + super().__init__(message) + self.code = code + + +#: upsd error tokens -> what the operator should actually do about them. +ERROR_HINTS = { + "ACCESS-DENIED": "upsd refused: this client is not in an ACL that allows it (upsd.conf `ACL`/" + "`LISTEN`, or the user lacks the privilege in upsd.users)", + "INVALID-PASSWORD": "wrong NUT_PASSWORD for NUT_USER (upsd.users)", + "INVALID-USERNAME": "NUT_USER is not defined in upsd.users", + "INVALID-ARGUMENT": "malformed command or unknown UPS", + "UNKNOWN-UPSNAME": "that UPS is not exported by upsd (check ups.conf driver instance name)", + "UNKNOWN-REQUEST": "this upsd does not implement that command", + "CMD-UNSUPP": "the driver does not implement that command", + "CMD-NOT-BLOCKING": "not a blocking command", + "INVALID-BITMAP": "bad mode bitmap", + "DATA-STALE": "the driver stopped updating this UPS (check nut-driver@ on the NUT host)", + "DRIVER-NOT-CONNECTED": "no driver is connected to that UPS right now", + "PERM-DENIED": "read-only variable, or the user lacks `actions` for that command in upsd.users", + "TOO-MANY-CLIENTS": "upsd MAXCONN reached", + "ALREADY-SET": "value already set", + "FEATURE-NOT-ENABLED": "feature disabled on the driver side", + "FEATURE-NOT-SUPPORTED": "driver does not support that feature", + "TOO-BUSY": "upsd is busy", + "INVALID-VALUE": "value out of range", +} + + +def _unquote(value: str) -> str: + if len(value) >= 2 and value[0] == '"' and value[-1] == '"': + return value[1:-1].replace('\\"', '"').replace("\\\\", "\\") + return value + + +class NutClient: + """One connection to upsd. Short-lived by design: connect, read, close. + + A persistent connection would be faster but hides restarts of upsd behind a stale + socket, and upsd has a per-client limit (MAXCONN) that a long-lived dashboard would + sit on. + """ + + def __init__(self, host: str, port: int = DEFAULT_PORT, *, timeout: float = 4.0, + username: str | None = None, password: str | None = None) -> None: + self.host = host + self.port = port + self.timeout = timeout + self.username = username + self.password = password + self._reader: asyncio.StreamReader | None = None + self._writer = None + + async def __aenter__(self) -> NutClient: + await self.connect() + return self + + async def __aexit__(self, *_exc: object) -> None: + await self.close() + + async def connect(self) -> None: + try: + self._reader, self._writer = await asyncio.wait_for( + asyncio.open_connection(self.host, self.port), self.timeout + ) + except (TimeoutError, OSError) as exc: + raise NutError( + f"cannot reach upsd at {self.host}:{self.port} ({type(exc).__name__}: {exc})", + code="UNREACHABLE", + ) from exc + + async def close(self) -> None: + if self._writer is None: + return + with contextlib.suppress(Exception): # logout is best effort; upsd may already be gone + if self.username and self.password: + await self._send("LOGOUT") + await self._read_line(expect_ok=False) + self._writer.close() + with contextlib.suppress(Exception): + await self._writer.wait_closed() + self._writer = None + self._reader = None + + # -- plumbing ------------------------------------------------------------ # + async def _send(self, command: str) -> None: + if self._writer is None: + raise NutError("not connected", code="NOCONNECTION") + self._writer.write((command + "\n").encode()) + await self._writer.drain() + + async def _read_line(self, *, expect_ok: bool = True) -> str: + if self._reader is None: + raise NutError("not connected", code="NOCONNECTION") + try: + raw = await asyncio.wait_for(self._reader.readline(), self.timeout) + except TimeoutError as exc: + raise NutError(f"upsd at {self.host}:{self.port} did not answer in " + f"{self.timeout:g}s", code="TIMEOUT") from exc + except (ConnectionResetError, BrokenPipeError, OSError) as exc: + # upsd dropping the socket (restart, MAXCONN, firewall) is a read failure with a + # name, not a traceback out of the poller + raise NutError(f"upsd at {self.host}:{self.port} dropped the connection " + f"({type(exc).__name__})", code="CLOSED") from exc + if not raw: + raise NutError("upsd closed the connection", code="CLOSED") + line = raw.decode(errors="replace").rstrip("\r\n") + if line.startswith("ERR"): + code = line.split(None, 2)[1] if len(line.split()) > 1 else "UNKNOWN" + raise NutError(f"upsd: {line} — {ERROR_HINTS.get(code, 'see upsd(8)')}", code=code) + if expect_ok and not line.startswith("OK"): + raise NutError(f"unexpected upsd reply: {line!r}", code="UNEXPECTED") + return line + + async def _list(self, command: str, kind: str) -> list[list[str]]: + """Run a LIST command and collect the lines between BEGIN/END.""" + begin = f"BEGIN LIST {kind}" + end = f"END LIST {kind}" + await self._send(command) + # LIST replies begin with `BEGIN LIST `, not `OK`, so this path must not + # demand an OK line — upsd would otherwise be "broken" on every read. + first = await self._read_line(expect_ok=False) + if not first.startswith(begin): + raise NutError(f"expected {begin!r}, got {first!r}", code="UNEXPECTED") + rows: list[list[str]] = [] + while True: + line = await self._read_line(expect_ok=False) + if line.startswith(end): + return rows + rows.append(_split_fields(line)) + if len(rows) > 4096: # a wedged upsd must not grow the heap + raise NutError(f"{kind} list too long", code="UNEXPECTED") + + async def login(self) -> bool: + """Authenticate for actions. Returns False when no credentials are configured.""" + if not self.username or not self.password: + return False + await self._send(f"USER {self.username}") + await self._read_line() + await self._send(f"PASS {self.password}") + await self._read_line() + return True + + # -- reads ---------------------------------------------------------------- # + async def list_ups(self) -> list[dict[str, str]]: + rows = await self._list("LIST UPS", "UPS") + out: list[dict[str, str]] = [] + for row in rows: + # UPS ["description"] + if len(row) >= 2 and row[0] == "UPS": + name = row[1] + desc = _unquote(" ".join(row[2:])) if len(row) > 2 else "" + out.append({"name": name, "description": desc}) + return out + + async def list_vars(self, ups: str) -> dict[str, str]: + """`VAR ` — the UPS name is field 1, the variable is field 2. + + Reading field 1 as the name returns the UPS name over and over and every lookup + misses; real upsd was the only way to catch that, so the fake in the tests speaks + the same shape. + """ + rows = await self._list(f"LIST VAR {_quote(ups)}", "VAR") + out: dict[str, str] = {} + for row in rows: + if len(row) >= 3 and row[0] == "VAR": + out[_unquote(row[2])] = _unquote(" ".join(row[3:])) + return out + + async def list_cmds(self, ups: str) -> list[str]: + """`CMD `.""" + rows = await self._list(f"LIST CMD {_quote(ups)}", "CMD") + return [_unquote(row[2]) for row in rows if len(row) >= 3 and row[0] == "CMD"] + + async def list_rw_vars(self, ups: str) -> list[str]: + """`RW `.""" + rows = await self._list(f"LIST RW {_quote(ups)}", "RW") + return [_unquote(row[2]) for row in rows if len(row) >= 3 and row[0] == "RW"] + + # -- writes ---------------------------------------------------------------- # + async def instcmd(self, ups: str, command: str) -> str: + """Send an instant command. upsd answers `OK Send shorthand command successful`.""" + if not (self.username and self.password): + raise NutError("instant commands need NUT_USER/NUT_PASSWORD with `actions` " + "in upsd.users", code="ACCESS-DENIED") + await self._send(f"INSTCMD {_quote(ups)} {_quote(command)}") + return await self._read_line() + + +def _quote(value: str) -> str: + """Quote a protocol argument. upsd splits on whitespace and understands C-style quoting.""" + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def _split_fields(line: str) -> list[str]: + """Split a protocol line into fields, honouring quoted spans (values contain spaces). + + Fields come back unquoted: upsd quotes every variable name and value, and a name that + still carries its quote characters (`"battery.charge"`) silently misses every lookup. + """ + fields: list[str] = [] + current: list[str] = [] + in_quotes = False + escaped = False + for char in line: + if escaped: + current.append(char) + escaped = False + elif in_quotes and char == "\\": + escaped = True + elif char == '"': + in_quotes = not in_quotes + current.append(char) + elif char == " " and not in_quotes: + if current: + fields.append("".join(current)) + current = [] + else: + current.append(char) + if current: + fields.append("".join(current)) + return [_unquote(field) for field in fields] + + +# --------------------------------------------------------------------------- # +# Variable mapping +# --------------------------------------------------------------------------- # +# Vendor drivers disagree about names (apc vs cyberpower vs usbhid-ups), so every field +# is a candidate list and an absent value is None, never 0. +VAR_MAP: dict[str, tuple[str, ...]] = { + "mfr": ("ups.mfr",), + "model": ("ups.model",), + "serial": ("ups.serial", "device.serial"), + "firmware": ("ups.firmware", "ups.firmware.aux"), + "driver": ("driver.name",), + "status": ("ups.status",), + "charger_status": ("battery.charger.status",), # NUT 2.8 rich status + "alarm": ("ups.alarm",), + "load_pct": ("ups.load", "ups.load.instead"), + "realpower": ("ups.realpower", "ups.power"), + "realpower_nominal": ("ups.realpower.nominal", "ups.power.nominal"), + "line_voltage": ("input.voltage", "input.voltage.fault"), + "line_voltage_nominal": ("input.voltage.nominal",), + "input_min": ("input.voltage.minimum", "input.low.line"), + "input_max": ("input.voltage.maximum", "input.high.line"), + "transfer_low": ("input.transfer.low",), + "transfer_high": ("input.transfer.high",), + "frequency": ("input.frequency", "input.frequency.nominal"), + "output_voltage": ("output.voltage",), + "output_current": ("output.current",), + "output_frequency": ("output.frequency",), + "charge_pct": ("battery.charge", "battery.capacity:typ"), + "battery_voltage": ("battery.voltage",), + "battery_voltage_nominal": ("battery.voltage.nominal",), + "battery_current": ("battery.current",), + "battery_temperature": ("battery.temperature",), + "battery_type": ("battery.type",), + "battery_runtime": ("battery.runtime", "battery.runtime.low"), + "battery_runtime_low": ("battery.runtime.low",), + "battery_date": ("battery.mfr.date", "battery.date"), + "test_result": ("battery.test.result", "ups.test.result"), + "test_date": ("battery.test.date", "ups.test.date"), + "start_battery": ("ups.start.battery",), + "ups_temperature": ("ups.temperature",), + "efficiency": ("ups.efficiency",), +} + +# Anything else worth showing verbatim, if the driver happens to export it. +EXTRA_VAR_PREFIXES = ("battery.test.", "input.transfer.", "output.", "ups.", "battery.") + +STATE_FLAGS = { + "OL": ("online", "ok", "On line — utility power is present"), + "OB": ("on battery", "warn", "Running on battery"), + "OB LB": ("low battery", "fail", "On battery and low — shutdown imminent"), + "LB": ("low battery", "fail", "Battery low"), + "RB": ("replace battery", "fail", "The UPS says the battery must be replaced"), + "CHRG": ("charging", "info", "Battery charging"), + "DISCHRG": ("discharging", "warn", "Battery discharging"), + "BYPASS": ("bypass", "warn", "Load is on bypass"), + "OFF": ("off", "fail", "Output is off"), + "NOBAT": ("no battery", "fail", "No battery detected"), + "NULL": ("unknown", "warn", "Driver reports no state"), + "TEST": ("testing", "info", "Self-test in progress"), + "CAL": ("calibrating", "warn", "Battery calibration — runtime estimates are unreliable"), + "OVER": ("overloaded", "fail", "Load exceeds the UPS rating"), + # FUB is the UPS telling us it decided to power off (scheduled shutdown, or the driver's + # own low-battery policy). Missing it reads as "no state" on the one status that matters. + "FUB": ("forced shutdown", "fail", "The UPS has decided to shut down (FUB)"), + "OTHER": ("unknown", "warn", "Driver reports an unclassifiable state"), + "TRIM": ("trimming", "info", "Trimming high input voltage"), + "BOOST": ("boosting", "info", "Boosting low input voltage"), +} + +CHARGER_STATUS = { + "ol": ("online", "ok"), + "ol trim": ("trimming", "info"), + "ol boost": ("boosting", "info"), + "chg": ("charging", "info"), + "dchg": ("discharging", "warn"), + "full": ("fully charged", "ok"), + "pover": ("overloaded", "fail"), + "shutd": ("shutting down", "fail"), + "ebypass": ("bypass", "warn"), + "ehazard": ("hazard", "fail"), + "rec": ("in recalibration", "warn"), + "test": ("testing", "info"), + "other": ("unknown", "warn"), +} + + +@dataclass +class UpsReading: + """One UPS, as reported by one poll.""" + + name: str + description: str = "" + vars: dict[str, str] = field(default_factory=dict) + commands: list[str] = field(default_factory=list) + writable: list[str] = field(default_factory=list) + ts: float = field(default_factory=time.time) + + def get(self, key: str) -> str | None: + for candidate in VAR_MAP.get(key, (key,)): + value = self.vars.get(candidate) + if value not in (None, "", "0.000000"): + return value + return None + + def number(self, key: str) -> float | None: + raw = self.get(key) + if raw is None: + return None + try: + return float(raw) + except ValueError: + return None + + @property + def status_flags(self) -> list[str]: + return (self.get("status") or "").split() + + def states(self) -> list[dict[str, str]]: + """Human state list. The 2.8 `battery.charger.status` wins over legacy ups.status + when both exist because it distinguishes trim/boost/full/recalibration.""" + out: list[dict[str, str]] = [] + charger = (self.get("charger_status") or "").strip().lower() + if charger and charger in CHARGER_STATUS: + label, severity = CHARGER_STATUS[charger] + out.append({"id": charger, "label": label, "severity": severity}) + raw = " ".join(self.status_flags) + if raw: + if raw in STATE_FLAGS: + label, severity, detail = STATE_FLAGS[raw] + out.append({"id": raw, "label": label, "severity": severity, "detail": detail}) + else: + for flag in self.status_flags: + if flag in STATE_FLAGS: + label, severity, detail = STATE_FLAGS[flag] + out.append({"id": flag, "label": label, "severity": severity, "detail": detail}) + return out + + @property + def on_battery(self) -> bool: + flags = set(self.status_flags) + if "OL" in flags: + return False + if "OB" in flags: + return True + return (self.get("charger_status") or "").lower() in ("dchg", "rec") + + @property + def alarms(self) -> list[str]: + raw = self.get("alarm") or "" + return [part.strip() for part in raw.split(",") if part.strip()] + + +# --------------------------------------------------------------------------- # +# Health assessment +# --------------------------------------------------------------------------- # +SEVERITY_ORDER = {"ok": 0, "info": 0, "unknown": 1, "warn": 2, "fail": 3} + + +def normalise_celsius(value: float | None) -> float | None: + """Turn a driver temperature into °C. + + NUT drivers disagree: usbhid-ups usually reports °C, while the powerware/mge heritage + and some CyberPower firmware report kelvin. 300 "°C" is not a melting battery, it is + 27 °C, and a health check that fails on it is worse than no check. Above 150 the value + can only be kelvin for a UPS battery, so convert; below that, trust the driver. + """ + if value is None: + return None + return value - 273.15 if value > 150 else value + + +def _check(name: str, status: str, detail: str, *, metric: str | None = None, + value: float | None = None) -> dict[str, Any]: + return {"name": name, "status": status, "detail": detail, "metric": metric, "value": value} + + +def assess(reading: UpsReading, *, now: float | None = None, + battery_replacement_years: float = 4.0, min_runtime_seconds: float = 180.0) -> dict[str, Any]: + """A list of explicit checks, not a mystery score. + + Every line says what it looked at and what it saw, because "health: 87" is the kind + of number an operator cannot act on. Absent data is `unknown`, never a pass. + """ + now = time.time() if now is None else now + checks: list[dict[str, Any]] = [] + + states = reading.states() + if not states: + checks.append(_check("Power path", "unknown", "ups.status is not exported")) + else: + worst = min(states, key=lambda s: -SEVERITY_ORDER.get(s["severity"], 0), + default=states[0]) + worst_sev = max((s["severity"] for s in states), key=lambda s: SEVERITY_ORDER.get(s, 0)) + checks.append(_check("Power path", worst_sev, ", ".join(s["label"] for s in states), + value=None)) + if worst_sev == "fail" and worst is not None: + checks[-1]["detail"] = worst.get("detail", worst["label"]) + + charge = reading.number("charge_pct") + if charge is None: + checks.append(_check("Charge", "unknown", "battery.charge is not exported")) + elif reading.on_battery: + # On battery this number only falls, and `LB` plus the runtime estimate already say + # whether that is urgent. Reporting it as an error here would double-alarm. + checks.append(_check("Charge", "warn" if charge < 30 else "ok", + f"{charge:.0f}% and discharging", value=charge)) + elif charge >= 99: + checks.append(_check("Charge", "ok", f"{charge:.0f}%", value=charge)) + elif charge >= 90: + # Plenty of drivers float at 90-94 % forever (chemistry, temperature, float voltage). + # Calling that "degraded" keeps the page permanently amber and teaches people to + # ignore the one badge that is supposed to mean something. + checks.append(_check("Charge", "ok", f"{charge:.0f}% — floating", value=charge)) + elif charge >= 50: + checks.append(_check("Charge", "warn", f"{charge:.0f}% — not reaching full charge", + value=charge)) + else: + checks.append(_check("Charge", "fail", f"{charge:.0f}% — barely charged", value=charge)) + + runtime = reading.number("battery_runtime") + if runtime is None: + checks.append(_check("Runtime estimate", "unknown", + "battery.runtime is not exported (the driver cannot estimate it)")) + elif runtime >= min_runtime_seconds * 2: + checks.append(_check("Runtime estimate", "ok", f"{_duration(runtime)} at the current load", + value=runtime)) + elif runtime >= min_runtime_seconds: + checks.append(_check("Runtime estimate", "warn", f"only {_duration(runtime)} left", + value=runtime)) + else: + checks.append(_check("Runtime estimate", "fail", + f"{_duration(runtime)} — too little to shut down cleanly", + value=runtime)) + + if "RB" in reading.status_flags: + checks.append(_check("Battery replacement", "fail", "The UPS flags RB — replace the battery")) + else: + age = _battery_age_years(reading, now) + if age is None: + checks.append(_check("Battery age", "unknown", + "no battery.mfr.date / battery.date from this driver")) + elif age >= battery_replacement_years: + checks.append(_check("Battery age", "warn", + f"{age:.1f} years — past the {battery_replacement_years:.0f}-year " + f"replacement window", value=age)) + else: + checks.append(_check("Battery age", "ok", f"{age:.1f} years", value=age)) + + result = reading.get("test_result") + if result is None: + checks.append(_check("Last self-test", "unknown", + "no self-test result reported — run a battery test")) + else: + passed = result.strip().lower() in ("ok", "passed", "pass", "completed") + failed = result.strip().lower() in ("failed", "fail", "error", "aborted", "test incomplete") + checks.append(_check("Last self-test", "ok" if passed else "fail" if failed else "warn", + f"{result}" + (f" ({reading.get('test_date')})" if reading.get("test_date") else ""))) + + raw_temp = reading.number("battery_temperature") + temp = normalise_celsius(raw_temp) + converted = temp is not None and raw_temp is not None and raw_temp > 150 + suffix = " (driver reports kelvin)" if converted else "" + if temp is None: + checks.append(_check("Battery temperature", "unknown", "not reported by this driver")) + elif temp >= 50: + checks.append(_check("Battery temperature", "fail", f"{temp:.1f} °C — VRLA damage risk{suffix}", + value=temp)) + elif temp >= 40: + checks.append(_check("Battery temperature", "warn", + f"{temp:.1f} °C — every +10 °C roughly halves VRLA life{suffix}", + value=temp)) + elif temp < -10: + checks.append(_check("Battery temperature", "warn", + f"{temp:.1f} °C looks wrong for a battery indoors", value=temp)) + else: + checks.append(_check("Battery temperature", "ok", f"{temp:.1f} °C{suffix}", value=temp)) + + load = reading.number("load_pct") + if load is not None: + status = "ok" if load < 70 else "warn" if load < 90 else "fail" + detail = f"{load:.0f}% of rated capacity" + power = reading.number("realpower") + nominal = reading.number("realpower_nominal") + if power is not None and nominal: + detail += f" ({power:.0f} W of {nominal:.0f} W)" + checks.append(_check("Load", status, detail, value=load)) + + alarms = reading.alarms + if alarms: + blocking = [a for a in alarms if re.search(r"(LB|RB|OVER|NOBAT|TEMP|SC)\b", a)] + checks.append(_check("Alarms", "fail" if blocking else "warn", ", ".join(alarms))) + + line = reading.number("line_voltage") + nominal = reading.number("line_voltage_nominal") + if line is not None and nominal: + drift = abs(line - nominal) / nominal * 100.0 + status = "ok" if drift <= 10 else "warn" if drift <= 15 else "fail" + checks.append(_check("Input voltage", status, + f"{line:.1f} V against {nominal:.0f} V nominal ({drift:.1f}%)", + value=line)) + low, high = reading.number("transfer_low"), reading.number("transfer_high") + if low is not None and high is not None: + checks[-1]["detail"] += f"; transfer at {low:.0f}-{high:.0f} V" + elif line is not None: + checks.append(_check("Input voltage", "ok", f"{line:.1f} V", value=line)) + + worst = max((SEVERITY_ORDER.get(c["status"], 0) for c in checks), default=0) + overall = {0: "healthy", 1: "unknown", 2: "degraded", 3: "at risk"}[worst] + return { + "overall": overall, + "severity": {0: "ok", 1: "unknown", 2: "warn", 3: "fail"}[worst], + "checks": checks, + "counts": { + name: sum(1 for c in checks if c["status"] == name) + for name in ("ok", "warn", "fail", "unknown") + }, + } + + +def _battery_age_years(reading: UpsReading, now: float) -> float | None: + raw = reading.get("battery_date") + if not raw: + return None + match = re.match(r"(\d{4})[-/](\d{1,2})(?:[-/](\d{1,2}))?", raw.strip()) + if not match: + return None + year, month, day = match.groups() + try: + created = int(year) * 365.25 + (int(month or 1) - 1) * 30.44 + (int(day or 1) - 1) + except ValueError: + return None + now_years = time.gmtime(now).tm_year * 365.25 + (time.gmtime(now).tm_yday - 1) + age = (now_years - created) / 365.25 + # A clock skew (battery.date defaults to 1970 or the future) is not a 56-year-old battery. + if age < 0 or age > 30: + return None + return age + + +def _duration(seconds: float) -> str: + seconds = max(0.0, seconds) + hours, remainder = divmod(int(seconds), 3600) + minutes, secs = divmod(remainder, 60) + if hours: + return f"{hours}h {minutes:02d}m" + if minutes: + return f"{minutes}m {secs:02d}s" + return f"{secs}s" + + +# --------------------------------------------------------------------------- # +# Instant commands (the app's only write path) +# --------------------------------------------------------------------------- # +RISK_SAFE = "safe" +RISK_CAUTION = "caution" +RISK_DANGEROUS = "dangerous" + +#: command -> label, risk, what it really does to the hardware behind the screen. +#: Anything not listed here is refused even if the driver advertises it. +COMMANDS: dict[str, dict[str, str]] = { + "test.battery.start.quick": { + "label": "Quick battery test", + "risk": RISK_CAUTION, + "help": "Transfers the load to battery for a few seconds. Safe on a healthy UPS, " + "but connected equipment is on battery for the duration.", + }, + "test.battery.start.deep": { + "label": "Deep battery test", + "risk": RISK_DANGEROUS, + "help": "Drains the battery until it is low or a runtime limit is hit. Everything on " + "this UPS runs on battery for minutes, and a worn battery may fail outright. " + "Run it deliberately, with an eye on the runtime number.", + }, + "test.battery.stop": {"label": "Stop battery test", "risk": RISK_SAFE, + "help": "Aborts a running battery test and returns to line power."}, + "test.panel.start": {"label": "Panel test", "risk": RISK_SAFE, + "help": "Lights and beeps the front panel for a few seconds."}, + "test.system.start": {"label": "Full system self-test", "risk": RISK_CAUTION, + "help": "Vendor self-test; behaviour varies by model and may move " + "the load to battery."}, + "beeper.enable": {"label": "Beeper on", "risk": RISK_SAFE, "help": "Re-enable audible alarms."}, + "beeper.disable": {"label": "Beeper off", "risk": RISK_SAFE, + "help": "Silences audible alarms. Faults are still reported in this dashboard."}, + "beeper.mute": {"label": "Beeper mute", "risk": RISK_SAFE, "help": "Silence the current alarm only."}, + "calibrate.start": { + "label": "Start calibration", + "risk": RISK_DANGEROUS, + "help": "Discharges the battery to recalibrate the charge gauge. Long and disruptive; " + "only do this after a full charge and when the load can tolerate it.", + }, + "calibrate.stop": {"label": "Stop calibration", "risk": RISK_SAFE, "help": "Ends calibration."}, + "reset.input.minmax": {"label": "Reset input min/max", "risk": RISK_SAFE, + "help": "Clears the recorded input voltage extremes."}, + "load.on": {"label": "Energise output", "risk": RISK_CAUTION, + "help": "Turns the outlets on."}, + "load.off": {"label": "Cut output power", "risk": RISK_DANGEROUS, + "help": "Kills power to everything plugged in, immediately, on battery or not."}, + "load.reboot": {"label": "Power-cycle outlets", "risk": RISK_DANGEROUS, + "help": "Cuts and restores output power. Everything attached reboots."}, + "shutdown.return": {"label": "Graceful shutdown, then restore", "risk": RISK_DANGEROUS, + "help": "Asks attached systems to shut down and powers back on when " + "utility returns."}, + "shutdown.standoff": {"label": "Graceful shutdown, stay off", "risk": RISK_DANGEROUS, + "help": "Shuts attached systems down and stays off."}, + "shutdown.stop": {"label": "Cancel shutdown", "risk": RISK_SAFE, + "help": "Aborts a pending shutdown sequence."}, +} + + +def command_advice(command: str) -> dict[str, str] | None: + return COMMANDS.get(command) + + +def available_commands(advertised: list[str], *, allow_actions: bool, + read_only: bool, has_credentials: bool, + on_battery: bool = False) -> list[dict[str, Any]]: + """What the UI may offer, with the reason when it may not. + + Filtering server-side (not just hiding buttons) is the point: the response says + `enabled: false` plus `reason`, so the page never offers a button that 403s. + """ + out: list[dict[str, Any]] = [] + for name in advertised: + advice = COMMANDS.get(name) + if advice is None: + continue + reason = None + if read_only: + reason = "the server is in DASH_READ_ONLY mode" + elif not allow_actions: + reason = "set NUT_ALLOW_ACTIONS=1 to enable UPS commands" + elif not has_credentials: + reason = "NUT_USER / NUT_PASSWORD must name a user with `actions` in upsd.users" + elif on_battery and advice["risk"] == RISK_DANGEROUS: + reason = "refusing a destructive command while the UPS is on battery" + out.append({ + "command": name, + "label": advice["label"], + "risk": advice["risk"], + "help": advice["help"], + "enabled": reason is None, + "reason": reason, + "confirm_text": None if advice["risk"] != RISK_DANGEROUS else "DANGER", + }) + order = {RISK_SAFE: 0, RISK_CAUTION: 1, RISK_DANGEROUS: 2} + out.sort(key=lambda item: (order[item["risk"]], item["label"])) + return out + + +def payload(reading: UpsReading, *, now: float | None = None, **assess_kwargs: Any) -> dict[str, Any]: + """The JSON one UPS card renders from.""" + now = time.time() if now is None else now + health = assess(reading, now=now, **assess_kwargs) + numbers = {key: reading.number(key) for key in VAR_MAP} + # charts must not plot 300 "°C"; the raw string stays visible in the variable dump + for key in ("battery_temperature", "ups_temperature"): + if numbers.get(key) is not None: + numbers[key] = normalise_celsius(numbers[key]) + return { + "name": reading.name, + "description": reading.description, + "identity": { + "mfr": reading.get("mfr"), + "model": reading.get("model"), + "serial": reading.get("serial"), + "firmware": reading.get("firmware"), + "driver": reading.get("driver"), + "battery_type": reading.get("battery_type"), + }, + "ts": reading.ts, + "status": reading.get("status"), + "charger_status": reading.get("charger_status"), + "states": reading.states(), + "alarms": reading.alarms, + "on_battery": reading.on_battery, + "numbers": numbers, + "health": health, + "advertised_commands": reading.commands, + "writable": reading.writable, + "extra": { + key: value for key, value in sorted(reading.vars.items()) + if key.startswith(EXTRA_VAR_PREFIXES) and key not in {v for vs in VAR_MAP.values() for v in vs} + }, + } + + +def history_samples(reading: UpsReading) -> dict[str, float]: + """The numeric series worth keeping a history of.""" + raw = { + "ups.charge_pct": "charge_pct", + "ups.battery_voltage": "battery_voltage", + "ups.battery_current": "battery_current", + "ups.battery_runtime_seconds": "battery_runtime", + "ups.battery_temperature_c": "battery_temperature", + "ups.line_voltage": "line_voltage", + "ups.output_voltage": "output_voltage", + "ups.output_current": "output_current", + "ups.output_frequency": "output_frequency", + "ups.frequency": "frequency", + "ups.load_pct": "load_pct", + "ups.realpower_w": "realpower", + "ups.temperature_c": "ups_temperature", + } + out: dict[str, float] = {} + for metric, key in raw.items(): + value = reading.number(key) + if value is None: + continue + if metric.endswith("_temperature_c") or metric == "ups.temperature_c": + value = normalise_celsius(value) + if value is not None: + out[metric] = value + return out diff --git a/src/dashboards/sources/nut_sim.py b/src/dashboards/sources/nut_sim.py new file mode 100644 index 0000000..ab77331 --- /dev/null +++ b/src/dashboards/sources/nut_sim.py @@ -0,0 +1,214 @@ +"""A *labelled* UPS simulator, for development when there is no upsd to talk to. + +Enable with `NUT_SIMULATE=1`. It exists so the UPS page can be built and screenshotted +before the hardware is cabled — and so the battery-test buttons can be exercised without +putting a real load on battery. It is never silent about what it is: the payload carries +`simulated: true` and the page shows a watermark on every simulated number. + +The model is deliberately small and physical rather than random: + * charge sits at float (100%) while on line, drains at a Peukert-ish rate when on battery + * runtime estimates scale with charge and fall off steeply with load (as they do for real + VRLA packs, which is why the number jumps around while the load moves) + * a quick battery test moves to OB for ~12s and then reports PASSED, so the whole + command → state change → event-log path is exercised +""" + +from __future__ import annotations + +import math +import time +from dataclasses import dataclass, field +from typing import Any + +from .nut import UpsReading + + +@dataclass +class SimUps: + """One fake UPS: nameplate values plus the state a driver would report.""" + + name: str + model: str + mfr: str + serial: str + nominal_va: float = 1500.0 + nominal_w: float = 900.0 + battery_cells: int = 2 # 2 x 12V blocks + battery_nominal: float = 24.0 + capacity_ah: float = 17.0 + load_w: float = 210.0 + charge: float = 100.0 + on_battery: bool = False + temperature_c: float = 27.0 + mfr_date: str = "2022-04-15" + test_result: str = "PASSED" + test_date: str = "2026-09-01" + firmware: str = "UPS 08.3 / IGR 1.00" + driver: str = "usbhid-ups" + _test_until: float = 0.0 + _deep_until: float = 0.0 + _beeper: bool = True + _alarms: list[str] = field(default_factory=list) + + def tick(self, now: float) -> None: + """Advance the model by wall-clock time since the last tick.""" + # a running self-test pulls the load from the wall, whichever test it is + self.on_battery = now < self._test_until or now < self._deep_until + + def step(self, dt: float) -> None: + dt = max(0.0, min(dt, 60.0)) + if self.on_battery: + # Ah drawn / capacity, with a mild Peukert exponent so a heavy load costs more + hours = dt / 3600.0 + load_fraction = max(0.05, self.load_w / self.nominal_w) + drawn = (self.load_w / self.battery_nominal) * hours * (load_fraction ** 0.15) + self.charge = max(0.0, self.charge - (drawn / self.capacity_ah) * 100.0) + elif self.charge < 100.0: + self.charge = min(100.0, self.charge + dt * (100.0 / 14400.0)) # ~4h recharge + self.temperature_c = 27.0 + 0.6 * math.sin(time.time() / 300.0) + self.charge / 400.0 + + def runtime_seconds(self) -> float: + """Battery runtime at the present load — the number the UPS itself estimates.""" + if not self.on_battery and self.charge >= 99.5: + # Full and on line: what a *fresh* pack would give at this load, which is what + # a UPS reports when it is idle on mains. + usable_ah = self.capacity_ah * 0.9 + else: + usable_ah = self.capacity_ah * (self.charge / 100.0) + amps = max(0.5, self.load_w / (self.battery_nominal * 0.9)) + load_penalty = max(0.35, 1.0 - (self.load_w / self.nominal_w) ** 2 * 0.45) + return (usable_ah / amps) * 3600.0 * load_penalty + + def vars(self, now: float) -> dict[str, str]: + load_pct = self.load_w / self.nominal_w * 100.0 + flags = ["OL"] if not self.on_battery else (["OB"] + (["LB"] if self.charge < 20 else [])) + if self.charge >= 99.5 and not self.on_battery: + flags.append("CHRG" if self.charge < 99.9 else "") + status = " ".join(part for part in flags if part) + charger = "ol" if not self.on_battery else ("dchg" if self.charge < 99.5 else "ol") + if self.charge >= 99.9 and not self.on_battery: + charger = "full" + volts = (self.battery_nominal * (0.86 + 0.14 * self.charge / 100.0) + if self.on_battery else self.battery_nominal * 1.045) + line = 121.4 + 1.1 * math.sin(now / 90.0) + out = { + "ups.mfr": self.mfr, + "ups.model": self.model, + "ups.serial": self.serial, + "ups.firmware": self.firmware, + "ups.status": status, + "battery.charger.status": charger, + "ups.load": f"{load_pct:.1f}", + "ups.realpower": f"{self.load_w:.1f}", + "ups.realpower.nominal": f"{self.nominal_w:.0f}", + "input.voltage": f"{line:.1f}", + "input.voltage.nominal": "120", + "input.frequency": f"{60.0 + 0.02 * math.sin(now / 40.0):.2f}", + "input.transfer.low": "92", + "input.transfer.high": "139", + "output.voltage": f"{line if not self.on_battery else 119.8:.1f}", + "output.current": f"{self.load_w / 120.0:.2f}", + "output.frequency": f"{60.0 + 0.02 * math.sin(now / 40.0):.2f}", + "battery.charge": f"{self.charge:.1f}", + "battery.voltage": f"{volts:.2f}", + "battery.voltage.nominal": f"{self.battery_nominal:.0f}", + "battery.current": f"{self.load_w / volts:.2f}", + "battery.temperature": f"{self.temperature_c + 273.15:.1f}", # drivers report K + "battery.type": "PbAcid: 12V / 9Ah / 17Ah sealed", + "battery.runtime": f"{self.runtime_seconds():.0f}", + "battery.runtime.low": "120", + "battery.mfr.date": self.mfr_date, + "battery.capacity": f"{self.capacity_ah:.0f}", + "battery.test.result": self.test_result, + "battery.test.date": self.test_date, + "driver.name": self.driver, + "driver.parameter.port": "auto", + "ups.id": self.name, + } + if self._alarms: + out["ups.alarm"] = ",".join(self._alarms) + return out + + def commands(self) -> list[str]: + return ["test.battery.start.quick", "test.battery.start.deep", "test.battery.stop", + "test.panel.start", "beeper.enable", "beeper.disable", "beeper.mute", + "calibrate.start", "calibrate.stop", "reset.input.minmax", + "load.on", "load.off", "shutdown.stop"] + + def run_command(self, command: str, now: float) -> str: + """Apply a command the way a driver would, so the UI's whole path is real.""" + if command == "test.battery.start.quick": + self._test_until = now + 12.0 + self.on_battery = True + self.test_date = time.strftime("%Y-%m-%d", time.gmtime(now)) + return "OK Start Quick Battery Test" + if command == "test.battery.start.deep": + self._deep_until = now + 90.0 + self.on_battery = True + return "OK Start Deep Battery Test" + if command in ("test.battery.stop", "calibrate.stop"): + self._test_until = self._deep_until = 0.0 + self.on_battery = False + return "OK Stop Test" + if command == "test.panel.start": + return "OK Start Panel Test" + if command == "beeper.enable": + self._beeper = True + return "OK Beeper enabled" + if command in ("beeper.disable", "beeper.mute"): + self._beeper = False + return "OK Beeper muted" + if command == "calibrate.start": + self._deep_until = now + 180.0 + self.on_battery = True + return "OK Start Calibration" + if command == "reset.input.minmax": + return "OK Input min/max reset" + if command == "load.on": + return "OK Load energised" + if command == "load.off": + return "OK Load switched off (simulated)" + if command == "shutdown.stop": + return "OK Shutdown cancelled" + raise ValueError(f"simulated UPS does not implement {command}") + + +class NutSimulator: + """Stands in for upsd: same shapes as NutClient reads, no network.""" + + def __init__(self, spec: str | None = None) -> None: + self.ups = [ + SimUps("cyberpower", "PR1500LCDRT2U", "Cyber Power Systems", "CPS-04213", + nominal_va=1500.0, nominal_w=900.0, load_w=210.0, capacity_ah=17.0, + mfr_date="2022-04-15"), + SimUps("apc", "Smart-UPS 1500", "APC", "AS-11902", + nominal_va=1500.0, nominal_w=865.0, load_w=140.0, capacity_ah=8.0, + mfr_date="2019-11-02", driver="usbhid-ups", + test_result="PASSED", test_date="2026-06-14"), + ] + self._last = time.time() + + async def read(self) -> list[UpsReading]: + now = time.time() + dt = now - self._last + self._last = now + for ups in self.ups: + ups.tick(now) + ups.step(dt) + return [ + UpsReading(name=ups.name, description=f"{ups.mfr} {ups.model} (simulated)", + vars=ups.vars(now), commands=ups.commands(), + writable=["beeper.enable"], ts=now) + for ups in self.ups + ] + + def command(self, ups_name: str, command: str) -> str: + for ups in self.ups: + if ups.name == ups_name: + return ups.run_command(command, time.time()) + raise KeyError(f"unknown simulated UPS {ups_name!r}") + + +def from_env(value: str | None) -> Any: + """`NUT_SIMULATE=1` -> a simulator; anything else -> None (talk to real upsd).""" + return NutSimulator() if (value or "").strip().lower() in ("1", "true", "yes", "on") else None diff --git a/src/dashboards/store.py b/src/dashboards/store.py new file mode 100644 index 0000000..24bfb4a --- /dev/null +++ b/src/dashboards/store.py @@ -0,0 +1,348 @@ +"""SQLite history: one sample table with three resolutions, plus an event log. + +Why SQLite and not Prometheus: the fleet is a handful of boxes, the poll resolution we +need is 1-10s, and a single file the container owns beats running a TSDB. Series are +addressed by `(source, host, metric, res, ts)`: + + source the dashboard family — "host", "llm", "ups" + host stable target key ("driveripper", "gufo", "apc-ups1") + metric dotted name, optionally `@stream` for per-device series + (cpu.usage_pct, cpu.usage_pct@core7, disk.used_pct@/srv, gpu.busy_pct@0) + res "raw" | "1m" | "1h" + +`maintain()` rolls raw up into 1m, 1m into 1h, and prunes past the retention window, so +the file stays bounded (a 3-host / 3-engine / 2-UPS fleet is a few hundred MB a year, +not a disk emergency). + +The connection is guarded by a lock and every method is synchronous: async callers wrap +them in `anyio.to_thread.run_sync`, sync FastAPI handlers run in the threadpool already. +""" + +from __future__ import annotations + +import json +import math +import sqlite3 +import threading +import time +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from pathlib import Path + +RES_RAW, RES_MIN, RES_HOUR = "raw", "1m", "1h" +MINUTE = 60.0 +HOUR = 3600.0 + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS sample ( + source TEXT NOT NULL, + host TEXT NOT NULL, + metric TEXT NOT NULL, + res TEXT NOT NULL, + ts REAL NOT NULL, + value REAL NOT NULL, + PRIMARY KEY (source, host, metric, res, ts) ON CONFLICT REPLACE +); + +CREATE TABLE IF NOT EXISTS event ( + id INTEGER PRIMARY KEY, + ts REAL NOT NULL, + source TEXT NOT NULL, + host TEXT NOT NULL, + kind TEXT NOT NULL, + severity TEXT NOT NULL, + message TEXT NOT NULL, + data TEXT +); +CREATE INDEX IF NOT EXISTS event_ts ON event (ts DESC); +CREATE INDEX IF NOT EXISTS event_source_ts ON event (source, ts DESC); + +CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT); +""" + + +@dataclass(frozen=True) +class Sample: + source: str + host: str + metric: str + value: float + ts: float + + +@dataclass(frozen=True) +class Event: + ts: float + source: str + host: str + kind: str + severity: str # info | warn | error + message: str + data: dict | None = None + + def as_json(self) -> dict: + return { + "ts": self.ts, + "source": self.source, + "host": self.host, + "kind": self.kind, + "severity": self.severity, + "message": self.message, + "data": self.data, + } + + +class SampleStore: + def __init__(self, path: str | Path, *, raw_keep_hours: float = 48, + minute_keep_days: float = 30, hour_keep_days: float = 400) -> None: + self.path = Path(path) + self.raw_keep_seconds = raw_keep_hours * HOUR + self.minute_keep_seconds = minute_keep_days * 24 * HOUR + self.hour_keep_seconds = hour_keep_days * 24 * HOUR + self._lock = threading.Lock() + if str(self.path) != ":memory:": + self.path.parent.mkdir(parents=True, exist_ok=True) + self._db = sqlite3.connect(str(self.path), check_same_thread=False) + self._db.execute("PRAGMA journal_mode=WAL") + self._db.execute("PRAGMA synchronous=NORMAL") # a lost sample is fine, corruption is not + self._db.execute("PRAGMA busy_timeout=5000") + self._db.executescript(_SCHEMA) + self._db.commit() + + def close(self) -> None: + with self._lock: + self._db.close() + + # -- writing --------------------------------------------------------------- # + def add(self, source: str, host: str, metric: str, value: float, ts: float | None = None) -> None: + self.add_many([Sample(source, host, metric, value, time.time() if ts is None else ts)]) + + def add_many(self, samples: Iterable[Sample]) -> int: + """Insert samples; NaN/inf are dropped (SQLite would store them and charts choke). + + Returns the number actually written so the caller can report it, never to fail + the poll over one bad value. + """ + rows = [ + (s.source, s.host, s.metric, RES_RAW, s.ts, s.value) + for s in samples + if isinstance(s.value, int | float) and math.isfinite(s.value) + ] + if not rows: + return 0 + with self._lock, self._db: + self._db.executemany("INSERT OR REPLACE INTO sample VALUES (?,?,?,?,?,?)", rows) + return len(rows) + + def log_event(self, event: Event) -> None: + with self._lock, self._db: + self._db.execute( + "INSERT INTO event (ts, source, host, kind, severity, message, data) VALUES (?,?,?,?,?,?,?)", + (event.ts, event.source, event.host, event.kind, event.severity, event.message, + json.dumps(event.data) if event.data else None), + ) + + # -- reading --------------------------------------------------------------- # + def query(self, source: str, host: str, metric: str, *, res: str = RES_RAW, + start: float = 0.0, end: float | None = None, limit: int = 2000, + tail: bool = False) -> list[tuple[float, float]]: + """(ts, value) ascending by time within [start, end]. + + `tail=True` keeps the *newest* `limit` rows instead of the oldest. A window query + that truncates at the head draws the left edge of the chart and silently drops + "now", which is the one point a dashboard has to have. + """ + end = time.time() if end is None else end + where = ("source=? AND host=? AND metric=? AND res=? AND ts>=? AND ts<=?") + args: tuple[object, ...] = (source, host, metric, res, start, end) + with self._lock: + if tail: + cur = self._db.execute( + f"SELECT ts, value FROM (SELECT ts, value FROM sample WHERE {where}" + " ORDER BY ts DESC LIMIT ?) ORDER BY ts ASC", + (*args, limit), + ) + else: + cur = self._db.execute( + f"SELECT ts, value FROM sample WHERE {where} ORDER BY ts ASC LIMIT ?", + (*args, limit), + ) + return cur.fetchall() # type: ignore[no-any-return] + + def pick_res(self, source: str, host: str, metric: str, span: float, max_points: int) -> str: + """Finest resolution that still fits in `max_points` for this window.""" + now = time.time() + start = now - span + finest_with_data: str | None = None + for res in (RES_RAW, RES_MIN, RES_HOUR): + with self._lock: + (count,) = self._db.execute( + "SELECT COUNT(*) FROM sample WHERE source=? AND host=? AND metric=? AND res=? AND ts>=?", + (source, host, metric, res, start), + ).fetchone() + if not count: + continue + if finest_with_data is None: + # too many points for the budget is fine — history() thins. Returning a + # resolution with *no* rows in the window reads as a dead sensor. + finest_with_data = res + if count <= max_points: + return res + return finest_with_data or RES_HOUR + + def history(self, source: str, host: str, metric: str, *, span: float = 3600.0, + max_points: int = 600) -> list[tuple[float, float]]: + """Windowed series, thinned to `max_points` so the browser never gets 100k rows.""" + res = self.pick_res(source, host, metric, span, max_points) + window_start = time.time() - span + points = self.query(source, host, metric, res=res, start=window_start, + limit=max_points * 4, tail=True) + if not points and res != RES_RAW: + # Rollups lag behind (they only cover buckets that are fully in the past, and a + # dashboard that has been up for ten minutes has none at all). Falling back to + # raw turns "no samples yet" into a real chart instead of a lie by omission. + points = self.query(source, host, metric, res=RES_RAW, start=window_start, + limit=max_points * 4, tail=True) + if len(points) <= max_points: + return points + stride = math.ceil(len(points) / max_points) + thinned = points[::stride] + if thinned[-1] != points[-1]: + thinned.append(points[-1]) # never lose "now" to the thinning + return thinned + + def series_names(self, source: str | None = None, host: str | None = None) -> list[str]: + sql = "SELECT DISTINCT metric FROM sample" + args: list[object] = [] + clauses = [] + if source: + clauses.append("source=?") + args.append(source) + if host: + clauses.append("host=?") + args.append(host) + if clauses: + sql += " WHERE " + " AND ".join(clauses) + with self._lock: + return [row[0] for row in self._db.execute(sql + " ORDER BY metric", args)] + + def hosts(self, source: str) -> list[str]: + with self._lock: + return [row[0] for row in self._db.execute( + "SELECT DISTINCT host FROM sample WHERE source=? ORDER BY host", (source,))] + + def recent_events(self, limit: int = 50, *, source: str | None = None, + since: float | None = None) -> list[Event]: + sql = "SELECT ts, source, host, kind, severity, message, data FROM event" + args: list[object] = [] + clauses: list[str] = [] + if source: + clauses.append("source=?") + args.append(source) + if since is not None: + clauses.append("ts>=?") + args.append(since) + if clauses: + sql += " WHERE " + " AND ".join(clauses) + sql += " ORDER BY ts DESC LIMIT ?" + args.append(limit) + out: list[Event] = [] + with self._lock: + for ts, src, host, kind, sev, msg, data in self._db.execute(sql, args): + out.append(Event(ts, src, host, kind, sev, msg, json.loads(data) if data else None)) + return out + + def count(self, *, res: str | None = None) -> int: + sql = "SELECT COUNT(*) FROM sample" + args: tuple[object, ...] = () + if res: + sql, args = "SELECT COUNT(*) FROM sample WHERE res=?", (res,) + with self._lock: + return int(self._db.execute(sql, args).fetchone()[0]) + + # -- maintenance ----------------------------------------------------------- # + def maintain(self, now: float | None = None) -> dict[str, int]: + """Roll up and prune. Safe to call every minute; each step is idempotent. + + A bucket is only rolled once its whole window is in the past (+60s slack), so a + rollup never averages a partial minute and then gets rewritten by a later one. + """ + now = time.time() if now is None else now + counts = {"to_minute": 0, "to_hour": 0, "pruned": 0} + with self._lock, self._db: + counts["to_minute"] = self._db.execute( + "INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)" + " SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)" + " FROM sample WHERE res=? AND ts < ? AND ts >= ?" + " GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)", + (RES_MIN, MINUTE, MINUTE, RES_RAW, now - MINUTE - 60, + now - self.raw_keep_seconds, MINUTE), + ).rowcount + counts["to_hour"] = self._db.execute( + "INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)" + " SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)" + " FROM sample WHERE res=? AND ts < ? AND ts >= ?" + " GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)", + (RES_HOUR, HOUR, HOUR, RES_MIN, now - HOUR - 60, + now - self.minute_keep_seconds, HOUR), + ).rowcount + counts["pruned"] = self._db.execute( + "DELETE FROM sample WHERE (res=? AND ts < ?) OR (res=? AND ts < ?) OR (res=? AND ts < ?)", + (RES_RAW, now - self.raw_keep_seconds, + RES_MIN, now - self.minute_keep_seconds, + RES_HOUR, now - self.hour_keep_seconds), + ).rowcount + self._db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_maintain', ?)", (str(now),)) + return counts + + def stats(self) -> dict[str, object]: + with self._lock: + by_res = {row[0]: row[1] for row in self._db.execute( + "SELECT res, COUNT(*) FROM sample GROUP BY res")} + (events,) = self._db.execute("SELECT COUNT(*) FROM event").fetchone() + last = self._db.execute("SELECT value FROM meta WHERE key='last_maintain'").fetchone() + size = 0 + if str(self.path) != ":memory:" and self.path.exists(): + size = self.path.stat().st_size + return { + "path": str(self.path), + "size_bytes": size, + "samples_by_res": by_res, + "samples_total": sum(by_res.values()), + "events": events, + "last_maintain": float(last[0]) if last else None, + } + + def delete_series(self, source: str, host: str, metric: str | None = None) -> int: + """Drop history for a series (or a whole host). Used by tests and by the + 'forget this host' escape hatch when a target is renamed for real.""" + sql = "DELETE FROM sample WHERE source=? AND host=?" + args: list[object] = [source, host] + if metric: + sql += " AND metric=?" + args.append(metric) + with self._lock, self._db: + return self._db.execute(sql, args).rowcount + + +def samples_from_flat(source: str, host: str, values: dict[str, float | None], + ts: float, *, prefix: str = "") -> list[Sample]: + """Flatten a parse result into samples, skipping absent values. + + `n/a` in the UI means "this exporter does not report it" — it must not become a + stored 0, which would look identical on a 30-day chart and lie about history. + """ + out: list[Sample] = [] + for key, value in values.items(): + if value is None or isinstance(value, dict | list | str | bool): + continue + out.append(Sample(source, host, f"{prefix}{key}", float(value), ts)) + return out + + +def select_window(span: float, allowed: Sequence[float]) -> float: + """Closest allowed window >= span (used to map ?range=1h onto the presets).""" + for candidate in sorted(allowed): + if span <= candidate: + return candidate + return max(allowed) diff --git a/src/dashboards/web/pages/index.html b/src/dashboards/web/pages/index.html new file mode 100644 index 0000000..863bfd5 --- /dev/null +++ b/src/dashboards/web/pages/index.html @@ -0,0 +1,128 @@ + + + + + +Homelab · overview + + + + +
+ ◆ Homelab + +
+ + --:--:-- + +
+
+ +
+
+
+

Overview

+
reading fleet…
+
+
+ +
+ +
+ +
+
+

System resources

+
+
+
Hosts answering
+
n/a
+
CPU average
+
n/a%
+
+
Memory
+
n/a
+
+
Worst disk
+
n/a%
+
+
GPUs
+
n/a
+
+
Open system dashboard →
+
+
+ +
+

LLM engines

+
+
+
Engines answering
+
n/a
+
Decode
+
n/atok/s
+
Prompt
+
n/atok/s
+
Active sessions
+
n/a
+
Queued
+
n/a
+
KV cache now
+
n/a%
+
+
+
Open LLM dashboard →
+
+
+ + +
+ +
+
+

Fleet

+
+ + + +
HostCPULoad 1Mem usedWorst diskGPU busyTempUp
+ +
+
+ +
+

Recent events

+ transitions only
+
  • loading…
+
+
+ +
+ + + every number is scraped from the box it belongs to · null renders as n/a +
+
+ + + + + + diff --git a/src/dashboards/web/pages/llm.html b/src/dashboards/web/pages/llm.html new file mode 100644 index 0000000..a0e3245 --- /dev/null +++ b/src/dashboards/web/pages/llm.html @@ -0,0 +1,121 @@ + + + + + +Homelab · LLM + + + + +
+ ◆ Homelab + +
+ + --:--:-- + + +
+
+ +
+
+
+

LLM engines

+
reading engines…
+
+
+
window
+
+ +
+ +
+
+
+
Engines answering
+
n/a
+
Decode (sum)
+
n/atok/s
+
Prompt (sum)
+
n/atok/s
+
Active sessions
+
n/a
+
Queued
+
n/a
+
Tokens lifetime
+
n/a
+
KV cache now
+
n/a%
+
+
+
+
+
+ +
+
+

Decode throughput

+ tok/s · every engine, summed slots
+
+
+
+

Prompt / prefill throughput

+ tok/s · newly computed tokens
+
+
+
+

KV / prefix cache pressure

% of cache in use
+
+
+
+

Sessions & queue

running vs waiting
+
+
+
+ +
+
+ +
+ +
+
+

gufo slots

+ one row per parallel slot
+
+ +
SlotStateTokensN promptN genCacheSpeed
+ +
+
+
+

Events

engine up / down
+
  • loading…
+
+
+ +
+ each cell carries its provenance: scraped gauge, Δ10s measured + counter, or lifetime estimate + + +
+
+ + + + + + diff --git a/src/dashboards/web/pages/system.html b/src/dashboards/web/pages/system.html new file mode 100644 index 0000000..ffb604d --- /dev/null +++ b/src/dashboards/web/pages/system.html @@ -0,0 +1,136 @@ + + + + + +Homelab · system + + + + +
+ ◆ Homelab + +
+ + --:--:-- + + +
+
+ +
+
+
+

System resources

+
reading hosts…
+
+
+
+ window +
+
+
+ +
+
+
+
Hosts answering
+
n/a
+
CPU average
+
n/a%
+
Load 1 (sum)
+
n/a
+
Memory in use
+
n/a
+
Worst filesystem
+
n/a%
+
Network in
+
n/a
+
Network out
+
n/a
+
GPU busy (avg)
+
n/a%
+
Power
+
n/a
+
+
+
+ +
+
+

CPU utilisation

% · per host
+
+
+
+

Memory used

% of MemAvailable basis
+
+
+
+

Disk I/O throughput

+ selected host · all physical devices summed
+
+
+
+

Network throughput

selected host · physical interfaces
+
+
+
+

GPU utilisation

one series per host GPU
+
+
+
+

Temperature

CPU package · °C
+
+
+
+ +
+
+

Detail

+ + + + + +
+
+
+
+
+
+
+ +
+

Per host

+ agent-reported values; rates measured over the window in each header
+
+ +
+ +
+

Events

unreachable / reachable / reboots
+
  • loading…
+
+ +
+ numbers are measured, not inferred: rates are Δcounter/Δt over the poll window + + +
+
+ + + + + + diff --git a/src/dashboards/web/pages/ups.html b/src/dashboards/web/pages/ups.html new file mode 100644 index 0000000..ac66a68 --- /dev/null +++ b/src/dashboards/web/pages/ups.html @@ -0,0 +1,101 @@ + + + + + +Homelab · UPS + + + + +
+ ◆ Homelab + +
+ + --:--:-- + + +
+
+ +
+
+
+

UPS & battery health

+
reading upsd…
+
+
+
window
+
+ +
+ +
+ + +
+
+

Battery charge

%
+
+
+
+

Runtime estimate

+ seconds of backup at the current load
+
+
+
+

Voltages

battery and incoming line
+
+
+
+

Load on the UPS

% of rated capacity
+
+
+
+ +
+

Power events

+ state changes, self-test results, commands sent
+
  • loading…
+
+ +
+ commands are the only write path in this app, and every one of them is confirmed here first + + +
+
+ + +
confirm

Run this command?

+
+

+

+ + +
+
+ + +
+ + + + + + + diff --git a/src/dashboards/web/static/app.js b/src/dashboards/web/static/app.js new file mode 100644 index 0000000..37e4fba --- /dev/null +++ b/src/dashboards/web/static/app.js @@ -0,0 +1,341 @@ +/* app.js — shared behaviour for every page: formatting, polling, theme, status. + * + * Two rules the whole UI follows: + * - `null`/`undefined` renders as `n/a`, never as `0`, `""`, or a leftover number. + * A dash that changes to another dash silently is how dashboards lie. + * - a stale value is shown *and labelled*: the dot goes amber/red, `stale 12s` appears, + * and the card dims. Hiding the failure is worse than the failure. + */ +'use strict'; + +const Dash = (() => { + const $ = (sel, root) => (root || document).querySelector(sel); + const $$ = (sel, root) => Array.from((root || document).querySelectorAll(sel)); + + // ---- formatting --------------------------------------------------------- + const BYTE_UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']; + + function fmtBytes(value, digits) { + if (value == null || !isFinite(value)) return null; + let v = Math.abs(value); + let i = 0; + while (v >= 1024 && i < BYTE_UNITS.length - 1) { v /= 1024; i += 1; } + const d = digits != null ? digits : (v >= 100 || i === 0 ? 0 : v >= 10 ? 1 : 2); + return { text: v.toFixed(d), unit: BYTE_UNITS[i] }; + } + + function bytes(value, digits) { + const parts = fmtBytes(value, digits); + return parts ? `${parts.text} ${parts.unit}` : naText; + } + + function rate(value, digits) { + const parts = fmtBytes(value, digits); + return parts ? `${parts.text} ${parts.unit}/s` : naText; + } + + function pct(value, digits) { + if (value == null || !isFinite(value)) return naText; + return `${value.toFixed(digits == null ? 1 : digits)}%`; + } + + function num(value, digits) { + if (value == null || !isFinite(value)) return naText; + const d = digits == null ? (Math.abs(value) >= 100 ? 0 : Math.abs(value) >= 10 ? 1 : 2) : digits; + return value.toFixed(d); + } + + function duration(seconds) { + if (seconds == null || !isFinite(seconds)) return naText; + const s = Math.max(0, seconds); + if (s < 90) return `${Math.round(s)}s`; + const m = s / 60; + if (m < 90) return `${m.toFixed(m < 10 ? 1 : 0)}m`; + const h = m / 60; + if (h < 48) return `${h.toFixed(h < 10 ? 1 : 0)}h`; + const d = h / 24; + if (d < 60) return `${d.toFixed(d < 10 ? 1 : 0)}d`; + return `${(d / 365.25).toFixed(1)}y`; + } + + function age(ts) { + if (ts == null) return 'never'; + const seconds = Math.max(0, Date.now() / 1000 - ts); + if (seconds < 2) return 'now'; + return duration(seconds).replace(/(h|m|s|d|y)$/, '$') + ' ago'; + } + + const naText = 'n/a'; + + // ---- DOM helpers -------------------------------------------------------- + /** Write text only when it changed: avoids killing transitions and focus. */ + function setTxt(el, value) { + if (!el) return; + const text = value == null ? naText : String(value); + if (el.textContent !== text) el.textContent = text; + el.classList.toggle('is-null', value == null); + el.classList.toggle('na', value == null); + } + + function setValue(el, value, digits, unit) { + if (!el) return; + const parent = el.parentElement; + if (value == null || !isFinite(value)) { + setTxt(el, naText); + if (parent) parent.classList.add('is-null'); + return; + } + if (parent) parent.classList.remove('is-null'); + const text = typeof value === 'string' ? value : num(value, digits); + setTxt(el, text); + const unitEl = el.nextElementSibling; + if (unit && unitEl && unitEl.classList.contains('unit')) setTxt(unitEl, unit); + } + + function setBar(el, value, kind) { + if (!el) return; + const fill = el.tagName === 'I' ? el : $('i', el); + const pctValue = value == null ? 0 : Math.max(0, Math.min(100, value)); + if (fill) fill.style.width = `${pctValue}%`; + const threshold = kind || severityFor(pctValue); + el.classList.remove('ok', 'warn', 'fail'); + el.classList.add(threshold); + } + + function severityFor(value, warnAt, failAt) { + const warn = warnAt == null ? 75 : warnAt; + const fail = failAt == null ? 90 : failAt; + if (value == null) return ''; + if (value >= fail) return 'fail'; + if (value >= warn) return 'warn'; + return 'ok'; + } + + function setDot(el, status) { + if (!el) return; + el.classList.remove('ok', 'warn', 'fail', 'idle'); + el.classList.add(status || 'idle'); + } + + function chip(text, kind, title) { + const span = document.createElement('span'); + span.className = 'chip' + (kind ? ' ' + kind : ''); + span.textContent = text; + if (title) span.title = title; + return span; + } + + // ---- networking --------------------------------------------------------- + async function fetchJSON(url, options) { + const response = await fetch(url, Object.assign({ headers: { accept: 'application/json' } }, + options || {})); + let body = null; + try { body = await response.json(); } catch (err) { body = null; } + if (!response.ok) { + const detail = body && body.detail + ? (typeof body.detail === 'string' ? body.detail : body.detail.detail || JSON.stringify(body.detail)) + : `${response.status} ${response.statusText}`; + const error = new Error(detail); + error.status = response.status; + error.body = body; + throw error; + } + return body; + } + + /** + * Poll `fn` every `ms`, skipping overlapping runs and pausing when the tab is hidden + * or the user hit pause. Errors are reported, never swallowed silently. + */ + function poll(fn, ms) { + let stopped = false; + let timer = null; + let inFlight = false; + const state = { + get paused() { return state._paused; }, + _paused: false, + interval: ms, + setInterval(next) { state.interval = next; if (timer) { clearTimeout(timer); tick(); } }, + setPaused(next) { + state._paused = next; + if (next) { if (timer) clearTimeout(timer); timer = null; } + else if (!timer) tick(); + }, + stop() { stopped = true; if (timer) clearTimeout(timer); }, + runNow() { tick(); }, + }; + async function tick() { + if (stopped || inFlight || state._paused || document.hidden) { schedule(); return; } + inFlight = true; + try { await fn(); } finally { inFlight = false; schedule(); } + } + function schedule() { + if (stopped || state._paused) return; + timer = setTimeout(tick, state.interval); + } + document.addEventListener('visibilitychange', () => { if (!document.hidden && !state._paused) tick(); }); + tick(); + return state; + } + + // ---- chrome ------------------------------------------------------------- + function initChrome() { + const root = document.documentElement; + const stored = localStorage.getItem('dash-theme'); + const prefersLight = window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches; + root.dataset.theme = stored || (prefersLight ? 'light' : 'dark'); + + const themeButton = $('#theme-toggle'); + if (themeButton) { + const paint = () => { + themeButton.textContent = root.dataset.theme === 'light' ? '☀ Light' : '☾ Dark'; + themeButton.setAttribute('aria-pressed', root.dataset.theme === 'light' ? 'true' : 'false'); + }; + themeButton.addEventListener('click', () => { + root.dataset.theme = root.dataset.theme === 'light' ? 'dark' : 'light'; + localStorage.setItem('dash-theme', root.dataset.theme); + paint(); + document.dispatchEvent(new CustomEvent('dash:theme')); + }); + paint(); + } + + const clock = $('#clock'); + if (clock) { + const tickClock = () => { + const now = new Date(); + const target = document.getElementById('sim-flag'); + clock.textContent = now.toLocaleTimeString(undefined, { hour12: false }); + void target; + }; + tickClock(); + setInterval(tickClock, 1000); + } + + const nav = $('#nav-status'); + if (nav) { + const jobs = [ + { key: 'system', label: 'hosts', href: '/system' }, + { key: 'llm', label: 'engines', href: '/llm' }, + { key: 'ups', label: 'UPS', href: '/ups' }, + ]; + nav.innerHTML = jobs.map((job) => ` + ${job.label}`).join(''); + const refreshStatus = async () => { + try { + const health = await fetchJSON('/api/health'); + for (const job of jobs) { + const dot = nav.querySelector(`[data-dot="${job.key}"]`); + const states = (health.sources || []).filter((s) => s.dashboard === job.key && s.configured); + if (!states.length) { setDot(dot, 'idle'); continue; } + const ok = states.filter((s) => s.ok).length; + setDot(dot, ok === states.length ? 'ok' : ok ? 'warn' : 'fail'); + dot.title = `${ok}/${states.length} ${job.label} answering`; + } + } catch (err) { + for (const job of jobs) setDot(nav.querySelector(`[data-dot="${job.key}"]`), 'fail'); + } + }; + refreshStatus(); + setInterval(refreshStatus, 15000); + } + } + + /** Range buttons (5m … 30d). Calls onChange(seconds, key). */ + function rangeControl(container, current, onChange, choices) { + const options = choices || [['5m', 300], ['15m', 900], ['1h', 3600], ['6h', 21600], + ['24h', 86400], ['7d', 604800], ['30d', 2592000]]; + let value = current || 3600; + container.className = 'gap'; + container.innerHTML = ''; + const buttons = options.map(([key, seconds]) => { + const button = document.createElement('button'); + button.className = 'ghost'; + button.type = 'button'; + button.textContent = key; + button.dataset.seconds = String(seconds); + button.setAttribute('aria-pressed', seconds === value ? 'true' : 'false'); + button.addEventListener('click', () => { + value = seconds; + buttons.forEach((b) => b.setAttribute('aria-pressed', + Number(b.dataset.seconds) === value ? 'true' : 'false')); + onChange(seconds, key); + }); + container.appendChild(button); + return button; + }); + return { get value() { return value; } }; + } + + let toastTimer = null; + function toast(message, kind) { + let host = $('#toasts'); + if (!host) { + host = document.createElement('div'); + host.id = 'toasts'; + host.style.cssText = 'position:fixed;right:1rem;bottom:1rem;z-index:60;display:grid;gap:.4rem'; + document.body.appendChild(host); + } + const div = document.createElement('div'); + div.className = 'banner ' + (kind || 'info'); + div.textContent = message; + host.appendChild(div); + setTimeout(() => div.remove(), kind === 'fail' ? 9000 : 4500); + if (toastTimer) clearTimeout(toastTimer); + toastTimer = setTimeout(() => { host.remove(); host = null; }, 10000); + } + + /** Pause button + Space, wired to a poll handle. */ + function bindPause(poller) { + const button = $('#pause'); + if (!button || !poller) return; + const paint = () => { + button.setAttribute('aria-pressed', poller.paused ? 'true' : 'false'); + button.textContent = poller.paused ? '▶ Paused' : '⏸ Pause'; + }; + button.addEventListener('click', () => { poller.setPaused(!poller.paused); paint(); }); + document.addEventListener('keydown', (event) => { + const tag = (event.target && event.target.tagName || '').toLowerCase(); + if (tag === 'input' || tag === 'textarea' || event.metaKey || event.ctrlKey) return; + if (event.code === 'Space') { event.preventDefault(); poller.setPaused(!poller.paused); paint(); } + if (event.key === 't' || event.key === 'T') { + const theme = $('#theme-toggle'); + if (theme) theme.click(); + } + }); + paint(); + } + + /** The "last poll" line every card header shows. */ + function staleness(el, state) { + if (!el) return; + if (!state) { setTxt(el, '—'); return; } + if (!state.configured) { setTxt(el, 'not configured'); return; } + if (!state.ok) { + setTxt(el, state.last_ok ? `stale ${duration(ageSeconds(state.last_ok))} · ${state.error || 'down'}` + : (state.error || 'never reached')); + setDot(el.parentElement && el.parentElement.querySelector('.dot'), 'fail'); + return; + } + const seconds = ageSeconds(state.last_ok); + setTxt(el, seconds > 30 ? `stale ${duration(seconds)}` : `updated ${duration(seconds)} ago`); + setDot(el.parentElement && el.parentElement.querySelector('.dot'), + seconds > 30 ? 'warn' : 'ok'); + } + + function ageSeconds(ts) { + return ts == null ? Infinity : Math.max(0, Date.now() / 1000 - ts); + } + + /** Cards whose data is older than the poll interval get dimmed + labelled. */ + function markStale(card, stale) { + if (!card) return; + card.classList.toggle('stale', !!stale); + } + + return { $, $$, fetchJSON, poll, fmtBytes, bytes, rate, pct, num, duration, age, setTxt, + setValue, setBar, setDot, chip, initChrome, rangeControl, toast, bindPause, + staleness, markStale, severityFor, ageSeconds, naText }; +})(); + +document.addEventListener('DOMContentLoaded', () => Dash.initChrome()); diff --git a/src/dashboards/web/static/charts.js b/src/dashboards/web/static/charts.js new file mode 100644 index 0000000..6ac1c4b --- /dev/null +++ b/src/dashboards/web/static/charts.js @@ -0,0 +1,378 @@ +/* charts.js — hand-rolled SVG time-series charts. No CDN, no chart library. + * + * Coordinate space is real pixels (viewBox = the element's measured size, no + * preserveAspectRatio="none"), which is what keeps axis text crisp: a stretched viewBox + * smears every glyph drawn inside it. Redraw happens on ResizeObserver, not on a timer. + * + * Deliberate behaviours: + * - A series with no points draws nothing and the chart says why. It never draws 0, + * because "no data" and "zero" are different claims. + * - The y-range comes from the data actually plotted, and the labels are generated from + * the same numbers the path used, so a label can never drift from the curve. + * - Gaps are gaps: a sample missing in the middle breaks the line rather than drawing a + * straight lie across the hole. + */ +'use strict'; + +const CHART_COLORS = ['--series-1', '--series-2', '--series-3', '--series-4', + '--series-5', '--series-6', '--series-7', '--series-8']; + +function cssVar(name) { + return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); +} + +/** Round a range to a "nice" tick step: 1, 2, 2.5, 5, 10 x 10^n. */ +function niceTicks(min, max, count) { + if (!isFinite(min) || !isFinite(max) || min === max) { + const base = isFinite(min) ? min : 0; + return { lo: base - 1, hi: base + 1, step: 1, values: [base - 1, base, base + 1] }; + } + const raw = (max - min) / Math.max(1, count); + const mag = Math.pow(10, Math.floor(Math.log10(Math.abs(raw) || 1))); + const norm = raw / mag; + const step = (norm <= 1 ? 1 : norm <= 2 ? 2 : norm <= 2.5 ? 2.5 : norm <= 5 ? 5 : 10) * mag; + const lo = Math.floor(min / step) * step; + const hi = Math.ceil(max / step) * step; + const values = []; + for (let v = lo; v <= hi + step * 0.5; v += step) values.push(Number(v.toFixed(10))); + return { lo, hi, step, values }; +} + +const DEFAULT_FORMAT = (v) => (Math.abs(v) >= 100 ? v.toFixed(0) : v.toFixed(1)); + +class LineChart { + /** + * container: element to fill (its width is used; height comes from opts.height) + * opts: {height, yFormat, xSpan, yMin, yMax, area, legend, showAxis, min, unit} + */ + constructor(container, opts) { + this.el = typeof container === 'string' ? document.querySelector(container) : container; + this.opts = Object.assign({ + height: 150, area: true, legend: true, showAxis: true, + yFormat: DEFAULT_FORMAT, xSpan: 3600, yMin: null, yMax: null, unit: '', + }, opts || {}); + this.series = []; + this.hoverTs = null; + this._build(); + if (typeof ResizeObserver !== 'undefined') { + this._ro = new ResizeObserver(() => this.draw()); + this._ro.observe(this.el); + } else { + window.addEventListener('resize', () => this.draw()); + } + } + + _build() { + this.el.classList.add('chart'); + this.el.style.height = this.opts.height + 'px'; + this.el.innerHTML = ''; + this.svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + this.svg.setAttribute('width', '100%'); + this.svg.setAttribute('height', this.opts.height); + this.el.appendChild(this.svg); + this.empty = document.createElement('div'); + this.empty.className = 'empty'; + this.el.appendChild(this.empty); + this.tooltip = document.createElement('div'); + this.tooltip.className = 'tooltip'; + this.el.appendChild(this.tooltip); + this.svg.addEventListener('mousemove', (e) => this._onMove(e)); + this.svg.addEventListener('mouseleave', () => { this.hoverTs = null; this._hideTip(); this._drawCursor(); }); + } + + /** series: [{name, points: [{ts, value}], color?, yMax?}] — replaces previous data. */ + setData(series) { + this.series = (series || []).map((s, i) => ({ + name: s.name, + color: s.color || cssVar(CHART_COLORS[i % CHART_COLORS.length]), + points: (s.points || []).filter((p) => p && Number.isFinite(p.value) && Number.isFinite(p.ts)) + .sort((a, b) => a.ts - b.ts), + })); + this.draw(); + return this; + } + + _layout() { + const w = Math.max(80, this.el.clientWidth); + const h = this.opts.height; + const pad = this.opts.showAxis ? { l: 44, r: 8, t: 8, b: 16 } : { l: 2, r: 2, t: 3, b: 3 }; + return { w, h, pad, plotW: Math.max(10, w - pad.l - pad.r), plotH: Math.max(10, h - pad.t - pad.b) }; + } + + _xExtent() { + const span = this.opts.xSpan; + const end = Date.now() / 1000; + return [end - span, end]; + } + + _yExtent() { + if (this.opts.yMin !== null && this.opts.yMax !== null) { + return [this.opts.yMin, this.opts.yMax]; + } + let lo = Infinity, hi = -Infinity; + for (const s of this.series) { + for (const p of s.points) { + if (p.value < lo) lo = p.value; + if (p.value > hi) hi = p.value; + } + } + if (!isFinite(lo)) return [0, 1]; + if (lo === hi) { const pad = Math.abs(lo) * 0.1 || 1; return [lo - pad, hi + pad]; } + const margin = (hi - lo) * 0.12; + lo = this.opts.yMin !== null ? this.opts.yMin : lo - margin; + hi = this.opts.yMax !== null ? this.opts.yMax : hi + margin; + return [lo, hi]; + } + + draw() { + if (!this.el.clientWidth) return; + const plotted = this.series.filter((s) => s.points.length); + const { w, h, pad, plotW, plotH } = this._layout(); + this.svg.setAttribute('viewBox', `0 0 ${w} ${h}`); + this.svg.innerHTML = ''; + if (!plotted.length) { + this.empty.textContent = 'no samples yet'; + this.empty.style.display = 'grid'; + return; + } + this.empty.style.display = 'none'; + const [x0, x1] = this._xExtent(); + const [ylo, yhi] = this._yExtent(); + const ticks = niceTicks(ylo, yhi, 4); + this.scales = { + x: (ts) => pad.l + ((ts - x0) / (x1 - x0)) * plotW, + y: (v) => pad.t + plotH - ((Math.min(Math.max(v, ticks.lo), ticks.hi) - ticks.lo) / (ticks.hi - ticks.lo)) * plotH, + ticks, x0, x1, ylo: ticks.lo, yhi: ticks.hi, + }; + const ns = 'http://www.w3.org/2000/svg'; + const g = (name) => { + const grp = document.createElementNS(ns, 'g'); + grp.setAttribute('class', name); + this.svg.appendChild(grp); + return grp; + }; + if (this.opts.showAxis) { + const grid = g('grid'); + for (const t of ticks.values) { + const y = this.scales.y(t); + if (y < pad.t - 1 || y > pad.t + plotH + 1) continue; + const line = document.createElementNS(ns, 'line'); + line.setAttribute('x1', pad.l); line.setAttribute('x2', pad.l + plotW); + line.setAttribute('y1', y); line.setAttribute('y2', y); + grid.appendChild(line); + const text = document.createElementNS(ns, 'text'); + text.setAttribute('x', pad.l - 6); text.setAttribute('y', y + 3.5); + text.setAttribute('text-anchor', 'end'); + text.textContent = this.opts.yFormat(t); + grid.appendChild(text); + } + const axis = g('axis'); + for (let i = 0; i <= 4; i++) { + const ts = x0 + ((x1 - x0) * i) / 4; + const x = this.scales.x(ts); + const text = document.createElementNS(ns, 'text'); + text.setAttribute('x', x); text.setAttribute('y', h - 3); + text.setAttribute('text-anchor', i === 0 ? 'start' : i === 4 ? 'end' : 'middle'); + text.textContent = shortTime(ts, x1 - x0); + axis.appendChild(text); + } + } + for (const s of plotted) { + const paths = buildPath(s.points, this.scales, plotH + pad.t); + if (this.opts.area && paths.length) { + for (const seg of paths) { + const area = document.createElementNS(ns, 'path'); + area.setAttribute('class', 'area'); + area.setAttribute('d', seg.area); + area.setAttribute('fill', s.color); + this.svg.appendChild(area); + } + } + for (const seg of paths) { + const line = document.createElementNS(ns, 'path'); + line.setAttribute('class', 'line'); + line.setAttribute('d', seg.line); + line.setAttribute('stroke', s.color); + this.svg.appendChild(line); + } + } + const cursor = document.createElementNS(ns, 'g'); + cursor.setAttribute('class', 'cursor'); + this.svg.appendChild(cursor); + this.cursorEl = cursor; + this._drawCursor(); + } + + _drawCursor() { + if (!this.cursorEl || !this.scales || this.hoverTs == null) { + if (this.cursorEl) this.cursorEl.innerHTML = ''; + return; + } + const { pad } = this._layout(); + const x = this.scales.x(this.hoverTs); + this.cursorEl.innerHTML = ''; + const line = document.createElementNS('http://www.w3.org/2000/svg', 'line'); + line.setAttribute('x1', x); line.setAttribute('x2', x); + line.setAttribute('y1', pad.t); line.setAttribute('y2', this.opts.height - pad.b); + this.cursorEl.appendChild(line); + } + + _onMove(event) { + if (!this.scales) return; + const rect = this.svg.getBoundingClientRect(); + const box = (this.svg.getAttribute('viewBox') || '0 0 1 1').split(' '); + const px = ((event.clientX - rect.left) / rect.width) * Number(box[2]); + const { pad, plotW } = this._layout(); + const { x0, x1 } = this.scales; + const frac = Math.min(1, Math.max(0, (px - pad.l) / plotW)); + const at = x0 + frac * (x1 - x0); + this.hoverTs = at; + this._drawCursor(); + this._showTip(at, event); + } + + _showTip(at, event) { + const rows = this.series.map((s) => { + const near = nearest(s.points, at); + if (!near) return null; + return `
` + + `${escapeHtml(s.name)}` + + `${this.opts.yFormat(near.value)}${this.opts.unit ? ' ' + this.opts.unit : ''}
`; + }).filter(Boolean); + if (!rows.length) { this._hideTip(); return; } + this.tooltip.innerHTML = `
${fullTime(at)}
${rows.join('')}`; + this.tooltip.style.opacity = '1'; + const rect = this.el.getBoundingClientRect(); + const left = Math.min(rect.width - 150, Math.max(4, event.clientX - rect.left + 12)); + this.tooltip.style.left = left + 'px'; + this.tooltip.style.top = '8px'; + } + + _hideTip() { this.tooltip.style.opacity = '0'; } +} + +/** Split points into contiguous runs; a gap > 3x the median step breaks the line. */ +function buildPath(points, scales, baseline) { + if (!points.length) return []; + const steps = []; + for (let i = 1; i < points.length; i++) steps.push(points[i].ts - points[i - 1].ts); + steps.sort((a, b) => a - b); + const median = steps.length ? steps[Math.floor(steps.length / 2)] : 0; + const maxGap = median > 0 ? median * 3 : Infinity; + const runs = []; + let current = [points[0]]; + for (let i = 1; i < points.length; i++) { + if (points[i].ts - points[i - 1].ts > maxGap) { runs.push(current); current = [points[i]]; } + else current.push(points[i]); + } + runs.push(current); + return runs.map((run) => { + let line = ''; + run.forEach((p, i) => { + const x = scales.x(p.ts).toFixed(1); + const y = scales.y(p.value).toFixed(1); + line += (i === 0 ? `M${x} ${y}` : ` L${x} ${y}`); + }); + const first = scales.x(run[0].ts).toFixed(1); + const last = scales.x(run[run.length - 1].ts).toFixed(1); + const area = `${line} L${last} ${baseline.toFixed(1)} L${first} ${baseline.toFixed(1)} Z`; + return { line, area }; + }); +} + +function nearest(points, ts) { + if (!points.length) return null; + let lo = 0, hi = points.length - 1; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (points[mid].ts < ts) lo = mid + 1; else hi = mid; + } + const a = points[Math.max(0, lo - 1)], b = points[lo]; + return Math.abs(a.ts - ts) < Math.abs(b.ts - ts) ? a : b; +} + +function shortTime(ts, span) { + const d = new Date(ts * 1000); + if (span > 86400 * 3) return d.toLocaleDateString(undefined, { month: 'short', day: 'numeric' }); + if (span > 86400) return d.toLocaleDateString(undefined, { weekday: 'short' }) + ' ' + + d.toLocaleTimeString(undefined, { hour: '2-digit' }); + return d.toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }); +} + +function fullTime(ts) { + return new Date(ts * 1000).toLocaleString(undefined, { + month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit', + }); +} + +function escapeHtml(value) { + return String(value).replace(/[&<>"']/g, (c) => ({ + '&': '&', '<': '<', '>': '>', '"': '"', "'": ''', + }[c])); +} + +/** Legend with the live last value per series, under the chart. */ +/** + * Legend with the live last value per series, under the chart. + * + * `showPeak` adds the window maximum too. Throughput is bursty: a legend that reads + * `0.0 tok/s` next to a chart that clearly went to 60 twice is read as a broken dashboard, + * even though "last poll was idle" is the truth. Saying both keeps it honest and legible. + */ +function renderLegend(container, series, format, showPeak) { + container.innerHTML = series.map((s, i) => { + const color = s.color || cssVar(CHART_COLORS[i % CHART_COLORS.length]); + const points = (s.points || []).filter((p) => Number.isFinite(p.value)); + const last = points[points.length - 1]; + let peakHtml = ''; + if (showPeak && points.length) { + const peak = points.reduce((best, p) => (p.value > best.value ? p : best), points[0]); + if (peak.value !== last.value && peak.value > 0) { + peakHtml = `peak ${format(peak.value)}`; + } + } + return `` + + `${escapeHtml(s.name)}` + + `${last ? format(last.value) : 'n/a'}${peakHtml}`; + }).join(''); +} + +/** Column heat bars for per-core usage: 48 cores read as a pattern, not a table. */ +function renderCores(container, values, format) { + const f = format || ((v) => (v == null ? 'n/a' : v.toFixed(0) + '%')); + container.innerHTML = (values || []).map((v, i) => { + const pct = v == null ? 0 : Math.max(0, Math.min(100, v)); + const hue = pct > 85 ? 'var(--fail)' : pct > 60 ? 'var(--warn)' : 'var(--accent)'; + return `` + + ``; + }).join(''); +} + +/** Small non-interactive sparkline for cards where a full chart would shout. */ +function sparkline(container, points, opts) { + const o = Object.assign({ height: 34, color: cssVar('--series-1'), min: null, max: null }, opts || {}); + const pts = (points || []).filter((p) => p && Number.isFinite(p.value)); + const w = Math.max(60, container.clientWidth || 160); + const h = o.height; + container.innerHTML = ''; + if (pts.length < 2) { + container.innerHTML = '
no history
'; + return; + } + const xs = pts.map((p) => p.ts), ys = pts.map((p) => p.value); + const x0 = Math.min(...xs), x1 = Math.max(...xs); + let lo = o.min !== null ? o.min : Math.min(...ys); + let hi = o.max !== null ? o.max : Math.max(...ys); + if (lo === hi) { const pad = Math.abs(lo) * 0.1 || 1; lo -= pad; hi += pad; } + const sx = (ts) => ((ts - x0) / (x1 - x0 || 1)) * (w - 2) + 1; + const sy = (v) => h - 2 - ((v - lo) / (hi - lo)) * (h - 4); + const d = pts.map((p, i) => `${i ? 'L' : 'M'}${sx(p.ts).toFixed(1)} ${sy(p.value).toFixed(1)}`).join(' '); + const svg = ` + + `; + container.innerHTML = svg; +} + +window.DashCharts = { LineChart, niceTicks, renderLegend, renderCores, sparkline, cssVar, + CHART_COLORS, DEFAULT_FORMAT, escapeHtml }; diff --git a/src/dashboards/web/static/dash.css b/src/dashboards/web/static/dash.css new file mode 100644 index 0000000..9f57f3a --- /dev/null +++ b/src/dashboards/web/static/dash.css @@ -0,0 +1,442 @@ +/* Homelab dashboards — design system. + * + * No CDN, no webfonts, no framework: this runs on a homelab network that may have no + * route to the internet, and a font that fails to load is a page that reflows in front + * of you. Everything here is system fonts and hand-written SVG. + * + * Two themes via [data-theme] on (persisted in localStorage), defaulting to the + * OS preference. All colours are tokens; nothing in a component hardcodes a colour. + */ + +:root { + --bg: #0d0f13; + --bg-raised: #14171e; + --bg-sunken: #0a0c0f; + --border: #232836; + --border-strong: #333b4d; + --text: #e7eaf0; + --text-muted: #8b94a7; + --text-faint: #5c6478; + --accent: #ff8a3d; + --accent-soft: rgba(255, 138, 61, 0.14); + --ok: #3fb950; + --ok-soft: rgba(63, 185, 80, 0.15); + --warn: #d29922; + --warn-soft: rgba(210, 153, 34, 0.16); + --fail: #f85149; + --fail-soft: rgba(248, 81, 73, 0.16); + --info: #58a6ff; + --info-soft: rgba(88, 166, 255, 0.15); + --series-1: #ff8a3d; + --series-2: #58a6ff; + --series-3: #3fb950; + --series-4: #bc8cff; + --series-5: #f778ba; + --series-6: #d29922; + --series-7: #39c5cf; + --series-8: #8b94a7; + --radius: 10px; + --radius-sm: 6px; + --shadow: 0 1px 2px rgba(0, 0, 0, 0.4), 0 8px 24px rgba(0, 0, 0, 0.25); + --mono: ui-monospace, "SF Mono", "JetBrains Mono", "Cascadia Mono", Menlo, Consolas, monospace; + --sans: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + --header-h: 58px; +} + +[data-theme="light"] { + --bg: #f4f5f8; + --bg-raised: #ffffff; + --bg-sunken: #e9ebf0; + --border: #dfe3ea; + --border-strong: #c3cad6; + --text: #16191f; + --text-muted: #5b6473; + --text-faint: #8a93a3; + --accent: #d2691e; + --accent-soft: rgba(210, 105, 30, 0.12); + --ok: #1a7f37; + --ok-soft: rgba(26, 127, 55, 0.12); + --warn: #9a6700; + --warn-soft: rgba(154, 103, 0, 0.12); + --fail: #cf222e; + --fail-soft: rgba(207, 34, 46, 0.1); + --info: #0969da; + --info-soft: rgba(9, 105, 218, 0.1); + --series-1: #d2691e; + --series-2: #0969da; + --series-3: #1a7f37; + --series-4: #8250df; + --series-5: #bf3989; + --series-6: #9a6700; + --series-7: #1b7c84; + --series-8: #5b6473; + --shadow: 0 1px 2px rgba(16, 24, 40, 0.06), 0 6px 20px rgba(16, 24, 40, 0.06); +} + +* { box-sizing: border-box; } + +html, body { height: 100%; } + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font: 14px/1.5 var(--sans); + -webkit-font-smoothing: antialiased; + font-variant-numeric: tabular-nums; +} + +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } + +:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 4px; } + +/* ---- header ------------------------------------------------------------- */ +.masthead { + position: sticky; + top: 0; + z-index: 40; + display: flex; + align-items: center; + gap: 1.1rem; + height: var(--header-h); + padding: 0 1.25rem; + background: color-mix(in srgb, var(--bg-raised) 92%, transparent); + backdrop-filter: blur(8px); + border-bottom: 1px solid var(--border); +} + +.brand { + display: flex; + align-items: center; + gap: .55rem; + font-weight: 650; + letter-spacing: .01em; + color: var(--text); + white-space: nowrap; +} +.brand:hover { text-decoration: none; } +.brand-mark { + width: 22px; height: 22px; + border-radius: 6px; + background: linear-gradient(140deg, var(--accent), color-mix(in srgb, var(--accent) 45%, var(--info))); + box-shadow: inset 0 0 0 1px rgba(255, 255, 255, .12); + display: grid; + place-items: center; + font-size: 12px; + color: #14171e; + font-weight: 800; +} + +nav.tabs { display: flex; gap: .15rem; margin-inline-start: .4rem; } +nav.tabs a { + padding: .38rem .7rem; + border-radius: var(--radius-sm); + color: var(--text-muted); + font-weight: 550; + display: flex; + align-items: center; + gap: .4rem; +} +nav.tabs a:hover { background: var(--bg-sunken); color: var(--text); text-decoration: none; } +nav.tabs a[aria-current="page"] { background: var(--accent-soft); color: var(--accent); } + +.masthead-right { margin-inline-start: auto; display: flex; align-items: center; gap: .55rem; } + +.clock { font-family: var(--mono); font-size: 12.5px; color: var(--text-muted); } +@media (max-width: 900px) { .clock { display: none; } } + +button.ghost { + background: transparent; + border: 1px solid var(--border); + color: var(--text-muted); + border-radius: var(--radius-sm); + padding: .32rem .6rem; + font: 550 12.5px var(--sans); + cursor: pointer; + display: inline-flex; + align-items: center; + gap: .35rem; +} +button.ghost:hover { border-color: var(--border-strong); color: var(--text); background: var(--bg-sunken); } +button.ghost[aria-pressed="true"] { border-color: var(--accent); color: var(--accent); background: var(--accent-soft); } + +/* ---- layout ------------------------------------------------------------- */ +main { padding: 1.15rem 1.25rem 3rem; max-width: 1680px; margin: 0 auto; } + +.page-head { + display: flex; + align-items: flex-end; + gap: 1rem; + flex-wrap: wrap; + margin-bottom: 1rem; +} +.page-head h1 { font-size: 1.15rem; margin: 0; font-weight: 650; letter-spacing: .01em; } +.page-head .sub { color: var(--text-muted); font-size: 12.5px; } +.page-head .spacer { margin-inline-start: auto; } + +.grid { display: grid; gap: .85rem; } +.grid.auto { grid-template-columns: repeat(auto-fit, minmax(min(360px, 100%), 1fr)); } +.grid.cols-2 { grid-template-columns: repeat(auto-fit, minmax(min(480px, 100%), 1fr)); } +.grid.cols-3 { grid-template-columns: repeat(auto-fit, minmax(min(300px, 100%), 1fr)); } +.grid.wide { grid-template-columns: minmax(0, 2fr) minmax(0, 1fr); } +@media (max-width: 1150px) { .grid.wide { grid-template-columns: 1fr; } } +.stack { display: grid; gap: .85rem; align-content: start; } + +/* ---- cards -------------------------------------------------------------- */ +.card { + background: var(--bg-raised); + border: 1px solid var(--border); + border-radius: var(--radius); + box-shadow: var(--shadow); + overflow: hidden; +} +.card > header { + display: flex; + align-items: center; + gap: .55rem; + padding: .6rem .8rem; + border-bottom: 1px solid var(--border); + background: color-mix(in srgb, var(--bg-raised) 70%, var(--bg-sunken)); +} +.card > header h2, .card > header h3 { + margin: 0; font-size: 12.5px; font-weight: 650; letter-spacing: .04em; text-transform: uppercase; + color: var(--text-muted); +} +.card > header .spacer { margin-inline-start: auto; } +.card > .body { padding: .8rem; } +.card > .body.tight { padding: .45rem .8rem .6rem; } +.card.flat { box-shadow: none; } + +.cardset { display: grid; gap: .85rem; } + +/* ---- metrics ------------------------------------------------------------ */ +.metrics { display: grid; gap: .7rem .9rem; grid-template-columns: repeat(auto-fit, minmax(104px, 1fr)); } +.metric { min-width: 0; } +.metric .label { + font-size: 10.5px; text-transform: uppercase; letter-spacing: .06em; color: var(--text-faint); + white-space: nowrap; overflow: hidden; text-overflow: ellipsis; +} +.metric .value { + font-family: var(--mono); font-size: 1.3rem; font-weight: 600; line-height: 1.25; + display: flex; align-items: baseline; gap: .22rem; row-gap: .1rem; + /* the provenance chip has to be able to drop under the number: with nowrap it painted + over the neighbouring column, which reads as a wrong value on the next metric */ + flex-wrap: wrap; +} +.metric .value > span, .metric .value > .unit { white-space: nowrap; } +.metric .value .unit { font-size: .62em; color: var(--text-muted); font-weight: 500; } +.metric .value.sm { font-size: 1.02rem; } +.metric .note { font-size: 11px; color: var(--text-faint); } +.metric .value.is-null { color: var(--text-faint); font-size: 1rem; } + +.prov { + font-family: var(--mono); font-size: 9.5px; letter-spacing: .02em; + color: var(--text-faint); border: 1px solid var(--border); border-radius: 999px; + padding: 0 .32rem; margin-inline-start: .28rem; vertical-align: 2px; white-space: nowrap; +} + +.na { color: var(--text-faint); } + +/* ---- bars, dots, chips -------------------------------------------------- */ +.bar { + position: relative; height: 7px; border-radius: 999px; background: var(--bg-sunken); + overflow: hidden; border: 1px solid var(--border); +} +.bar > i { + position: absolute; inset: 0 auto 0 0; width: 0; border-radius: 999px; + background: var(--accent); transition: width .35s ease; +} +.bar.ok > i { background: var(--ok); } +.bar.warn > i { background: var(--warn); } +.bar.fail > i { background: var(--fail); } +.bar.thin { height: 5px; } +.bar.tall { height: 10px; } + +.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--text-faint); flex: none; } +.dot.ok { background: var(--ok); box-shadow: 0 0 0 3px var(--ok-soft); } +.dot.warn { background: var(--warn); box-shadow: 0 0 0 3px var(--warn-soft); } +.dot.fail { background: var(--fail); box-shadow: 0 0 0 3px var(--fail-soft); } +.dot.idle { background: var(--text-faint); } + +.chip { + display: inline-flex; align-items: center; gap: .3rem; + font-size: 11px; font-weight: 600; letter-spacing: .02em; + padding: .16rem .45rem; border-radius: 999px; + background: var(--bg-sunken); border: 1px solid var(--border); color: var(--text-muted); + white-space: nowrap; +} +.chip.ok { color: var(--ok); background: var(--ok-soft); border-color: color-mix(in srgb, var(--ok) 35%, transparent); } +.chip.warn { color: var(--warn); background: var(--warn-soft); border-color: color-mix(in srgb, var(--warn) 35%, transparent); } +.chip.fail { color: var(--fail); background: var(--fail-soft); border-color: color-mix(in srgb, var(--fail) 35%, transparent); } +.chip.info { color: var(--info); background: var(--info-soft); border-color: color-mix(in srgb, var(--info) 35%, transparent); } +.chip.accent { color: var(--accent); background: var(--accent-soft); border-color: color-mix(in srgb, var(--accent) 35%, transparent); } + +.banner { + display: flex; gap: .55rem; align-items: center; + padding: .5rem .7rem; border-radius: var(--radius-sm); + background: var(--bg-sunken); border: 1px solid var(--border); + color: var(--text-muted); font-size: 12.5px; +} +.banner.warn { background: var(--warn-soft); border-color: color-mix(in srgb, var(--warn) 35%, transparent); color: var(--warn); } +.banner.fail { background: var(--fail-soft); border-color: color-mix(in srgb, var(--fail) 35%, transparent); color: var(--fail); } +.banner.ok { background: var(--ok-soft); border-color: color-mix(in srgb, var(--ok) 35%, transparent); color: var(--ok); } +.banner.info { background: var(--info-soft); border-color: color-mix(in srgb, var(--info) 30%, transparent); color: var(--info); } +.banner code { font-family: var(--mono); font-size: 11.5px; background: var(--bg-sunken); padding: 0 .25rem; border-radius: 3px; } + +/* simulated data is watermarked everywhere it appears */ +.sim-badge { + font: 700 10px/1 var(--mono); letter-spacing: .1em; text-transform: uppercase; + color: var(--warn); background: var(--warn-soft); + border: 1px dashed color-mix(in srgb, var(--warn) 55%, transparent); + padding: .2rem .4rem; border-radius: 4px; +} + +/* ---- tables ------------------------------------------------------------- */ +table.data { width: 100%; border-collapse: collapse; font-size: 12.5px; } +table.data th, table.data td { padding: .34rem .5rem; text-align: right; white-space: nowrap; } +table.data th:first-child, table.data td:first-child { text-align: left; } +table.data thead th { + font-size: 10.5px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint); + border-bottom: 1px solid var(--border); position: sticky; top: 0; background: var(--bg-raised); +} +table.data tbody tr + tr td { border-top: 1px solid color-mix(in srgb, var(--border) 55%, transparent); } +table.data tbody tr:hover td { background: var(--bg-sunken); } +table.data td.name { font-weight: 600; } +table.data .num { font-family: var(--mono); } +table.data .mute { color: var(--text-muted); } +table.data.scroll { display: block; overflow: auto; max-height: 320px; } + +/* ---- charts ------------------------------------------------------------- */ +.chart { position: relative; width: 100%; } +.chart svg { display: block; width: 100%; height: 100%; overflow: visible; } +.chart .axis text { fill: var(--text-faint); font-family: var(--mono); font-size: 10px; } +.chart .axis line, .chart .grid line { stroke: var(--border); stroke-width: 1; } +.chart .grid line { stroke-dasharray: 2 4; opacity: .75; } +.chart .zero line { stroke: var(--border-strong); } +.chart .line { fill: none; stroke-width: 1.75; stroke-linejoin: round; stroke-linecap: round; } +.chart .area { opacity: .16; } +.chart .cursor line { stroke: var(--text-faint); stroke-width: 1; stroke-dasharray: 3 3; } +.chart .empty { + position: absolute; inset: 0; display: grid; place-items: center; + color: var(--text-faint); font-size: 12px; pointer-events: none; +} +.legend { display: flex; flex-wrap: wrap; gap: .35rem .8rem; font-size: 11.5px; color: var(--text-muted); } +.legend .item { display: inline-flex; align-items: center; gap: .35rem; } +.legend .swatch { width: 9px; height: 9px; border-radius: 2px; } +.legend .lv { font-family: var(--mono); color: var(--text); } + +.tooltip { + position: absolute; z-index: 30; pointer-events: none; opacity: 0; + background: color-mix(in srgb, var(--bg-raised) 95%, transparent); + border: 1px solid var(--border-strong); border-radius: var(--radius-sm); + box-shadow: var(--shadow); padding: .35rem .5rem; font-size: 11.5px; min-width: 118px; + transition: opacity .12s ease; +} +.tooltip .ts { color: var(--text-faint); font-family: var(--mono); font-size: 10.5px; margin-bottom: .18rem; } +.tooltip .row { display: flex; align-items: center; gap: .35rem; } +.tooltip .row .val { margin-inline-start: auto; font-family: var(--mono); } + +/* per-core usage heat bars */ +.cores { display: grid; grid-template-columns: repeat(auto-fill, minmax(9px, 1fr)); gap: 2px; } +.cores .core { height: 22px; border-radius: 2px; background: var(--bg-sunken); position: relative; } +.cores .core > i { position: absolute; bottom: 0; left: 0; right: 0; background: var(--accent); border-radius: 2px; } + +/* ---- lists -------------------------------------------------------------- */ +ul.events { list-style: none; margin: 0; padding: 0; display: grid; gap: .1rem; } +ul.events li { + display: grid; grid-template-columns: 62px 1fr; gap: .55rem; align-items: baseline; + padding: .3rem .1rem; border-bottom: 1px solid color-mix(in srgb, var(--border) 50%, transparent); + font-size: 12.5px; +} +ul.events li:last-child { border-bottom: 0; } +ul.events time { font-family: var(--mono); font-size: 11px; color: var(--text-faint); } +ul.events .sev { font-weight: 600; } +ul.events .sev.info { color: var(--text-muted); } +ul.events .sev.warn { color: var(--warn); } +ul.events .sev.error { color: var(--fail); } + +ul.checks { list-style: none; margin: 0; padding: 0; display: grid; gap: .4rem; } +ul.checks li { display: grid; grid-template-columns: 18px 1fr; gap: .5rem; align-items: start; } +ul.checks .what { font-weight: 600; font-size: 12.5px; } +ul.checks .detail { color: var(--text-muted); font-size: 12px; } +ul.checks .mark { font-family: var(--mono); font-size: 12px; text-align: center; } +ul.checks li.ok .mark { color: var(--ok); } +ul.checks li.warn .mark { color: var(--warn); } +ul.checks li.fail .mark { color: var(--fail); } +ul.checks li.unknown .mark { color: var(--text-faint); } + +.kv { display: grid; grid-template-columns: max-content 1fr; gap: .18rem .8rem; font-size: 12.5px; } +.kv dt { color: var(--text-faint); } +.kv dd { margin: 0; font-family: var(--mono); text-align: right; overflow-wrap: anywhere; } + +/* ---- buttons ------------------------------------------------------------ */ +button.action { + font: 600 12.5px var(--sans); + padding: .42rem .7rem; border-radius: var(--radius-sm); + border: 1px solid var(--border-strong); background: var(--bg-sunken); color: var(--text); + cursor: pointer; +} +button.action:hover:not(:disabled) { border-color: var(--accent); color: var(--accent); } +button.action:disabled { opacity: .5; cursor: not-allowed; } +button.action.caution { border-color: color-mix(in srgb, var(--warn) 45%, var(--border-strong)); } +button.action.dangerous { border-color: color-mix(in srgb, var(--fail) 45%, var(--border-strong)); color: var(--fail); } +button.action.dangerous:hover:not(:disabled) { background: var(--fail-soft); border-color: var(--fail); color: var(--fail); } + +fieldset.commands { border: 1px solid var(--border); border-radius: var(--radius-sm); padding: .6rem .7rem; margin: 0; } +fieldset.commands legend { font-size: 11px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint); padding: 0 .3rem; } +fieldset.commands .row { display: flex; flex-wrap: wrap; gap: .4rem; } + +input[type="text"].confirm { + font-family: var(--mono); background: var(--bg-sunken); color: var(--text); + border: 1px solid var(--fail); border-radius: var(--radius-sm); padding: .35rem .5rem; width: 11ch; +} + +/* ---- dialog ------------------------------------------------------------- */ +dialog { + border: 1px solid var(--border-strong); border-radius: var(--radius); background: var(--bg-raised); + color: var(--text); padding: 0; max-width: 34rem; width: calc(100% - 2rem); box-shadow: var(--shadow); +} +dialog::backdrop { background: rgba(0, 0, 0, .55); } +dialog h2 { margin: 0; font-size: 1rem; } +dialog > header { padding: .8rem .9rem; border-bottom: 1px solid var(--border); display: flex; gap: .5rem; align-items: center; } +dialog .body { padding: .9rem; font-size: 13px; color: var(--text-muted); } +dialog footer { padding: .7rem .9rem; border-top: 1px solid var(--border); display: flex; gap: .5rem; justify-content: flex-end; } + +/* ---- misc --------------------------------------------------------------- */ +.muted { color: var(--text-muted); } +.faint { color: var(--text-faint); } +.mono { font-family: var(--mono); } +.nowrap { white-space: nowrap; } +.right { text-align: right; } +.center { text-align: center; } +.gap { display: flex; gap: .5rem; align-items: center; flex-wrap: wrap; } +/* `[hidden]` must beat any display set on the same element — .banner is display:flex, so a + * hidden banner used to occupy a grey strip above the KPIs on every card. */ +[hidden] { display: none !important; } + +.vh { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); } +.wrap { overflow-wrap: anywhere; } + +.empty-state { padding: 2rem 1rem; text-align: center; color: var(--text-muted); } +.empty-state h3 { margin: 0 0 .4rem; font-size: 1rem; color: var(--text); } +.empty-state code { font-family: var(--mono); font-size: 12px; background: var(--bg-sunken); padding: .1rem .3rem; border-radius: 4px; } +.empty-state ul { text-align: left; display: inline-block; margin: .6rem 0 0; } + +.stale { opacity: .62; } +.stale-flag { color: var(--warn); font-size: 11px; } + +select, input[type="text"] { + background: var(--bg-sunken); color: var(--text); border: 1px solid var(--border); + border-radius: var(--radius-sm); padding: .3rem .45rem; font: 500 12.5px var(--sans); +} +label.field { font-size: 11px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint); } + +footer.pagefoot { + margin-top: 1.6rem; padding-top: .8rem; border-top: 1px solid var(--border); + color: var(--text-faint); font-size: 11.5px; display: flex; gap: 1rem; flex-wrap: wrap; +} + +@media (prefers-reduced-motion: reduce) { + * { transition: none !important; animation: none !important; } +} diff --git a/src/dashboards/web/static/page-index.js b/src/dashboards/web/static/page-index.js new file mode 100644 index 0000000..a28d424 --- /dev/null +++ b/src/dashboards/web/static/page-index.js @@ -0,0 +1,171 @@ +/* Overview page: one /api/summary call, plus the config-driven setup hints. */ +'use strict'; + +const { $, fetchJSON, setTxt, setValue, setBar, setDot, bytes, pct, num, duration, poll } = Dash; + +async function loadConfig() { + try { + const config = await fetchJSON('/api/config'); + const banners = []; + if (!config.system.enabled) { + banners.push(['warn', 'System dashboard has no targets — set HOST_TARGETS ' + + '(run agent/host_agent.py on each host first).']); + } + if (config.system.target_problems && config.system.target_problems.length) { + banners.push(['fail', 'Bad HOST_TARGETS entries: ' + + config.system.target_problems.map((p) => `${DashCharts.escapeHtml(p)}`).join(' ')]); + } + if (!config.llm.gufo.enabled && !config.llm.vllm.enabled) { + banners.push(['warn', 'No LLM engines configured — set GUFO_BASE_URL and/or ' + + 'VLLM_BASE_URL.']); + } + if (!config.ups.enabled) { + banners.push(['warn', 'No NUT server configured — set NUT_HOST (upsd must ' + + 'have a LISTEN line).']); + } else if (!config.ups.actions_enabled) { + banners.push(['info', 'UPS commands are off — set NUT_ALLOW_ACTIONS=1 to enable them.']); + } else if (!config.ups.auth && !config.ups.simulated) { + // the simulator needs no credentials: asking for them there sends the operator fixing + // something that is not broken + banners.push(['info', 'UPS commands need credentials: set NUT_USER/' + + 'NUT_PASSWORD with actions in upsd.users.']); + } + if (config.ups.simulated) { + banners.push(['warn', 'SIMULATED UPS DATA. NUT_SIMULATE=1 is set: ' + + 'the UPS numbers are a development model, not your batteries.']); + } + const host = $('#banners'); + host.innerHTML = banners.map(([kind, html]) => + ``).join(''); + $('#foot-versions').textContent = + `dashboards ${config.app.version} · poll system ${config.system.interval_seconds}s · ` + + `llm ${config.llm.interval_seconds}s · ups ${config.ups.interval_seconds}s · ` + + `uptime ${duration(config.app.started_at ? Date.now() / 1000 - config.app.started_at : null)}` + + (config.app.read_only ? ' · read-only' : ''); + return config; + } catch (err) { + return null; + } +} + +function renderEvents(events) { + const list = $('#events'); + if (!events || !events.length) { + list.innerHTML = '
  • Nothing has changed ' + + 'since this server started — the log only records transitions.
    • '; + return; + } + list.innerHTML = events.map((event) => { + const when = new Date(event.ts * 1000).toLocaleTimeString(undefined, + { hour: '2-digit', minute: '2-digit', hour12: false }); + return `
  • ${event.host} ` + + `${DashCharts.escapeHtml(event.message.replace(event.host + ':', '').trim() || event.kind)}
    • `; + }).join(''); +} + +function renderSummary(summary) { + const system = summary.system || {}; + const fleet = system.fleet || {}; + setDot($('#dot-system'), system.enabled ? (fleet.hosts_live === fleet.hosts_total ? 'ok' + : fleet.hosts_live ? 'warn' : 'fail') : 'idle'); + setTxt($('#sys-hosts-live'), fleet.hosts_live == null ? 'n/a' : String(fleet.hosts_live)); + setTxt($('#sys-hosts-total'), fleet.hosts_total == null ? '' : `of ${fleet.hosts_total}`); + setValue($('#sys-cpu'), fleet.cpu_usage_pct_avg, 0); + setBar($('#sys-cpu').closest('.metric').querySelector('.bar'), fleet.cpu_usage_pct_avg); + if (fleet.memory_used_bytes != null) { + const used = Dash.fmtBytes(fleet.memory_used_bytes, 0); + const total = Dash.fmtBytes(fleet.memory_total_bytes, 0); + // "200 / 253 GiB", not "200 GiB of 253 GiB": the card column is narrow and the long form + // ran into the worst-disk number next to it + setTxt($('#sys-mem'), total ? `${used.text}/${total.text}` : used.text); + setTxt($('#sys-mem-unit'), used.unit); + } else { + setTxt($('#sys-mem'), 'n/a'); + setTxt($('#sys-mem-unit'), ''); + } + setBar($('#sys-mem').closest('.metric').querySelector('.bar'), fleet.memory_used_pct); + setValue($('#sys-disk'), fleet.disk_worst_used_pct, 0); + setBar($('#sys-disk').closest('.metric').querySelector('.bar'), fleet.disk_worst_used_pct, + Dash.severityFor(fleet.disk_worst_used_pct, 80, 92)); + setTxt($('#sys-gpus'), fleet.gpu_count == null ? 'n/a' + : `${fleet.gpu_count}${fleet.gpu_busy_pct_avg != null ? ' · ' + num(fleet.gpu_busy_pct_avg, 0) + '%' : ''}`); + + const llm = summary.llm || {}; + const engines = llm.fleet || {}; + setDot($('#dot-llm'), !llm.enabled ? 'idle' + : (engines.engines_live === engines.engines_total ? 'ok' : engines.engines_live ? 'warn' : 'fail')); + setTxt($('#llm-live'), engines.engines_live == null ? 'n/a' : String(engines.engines_live)); + setTxt($('#llm-total'), engines.engines_total == null ? '' : `of ${engines.engines_total}`); + setValue($('#llm-decode'), engines.decode_tokens_per_second, 1); + setValue($('#llm-prompt'), engines.prompt_tokens_per_second, 0); + setTxt($('#llm-active'), engines.requests_active == null ? 'n/a' : num(engines.requests_active, 0)); + setTxt($('#llm-waiting'), engines.requests_waiting == null ? 'n/a' : num(engines.requests_waiting, 0)); + setValue($('#llm-kv'), engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100, 0); + setBar($('#llm-kv').closest('.metric').querySelector('.bar'), + engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100, + Dash.severityFor(engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100, 75, 90)); + + const ups = summary.ups || {}; + setDot($('#dot-ups'), !ups.enabled ? 'idle' + : (ups.ups || []).some((u) => u.overall === 'at risk') ? 'fail' + : (ups.ups || []).some((u) => u.overall !== 'healthy') ? 'warn' : 'ok'); + const list = $('#ups-list'); + if (!ups.enabled) { + list.innerHTML = '
    No NUT server configured.
    '; + } else if (!(ups.ups || []).length) { + list.innerHTML = '
    No UPS answering from this upsd yet.
    '; + } else { + list.innerHTML = ups.ups.map((item) => { + const kind = item.overall === 'healthy' ? 'ok' : item.overall === 'at risk' ? 'fail' + : item.overall === 'degraded' ? 'warn' : ''; + return `
    + ${DashCharts.escapeHtml(item.name)} + ${DashCharts.escapeHtml(item.status || 'no state')} + ${ups.simulated ? 'sim' : ''} + ${item.charge_pct == null ? 'n/a' : num(item.charge_pct, 0) + '%'} · + ${item.load_pct == null ? 'n/a' : num(item.load_pct, 0) + '% load'} · + ${item.runtime_seconds == null ? 'n/a' : duration(item.runtime_seconds) + ' runtime'} +
    `; + }).join(''); + } + + const rows = system.hosts || []; + const tbody = $('#fleet-table tbody'); + $('#fleet-empty').hidden = rows.length > 0; + if (!rows.length) { tbody.innerHTML = ''; return; } + tbody.innerHTML = rows.map((host) => { + const gpus = (host.gpu_busy_pct || []).filter((v) => v != null); + return ` + ${DashCharts.escapeHtml(host.name)} + ${host.cpu_usage_pct == null ? 'n/a' : num(host.cpu_usage_pct, 0) + '%'} + ${host.load1 == null ? 'n/a' : num(host.load1, 2)} + ${host.mem_used_pct == null ? 'n/a' : num(host.mem_used_pct, 0) + '%'} + ${host.disk_worst_used_pct == null ? 'n/a' : num(host.disk_worst_used_pct, 0) + '%'} + ${gpus.length ? num(gpus[0], 0) + '%' : '—'} + ${host.temp_c == null ? 'n/a' : num(host.temp_c, 0) + '°C'} + ${host.uptime_seconds == null ? 'n/a' : duration(host.uptime_seconds)} + `; + }).join(''); +} + +async function refresh() { + try { + const summary = await fetchJSON('/api/summary'); + const total = (summary.system.hosts || []).length; + const live = summary.system.fleet ? summary.system.fleet.hosts_live : 0; + const engines = summary.llm.fleet ? `${summary.llm.fleet.engines_live}/${summary.llm.fleet.engines_total} engines` : 'no engines'; + const ups = summary.ups.ups ? `${summary.ups.ups.length} UPS` : 'no UPS'; + $('#summary-line').textContent = `${live}/${total} hosts · ${engines} · ${ups} · ` + + `${(summary.recent_events || []).length} recent events`; + renderSummary(summary); + renderEvents(summary.recent_events); + } catch (err) { + $('#summary-line').textContent = `summary failed: ${err.message}`; + } +} + +document.addEventListener('DOMContentLoaded', async () => { + await loadConfig(); + await refresh(); + poll(refresh, 10000); +}); diff --git a/src/dashboards/web/static/page-llm.js b/src/dashboards/web/static/page-llm.js new file mode 100644 index 0000000..36222e1 --- /dev/null +++ b/src/dashboards/web/static/page-llm.js @@ -0,0 +1,382 @@ +/* LLM dashboard. + * + * Every throughput cell shows where its number came from, because "78 tok/s" means + * different things depending on whether it was scraped from an aggregate gauge, measured as + * Δtokens/Δt across all slots, or estimated from TPOT over the process lifetime. The tag is + * generated from the cell's `source`, never hardcoded in markup, so a parser change cannot + * leave a stale label behind. + */ +'use strict'; + +const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, pct, duration, poll, + severityFor } = Dash; + +const state = { range: 3600, panels: new Map(), charts: {}, poller: null, pollMs: 2000 }; + +const CELLS = [ + { key: 'decode_tokens_per_second', label: 'Decode', unit: 'tok/s', digits: 1, + tip: 'Aggregate generation speed across every busy slot.' }, + { key: 'prompt_tokens_per_second', label: 'Prompt', unit: 'tok/s', digits: 1, + tip: 'Prefill speed for newly computed tokens.' }, + { key: 'sessions', label: 'Sessions', unit: 'active', digits: 0 }, + { key: 'queued', label: 'Queued', unit: 'waiting', digits: 0 }, + { key: 'cache', label: 'KV cache', unit: '%', digits: 0, bar: true }, + { key: 'ttft_mean', label: 'TTFT mean', unit: 'ms', digits: 0 }, + { key: 'ttft_p95', label: 'TTFT p95', unit: 'ms', digits: 0 }, + { key: 'tpot_mean', label: 'TPOT mean', unit: 'ms', digits: 1 }, + { key: 'queue_mean', label: 'Queue mean', unit: 'ms', digits: 0 }, + { key: 'cache_hit', label: 'Prefix cache hit', unit: '%', digits: 0 }, + { key: 'spec_accept', label: 'Spec accept', unit: '%', digits: 0 }, + { key: 'tokens_total', label: 'Tokens lifetime', unit: '', digits: 0, human: true }, + { key: 'preemptions', label: 'Preemptions', unit: '', digits: 0 }, + { key: 'device_lost', label: 'GPU context lost', unit: '', digits: 0 }, +]; + +function cellValue(engine, metrics, sessions, key) { + const get = (k) => (metrics[k] && metrics[k].value != null ? metrics[k] : null); + switch (key) { + case 'sessions': { + const value = sessions && sessions.active != null ? sessions.active : null; + return { value, source: 'session count', derived: false }; + } + case 'queued': { + const cell = get('requests_deferred') || get('num_requests_waiting'); + return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null }; + } + case 'cache': { + const cell = get('kv_cache_usage_ratio') || get('gpu_cache_usage'); + return cell ? { value: cell.value * 100, source: cell.source, derived: false } : { value: null }; + } + case 'ttft_mean': { + const cell = get('time_to_first_token_seconds__mean'); + return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null }; + } + case 'ttft_p95': { + const cell = get('time_to_first_token_seconds__p95'); + return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null }; + } + case 'tpot_mean': { + const cell = get('time_per_output_token_seconds__mean'); + return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null }; + } + case 'queue_mean': { + const cell = get('request_queue_time_seconds__mean'); + return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null }; + } + case 'cache_hit': { + const cell = get('prefix_cache_hit_ratio') || get('prompt_cache_hit_ratio'); + return cell ? { value: cell.value * 100, source: cell.source, derived: true } : { value: null }; + } + case 'spec_accept': { + const cell = get('spec_accept_rate'); + return cell ? { value: cell.value * 100, source: cell.source, derived: true } : { value: null }; + } + case 'tokens_total': { + const cell = get('tokens_total'); + return cell ? { value: cell.value, source: cell.source, derived: true } : { value: null }; + } + case 'preemptions': { + const cell = get('num_preemptions_total'); + return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null }; + } + case 'device_lost': { + const cell = get('device_lost_total'); + return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null }; + } + default: { + const cell = get(key); + return cell ? { value: cell.value, source: cell.source, derived: cell.derived, + window: cell.window } : { value: null }; + } + } +} + +/** The provenance tag: Δ10.0s for measured rates, the formula for estimates, nothing for a + * plain scraped gauge (which would only add noise). */ +function provenance(cell) { + if (!cell || !cell.source) return ''; + if (!String(cell.source).startsWith('computed:')) return ''; + if (cell.window) return `Δ${cell.window.toFixed(1)}s`; + const text = String(cell.source).replace('computed:', ''); + return text.length > 16 ? text.slice(0, 15) + '…' : text; +} + +function humanNumber(value) { + if (value == null || !isFinite(value)) return null; + if (value >= 1e9) return { text: (value / 1e9).toFixed(2), unit: 'B tok' }; + if (value >= 1e6) return { text: (value / 1e6).toFixed(2), unit: 'M tok' }; + if (value >= 1e3) return { text: (value / 1e3).toFixed(1), unit: 'k tok' }; + return { text: value.toFixed(0), unit: 'tok' }; +} + +function enginePanel(engine) { + const card = document.createElement('article'); + card.className = 'card'; + card.dataset.engine = engine; + card.innerHTML = ` +
    + +

    ${DashCharts.escapeHtml(engine)}

    + + + +
    +
    + +
    ${CELLS.map((cell) => ` +
    +
    ${cell.label}
    +
    n/a${cell.unit}
    + ${cell.bar ? '
    ' : ''} +
    `).join('')}
    +
    +
    +
    `; + return card; +} + +function renderPanel(card, enginePayload, stateRow) { + const metrics = enginePayload.metrics || {}; + const sessions = enginePayload.sessions || {}; + const q = (sel) => card.querySelector(sel); + + const dot = q('.dot'); + setDot(dot, stateRow && stateRow.ok ? (Dash.ageSeconds(stateRow.last_ok) > 15 ? 'warn' : 'ok') : 'fail'); + setTxt(q('.updated'), stateRow ? (stateRow.last_ok == null + ? (stateRow.error || 'never reached') + : (Dash.ageSeconds(stateRow.last_ok) > 15 ? `stale ${duration(Dash.ageSeconds(stateRow.last_ok))}` + : `updated ${duration(Dash.ageSeconds(stateRow.last_ok))}`)) : '—'); + Dash.markStale(card, stateRow && (!stateRow.ok || Dash.ageSeconds(stateRow.last_ok) > 15)); + + const banner = q('.banner.err'); + if (stateRow && !stateRow.ok) { + banner.hidden = false; + banner.className = 'banner fail err'; + banner.textContent = stateRow.error || 'engine is not answering'; + } else { + banner.hidden = true; + } + + for (const cell of CELLS) { + const wrapper = q(`[data-cell="${cell.key}"]`); + if (!wrapper) continue; + const value = cellValue(enginePayload.engine, metrics, sessions, cell.key); + const valueEl = wrapper.querySelector('[data-v]'); + const unitEl = wrapper.querySelector('.unit'); + if (value.value == null) { + setTxt(valueEl, 'n/a'); + setTxt(unitEl, ''); // "n/a %" reads like a measurement of nothing + wrapper.classList.add('is-null'); + } else { + wrapper.classList.remove('is-null'); + if (cell.human) { + const parts = humanNumber(value.value); + setTxt(valueEl, parts ? parts.text : 'n/a'); + setTxt(unitEl, parts ? parts.unit : ''); + } else { + setTxt(valueEl, num(value.value, cell.digits)); + setTxt(unitEl, cell.unit); + } + } + let tag = wrapper.querySelector('.prov'); + const label = provenance(value); + if (label) { + if (!tag) { + tag = document.createElement('span'); + tag.className = 'prov'; + wrapper.querySelector('.value').appendChild(tag); + } + setTxt(tag, label); + tag.title = `${value.source || ''}${value.window ? ` · measured over ${value.window.toFixed(1)}s` : ''}`; + } else if (tag) { + tag.remove(); + } + if (cell.bar) setBar(wrapper.querySelector('.bar'), value.value, + severityFor(value.value, 75, 90)); + if (cell.key === 'device_lost' && value.value != null && value.value > 0) { + wrapper.querySelector('.value').classList.add('is-null'); + wrapper.classList.add('fail'); + wrapper.title = `GPU context has been lost ${value.value} times — an incident, not a stat`; + } + } + + const note = q('.sessions-note'); + setTxt(note, sessions.detail || `${sessions.active == null ? 'n/a' : sessions.active} active sessions`); + const upstream = q('.chip.upstream'); + if (enginePayload.upstream) { + upstream.hidden = false; + upstream.textContent = enginePayload.upstream.replace(/^https?:\/\//, ''); + upstream.title = enginePayload.upstream; + } + setTxt(q('.debug'), enginePayload.unparsed != null + ? `${enginePayload.series_count} series · ${enginePayload.unparsed} unparsed` + : ''); +} + +function renderSlots(payloads) { + const gufo = payloads.find((p) => p.engine === 'gufo'); + const rows = gufo && gufo.sessions ? gufo.sessions.list || [] : []; + const tbody = $('#slots-table tbody'); + $('#slots-empty').hidden = rows.length > 0; + $('#slots-card').hidden = !gufo; + tbody.innerHTML = rows.map((slot) => { + const id = slot.id != null ? slot.id : slot.slot != null ? slot.slot : '—'; + const stateText = slot.state || (slot.idle === true ? 'idle' : slot.idle === false ? 'busy' : '—'); + const withText = (slot.with_messages && slot.with_messages.length) || slot.with_message; + const nPrompt = slot.n_prompt != null ? slot.n_prompt : (slot.prompt_tokens != null ? slot.prompt_tokens : null); + const nPredict = slot.n_predict != null ? slot.n_predict : (slot.generated_tokens != null ? slot.generated_tokens : null); + const cache = slot.cache != null ? slot.cache : slot.cache_usable; + const speed = slot.decode_per_second || slot.tokens_per_second || null; + const cell = (value, digits) => `${value == null ? '—' : num(value, digits == null ? 0 : digits)}`; + return ` + ${DashCharts.escapeHtml(id)} + ${stateText === 'busy' || stateText === 'processing' ? 'busy' + : stateText === 'idle' ? 'idle' + : `${DashCharts.escapeHtml(stateText)}`} + ${withText ? 'has msg' : '—'} + ${cell(nPrompt)}${cell(nPredict)}${cell(cache)}${cell(speed, 1)}`; + }).join(''); +} + +function makeCharts() { + const common = { height: 165, xSpan: state.range }; + state.charts.decode = new DashCharts.LineChart('#chart-decode', Object.assign({}, common, { yMin: 0 })); + state.charts.prompt = new DashCharts.LineChart('#chart-prompt', Object.assign({}, common, { yMin: 0 })); + state.charts.kv = new DashCharts.LineChart('#chart-kv', Object.assign({}, common, + { yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) })); + state.charts.queue = new DashCharts.LineChart('#chart-queue', Object.assign({}, common, + { yMin: 0, yFormat: (v) => num(v, 0) })); +} + +function pointsFor(response, engine, metric) { + const found = (response.series || []).find((s) => s.host === engine && s.metric === `llm.${metric}`); + return found ? found.points.map((p) => ({ ts: p.ts, value: p.value })) : []; +} + +async function refreshCharts(engineNames) { + if (!engineNames.length) return; + const list = engineNames.join(','); + const [decode, prompt, queue] = await Promise.all([ + fetchJSON(`/api/llm/history?engines=${list}&metrics=decode_tokens_per_second&range=${state.range}`) + .catch(() => null), + fetchJSON(`/api/llm/history?engines=${list}&metrics=prompt_tokens_per_second&range=${state.range}`) + .catch(() => null), + fetchJSON(`/api/llm/history?engines=${list}&metrics=requests_running,requests_waiting,` + + `kv_cache_usage&range=${state.range}`).catch(() => null), + ]); + if (decode) { + const series = engineNames.map((engine) => ({ name: engine, + points: pointsFor(decode, engine, 'decode_tokens_per_second') })); + state.charts.decode.setData(series); + DashCharts.renderLegend($('#legend-decode'), series, (v) => num(v, 1) + ' tok/s', true); + } + if (prompt) { + const series = engineNames.map((engine) => ({ name: engine, + points: pointsFor(prompt, engine, 'prompt_tokens_per_second') })); + state.charts.prompt.setData(series); + DashCharts.renderLegend($('#legend-prompt'), series, (v) => num(v, 0) + ' tok/s', true); + } + if (queue) { + const series = []; + for (const engine of engineNames) { + series.push({ name: `${engine} running`, points: pointsFor(queue, engine, 'requests_running') }); + series.push({ name: `${engine} waiting`, points: pointsFor(queue, engine, 'requests_waiting') }); + } + state.charts.queue.setData(series); + DashCharts.renderLegend($('#legend-queue'), series, (v) => num(v, 0)); + const kv = engineNames.map((engine) => ({ name: engine, + points: pointsFor(queue, engine, 'kv_cache_usage').map((p) => ({ ts: p.ts, value: p.value * 100 })) })); + state.charts.kv.setData(kv); + DashCharts.renderLegend($('#legend-kv'), kv, (v) => num(v, 1) + '%', true); + } +} + +function renderFleet(fleet) { + if (!fleet) return; + setTxt($('#f-live'), fleet.engines_live == null ? 'n/a' : String(fleet.engines_live)); + setTxt($('#f-total'), fleet.engines_total == null ? '' : `of ${fleet.engines_total}`); + setValue($('#f-decode'), fleet.decode_tokens_per_second, 1); + setValue($('#f-prompt'), fleet.prompt_tokens_per_second, 0); + setTxt($('#f-active'), fleet.requests_active == null ? 'n/a' : num(fleet.requests_active, 0)); + setTxt($('#f-waiting'), fleet.requests_waiting == null ? 'n/a' : num(fleet.requests_waiting, 0)); + const tokens = humanNumber(fleet.tokens_total); + setTxt($('#f-tokens'), tokens ? tokens.text : 'n/a'); + $('#f-tokens').nextElementSibling && setTxt($('#f-tokens').nextElementSibling, tokens ? tokens.unit : ''); + const kv = fleet.kv_cache_usage_max == null ? null : fleet.kv_cache_usage_max * 100; + setValue($('#f-kv'), kv, 0); + setBar($('#f-kv').closest('.metric').querySelector('.bar'), kv, severityFor(kv, 75, 90)); + $('#seam-note').textContent = fleet.seams || ''; +} + +async function refresh() { + let snapshot; + try { + snapshot = await fetchJSON('/api/llm/snapshot'); + } catch (err) { + $('#sub').textContent = err.message; + return; + } + const rows = {}; + (snapshot.states || []).forEach((row) => { rows[row.label.toLowerCase()] = row; }); + const payloads = snapshot.engines || []; + const names = payloads.map((p) => p.engine); + payloads.forEach((payload) => { + let card = state.panels.get(payload.engine); + if (!card) { + card = enginePanel(payload.engine); + state.panels.set(payload.engine, card); + $('#engines').appendChild(card); + } + renderPanel(card, payload, rows[payload.engine] || + (snapshot.states || []).find((row) => row.key.endsWith(payload.engine))); + }); + $('#engines-empty').hidden = payloads.length > 0; + renderFleet(snapshot.fleet); + renderSlots(payloads); + const fleet = snapshot.fleet || {}; + $('#sub').textContent = `${fleet.engines_live || 0}/${fleet.engines_total || 0} engines · ` + + `${fleet.requests_active == null ? 'n/a' : num(fleet.requests_active, 0)} active · ` + + `window ${duration(state.range)}`; + $('#foot-note').textContent = `poll ${Math.round(state.pollMs / 1000)}s · rates over the ` + + `Δ window in each tag`; + await refreshCharts(names); +} + +async function refreshEvents() { + try { + const data = await fetchJSON('/api/events?limit=20&source=llm'); + const list = $('#events'); + if (!(data.events || []).length) { + list.innerHTML = '
  • No engine transitions yet.
    • '; + return; + } + list.innerHTML = data.events.map((event) => `
  • ${DashCharts.escapeHtml(event.host)} ` + + `${DashCharts.escapeHtml(event.message)}
    • `).join(''); + } catch (err) { /* next poll */ } +} + +document.addEventListener('DOMContentLoaded', async () => { + makeCharts(); + try { + const config = await fetchJSON('/api/config'); + state.pollMs = config.llm.poll_ms || 2000; + const banners = []; + if (!config.llm.gufo.enabled && !config.llm.vllm.enabled) { + banners.push(['warn', 'Neither engine is configured.']); + } + if (config.ups.simulated) banners.push(['warn', 'UPS numbers on other pages are simulated.']); + $('#banners').innerHTML = banners.map(([kind, html]) => ``) + .join(''); + } catch (err) { /* not fatal */ } + + Dash.rangeControl($('#range'), state.range, (seconds) => { + state.range = seconds; + Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = seconds; }); + refresh(); + }); + + state.poller = poll(async () => { await refresh(); await refreshEvents(); }, state.pollMs); + Dash.bindPause(state.poller); +}); diff --git a/src/dashboards/web/static/page-system.js b/src/dashboards/web/static/page-system.js new file mode 100644 index 0000000..207ceb5 --- /dev/null +++ b/src/dashboards/web/static/page-system.js @@ -0,0 +1,503 @@ +/* System dashboard page. + * + * Host cards are created once and patched in place (keyed by host name) — rebuilding the + * DOM every 10s kills hover, transitions, and any scroll position inside the tables. + * Charts get their history from /api/system/history: one request per chart, N hosts each. + */ +'use strict'; + +const { $, $$, fetchJSON, setTxt, setValue, setBar, setDot, bytes, pct, num, duration, poll, + staleness, markStale, severityFor } = Dash; + +const state = { + range: 3600, + detailHost: null, + detailMetric: 'cpu.usage_pct', + hosts: new Map(), // name -> card element + charts: {}, + poller: null, + pollMs: 10000, + metricList: [], +}; + +const DETAIL_METRICS = [ + ['cpu.usage_pct', 'CPU utilisation', 'pct'], + ['cpu.user_pct', 'CPU user', 'pct'], + ['cpu.system_pct', 'CPU system', 'pct'], + ['cpu.iowait_pct', 'CPU iowait', 'pct'], + ['cpu.steal_pct', 'CPU steal', 'pct'], + ['cpu.load1', 'Load average (1m)', 'num'], + ['cpu.load15', 'Load average (15m)', 'num'], + ['cpu.freq_mhz', 'CPU frequency', 'mhz'], + ['cpu.context_switches_per_s', 'Context switches', 'num'], + ['mem.used_pct', 'Memory used', 'pct'], + ['mem.used_bytes', 'Memory used', 'bytes'], + ['mem.available_bytes', 'Memory available', 'bytes'], + ['mem.swap_used_pct', 'Swap used', 'pct'], + ['disk.used_pct', 'Filesystem used (worst)', 'pct'], + ['disk.read_bytes_per_s', 'Disk read', 'bytes'], + ['disk.write_bytes_per_s', 'Disk write', 'bytes'], + ['disk.util_pct', 'Disk busy', 'pct'], + ['net.rx_bytes_per_s', 'Network in', 'bytes'], + ['net.tx_bytes_per_s', 'Network out', 'bytes'], + ['temp.cpu_c', 'CPU temperature', 'temp'], + ['temp.nvme_max_c', 'NVMe temperature', 'temp'], + ['host.uptime_seconds', 'Uptime', 'dur'], + ['host.processes', 'Process count', 'num'], +]; + +const FORMATS = { + pct: { fn: (v) => num(v, 1), unit: '%', axis: (v) => num(v, 0) }, + bytes: { fn: (v) => bytes(v, 1), unit: 'B/s', axis: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text + + ((Dash.fmtBytes(v, 0) || { unit: '' }).unit) }, + mhz: { fn: (v) => num(v, 0) + ' MHz', unit: 'MHz', axis: (v) => num(v, 0) }, + temp: { fn: (v) => num(v, 1) + ' °C', unit: '°C', axis: (v) => num(v, 0) }, + dur: { fn: (v) => duration(v), unit: '', axis: (v) => duration(v) }, + num: { fn: (v) => num(v, 1), unit: '', axis: (v) => num(v, 0) }, +}; + +function formatFor(kind) { return FORMATS[kind] || FORMATS.num; } + +function metricKind(name) { + if (name.endsWith('_pct')) return 'pct'; + if (name.endsWith('_bytes') || name.includes('bytes_per_s')) return 'bytes'; + if (name.includes('_c') && name.startsWith('temp.')) return 'temp'; + if (name === 'host.uptime_seconds') return 'dur'; + return 'num'; +} + +// ---- host cards ----------------------------------------------------------- +function hostCard(name) { + const card = document.createElement('article'); + card.className = 'card'; + card.dataset.host = name; + card.innerHTML = ` +
    + +

    + + + +
    +
    +
    +
    CPU
    n/a%
    +
    +
    Load 1 / 15
    n/a
    +
    CPU temp
    n/a°C
    +
    Clock
    n/aGHz
    +
    Memory
    n/a
    +
    +
    Worst fs
    n/a
    +
    +
    +
    +
    Per-core
    +
    +
    +
    +
    Disk read
    n/a
    +
    Disk write
    n/a
    +
    Disk busy
    n/a%
    +
    Net in
    n/a
    +
    Net out
    n/a
    +
    Up
    n/a
    +
    +
    +
    + Filesystems & processes +
    +
    +
    `; + return card; +} + +function renderHostCard(card, host) { + const q = (key) => card.querySelector(`[data-k="${key}"]`); + const stateRow = host.state || {}; + card.querySelector('h3').textContent = host.name; + const osChip = card.querySelector('.os'); + const osName = (host.info || {}).os_pretty || (host.info || {}).os_id; + if (osName) { osChip.hidden = false; osChip.textContent = osName; } + setDot(card.querySelector('.dot'), stateRow.ok === false ? 'fail' + : (Dash.ageSeconds(stateRow.last_ok) > 3 * (host.window || 10) ? 'warn' : 'ok')); + setTxt(card.querySelector('.updated'), stalenessText(stateRow)); + markStale(card, stateRow.ok === false || Dash.ageSeconds(stateRow.last_ok) > 5 * (host.window || 10)); + + const cpu = host.cpu || {}; + setValue(q('cpu'), cpu.usage_pct, 0); + setBar(card.querySelector('[data-bar="cpu"]'), cpu.usage_pct, severityFor(cpu.usage_pct, 75, 90)); + setTxt(q('load'), cpu.load1 == null ? 'n/a' : `${num(cpu.load1, 2)} / ${num(cpu.load15, 2)}`); + setValue(q('temp'), cpu.temp_c, 1); + setValue(q('freq'), cpu.freq_mhz == null ? null : cpu.freq_mhz / 1000, 2); + + const mem = host.memory || {}; + const memText = Dash.fmtBytes(mem.used_bytes, 1); + const memTotal = Dash.fmtBytes(mem.total_bytes, 0); + setTxt(q('mem'), memText ? memText.text : 'n/a'); + setTxt(q('mem').nextElementSibling, memText ? `${memText.unit}/${memTotal ? memTotal.text + memTotal.unit : ''}` : ''); + setBar(card.querySelector('[data-bar="mem"]'), mem.used_pct, severityFor(mem.used_pct, 80, 92)); + + const disks = host.disks || {}; + setTxt(q('disk'), disks.worst_used_pct == null ? 'n/a' : num(disks.worst_used_pct, 0) + '%'); + setTxt(q('disk').nextElementSibling, disks.worst_mount ? ' ' + disks.worst_mount : ''); + setBar(card.querySelector('[data-bar="disk"]'), disks.worst_used_pct, severityFor(disks.worst_used_pct, 80, 92)); + + DashCharts.renderCores(q('cores'), cpu.core_usage_pct || []); + + setValue(q('dr'), disks.read_bytes_per_s == null ? null : disks.read_bytes_per_s, null); + if (disks.read_bytes_per_s != null) { + const parts = Dash.fmtBytes(disks.read_bytes_per_s, 1); + setTxt(q('dr'), parts.text); setTxt(q('dr').nextElementSibling, parts.unit + '/s'); + const w = Dash.fmtBytes(disks.write_bytes_per_s, 1); + setTxt(q('dw'), w ? w.text : 'n/a'); setTxt(q('dw').nextElementSibling, w ? w.unit + '/s' : ''); + } else { setTxt(q('dr'), 'n/a'); setTxt(q('dw'), 'n/a'); } + setValue(q('dutil'), disks.util_pct, 0); + const rx = Dash.fmtBytes((host.network || {}).rx_bytes_per_s, 1); + const tx = Dash.fmtBytes((host.network || {}).tx_bytes_per_s, 1); + setTxt(q('rx'), rx ? rx.text : 'n/a'); setTxt(q('rx').nextElementSibling, rx ? rx.unit + '/s' : ''); + setTxt(q('tx'), tx ? tx.text : 'n/a'); setTxt(q('tx').nextElementSibling, tx ? tx.unit + '/s' : ''); + setTxt(q('uptime'), duration((host.info || {}).uptime_seconds)); + + // GPUs: one row each, with both memory pools named (an APU's model lives in GTT) + const gpus = host.gpus || []; + const gpuHost = q('gpus'); + if (!gpus.length) { + gpuHost.innerHTML = '
    No GPU reported by this agent
    '; + } else { + gpuHost.innerHTML = gpus.map((gpu) => ` +
    +
    + ${DashCharts.escapeHtml(gpu.model || 'GPU')} + + ${gpu.busy_pct == null ? 'n/a' : num(gpu.busy_pct, 0) + '% busy'} · + ${gpu.power_watts == null ? 'power n/a' : num(gpu.power_watts, 0) + ' W'} · + ${gpu.temp_c == null ? 'temp n/a' : num(gpu.temp_c, 0) + '°C'} +
    +
    +
    + VRAM ${gpu.vram_used_bytes == null ? 'n/a' : bytes(gpu.vram_used_bytes, 0)} + ${gpu.vram_total_bytes ? 'of ' + bytes(gpu.vram_total_bytes, 0) : ''} + shared (GTT) ${gpu.gtt_used_bytes == null ? 'n/a' : bytes(gpu.gtt_used_bytes, 0)} + ${gpu.mem_used_pct == null ? '' : ' · pool ' + num(gpu.mem_used_pct, 0) + '%'} +
    +
    +
    `).join(''); + } + + const filesystems = (disks.filesystems || []).filter((fs) => fs.used_pct != null); + const procs = host.processes || {}; + const details = q('details'); + details.innerHTML = ` + + + ${filesystems.map((fs) => ` + + + + + `).join('') || + ''} +
    MountTypeUsedFreeTotal
    ${DashCharts.escapeHtml(fs.mountpoint)}${DashCharts.escapeHtml(fs.fstype)}${num(fs.used_pct, 1)}%${bytes(fs.free_bytes, 0)}${bytes(fs.total_bytes, 0)}
    no filesystems reported
    +
    +
    Top by CPU
    + ${(procs.top_cpu || []).slice(0, 5).map((p) => ` + + `).join('') + || ''}
    ${DashCharts.escapeHtml(p.name)}${num(p.cpu_pct, 0)}%pid ${p.pid}
    needs two samples
    +
    Top by RSS
    + ${(procs.top_rss || []).slice(0, 5).map((p) => ` + + `).join('') + || ''}
    ${DashCharts.escapeHtml(p.name)}${bytes(p.rss_bytes, 0)}pid ${p.pid}
    —
    +
    + ${(host.agent && host.agent.errors && host.agent.errors.length) + ? `` + : ''} + ${(host.degraded && host.degraded.length) + ? `` + : ''}`; +} + +function stalenessText(row) { + if (!row || !row.configured) return 'not configured'; + if (row.last_ok == null) return row.error || 'never reached'; + const seconds = Dash.ageSeconds(row.last_ok); + return seconds > 30 ? `stale ${duration(seconds)}` : `updated ${seconds < 2 ? 'now' : duration(seconds)}`; +} + +// ---- charts --------------------------------------------------------------- +function makeCharts() { + const common = { height: 165, xSpan: state.range }; + state.charts.cpu = new DashCharts.LineChart('#chart-cpu', Object.assign({}, common, { + yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) })); + state.charts.mem = new DashCharts.LineChart('#chart-mem', Object.assign({}, common, { + yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) })); + state.charts.disk = new DashCharts.LineChart('#chart-disk', Object.assign({}, common, { + yMin: 0, yFormat: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text })); + state.charts.net = new DashCharts.LineChart('#chart-net', Object.assign({}, common, { + yMin: 0, yFormat: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text })); + state.charts.gpu = new DashCharts.LineChart('#chart-gpu', Object.assign({}, common, { + yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) })); + state.charts.temp = new DashCharts.LineChart('#chart-temp', Object.assign({}, common, { + yFormat: (v) => num(v, 0) })); + state.charts.detail = new DashCharts.LineChart('#chart-detail', Object.assign({}, common, { + height: 200, yFormat: (v) => num(v, 1) })); +} + +function setChartSpan(span) { + Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = span; }); +} + +function seriesFor(response, metric) { + return (response.series || []).filter((s) => s.metric === metric).map((s) => ({ + name: s.host, points: s.points.map((p) => ({ ts: p.ts, value: p.value })), + })); +} + +async function refreshCharts(snapshot) { + const hostNames = (snapshot.hosts || []).map((h) => h.name); + const detailHost = state.detailHost || hostNames[0] || null; + const span = state.range; + const requests = { + cpu: fetchJSON(`/api/system/history?metrics=cpu.usage_pct&range=${span}`).catch(() => null), + mem: fetchJSON(`/api/system/history?metrics=mem.used_pct&range=${span}`).catch(() => null), + temp: fetchJSON(`/api/system/history?metrics=temp.cpu_c&range=${span}`).catch(() => null), + disk: detailHost ? fetchJSON(`/api/system/history?hosts=${encodeURIComponent(detailHost)}&` + + `metrics=disk.read_bytes_per_s,disk.write_bytes_per_s&range=${span}`).catch(() => null) : null, + net: detailHost ? fetchJSON(`/api/system/history?hosts=${encodeURIComponent(detailHost)}&` + + `metrics=net.rx_bytes_per_s,net.tx_bytes_per_s&range=${span}`).catch(() => null) : null, + }; + const [cpu, mem, temp, disk, net] = await Promise.all([requests.cpu, requests.mem, + requests.temp, requests.disk, requests.net]); + + if (cpu) { + const series = seriesFor(cpu, 'cpu.usage_pct'); + state.charts.cpu.setData(series); + DashCharts.renderLegend($('#legend-cpu'), series, (v) => num(v, 1) + '%', true); + } + if (mem) { + const series = seriesFor(mem, 'mem.used_pct'); + state.charts.mem.setData(series); + DashCharts.renderLegend($('#legend-mem'), series, (v) => num(v, 1) + '%'); + } + if (temp) { + const series = seriesFor(temp, 'temp.cpu_c'); + state.charts.temp.setData(series); + DashCharts.renderLegend($('#legend-temp'), series, (v) => num(v, 1) + '°C', true); + } + if (disk) { + const read = (disk.series.find((s) => s.metric === 'disk.read_bytes_per_s') || { points: [] }) + .points.map((p) => ({ ts: p.ts, value: p.value })); + const write = (disk.series.find((s) => s.metric === 'disk.write_bytes_per_s') || { points: [] }) + .points.map((p) => ({ ts: p.ts, value: p.value })); + const series = [{ name: 'read', points: read }, { name: 'write', points: write }]; + state.charts.disk.setData(series); + DashCharts.renderLegend($('#legend-disk'), series, (v) => bytes(v, 1) + '/s', true); + } + if (net) { + const rx = (net.series.find((s) => s.metric === 'net.rx_bytes_per_s') || { points: [] }) + .points.map((p) => ({ ts: p.ts, value: p.value })); + const tx = (net.series.find((s) => s.metric === 'net.tx_bytes_per_s') || { points: [] }) + .points.map((p) => ({ ts: p.ts, value: p.value })); + const series = [{ name: 'in', points: rx }, { name: 'out', points: tx }]; + state.charts.net.setData(series); + DashCharts.renderLegend($('#legend-net'), series, (v) => bytes(v, 1) + '/s', true); + } + await refreshGpuChart(hostNames); + await refreshDetailChart(); +} + +async function refreshGpuChart(hostNames) { + if (!hostNames.length) { + state.charts.gpu.setData([]); + $('#legend-gpu').innerHTML = 'no hosts'; + return; + } + const series = []; + for (const host of hostNames) { + // per-stream names are discovered, not guessed: gpu.busy_pct@0, @1, ... + const found = await fetchJSON(`/api/metrics?source=host&host=${encodeURIComponent(host)}`) + .catch(() => ({ metrics: [] })); + const names = (found.metrics || []).filter((m) => m.startsWith('gpu.busy_pct@')); + for (const name of names) { + const data = await fetchJSON(`/api/history?source=host&host=${encodeURIComponent(host)}` + + `&metric=${encodeURIComponent(name)}&range=${state.range}`).catch(() => null); + if (data) series.push({ name: `${host} ${name.split('@')[1]}`, + points: data.points.map((p) => ({ ts: p.ts, value: p.value })) }); + } + } + state.charts.gpu.setData(series); + DashCharts.renderLegend($('#legend-gpu'), series, (v) => num(v, 0) + '%', true); +} + +async function refreshDetailChart() { + const host = state.detailHost; + if (!host) { state.charts.detail.setData([]); return; } + const kind = metricKind(state.detailMetric); + const format = formatFor(kind); + const data = await fetchJSON(`/api/history?source=host&host=${encodeURIComponent(host)}` + + `&metric=${encodeURIComponent(state.detailMetric)}&range=${state.range}`).catch(() => null); + if (!data) return; + const series = [{ name: state.detailMetric, + points: data.points.map((p) => ({ ts: p.ts, value: p.value })) }]; + state.charts.detail.opts.yFormat = format.axis; + state.charts.detail.opts.yMin = state.detailMetric.endsWith('_pct') ? 0 : null; + state.charts.detail.opts.yMax = state.detailMetric.endsWith('_pct') ? 100 : null; + state.charts.detail.setData(series); + DashCharts.renderLegend($('#legend-detail'), series, format.fn); + const last = series[0].points.slice(-1)[0]; + $('#detail-note').textContent = `${series[0].points.length} samples over ${duration(state.range)} · ` + + `last ${last ? format.fn(last.value) : 'n/a'} · ${state.range <= 86400 ? 'raw resolution' : 'rolled up'}`; +} + +async function populateDetailMetrics(host) { + const select = $('#detail-metric'); + const found = await fetchJSON(`/api/metrics?source=host&host=${encodeURIComponent(host)}`) + .catch(() => ({ metrics: [] })); + const curated = new Set(DETAIL_METRICS.map(([name]) => name)); + const extras = (found.metrics || []).filter((m) => !curated.has(m)).sort(); + select.innerHTML = ''; + const groupMain = document.createElement('optgroup'); + groupMain.label = 'Common'; + DETAIL_METRICS.forEach(([name, label]) => { + const option = document.createElement('option'); + option.value = name; option.textContent = label; + groupMain.appendChild(option); + }); + select.appendChild(groupMain); + if (extras.length) { + const groupExtra = document.createElement('optgroup'); + groupExtra.label = `Per-device (${extras.length})`; + extras.forEach((name) => { + const option = document.createElement('option'); + option.value = name; option.textContent = name; + groupExtra.appendChild(option); + }); + select.appendChild(groupExtra); + } + select.value = DETAIL_METRICS.some(([name]) => name === state.detailMetric) + ? state.detailMetric : (extras[0] || DETAIL_METRICS[0][0]); + state.detailMetric = select.value || DETAIL_METRICS[0][0]; +} + +// ---- snapshot ------------------------------------------------------------- +function renderFleet(fleet) { + if (!fleet) return; + setTxt($('#f-hosts'), fleet.hosts_live == null ? 'n/a' : String(fleet.hosts_live)); + setTxt($('#f-hosts-of'), fleet.hosts_total == null ? '' : `of ${fleet.hosts_total}`); + setValue($('#f-cpu'), fleet.cpu_usage_pct_avg, 0); + setValue($('#f-load'), fleet.load1_total, 2); + const used = Dash.fmtBytes(fleet.memory_used_bytes, 0); + const total = Dash.fmtBytes(fleet.memory_total_bytes, 0); + setTxt($('#f-mem'), used ? used.text : 'n/a'); + setTxt($('#f-mem-of'), used && total ? `${used.unit} of ${total.text} ${total.unit}` : ''); + setValue($('#f-disk'), fleet.disk_worst_used_pct, 0); + const rx = Dash.fmtBytes(fleet.net_rx_bytes_per_s, 1); + const tx = Dash.fmtBytes(fleet.net_tx_bytes_per_s, 1); + setTxt($('#f-rx'), rx ? rx.text : 'n/a'); setTxt($('#f-rx-unit'), rx ? rx.unit + '/s' : ''); + setTxt($('#f-tx'), tx ? tx.text : 'n/a'); setTxt($('#f-tx-unit'), tx ? tx.unit + '/s' : ''); + setValue($('#f-gpu'), fleet.gpu_busy_pct_avg, 0); +} + +async function refresh() { + let snapshot; + try { + snapshot = await fetchJSON('/api/system/snapshot'); + } catch (err) { + $('#sub').textContent = err.message; + return; + } + const states = {}; + (snapshot.states || []).forEach((row) => { states[row.key.split(':')[1]] = row; }); + const hostNames = []; + (snapshot.hosts || []).forEach((host) => { + hostNames.push(host.name); + host.state = states[host.name]; + host.window = snapshot.window_seconds; + let card = state.hosts.get(host.name); + if (!card) { + card = hostCard(host.name); + state.hosts.set(host.name, card); + $('#hosts').appendChild(card); + card.addEventListener('click', () => { + if (state.detailHost !== host.name) { + state.detailHost = host.name; + $('#detail-host').value = host.name; + populateDetailMetrics(host.name).then(refreshDetailChart); + } + }); + } + renderHostCard(card, host); + }); + $('#hosts-empty').hidden = hostNames.length > 0; + + const hostSelect = $('#detail-host'); + if (hostSelect.dataset.names !== hostNames.join(',')) { + hostSelect.dataset.names = hostNames.join(','); + hostSelect.innerHTML = hostNames.map((name) => + ``).join(''); + if (!state.detailHost || !hostNames.includes(state.detailHost)) { + state.detailHost = hostNames[0] || null; + } + if (state.detailHost) hostSelect.value = state.detailHost; + if (state.detailHost) await populateDetailMetrics(state.detailHost); + } + + renderFleet(snapshot.fleet); + const fleet = snapshot.fleet || {}; + $('#sub').textContent = `${fleet.hosts_live || 0}/${fleet.hosts_total || 0} hosts answering · ` + + `${fleet.cpu_cores || 0} cores · ${fleet.gpu_count || 0} GPUs · ` + + `window ${duration(state.range)}`; + $('#foot-note').textContent = `rates over Δ${snapshot.window_seconds}s · ` + + `poll ${Math.round(state.pollMs / 1000)}s`; + await refreshCharts(snapshot); +} + +async function refreshEvents() { + try { + const data = await fetchJSON('/api/events?limit=25&source=host'); + const list = $('#events'); + if (!(data.events || []).length) { + list.innerHTML = '
  • No host transitions ' + + 'recorded yet — this log fills when a host goes away, comes back, or reboots.
    • '; + return; + } + list.innerHTML = data.events.map((event) => `
  • ${DashCharts.escapeHtml(event.host)} ` + + `${DashCharts.escapeHtml(event.message)}
    • `).join(''); + } catch (err) { /* the next poll will try again */ } +} + +document.addEventListener('DOMContentLoaded', async () => { + makeCharts(); + let pollMs = 10000; + try { + const config = await fetchJSON('/api/config'); + pollMs = config.system.poll_ms || 10000; + if (!config.system.enabled) $('#sub').textContent = config.system.target_problems.length + ? `bad HOST_TARGETS: ${config.system.target_problems.join('; ')}` + : 'no hosts configured — set HOST_TARGETS'; + } catch (err) { /* config is nice-to-have */ } + state.pollMs = pollMs; + + Dash.rangeControl($('#range'), state.range, (seconds) => { + state.range = seconds; + setChartSpan(seconds); + refresh(); + }); + + $('#detail-host').addEventListener('change', async (event) => { + state.detailHost = event.target.value; + await populateDetailMetrics(state.detailHost); + await refreshDetailChart(); + }); + $('#detail-metric').addEventListener('change', async (event) => { + state.detailMetric = event.target.value; + await refreshDetailChart(); + }); + + state.poller = poll(async () => { await refresh(); await refreshEvents(); }, state.pollMs); + Dash.bindPause(state.poller); +}); diff --git a/src/dashboards/web/static/page-ups.js b/src/dashboards/web/static/page-ups.js new file mode 100644 index 0000000..2a83f30 --- /dev/null +++ b/src/dashboards/web/static/page-ups.js @@ -0,0 +1,358 @@ +/* UPS dashboard: health assessment, NUT instant commands, history. + * + * The health list is deliberately explicit checks rather than a score: every row names the + * variable it read and what it saw, because "health: 87" is not actionable. The backend + * owns that logic (sources/nut.py::assess) so the same rules apply in tests and API clients. + * + * Commands are the only write path in the product. They are offered only when the backend + * says `enabled`, and anything that can cut power requires typing DANGER here *and* the + * server re-checks the same gate — a hidden button is not a safety control. + */ +'use strict'; + +const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, duration, poll, severityFor } = Dash; + +const state = { range: 86400, cards: new Map(), charts: {}, poller: null, pollMs: 5000, + simulated: false, pending: null }; + +const MARKS = { ok: '✓', warn: '!', fail: '✕', unknown: '?' }; + +/** + * Runtime left as "1h 32m", not "1.5h". The health check next to it spells the same number + * out in h+m; two formats for one value invites a misread while the UPS is discharging. + */ +function runtimeText(seconds) { + if (seconds == null || !isFinite(seconds)) return 'n/a'; + if (seconds < 0) return 'on battery'; + const total = Math.round(seconds); + if (total < 120) return `${total}s`; + const minutes = Math.round(total / 60); + if (minutes < 60) return `${minutes}m`; + return `${Math.floor(minutes / 60)}h ${String(minutes % 60).padStart(2, '0')}m`; +} + +function upsCard(name) { + const card = document.createElement('article'); + card.className = 'card'; + card.dataset.ups = name; + card.innerHTML = ` +
    + +

    + + + + unknown + +
    +
    +
    +
    +
    Charge
    +
    n/a%
    +
    +
    Runtime left
    +
    n/a
    +
    Load
    +
    n/a%
    +
    +
    Line in
    +
    n/aV
    +
    Out
    +
    n/aV
    +
    Battery
    +
    n/aV
    +
    Battery A
    +
    n/aA
    +
    Battery °C
    +
    n/a°C
    +
    +
    +
    + Health checks
    +
      +
      +
      +
      + NUT variables +
      +
      +
      `; + return card; +} + +function renderCard(card, item, stateRow) { + const q = (key) => card.querySelector(`[data-k="${key}"]`); + card.querySelector('h3').textContent = item.name; + const model = [item.identity && item.identity.mfr, item.identity && item.identity.model] + .filter(Boolean).join(' '); + const modelChip = card.querySelector('.model'); + if (model) { modelChip.hidden = false; modelChip.textContent = model; } + card.querySelector('.sim-badge').hidden = !state.simulated; + + const overall = (item.health && item.health.overall) || 'unknown'; + const healthChip = card.querySelector('.health'); + healthChip.textContent = overall; + healthChip.className = 'chip health ' + ({ healthy: 'ok', degraded: 'warn', 'at risk': 'fail', + unknown: '' }[overall] || ''); + const severity = stateRow && !stateRow.ok ? 'fail' + : (item.health && item.health.severity) || 'idle'; + setDot(card.querySelector('.dot'), severity === 'ok' ? 'ok' : severity === 'warn' ? 'warn' + : severity === 'fail' ? 'fail' : 'idle'); + setTxt(card.querySelector('.updated'), stateRow && stateRow.last_ok != null + ? (Dash.ageSeconds(stateRow.last_ok) > 30 ? `stale ${duration(Dash.ageSeconds(stateRow.last_ok))}` + : `updated ${duration(Dash.ageSeconds(stateRow.last_ok))}`) + : (stateRow && stateRow.error) || 'never reached'); + + const states = card.querySelector('.states'); + states.innerHTML = (item.states || []).map((s) => { + const kind = s.severity === 'ok' ? 'ok' : s.severity === 'warn' ? 'warn' + : s.severity === 'fail' ? 'fail' : 'info'; + return `${ + DashCharts.escapeHtml(s.label)}`; + }).join('') + (item.status ? `${ + DashCharts.escapeHtml(item.status)}` : '') + + (item.alarms || []).map((alarm) => `alarm: ${ + DashCharts.escapeHtml(alarm)}`).join(''); + + const n = item.numbers || {}; + const cell = (key) => q(key); + setValue(cell('charge'), n.charge_pct, 0); + setBar(card.querySelector('[data-bar="charge"]'), n.charge_pct, + n.charge_pct == null ? '' : n.charge_pct >= 95 ? 'ok' : n.charge_pct >= 60 ? 'warn' : 'fail'); + setTxt(cell('runtime'), runtimeText(n.battery_runtime)); + setValue(cell('load'), n.load_pct, 1); + setBar(card.querySelector('[data-bar="load"]'), n.load_pct, severityFor(n.load_pct, 70, 90)); + setValue(cell('line'), n.line_voltage, 1); + setValue(cell('out'), n.output_voltage, 1); + setValue(cell('batt'), n.battery_voltage, 2); + setValue(cell('amps'), n.battery_current, 2); + setValue(cell('temp'), n.battery_temperature, 1); + + const health = item.health || { checks: [] }; + q('checks').innerHTML = (health.checks || []).map((check) => ` +
    • + ${MARKS[check.status] || '?'} + ${DashCharts.escapeHtml(check.name)}
      + ${DashCharts.escapeHtml(check.detail)}       +
      • `).join('') || '
    • ?No checks available
      • '; + + const commands = item.commands || []; + const groups = [['safe', 'Diagnostics'], ['caution', 'Affects the load briefly'], + ['dangerous', 'Can cut power']]; + const host = q('commands'); + host.innerHTML = groups.map(([risk, legend]) => { + const inGroup = commands.filter((command) => command.risk === risk); + if (!inGroup.length) return ''; + return `
      ${legend} +
      ${inGroup.map((command) => ` + `).join('')}
      `; + }).join('') || '
      ' + + 'The driver advertises no commands we know how to send.
      '; + host.querySelectorAll('button[data-command]').forEach((button) => { + button.addEventListener('click', () => openCommandDialog(item.name, button.dataset.command, + commands)); + }); + + const extra = item.extra || {}; + const entries = Object.entries(extra); + q('raw').innerHTML = `${entries.map(([key, value]) => + ` + `).join('') + || ''}
      ${DashCharts.escapeHtml(key)}${DashCharts.escapeHtml(value)}
      no extra variables
      ` + + `
      driver ${ + DashCharts.escapeHtml((item.identity || {}).driver || '?')} · firmware ${ + DashCharts.escapeHtml((item.identity || {}).firmware || '?')} · serial ${ + DashCharts.escapeHtml((item.identity || {}).serial || '?')} · ${ + (item.advertised_commands || []).length} commands advertised
      `; +} + +// ---- command dialog ------------------------------------------------------- +function openCommandDialog(upsName, commandName, commands) { + const command = commands.find((c) => c.command === commandName); + if (!command) return; + const dialog = $('#confirm-dialog'); + const dangerous = command.risk === 'dangerous'; + $('#dialog-title').textContent = command.label; + $('#dialog-help').textContent = command.help; + $('#dialog-target').textContent = `UPS ${upsName} · ${command.command} · risk: ${command.risk}` + + (state.simulated ? ' · this UPS is simulated' : ''); + const riskChip = $('#dialog-risk'); + riskChip.textContent = command.risk; + riskChip.className = 'chip ' + (command.risk === 'dangerous' ? 'fail' + : command.risk === 'caution' ? 'warn' : 'ok'); + $('#dialog-confirm-wrap').hidden = !dangerous; + $('#dialog-input-wrap').hidden = !dangerous; + $('#dialog-input').value = ''; + const run = $('#dialog-run'); + run.classList.toggle('dangerous', dangerous); + run.disabled = dangerous; + state.pending = { ups: upsName, command: commandName, dangerous }; + dialog.showModal(); + if (dangerous) $('#dialog-input').focus(); +} + +async function runPending() { + const pending = state.pending; + if (!pending) return; + const body = { ups: pending.ups, command: pending.command }; + if (pending.dangerous) body.confirm = $('#dialog-input').value.trim(); + $('#confirm-dialog').close(); + try { + const result = await fetchJSON('/api/ups/command', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + Dash.toast(`sent ${result.command} to ${result.ups}: ${result.detail}`, 'ok'); + } catch (err) { + Dash.toast(`refused: ${err.message}`, 'fail'); + } + state.pending = null; + refresh(); +} + +// ---- charts --------------------------------------------------------------- +function makeCharts() { + const common = { height: 160, xSpan: state.range }; + state.charts.charge = new DashCharts.LineChart('#chart-charge', Object.assign({}, common, + { yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) })); + state.charts.runtime = new DashCharts.LineChart('#chart-runtime', Object.assign({}, common, + { yMin: 0, yFormat: (v) => duration(v) })); + state.charts.voltage = new DashCharts.LineChart('#chart-voltage', Object.assign({}, common, + { yFormat: (v) => num(v, 1) })); + state.charts.load = new DashCharts.LineChart('#chart-load', Object.assign({}, common, + { yMin: 0, yFormat: (v) => num(v, 0) })); +} + +function points(response, upsName, metric) { + const found = (response.series || []).find((s) => s.host === upsName && s.metric === metric); + return found ? found.points.map((p) => ({ ts: p.ts, value: p.value })) : []; +} + +async function refreshCharts(names) { + if (!names.length) return; + const list = names.join(','); + const metrics = 'ups.charge_pct,ups.battery_runtime_seconds,ups.battery_voltage,' + + 'ups.line_voltage,ups.load_pct'; + const data = await fetchJSON(`/api/ups/history?ups=${encodeURIComponent(list)}&metrics=${metrics}` + + `&range=${state.range}`).catch(() => null); + if (!data) return; + const build = (metric, scale) => names.map((name) => ({ + name, points: points(data, name, metric).map((p) => (scale ? { ts: p.ts, value: scale(p.value) } : p)), + })); + const charge = build('ups.charge_pct'); + state.charts.charge.setData(charge); + DashCharts.renderLegend($('#legend-charge'), charge, (v) => num(v, 0) + '%'); + const runtime = build('ups.battery_runtime_seconds'); + state.charts.runtime.setData(runtime); + DashCharts.renderLegend($('#legend-runtime'), runtime, (v) => duration(v), true); + const battery = build('ups.battery_voltage'); + const line = build('ups.line_voltage'); + const voltage = battery.map((series, index) => ({ name: `${series.name} battery`, + points: series.points })).concat(line.map((series, index) => ({ name: `${series.name} line`, + points: series.points, color: undefined }))); + state.charts.voltage.setData(voltage); + DashCharts.renderLegend($('#legend-voltage'), voltage, (v) => num(v, 1) + ' V'); + const load = build('ups.load_pct'); + state.charts.load.setData(load); + DashCharts.renderLegend($('#legend-load'), load, (v) => num(v, 0) + '%', true); +} + +async function refresh() { + let snapshot; + try { + snapshot = await fetchJSON('/api/ups/snapshot'); + } catch (err) { + $('#sub').textContent = err.message; + return; + } + state.simulated = !!snapshot.simulated; + const stateRow = (snapshot.states || [])[0]; + const items = snapshot.ups || []; + items.forEach((item) => { + let card = state.cards.get(item.name); + if (!card) { + card = upsCard(item.name); + state.cards.set(item.name, card); + $('#ups-cards').appendChild(card); + } + renderCard(card, item, stateRow); + }); + $('#ups-empty').hidden = items.length > 0; + + const banners = []; + if (state.simulated) { + banners.push(['warn', 'SIMULATED DATA. NUT_SIMULATE=1 — these are ' + + 'modelled UPSes for development, not your batteries. Unset it to read real upsd.']); + } + if (!snapshot.actions_enabled) { + banners.push(['info', 'Command buttons are disabled. Set NUT_ALLOW_ACTIONS=1 plus ' + + 'NUT_USER/NUT_PASSWORD for a user with actions in ' + + 'upsd.users.']); + } + if (stateRow && !stateRow.ok && stateRow.error) { + banners.push(['fail', `upsd: ${DashCharts.escapeHtml(stateRow.error)}`]); + } + $('#banners').innerHTML = banners.map(([kind, html]) => ``) + .join(''); + + // "0 healthy" reads like an outage when every UPS is merely past its replacement window; + // name the states that are actually present instead of counting the absence of one. + const counts = { healthy: 0, degraded: 0, 'at risk': 0, unknown: 0 }; + items.forEach((item) => { counts[(item.health && item.health.overall) || 'unknown'] += 1; }); + const breakdown = Object.entries(counts).filter(([, n]) => n > 0) + .map(([name, n]) => `${n} ${name}`).join(', ') || 'none reporting'; + $('#sub').textContent = `${items.length} UPS${items.length === 1 ? '' : 'es'} · ${breakdown} · ` + + (snapshot.server ? `upsd ${snapshot.server} · ` : '') + `window ${duration(state.range)}`; + $('#foot-note').textContent = `poll ${Math.round(state.pollMs / 1000)}s · ` + + `${items.reduce((total, item) => total + (item.advertised_commands || []).length, 0)} commands advertised`; + await refreshCharts(items.map((item) => item.name)); + await refreshEvents(); +} + +async function refreshEvents() { + try { + const data = await fetchJSON('/api/events?limit=40&source=ups'); + const list = $('#events'); + if (!(data.events || []).length) { + list.innerHTML = '
    • No power events yet — ' + + 'state changes, self-test results, and commands you send all land here.
      • '; + return; + } + list.innerHTML = data.events.map((event) => `
    • ${DashCharts.escapeHtml(event.host)} ` + + `${DashCharts.escapeHtml(event.message.replace(event.host + ':', '').trim())}` + + `${event.data && event.data.runtime_seconds ? ` (${ + duration(event.data.runtime_seconds)} runtime)` : ''}
      • `).join(''); + } catch (err) { /* next poll */ } +} + +document.addEventListener('DOMContentLoaded', async () => { + makeCharts(); + try { + const config = await fetchJSON('/api/config'); + state.pollMs = config.ups.poll_ms || 5000; + } catch (err) { /* default poll rate */ } + + Dash.rangeControl($('#range'), state.range, (seconds) => { + state.range = seconds; + Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = seconds; }); + refresh(); + }, [['1h', 3600], ['6h', 21600], ['24h', 86400], ['7d', 604800], ['30d', 2592000], + ['90d', 7776000]]); + + $('#dialog-cancel').addEventListener('click', () => { $('#confirm-dialog').close(); state.pending = null; }); + $('#dialog-run').addEventListener('click', runPending); + $('#dialog-input').addEventListener('input', (event) => { + $('#dialog-run').disabled = event.target.value.trim() !== 'DANGER'; + }); + + state.poller = poll(refresh, state.pollMs); + Dash.bindPause(state.poller); +}); diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..419e2ba --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,71 @@ +"""Shared fixtures: real captures for shape, fakes for behaviour. + +Anything that must match an upstream (an agent snapshot, an engine's /metrics text, the NUT +wire protocol) is asserted against a verbatim capture in tests/fixtures/. Everything that +needs a *failure* — a dead host, a refused command, a counter reset — is a fake here. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from dashboards.rates import RateEstimator # noqa: E402 +from dashboards.store import SampleStore # noqa: E402 + +FIXTURES = Path(__file__).resolve().parent / "fixtures" + + +@pytest.fixture +def db(tmp_path: Path) -> Any: + store = SampleStore(tmp_path / "t.sqlite3") + try: + yield store + finally: + store.close() + + +@pytest.fixture +def store(db: Any) -> Any: + """Alias: some tests read better with `store`, others with `db`. Same object.""" + return db + + +@pytest.fixture +def agent_snapshot() -> dict[str, Any]: + """A real driveripper snapshot (Threadripper + R9700, 48 logical CPUs).""" + return json.loads((FIXTURES / "agent_driveripper.json").read_text()) + + +@pytest.fixture +def agent_snapshots() -> dict[str, dict[str, Any]]: + """Every captured host, keyed by name — fleet code paths need >1 host to mean anything.""" + out: dict[str, dict[str, Any]] = {} + for path in sorted(FIXTURES.glob("agent_*.json")): + out[path.stem.removeprefix("agent_")] = json.loads(path.read_text()) + return out + + +@pytest.fixture +def metrics_text() -> Any: + """metrics_text('gufo') / metrics_text('vllm') -> the captured exposition verbatim.""" + def load(engine: str) -> str: + path = FIXTURES / f"metrics_{engine}.txt" + if not path.exists(): + pytest.skip(f"missing fixture {path.name} — run scripts/capture-fixtures.sh --with-llm") + return path.read_text() + + return load + + +@pytest.fixture +def rates() -> RateEstimator: + return RateEstimator(span=10.0) diff --git a/tests/fixtures/agent_deskwork.json b/tests/fixtures/agent_deskwork.json new file mode 100644 index 0000000..04cd135 --- /dev/null +++ b/tests/fixtures/agent_deskwork.json @@ -0,0 +1 @@ +{"ok": true, "schema": 1, "agent_version": "1.0.0", "ts": 1792178443.0380626, "duration_ms": 16.42, "host": {"hostname": "deskwork", "os_pretty": "Fedora Linux 43 (Server Edition)", "os_id": "fedora", "kernel": "7.2.10-100.fc43.x86_64", "arch": "x86_64", "python": "3.14.8", "cpu_model": "AMD RYZEN AI MAX+ 395 w/ Radeon 8060S", "cpu_cores_logical": 32, "cpu_cores_physical": 16, "memory_total_bytes": 134296711168.0, "boot_ts": 1791966026.2263377, "uptime_seconds": 212416.79, "container": false, "agent_version": "1.0.0"}, "cpu": {"modes": {"user": 11853771.0, "nice": 30631.0, "system": 7800223.0, "idle": 658608117.0, "iowait": 1278478.0, "irq": 272815.0, "softirq": 171965.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 680016000.0}, "cores": [{"core": 0, "modes": {"user": 161241.0, "nice": 2482.0, "system": 181664.0, "idle": 20758034.0, "iowait": 84170.0, "irq": 10154.0, "softirq": 98025.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21295770.0}}, {"core": 1, "modes": {"user": 3036916.0, "nice": 3899.0, "system": 1999277.0, "idle": 16097736.0, "iowait": 78610.0, "irq": 21817.0, "softirq": 11019.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21249274.0}}, {"core": 2, "modes": {"user": 27477.0, "nice": 194.0, "system": 38504.0, "idle": 21105062.0, "iowait": 74358.0, "irq": 5205.0, "softirq": 4463.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21255263.0}}, {"core": 3, "modes": {"user": 40264.0, "nice": 322.0, "system": 50965.0, "idle": 21074920.0, "iowait": 78178.0, "irq": 5379.0, "softirq": 2653.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21252681.0}}, {"core": 4, "modes": {"user": 12339.0, "nice": 50.0, "system": 18672.0, "idle": 21143856.0, "iowait": 73655.0, "irq": 3131.0, "softirq": 960.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21252663.0}}, {"core": 5, "modes": {"user": 2987777.0, "nice": 5267.0, "system": 2058104.0, "idle": 16093690.0, "iowait": 78318.0, "irq": 19761.0, "softirq": 6254.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21249171.0}}, {"core": 6, "modes": {"user": 27123.0, "nice": 157.0, "system": 40320.0, "idle": 21106702.0, "iowait": 73832.0, "irq": 4308.0, "softirq": 1707.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21254149.0}}, {"core": 7, "modes": {"user": 54218.0, "nice": 600.0, "system": 67005.0, "idle": 21046582.0, "iowait": 77817.0, "irq": 4718.0, "softirq": 2139.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21253079.0}}, {"core": 8, "modes": {"user": 9116.0, "nice": 54.0, "system": 9701.0, "idle": 21214665.0, "iowait": 4241.0, "irq": 3408.0, "softirq": 1452.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242637.0}}, {"core": 9, "modes": {"user": 2678.0, "nice": 13.0, "system": 5004.0, "idle": 21229457.0, "iowait": 3203.0, "irq": 1937.0, "softirq": 581.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242873.0}}, {"core": 10, "modes": {"user": 8878.0, "nice": 57.0, "system": 10567.0, "idle": 21213579.0, "iowait": 4191.0, "irq": 3808.0, "softirq": 1295.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242375.0}}, {"core": 11, "modes": {"user": 5209.0, "nice": 19.0, "system": 11177.0, "idle": 21122513.0, "iowait": 3935.0, "irq": 92258.0, "softirq": 2458.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21237569.0}}, {"core": 12, "modes": {"user": 5711.0, "nice": 29.0, "system": 9129.0, "idle": 21218831.0, "iowait": 4074.0, "irq": 3661.0, "softirq": 986.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242421.0}}, {"core": 13, "modes": {"user": 9500.0, "nice": 55.0, "system": 11420.0, "idle": 21211495.0, "iowait": 4527.0, "irq": 4075.0, "softirq": 1374.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242446.0}}, {"core": 14, "modes": {"user": 19547.0, "nice": 225.0, "system": 17679.0, "idle": 21191182.0, "iowait": 5290.0, "irq": 5812.0, "softirq": 2263.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21241998.0}}, {"core": 15, "modes": {"user": 5107.0, "nice": 18.0, "system": 8122.0, "idle": 21220032.0, "iowait": 4678.0, "irq": 3648.0, "softirq": 1318.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242923.0}}, {"core": 16, "modes": {"user": 39840.0, "nice": 1336.0, "system": 55418.0, "idle": 21072844.0, "iowait": 77726.0, "irq": 4164.0, "softirq": 1907.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21253235.0}}, {"core": 17, "modes": {"user": 2740774.0, "nice": 7235.0, "system": 1497928.0, "idle": 16925780.0, "iowait": 71401.0, "irq": 14355.0, "softirq": 3007.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21260480.0}}, {"core": 18, "modes": {"user": 12430.0, "nice": 81.0, "system": 20880.0, "idle": 21141310.0, "iowait": 74870.0, "irq": 3264.0, "softirq": 945.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21253780.0}}, {"core": 19, "modes": {"user": 13385.0, "nice": 91.0, "system": 24405.0, "idle": 21147926.0, "iowait": 73823.0, "irq": 5355.0, "softirq": 1108.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21266093.0}}, {"core": 20, "modes": {"user": 7403.0, "nice": 34.0, "system": 14486.0, "idle": 21153607.0, "iowait": 74026.0, "irq": 2564.0, "softirq": 524.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21252644.0}}, {"core": 21, "modes": {"user": 2561133.0, "nice": 7978.0, "system": 1548359.0, "idle": 17036167.0, "iowait": 71516.0, "irq": 24116.0, "softirq": 18818.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21268087.0}}, {"core": 22, "modes": {"user": 14788.0, "nice": 66.0, "system": 23346.0, "idle": 21136484.0, "iowait": 74584.0, "irq": 3438.0, "softirq": 1115.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21253821.0}}, {"core": 23, "modes": {"user": 18139.0, "nice": 245.0, "system": 32400.0, "idle": 21134461.0, "iowait": 74471.0, "irq": 5549.0, "softirq": 1292.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21266557.0}}, {"core": 24, "modes": {"user": 4533.0, "nice": 16.0, "system": 6462.0, "idle": 21224318.0, "iowait": 3992.0, "irq": 2693.0, "softirq": 854.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242868.0}}, {"core": 25, "modes": {"user": 3527.0, "nice": 8.0, "system": 5370.0, "idle": 21228521.0, "iowait": 3431.0, "irq": 1972.0, "softirq": 444.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21243273.0}}, {"core": 26, "modes": {"user": 4101.0, "nice": 9.0, "system": 5766.0, "idle": 21226447.0, "iowait": 4130.0, "irq": 1984.0, "softirq": 489.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242926.0}}, {"core": 27, "modes": {"user": 2828.0, "nice": 9.0, "system": 4579.0, "idle": 21229770.0, "iowait": 3729.0, "irq": 1797.0, "softirq": 353.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21243065.0}}, {"core": 28, "modes": {"user": 3267.0, "nice": 10.0, "system": 5105.0, "idle": 21228103.0, "iowait": 4192.0, "irq": 1785.0, "softirq": 398.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242860.0}}, {"core": 29, "modes": {"user": 4054.0, "nice": 13.0, "system": 5645.0, "idle": 21226362.0, "iowait": 4291.0, "irq": 2068.0, "softirq": 453.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242886.0}}, {"core": 30, "modes": {"user": 7104.0, "nice": 37.0, "system": 7378.0, "idle": 21220463.0, "iowait": 4708.0, "irq": 2458.0, "softirq": 755.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21242903.0}}, {"core": 31, "modes": {"user": 3347.0, "nice": 7.0, "system": 5370.0, "idle": 21227203.0, "iowait": 4496.0, "irq": 2157.0, "softirq": 542.0, "steal": 0.0, "guest": 0.0, "guest_nice": 0.0, "total": 21243122.0}}], "clock_ticks_per_second": 100, "load1": 1.92, "load5": 1.8, "load15": 2.02, "procs_running": 1.0, "procs_blocked": 0.0, "ctxt_total": 523554930.0, "processes_total": 1168739.0, "freq_mhz": 2185.16434375, "freq_max_mhz": 5187.5}, "memory": {"total_bytes": 134296711168.0, "free_bytes": 2958434304.0, "available_bytes": 12516724736.0, "used_bytes": 121779986432.0, "buffers_bytes": 32768.0, "cached_bytes": 10943029248.0, "dirty_bytes": 1863680.0, "swap_total_bytes": 0.0, "swap_used_bytes": 0.0, "committed_bytes": 18375344128.0}, "filesystems": [{"device": "/dev/nvme0n1p3", "mountpoint": "/", "fstype": "btrfs", "total_bytes": 998043025408, "used_bytes": 740790976512, "free_bytes": 251342389248, "used_pct": 74.6664714722637, "read_only": false}, {"device": "/dev/nvme0n1p2", "mountpoint": "/boot", "fstype": "btrfs", "total_bytes": 1073741824, "used_bytes": 765698048, "free_bytes": 191717376, "used_pct": 79.97552878362653, "read_only": false}, {"device": "/dev/nvme0n1p1", "mountpoint": "/boot/efi", "fstype": "vfat", "total_bytes": 1071628288, "used_bytes": 7868416, "free_bytes": 1063759872, "used_pct": 0.73424862782271, "read_only": false}, {"device": "/dev/nvme0n1p3", "mountpoint": "/btrfs", "fstype": "btrfs", "total_bytes": 998043025408, "used_bytes": 740790976512, "free_bytes": 251342389248, "used_pct": 74.6664714722637, "read_only": false}, {"device": "/dev/nvme0n1p3", "mountpoint": "/home", "fstype": "btrfs", "total_bytes": 998043025408, "used_bytes": 740790976512, "free_bytes": 251342389248, "used_pct": 74.6664714722637, "read_only": false}], "disk_io": {"devices": [{"name": "nvme0n1", "model": "PCIe SSD", "rotational": false, "size_bytes": 1000204886016.0, "reads_total": 64020120.0, "writes_total": 1328401.0, "read_bytes_total": 977963038720.0, "write_bytes_total": 53677868032.0, "in_flight": 0.0, "io_time_ms_total": 25161781.0, "weighted_io_ms_total": 49058985.0, "discard_bytes_total": 32813576192.0}], "device_names": ["nvme0n1"]}, "network": {"interfaces": [{"name": "enp191s0", "rx_bytes_total": 3097623492.0, "rx_packets_total": 6408628.0, "rx_errors_total": 0.0, "rx_dropped_total": 13.0, "tx_bytes_total": 3711578389.0, "tx_packets_total": 4898468.0, "tx_errors_total": 0.0, "tx_dropped_total": 0.0, "speed_mbps": 5000.0}]}, "gpus": [{"index": 0, "card": "card1", "vendor": "AMD", "model": "Radeon 8060S Graphics", "model_full": "Advanced Micro Devices, Inc. [AMD/ATI] Strix Halo [Radeon Graphics / Radeon 8050S Graphics / Radeon 8060S Graphics]", "pci_slot": "0000:c1:00.0", "pci_address": "1002:1586", "driver": "amdgpu", "busy_pct": 62.0, "vram_total_bytes": 536870912.0, "vram_used_bytes": 180658176.0, "gtt_total_bytes": 133143986176.0, "gtt_used_bytes": 104393768960.0, "power_watts": 70.217, "temps": {"edge": 56.0}, "clocks": {"sclk": 2006.0}}], "temperatures": [{"kind": "hwmon", "chip": "acpitz", "label": "acpitz", "celsius": 54.8}, {"kind": "hwmon", "chip": "acpitz", "label": "acpitz", "celsius": 56.8}, {"kind": "hwmon", "chip": "acpitz", "label": "acpitz", "celsius": 53.8}, {"kind": "hwmon", "chip": "acpitz", "label": "acpitz", "celsius": 83.8}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 51.85}, {"kind": "hwmon", "chip": "amdgpu", "label": "edge", "celsius": 56.0}, {"kind": "hwmon", "chip": "k10temp", "label": "Tctl", "celsius": 80.125}, {"kind": "hwmon", "chip": "r8169_0_bf00:00", "label": "r8169_0_bf00:00", "celsius": 68.0}, {"kind": "hwmon", "chip": "cros_ec", "label": "mainboard_power@4d", "celsius": 54.85}, {"kind": "hwmon", "chip": "cros_ec", "label": "mainboard_memory@4d", "celsius": 56.85}, {"kind": "hwmon", "chip": "cros_ec", "label": "mainboard_ambient@4d", "celsius": 53.85}, {"kind": "hwmon", "chip": "cros_ec", "label": "cpu@4c", "celsius": 83.85}, {"kind": "hwmon", "chip": "cros_ec", "label": "cpu_virtual@4c", "celsius": 75.85}, {"kind": "thermal", "chip": "acpitz", "label": "acpitz", "celsius": 54.8}, {"kind": "thermal", "chip": "acpitz", "label": "acpitz", "celsius": 56.8}, {"kind": "thermal", "chip": "acpitz", "label": "acpitz", "celsius": 53.8}, {"kind": "thermal", "chip": "acpitz", "label": "acpitz", "celsius": 83.8}], "cpu_temperature_celsius": 80.125, "processes": {"total": 495, "top_rss": [{"pid": 1032371, "name": "gufo", "rss_bytes": 15113818112.0}, {"pid": 1240, "name": "osbuild-compose", "rss_bytes": 397737984.0}, {"pid": 801, "name": "systemd-journal", "rss_bytes": 95752192.0}, {"pid": 1248, "name": "osbuild-worker", "rss_bytes": 52178944.0}, {"pid": 1412, "name": "rsyslogd", "rss_bytes": 51032064.0}], "top_cpu": [{"pid": 1164998, "name": "python3", "cpu_pct": 3.107202572109063}, {"pid": 1, "name": "systemd", "cpu_pct": 0.0}, {"pid": 2, "name": "kthreadd", "cpu_pct": 0.0}, {"pid": 3, "name": "pool_workqueue_release", "cpu_pct": 0.0}, {"pid": 4, "name": "kworker/R-rcu_gp", "cpu_pct": 0.0}]}, "errors": []} \ No newline at end of file diff --git a/tests/fixtures/agent_driveripper.json b/tests/fixtures/agent_driveripper.json new file mode 100644 index 0000000..5bdc00e --- /dev/null +++ b/tests/fixtures/agent_driveripper.json @@ -0,0 +1 @@ +{"ok": true, "schema": 1, "agent_version": "1.0.0", "ts": 1792178442.9119747, "duration_ms": 251.12, "host": {"hostname": "driveripper", "os_pretty": "Fedora Linux 43 (Server Edition)", "os_id": "fedora", "kernel": "7.2.10-100.fc43.x86_64", "arch": "x86_64", "python": "3.14.8", "cpu_model": "AMD Ryzen Threadripper 3960X 24-Core Processor", "cpu_cores_logical": 48, "cpu_cores_physical": 24, "memory_total_bytes": 134888374272.0, "boot_ts": 1791967372.373984, "uptime_seconds": 211070.29, "container": false, "agent_version": "1.0.0"}, "cpu": {"modes": {"user": 42016295.0, "nice": 189998.0, "system": 8777510.0, "idle": 966147625.0, "iowait": 4456512.0, "irq": 1500827.0, "softirq": 2098659.0, "steal": 0.0, "guest": 13679302.0, "guest_nice": 0.0, "total": 1025187426.0}, "cores": [{"core": 0, "modes": {"user": 126139.0, "nice": 445.0, "system": 85877.0, "idle": 18064187.0, "iowait": 53148.0, "irq": 23288.0, "softirq": 1521273.0, "steal": 0.0, "guest": 88450.0, "guest_nice": 0.0, "total": 19874357.0}}, {"core": 1, "modes": {"user": 71526.0, "nice": 44.0, "system": 73655.0, "idle": 21016592.0, "iowait": 50834.0, "irq": 13838.0, "softirq": 71383.0, "steal": 0.0, "guest": 62404.0, "guest_nice": 0.0, "total": 21297872.0}}, {"core": 2, "modes": {"user": 67110.0, "nice": 39.0, "system": 70862.0, "idle": 21147652.0, "iowait": 33124.0, "irq": 13514.0, "softirq": 16695.0, "steal": 0.0, "guest": 59082.0, "guest_nice": 0.0, "total": 21348996.0}}, {"core": 3, "modes": {"user": 70935.0, "nice": 301.0, "system": 59946.0, "idle": 21135614.0, "iowait": 38705.0, "irq": 10905.0, "softirq": 10821.0, "steal": 0.0, "guest": 46288.0, "guest_nice": 0.0, "total": 21327227.0}}, {"core": 4, "modes": {"user": 57249.0, "nice": 210.0, "system": 55958.0, "idle": 21159374.0, "iowait": 32689.0, "irq": 12540.0, "softirq": 11902.0, "steal": 0.0, "guest": 39443.0, "guest_nice": 0.0, "total": 21329922.0}}, {"core": 5, "modes": {"user": 58080.0, "nice": 184.0, "system": 57139.0, "idle": 21161135.0, "iowait": 28717.0, "irq": 10580.0, "softirq": 7071.0, "steal": 0.0, "guest": 41179.0, "guest_nice": 0.0, "total": 21322906.0}}, {"core": 6, "modes": {"user": 1904259.0, "nice": 10856.0, "system": 471662.0, "idle": 18719522.0, "iowait": 235331.0, "irq": 50916.0, "softirq": 20023.0, "steal": 0.0, "guest": 910541.0, "guest_nice": 0.0, "total": 21412569.0}}, {"core": 7, "modes": {"user": 706993.0, "nice": 2800.0, "system": 341621.0, "idle": 20086752.0, "iowait": 253579.0, "irq": 43943.0, "softirq": 18244.0, "steal": 0.0, "guest": 521222.0, "guest_nice": 0.0, "total": 21453932.0}}, {"core": 8, "modes": {"user": 345070.0, "nice": 706.0, "system": 248462.0, "idle": 20582601.0, "iowait": 219176.0, "irq": 38008.0, "softirq": 15147.0, "steal": 0.0, "guest": 284385.0, "guest_nice": 0.0, "total": 21449170.0}}, {"core": 9, "modes": {"user": 739607.0, "nice": 3543.0, "system": 297352.0, "idle": 20160554.0, "iowait": 178579.0, "irq": 37992.0, "softirq": 16006.0, "steal": 0.0, "guest": 463031.0, "guest_nice": 0.0, "total": 21433633.0}}, {"core": 10, "modes": {"user": 319598.0, "nice": 576.0, "system": 220699.0, "idle": 20704640.0, "iowait": 174792.0, "irq": 35062.0, "softirq": 13999.0, "steal": 0.0, "guest": 262724.0, "guest_nice": 0.0, "total": 21469366.0}}, {"core": 11, "modes": {"user": 183972.0, "nice": 201.0, "system": 165632.0, "idle": 20930012.0, "iowait": 136679.0, "irq": 29998.0, "softirq": 11144.0, "steal": 0.0, "guest": 159073.0, "guest_nice": 0.0, "total": 21457638.0}}, {"core": 12, "modes": {"user": 6460539.0, "nice": 23732.0, "system": 434400.0, "idle": 14136544.0, "iowait": 127154.0, "irq": 98276.0, "softirq": 13146.0, "steal": 0.0, "guest": 970907.0, "guest_nice": 0.0, "total": 21293791.0}}, {"core": 13, "modes": {"user": 5784389.0, "nice": 23567.0, "system": 435213.0, "idle": 14825058.0, "iowait": 107652.0, "irq": 89378.0, "softirq": 11669.0, "steal": 0.0, "guest": 980884.0, "guest_nice": 0.0, "total": 21276926.0}}, {"core": 14, "modes": {"user": 2065064.0, "nice": 11761.0, "system": 389000.0, "idle": 18665039.0, "iowait": 208642.0, "irq": 57270.0, "softirq": 16732.0, "steal": 0.0, "guest": 1337818.0, "guest_nice": 0.0, "total": 21413508.0}}, {"core": 15, "modes": {"user": 70716.0, "nice": 340.0, "system": 61129.0, "idle": 21139690.0, "iowait": 34436.0, "irq": 13849.0, "softirq": 11386.0, "steal": 0.0, "guest": 46381.0, "guest_nice": 0.0, "total": 21331546.0}}, {"core": 16, "modes": {"user": 58765.0, "nice": 233.0, "system": 58070.0, "idle": 21172879.0, "iowait": 30360.0, "irq": 11337.0, "softirq": 7622.0, "steal": 0.0, "guest": 41789.0, "guest_nice": 0.0, "total": 21339266.0}}, {"core": 17, "modes": {"user": 59711.0, "nice": 186.0, "system": 58034.0, "idle": 21162859.0, "iowait": 26875.0, "irq": 11916.0, "softirq": 9270.0, "steal": 0.0, "guest": 42781.0, "guest_nice": 0.0, "total": 21328851.0}}, {"core": 18, "modes": {"user": 383264.0, "nice": 1166.0, "system": 194529.0, "idle": 20696198.0, "iowait": 115701.0, "irq": 29500.0, "softirq": 12075.0, "steal": 0.0, "guest": 263561.0, "guest_nice": 0.0, "total": 21432433.0}}, {"core": 19, "modes": {"user": 183542.0, "nice": 270.0, "system": 146997.0, "idle": 20977291.0, "iowait": 107811.0, "irq": 27116.0, "softirq": 10453.0, "steal": 0.0, "guest": 156273.0, "guest_nice": 0.0, "total": 21453480.0}}, {"core": 20, "modes": {"user": 113566.0, "nice": 119.0, "system": 112088.0, "idle": 21093462.0, "iowait": 80858.0, "irq": 22817.0, "softirq": 8441.0, "steal": 0.0, "guest": 99716.0, "guest_nice": 0.0, "total": 21431351.0}}, {"core": 21, "modes": {"user": 239401.0, "nice": 1117.0, "system": 137635.0, "idle": 20920342.0, "iowait": 76403.0, "irq": 23011.0, "softirq": 9512.0, "steal": 0.0, "guest": 166560.0, "guest_nice": 0.0, "total": 21407421.0}}, {"core": 22, "modes": {"user": 122075.0, "nice": 120.0, "system": 106657.0, "idle": 21076046.0, "iowait": 69315.0, "irq": 20778.0, "softirq": 7912.0, "steal": 0.0, "guest": 105407.0, "guest_nice": 0.0, "total": 21402903.0}}, {"core": 23, "modes": {"user": 80291.0, "nice": 48.0, "system": 83872.0, "idle": 21151008.0, "iowait": 51817.0, "irq": 18415.0, "softirq": 10818.0, "steal": 0.0, "guest": 70762.0, "guest_nice": 0.0, "total": 21396269.0}}, {"core": 24, "modes": {"user": 175297.0, "nice": 1680.0, "system": 105298.0, "idle": 21066756.0, "iowait": 43502.0, "irq": 18506.0, "softirq": 5682.0, "steal": 0.0, "guest": 108632.0, "guest_nice": 0.0, "total": 21416721.0}}, {"core": 25, "modes": {"user": 84571.0, "nice": 66.0, "system": 81762.0, "idle": 21148518.0, "iowait": 39426.0, "irq": 14707.0, "softirq": 4397.0, "steal": 0.0, "guest": 75293.0, "guest_nice": 0.0, "total": 21373447.0}}, {"core": 26, "modes": {"user": 83854.0, "nice": 55.0, "system": 81041.0, "idle": 21145212.0, "iowait": 35238.0, "irq": 14358.0, "softirq": 5366.0, "steal": 0.0, "guest": 74874.0, "guest_nice": 0.0, "total": 21365124.0}}, {"core": 27, "modes": {"user": 88913.0, "nice": 708.0, "system": 62480.0, "idle": 21129089.0, "iowait": 30724.0, "irq": 13502.0, "softirq": 10097.0, "steal": 0.0, "guest": 47660.0, "guest_nice": 0.0, "total": 21335513.0}}, {"core": 28, "modes": {"user": 92443.0, "nice": 564.0, "system": 65642.0, "idle": 21141200.0, "iowait": 29522.0, "irq": 12084.0, "softirq": 6197.0, "steal": 0.0, "guest": 52985.0, "guest_nice": 0.0, "total": 21347652.0}}, {"core": 29, "modes": {"user": 93250.0, "nice": 720.0, "system": 68170.0, "idle": 21127517.0, "iowait": 31170.0, "irq": 13215.0, "softirq": 8944.0, "steal": 0.0, "guest": 56494.0, "guest_nice": 0.0, "total": 21342986.0}}, {"core": 30, "modes": {"user": 1429759.0, "nice": 17452.0, "system": 456266.0, "idle": 19306159.0, "iowait": 175800.0, "irq": 45954.0, "softirq": 15498.0, "steal": 0.0, "guest": 706190.0, "guest_nice": 0.0, "total": 21446888.0}}, {"core": 31, "modes": {"user": 577489.0, "nice": 2332.0, "system": 316585.0, "idle": 20305261.0, "iowait": 190837.0, "irq": 40930.0, "softirq": 15210.0, "steal": 0.0, "guest": 424370.0, "guest_nice": 0.0, "total": 21448644.0}}, {"core": 32, "modes": {"user": 326707.0, "nice": 575.0, "system": 247484.0, "idle": 20650571.0, "iowait": 173216.0, "irq": 37279.0, "softirq": 13830.0, "steal": 0.0, "guest": 269446.0, "guest_nice": 0.0, "total": 21449662.0}}, {"core": 33, "modes": {"user": 494472.0, "nice": 4717.0, "system": 263727.0, "idle": 20513821.0, "iowait": 137733.0, "irq": 35240.0, "softirq": 12566.0, "steal": 0.0, "guest": 326715.0, "guest_nice": 0.0, "total": 21462276.0}}, {"core": 34, "modes": {"user": 281997.0, "nice": 520.0, "system": 210975.0, "idle": 20806443.0, "iowait": 136259.0, "irq": 33939.0, "softirq": 12460.0, "steal": 0.0, "guest": 232930.0, "guest_nice": 0.0, "total": 21482593.0}}, {"core": 35, "modes": {"user": 190712.0, "nice": 200.0, "system": 171306.0, "idle": 20931402.0, "iowait": 118705.0, "irq": 30121.0, "softirq": 10810.0, "steal": 0.0, "guest": 165142.0, "guest_nice": 0.0, "total": 21453256.0}}, {"core": 36, "modes": {"user": 7749733.0, "nice": 33028.0, "system": 506830.0, "idle": 12774374.0, "iowait": 96596.0, "irq": 110531.0, "softirq": 11002.0, "steal": 0.0, "guest": 1197135.0, "guest_nice": 0.0, "total": 21282094.0}}, {"core": 37, "modes": {"user": 7795253.0, "nice": 33772.0, "system": 513719.0, "idle": 12724176.0, "iowait": 76888.0, "irq": 109045.0, "softirq": 9750.0, "steal": 0.0, "guest": 1118640.0, "guest_nice": 0.0, "total": 21262603.0}}, {"core": 38, "modes": {"user": 1040677.0, "nice": 5689.0, "system": 311784.0, "idle": 19879702.0, "iowait": 150681.0, "irq": 44564.0, "softirq": 14314.0, "steal": 0.0, "guest": 683286.0, "guest_nice": 0.0, "total": 21447411.0}}, {"core": 39, "modes": {"user": 83925.0, "nice": 497.0, "system": 63856.0, "idle": 21141563.0, "iowait": 28979.0, "irq": 12348.0, "softirq": 7705.0, "steal": 0.0, "guest": 49813.0, "guest_nice": 0.0, "total": 21338873.0}}, {"core": 40, "modes": {"user": 93389.0, "nice": 529.0, "system": 66671.0, "idle": 21138070.0, "iowait": 30274.0, "irq": 13137.0, "softirq": 8323.0, "steal": 0.0, "guest": 54564.0, "guest_nice": 0.0, "total": 21350393.0}}, {"core": 41, "modes": {"user": 102268.0, "nice": 658.0, "system": 69478.0, "idle": 21116674.0, "iowait": 30981.0, "irq": 14626.0, "softirq": 11351.0, "steal": 0.0, "guest": 57998.0, "guest_nice": 0.0, "total": 21346036.0}}, {"core": 42, "modes": {"user": 268157.0, "nice": 1394.0, "system": 168903.0, "idle": 20877276.0, "iowait": 91848.0, "irq": 28342.0, "softirq": 10185.0, "steal": 0.0, "guest": 188686.0, "guest_nice": 0.0, "total": 21446105.0}}, {"core": 43, "modes": {"user": 171471.0, "nice": 209.0, "system": 143761.0, "idle": 21010525.0, "iowait": 89707.0, "irq": 27303.0, "softirq": 9793.0, "steal": 0.0, "guest": 147673.0, "guest_nice": 0.0, "total": 21452769.0}}, {"core": 44, "modes": {"user": 122658.0, "nice": 99.0, "system": 117906.0, "idle": 21071884.0, "iowait": 78187.0, "irq": 23634.0, "softirq": 8202.0, "steal": 0.0, "guest": 108219.0, "guest_nice": 0.0, "total": 21422570.0}}, {"core": 45, "modes": {"user": 187041.0, "nice": 1814.0, "system": 121189.0, "idle": 21025406.0, "iowait": 60137.0, "irq": 22892.0, "softirq": 8879.0, "steal": 0.0, "guest": 126340.0, "guest_nice": 0.0, "total": 21427358.0}}, {"core": 46, "modes": {"user": 119767.0, "nice": 85.0, "system": 106368.0, "idle": 21101963.0, "iowait": 57268.0, "irq": 21445.0, "softirq": 7657.0, "steal": 0.0, "guest": 104908.0, "guest_nice": 0.0, "total": 21414553.0}}, {"core": 47, "modes": {"user": 90609.0, "nice": 44.0, "system": 89796.0, "idle": 21128988.0, "iowait": 50432.0, "irq": 18856.0, "softirq": 7672.0, "steal": 0.0, "guest": 80629.0, "guest_nice": 0.0, "total": 21386397.0}}], "clock_ticks_per_second": 100, "load1": 1.83, "load5": 2.27, "load15": 2.53, "procs_running": 3.0, "procs_blocked": 0.0, "ctxt_total": 7541233706.0, "processes_total": 2786499.0, "freq_mhz": 3768.7093958333335, "freq_max_mhz": 4572.218}, "memory": {"total_bytes": 134888374272.0, "free_bytes": 2936516608.0, "available_bytes": 42943569920.0, "used_bytes": 91944804352.0, "buffers_bytes": 1019904.0, "cached_bytes": 41681354752.0, "dirty_bytes": 8601600.0, "swap_total_bytes": 0.0, "swap_used_bytes": 0.0, "committed_bytes": 148763418624.0}, "filesystems": [{"device": "/dev/mapper/luks-00000000-0000-4000-8000-000000000001", "mountpoint": "/", "fstype": "btrfs", "total_bytes": 247893852160, "used_bytes": 25345941504, "free_bytes": 217773121536, "used_pct": 10.425320494028833, "read_only": false}, {"device": "/dev/nvme0n1p2", "mountpoint": "/boot", "fstype": "ext4", "total_bytes": 1020702720, "used_bytes": 466849792, "free_bytes": 483389440, "used_pct": 49.129711369357565, "read_only": false}, {"device": "/dev/nvme0n1p1", "mountpoint": "/boot/efi", "fstype": "vfat", "total_bytes": 1071628288, "used_bytes": 7864320, "free_bytes": 1063763968, "used_pct": 0.7338664057363891, "read_only": false}, {"device": "/dev/mapper/luks-backup0-00000000-0000-4000-8000-000000000002", "mountpoint": "/btrfs/backup0", "fstype": "btrfs", "total_bytes": 18000189063168, "used_bytes": 11459991527424, "free_bytes": 6520108589056, "used_pct": 63.73708407174067, "read_only": false}, {"device": "/dev/mapper/luks-00000000-0000-4000-8000-000000000001", "mountpoint": "/btrfs/boot", "fstype": "btrfs", "total_bytes": 247893852160, "used_bytes": 25345941504, "free_bytes": 217773121536, "used_pct": 10.425320494028833, "read_only": false}, {"device": "/dev/dm-1", "mountpoint": "/btrfs/pool0", "fstype": "btrfs", "total_bytes": 19003100626944, "used_bytes": 12912565084160, "free_bytes": 5277774553088, "used_pct": 70.98583831672497, "read_only": false}, {"device": "/dev/dm-4", "mountpoint": "/btrfs/pool1", "fstype": "btrfs", "total_bytes": 4000749453312, "used_bytes": 2880776658944, "free_bytes": 980471013376, "used_pct": 74.6074042231305, "read_only": false}, {"device": "/dev/mapper/luks-00000000-0000-4000-8000-000000000001", "mountpoint": "/home", "fstype": "btrfs", "total_bytes": 247893852160, "used_bytes": 25345941504, "free_bytes": 217773121536, "used_pct": 10.425320494028833, "read_only": false}, {"device": "/dev/dm-4", "mountpoint": "/home/ai", "fstype": "btrfs", "total_bytes": 4000749453312, "used_bytes": 2880776658944, "free_bytes": 980471013376, "used_pct": 74.6074042231305, "read_only": false}, {"device": "/dev/dm-1", "mountpoint": "/srv/archive", "fstype": "btrfs", "total_bytes": 19003100626944, "used_bytes": 12912565084160, "free_bytes": 5277774553088, "used_pct": 70.98583831672497, "read_only": false}, {"device": "/dev/mapper/luks-backup0-00000000-0000-4000-8000-000000000002", "mountpoint": "/srv/backup", "fstype": "btrfs", "total_bytes": 18000189063168, "used_bytes": 11459991527424, "free_bytes": 6520108589056, "used_pct": 63.73708407174067, "read_only": false}, {"device": "/dev/mapper/luks-backup0-00000000-0000-4000-8000-000000000002", "mountpoint": "/srv/borg", "fstype": "btrfs", "total_bytes": 18000189063168, "used_bytes": 11459991527424, "free_bytes": 6520108589056, "used_pct": 63.73708407174067, "read_only": false}, {"device": "/dev/dm-1", "mountpoint": "/srv/smb/pool0", "fstype": "btrfs", "total_bytes": 19003100626944, "used_bytes": 12912565084160, "free_bytes": 5277774553088, "used_pct": 70.98583831672497, "read_only": false}, {"device": "/dev/dm-4", "mountpoint": "/srv/smb/pool1", "fstype": "btrfs", "total_bytes": 4000749453312, "used_bytes": 2880776658944, "free_bytes": 980471013376, "used_pct": 74.6074042231305, "read_only": false}, {"device": "/dev/dm-1", "mountpoint": "/srv/vm/pool0", "fstype": "btrfs", "total_bytes": 19003100626944, "used_bytes": 12912565084160, "free_bytes": 5277774553088, "used_pct": 70.98583831672497, "read_only": false}, {"device": "/dev/dm-4", "mountpoint": "/srv/vm/pool1", "fstype": "btrfs", "total_bytes": 4000749453312, "used_bytes": 2880776658944, "free_bytes": 980471013376, "used_pct": 74.6074042231305, "read_only": false}, {"device": "/dev/mapper/luks-00000000-0000-4000-8000-000000000001", "mountpoint": "/var", "fstype": "btrfs", "total_bytes": 247893852160, "used_bytes": 25345941504, "free_bytes": 217773121536, "used_pct": 10.425320494028833, "read_only": false}], "disk_io": {"devices": [{"name": "nvme0n1", "model": "Samsung SSD 970 EVO Plus 250GB", "rotational": false, "size_bytes": 250059350016.0, "reads_total": 1472296.0, "writes_total": 2133613.0, "read_bytes_total": 62344609280.0, "write_bytes_total": 47524078592.0, "in_flight": 0.0, "io_time_ms_total": 337240.0, "weighted_io_ms_total": 3505695.0, "discard_bytes_total": 32976027648.0}, {"name": "nvme1n1", "model": "Samsung SSD 980 PRO 2TB", "rotational": false, "size_bytes": 2000398934016.0, "reads_total": 4174884.0, "writes_total": 102358280.0, "read_bytes_total": 267655655936.0, "write_bytes_total": 1535029579776.0, "in_flight": 0.0, "io_time_ms_total": 31605390.0, "weighted_io_ms_total": 91173591.0, "discard_bytes_total": 193970274304.0}, {"name": "nvme2n1", "model": "WDS100T1X0E-00AFY0", "rotational": false, "size_bytes": 1000204886016.0, "reads_total": 1620198.0, "writes_total": 20480772.0, "read_bytes_total": 130020225536.0, "write_bytes_total": 263331745792.0, "in_flight": 0.0, "io_time_ms_total": 1319939.0, "weighted_io_ms_total": 3988469.0, "discard_bytes_total": 64095707136.0}, {"name": "nvme3n1", "model": "Samsung SSD 970 EVO 1TB", "rotational": false, "size_bytes": 1000204886016.0, "reads_total": 1530350.0, "writes_total": 19907249.0, "read_bytes_total": 124843725312.0, "write_bytes_total": 257520640000.0, "in_flight": 0.0, "io_time_ms_total": 15065347.0, "weighted_io_ms_total": 36382747.0, "discard_bytes_total": 64154402816.0}, {"name": "nvme4n1", "model": "WDBRPG0010BNC-WRSN", "rotational": false, "size_bytes": 1000204886016.0, "reads_total": 2109282.0, "writes_total": 56417206.0, "read_bytes_total": 138510131712.0, "write_bytes_total": 819240173568.0, "in_flight": 0.0, "io_time_ms_total": 2273421.0, "weighted_io_ms_total": 7067556.0, "discard_bytes_total": 64178151424.0}, {"name": "nvme5n1", "model": "WDBRPG0010BNC-WRSN", "rotational": false, "size_bytes": 1000204886016.0, "reads_total": 2066780.0, "writes_total": 50339464.0, "read_bytes_total": 143692370432.0, "write_bytes_total": 730960179200.0, "in_flight": 0.0, "io_time_ms_total": 2280481.0, "weighted_io_ms_total": 8123239.0, "discard_bytes_total": 68713123840.0}, {"name": "nvme6n1", "model": "WDBRPG0010BNC-WRSN", "rotational": false, "size_bytes": 1000204886016.0, "reads_total": 2138436.0, "writes_total": 43107415.0, "read_bytes_total": 140699124224.0, "write_bytes_total": 593777307648.0, "in_flight": 0.0, "io_time_ms_total": 2122096.0, "weighted_io_ms_total": 6810641.0, "discard_bytes_total": 89339125760.0}, {"name": "nvme7n1", "model": "Samsung SSD 980 PRO 1TB", "rotational": false, "size_bytes": 1000204886016.0, "reads_total": 1194570.0, "writes_total": 33513454.0, "read_bytes_total": 111336550912.0, "write_bytes_total": 506137415680.0, "in_flight": 0.0, "io_time_ms_total": 17784769.0, "weighted_io_ms_total": 36251661.0, "discard_bytes_total": 57250082816.0}, {"name": "sda", "model": "ST8000VN004-2M21", "rotational": true, "size_bytes": 8001563222016.0, "reads_total": 108770.0, "writes_total": 65661.0, "read_bytes_total": 1797992960.0, "write_bytes_total": 3539456000.0, "in_flight": 0.0, "io_time_ms_total": 291278.0, "weighted_io_ms_total": 1198041.0, "discard_bytes_total": 0.0}, {"name": "sdb", "model": "ST2000VN004-2E41", "rotational": true, "size_bytes": 2000398934016.0, "reads_total": 454.0, "writes_total": 3327.0, "read_bytes_total": 5419520.0, "write_bytes_total": 9117696.0, "in_flight": 0.0, "io_time_ms_total": 50206.0, "weighted_io_ms_total": 99975.0, "discard_bytes_total": 0.0}, {"name": "sdc", "model": "ST6000VN001-2BB1", "rotational": true, "size_bytes": 6001175126016.0, "reads_total": 45289.0, "writes_total": 60947.0, "read_bytes_total": 754860544.0, "write_bytes_total": 3112624128.0, "in_flight": 0.0, "io_time_ms_total": 302255.0, "weighted_io_ms_total": 1780126.0, "discard_bytes_total": 0.0}, {"name": "sdd", "model": "ST2000VN004-2E41", "rotational": true, "size_bytes": 2000398934016.0, "reads_total": 953.0, "writes_total": 3342.0, "read_bytes_total": 11682304.0, "write_bytes_total": 9383936.0, "in_flight": 0.0, "io_time_ms_total": 51134.0, "weighted_io_ms_total": 99811.0, "discard_bytes_total": 0.0}, {"name": "sde", "model": "WDC WD180EDGZ-11", "rotational": true, "size_bytes": 18000207937536.0, "reads_total": 78058.0, "writes_total": 21314.0, "read_bytes_total": 2917753344.0, "write_bytes_total": 2479779840.0, "in_flight": 0.0, "io_time_ms_total": 539973.0, "weighted_io_ms_total": 1291278.0, "discard_bytes_total": 0.0}, {"name": "sdf", "model": "ST6000VN001-2BB1", "rotational": true, "size_bytes": 6001175126016.0, "reads_total": 5949.0, "writes_total": 27315.0, "read_bytes_total": 95121920.0, "write_bytes_total": 561295360.0, "in_flight": 0.0, "io_time_ms_total": 100712.0, "weighted_io_ms_total": 1109422.0, "discard_bytes_total": 0.0}, {"name": "sdg", "model": "WDC WD40EZRZ-00G", "rotational": true, "size_bytes": 4000787030016.0, "reads_total": 97563.0, "writes_total": 21941.0, "read_bytes_total": 1588789760.0, "write_bytes_total": 397709312.0, "in_flight": 0.0, "io_time_ms_total": 479846.0, "weighted_io_ms_total": 1454640.0, "discard_bytes_total": 0.0}, {"name": "sdh", "model": "ST10000DM0004-1Z", "rotational": true, "size_bytes": 10000831348736.0, "reads_total": 43047.0, "writes_total": 74186.0, "read_bytes_total": 799883776.0, "write_bytes_total": 5033156608.0, "in_flight": 0.0, "io_time_ms_total": 3288372.0, "weighted_io_ms_total": 1682399.0, "discard_bytes_total": 0.0}], "device_names": ["nvme0n1", "nvme1n1", "nvme2n1", "nvme3n1", "nvme4n1", "nvme5n1", "nvme6n1", "nvme7n1", "sda", "sdb", "sdc", "sdd", "sde", "sdf", "sdg", "sdh"]}, "network": {"interfaces": [{"name": "enp3s0", "rx_bytes_total": 27160036011.0, "rx_packets_total": 34075414.0, "rx_errors_total": 0.0, "rx_dropped_total": 70636.0, "tx_bytes_total": 9179329815.0, "tx_packets_total": 19634372.0, "tx_errors_total": 0.0, "tx_dropped_total": 0.0, "speed_mbps": 10000.0}, {"name": "enp69s0", "rx_bytes_total": 0.0, "rx_packets_total": 0.0, "rx_errors_total": 0.0, "rx_dropped_total": 0.0, "tx_bytes_total": 0.0, "tx_packets_total": 0.0, "tx_errors_total": 0.0, "tx_dropped_total": 0.0, "speed_mbps": null}, {"name": "enp70s0", "rx_bytes_total": 0.0, "rx_packets_total": 0.0, "rx_errors_total": 0.0, "rx_dropped_total": 0.0, "tx_bytes_total": 0.0, "tx_packets_total": 0.0, "tx_errors_total": 0.0, "tx_dropped_total": 0.0, "speed_mbps": null}, {"name": "wlo2", "rx_bytes_total": 0.0, "rx_packets_total": 0.0, "rx_errors_total": 0.0, "rx_dropped_total": 0.0, "tx_bytes_total": 0.0, "tx_packets_total": 0.0, "tx_errors_total": 0.0, "tx_dropped_total": 0.0, "speed_mbps": null}]}, "gpus": [{"index": 0, "card": "card1", "vendor": "AMD", "model": "Radeon AI PRO R9700", "model_full": "Advanced Micro Devices, Inc. [AMD/ATI] Navi 48 [Radeon AI PRO R9700]", "pci_slot": "0000:4d:00.0", "pci_address": "1002:7551", "driver": "amdgpu", "busy_pct": 3.0, "vram_total_bytes": 34208743424.0, "vram_used_bytes": 32755552256.0, "gtt_total_bytes": 67444187136.0, "gtt_used_bytes": 568807424.0, "power_watts": 24.0, "temps": {"edge": 42.0, "junction": 43.0, "mem": 42.0}, "clocks": {"sclk": 25.0, "mclk": 96.0}}], "temperatures": [{"kind": "hwmon", "chip": "acpitz", "label": "acpitz", "celsius": 16.8}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 49.85}, {"kind": "hwmon", "chip": "k10temp", "label": "Tctl", "celsius": 67.375}, {"kind": "hwmon", "chip": "k10temp", "label": "Tccd1", "celsius": 43.75}, {"kind": "hwmon", "chip": "k10temp", "label": "Tccd3", "celsius": 57.5}, {"kind": "hwmon", "chip": "k10temp", "label": "Tccd5", "celsius": 68.0}, {"kind": "hwmon", "chip": "k10temp", "label": "Tccd7", "celsius": 48.25}, {"kind": "hwmon", "chip": "iwlwifi_1", "label": "iwlwifi_1", "celsius": 45.0}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 50.85}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 49.85}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 51.85}, {"kind": "hwmon", "chip": "nvme", "label": "Sensor 1", "celsius": 51.85}, {"kind": "hwmon", "chip": "nvme", "label": "Sensor 2", "celsius": 53.85}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 48.85}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 57.85}, {"kind": "hwmon", "chip": "nvme", "label": "Sensor 1", "celsius": 57.85}, {"kind": "hwmon", "chip": "nvme", "label": "Sensor 2", "celsius": 74.85}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 55.85}, {"kind": "hwmon", "chip": "nvme", "label": "Sensor 1", "celsius": 55.85}, {"kind": "hwmon", "chip": "nvme", "label": "Sensor 2", "celsius": 70.85}, {"kind": "hwmon", "chip": "nvme", "label": "Composite", "celsius": 48.85}, {"kind": "hwmon", "chip": "nvme", "label": "Sensor 1", "celsius": 48.85}, {"kind": "hwmon", "chip": "nvme", "label": "Sensor 2", "celsius": 53.85}, {"kind": "hwmon", "chip": "amdgpu", "label": "edge", "celsius": 42.0}, {"kind": "hwmon", "chip": "amdgpu", "label": "junction", "celsius": 43.0}, {"kind": "hwmon", "chip": "amdgpu", "label": "mem", "celsius": 42.0}, {"kind": "thermal", "chip": "acpitz", "label": "acpitz", "celsius": 16.8}, {"kind": "thermal", "chip": "iwlwifi_1", "label": "iwlwifi_1", "celsius": 45.0}], "cpu_temperature_celsius": 68.0, "processes": {"total": 991, "top_rss": [{"pid": 7252, "name": "qemu-system-x86", "rss_bytes": 8633606144.0}, {"pid": 8095, "name": "qemu-system-x86", "rss_bytes": 8625516544.0}, {"pid": 1567838, "name": "qemu-system-x86", "rss_bytes": 4339527680.0}, {"pid": 7677, "name": "qemu-system-x86", "rss_bytes": 4332146688.0}, {"pid": 7398, "name": "qemu-system-x86", "rss_bytes": 4332130304.0}], "top_cpu": [{"pid": 2781693, "name": "python3", "cpu_pct": 10058.282973621104}, {"pid": 7252, "name": "qemu-system-x86", "cpu_pct": 11.198514982692393}, {"pid": 7677, "name": "qemu-system-x86", "cpu_pct": 5.399284009512403}, {"pid": 7482, "name": "qemu-system-x86", "cpu_pct": 2.1997083001717197}, {"pid": 8095, "name": "qemu-system-x86", "cpu_pct": 2.099721559254823}]}, "errors": []} \ No newline at end of file diff --git a/tests/fixtures/agent_gateway.json b/tests/fixtures/agent_gateway.json new file mode 100644 index 0000000..abada65 --- /dev/null +++ b/tests/fixtures/agent_gateway.json @@ -0,0 +1 @@ +{"ok": true, "schema": 1, "agent_version": "1.0.0", "ts": 1792178443.0656452, "duration_ms": 10.22, "host": {"hostname": "gateway", "os_pretty": "AlmaLinux 10.2 (Lavender Lion)", "os_id": "almalinux", "kernel": "6.12.0-211.65.1.el10_2.x86_64_v2", "arch": "x86_64", "python": "3.12.15", "cpu_model": "AMD Ryzen Threadripper 3960X 24-Core Processor", "cpu_cores_logical": 2, "cpu_cores_physical": 2, "memory_total_bytes": 2051493888.0, "boot_ts": 1791967457.1014743, "uptime_seconds": 210985.95, "container": false, "agent_version": "1.0.0"}, "cpu": {"modes": {"user": 561498.0, "nice": 2756.0, "system": 122750.0, "idle": 41118362.0, "iowait": 290460.0, "irq": 24373.0, "softirq": 46665.0, "steal": 4918.0, "guest": 0.0, "guest_nice": 0.0, "total": 42171782.0}, "cores": [{"core": 0, "modes": {"user": 285993.0, "nice": 1771.0, "system": 60082.0, "idle": 20538975.0, "iowait": 143245.0, "irq": 14642.0, "softirq": 38622.0, "steal": 3640.0, "guest": 0.0, "guest_nice": 0.0, "total": 21086970.0}}, {"core": 1, "modes": {"user": 275504.0, "nice": 985.0, "system": 62667.0, "idle": 20579386.0, "iowait": 147214.0, "irq": 9730.0, "softirq": 8043.0, "steal": 1277.0, "guest": 0.0, "guest_nice": 0.0, "total": 21084806.0}}], "clock_ticks_per_second": 100, "load1": 0.01, "load5": 0.07, "load15": 0.08, "procs_running": 1.0, "procs_blocked": 2.0, "ctxt_total": 64391428.0, "processes_total": 343402.0, "freq_mhz": null, "freq_max_mhz": null}, "memory": {"total_bytes": 2051493888.0, "free_bytes": 172544000.0, "available_bytes": 1400086528.0, "used_bytes": 651407360.0, "buffers_bytes": 106156032.0, "cached_bytes": 1315049472.0, "dirty_bytes": 2158592.0, "swap_total_bytes": 0.0, "swap_used_bytes": 0.0, "committed_bytes": 720392192.0}, "filesystems": [{"device": "/dev/vda4", "mountpoint": "/", "fstype": "ext4", "total_bytes": 9197531136, "used_bytes": 5252460544, "free_bytes": 3455754240, "used_pct": 60.316157493618384, "read_only": false}, {"device": "/dev/vda3", "mountpoint": "/boot", "fstype": "ext4", "total_bytes": 1020702720, "used_bytes": 272543744, "free_bytes": 677695488, "used_pct": 28.68159246870582, "read_only": false}, {"device": "/dev/vda2", "mountpoint": "/boot/efi", "fstype": "vfat", "total_bytes": 209489920, "used_bytes": 9560064, "free_bytes": 199929856, "used_pct": 4.56349594290742, "read_only": false}], "disk_io": {"devices": [{"name": "vda", "model": null, "rotational": true, "size_bytes": 10737418240.0, "reads_total": 14464.0, "writes_total": 1989589.0, "read_bytes_total": 645659136.0, "write_bytes_total": 22551089664.0, "in_flight": 0.0, "io_time_ms_total": 2882080.0, "weighted_io_ms_total": 6688324.0, "discard_bytes_total": 0.0}], "device_names": ["vda"]}, "network": {"interfaces": [{"name": "eth0", "rx_bytes_total": 14580699012.0, "rx_packets_total": 22626423.0, "rx_errors_total": 0.0, "rx_dropped_total": 287.0, "tx_bytes_total": 10807075678.0, "tx_packets_total": 10084825.0, "tx_errors_total": 0.0, "tx_dropped_total": 0.0, "speed_mbps": null}]}, "gpus": [], "temperatures": [], "cpu_temperature_celsius": null, "processes": {"total": 142, "top_rss": [{"pid": 592, "name": "systemd-journal", "rss_bytes": 105857024.0}, {"pid": 1182, "name": "haproxy", "rss_bytes": 99045376.0}, {"pid": 723, "name": "firewalld", "rss_bytes": 45318144.0}, {"pid": 1091, "name": "polkitd", "rss_bytes": 41594880.0}, {"pid": 1, "name": "systemd", "rss_bytes": 37564416.0}], "top_cpu": [{"pid": 342784, "name": "python3", "cpu_pct": 2.680722502695548}, {"pid": 1, "name": "systemd", "cpu_pct": 0.0}, {"pid": 2, "name": "kthreadd", "cpu_pct": 0.0}, {"pid": 3, "name": "pool_workqueue_release", "cpu_pct": 0.0}, {"pid": 4, "name": "kworker/R-rcu_gp", "cpu_pct": 0.0}]}, "errors": []} \ No newline at end of file diff --git a/tests/fixtures/metrics_gufo.txt b/tests/fixtures/metrics_gufo.txt new file mode 100644 index 0000000..cf3223b --- /dev/null +++ b/tests/fixtures/metrics_gufo.txt @@ -0,0 +1,45 @@ +# HELP gufo_device_lost_total Confirmed GPU context losses +# TYPE gufo_device_lost_total counter +gufo_device_lost_total 0 +# HELP llamacpp:prompt_tokens_total Total prompt tokens processed, excluding cache hits +# TYPE llamacpp:prompt_tokens_total counter +llamacpp:prompt_tokens_total 1350666 +# HELP llamacpp:prompt_tokens_cached_total Total prompt tokens reused from cache +# TYPE llamacpp:prompt_tokens_cached_total counter +llamacpp:prompt_tokens_cached_total 124529105 +# HELP llamacpp:prompt_seconds_total Prompt process time +# TYPE llamacpp:prompt_seconds_total counter +llamacpp:prompt_seconds_total 1478.1443332270005 +# HELP llamacpp:tokens_predicted_total Total tokens generated +# TYPE llamacpp:tokens_predicted_total counter +llamacpp:tokens_predicted_total 877336 +# HELP llamacpp:tokens_predicted_seconds_total Predict process time +# TYPE llamacpp:tokens_predicted_seconds_total counter +llamacpp:tokens_predicted_seconds_total 26565.247833160014 +# HELP llamacpp:n_tokens_max Largest observed n_tokens. +# TYPE llamacpp:n_tokens_max counter +llamacpp:n_tokens_max 252622 +# HELP llamacpp:spec_decode_num_drafts_total Total speculative verification rounds +# TYPE llamacpp:spec_decode_num_drafts_total counter +llamacpp:spec_decode_num_drafts_total 291738 +# HELP llamacpp:spec_decode_num_draft_tokens_total Total draft tokens proposed +# TYPE llamacpp:spec_decode_num_draft_tokens_total counter +llamacpp:spec_decode_num_draft_tokens_total 789028 +# HELP llamacpp:spec_decode_num_accepted_tokens_total Total draft tokens accepted +# TYPE llamacpp:spec_decode_num_accepted_tokens_total counter +llamacpp:spec_decode_num_accepted_tokens_total 578514 +# HELP llamacpp:prompt_tokens_seconds Prompt processing speed in tokens per second +# TYPE llamacpp:prompt_tokens_seconds gauge +llamacpp:prompt_tokens_seconds 160.47362399350274 +# HELP llamacpp:predicted_tokens_seconds Generation speed in tokens per second +# TYPE llamacpp:predicted_tokens_seconds gauge +llamacpp:predicted_tokens_seconds 42.621031668072597 +# HELP llamacpp:requests_processing Number of admitted requests, including cache preparation +# TYPE llamacpp:requests_processing gauge +llamacpp:requests_processing 0 +# HELP llamacpp:requests_deferred Number of requests waiting for a session +# TYPE llamacpp:requests_deferred gauge +llamacpp:requests_deferred 0 +# HELP llamacpp:kv_cache_usage_ratio In-flight prompt and generated tokens over sessions times context; excludes retained cache +# TYPE llamacpp:kv_cache_usage_ratio gauge +llamacpp:kv_cache_usage_ratio 0 diff --git a/tests/fixtures/metrics_vllm.txt b/tests/fixtures/metrics_vllm.txt new file mode 100644 index 0000000..f8dea79 --- /dev/null +++ b/tests/fixtures/metrics_vllm.txt @@ -0,0 +1,662 @@ +# HELP python_gc_objects_collected_total Objects collected during gc +# TYPE python_gc_objects_collected_total counter +python_gc_objects_collected_total{generation="0"} 166913.0 +python_gc_objects_collected_total{generation="1"} 27377.0 +python_gc_objects_collected_total{generation="2"} 1409.0 +# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC +# TYPE python_gc_objects_uncollectable_total counter +python_gc_objects_uncollectable_total{generation="0"} 0.0 +python_gc_objects_uncollectable_total{generation="1"} 0.0 +python_gc_objects_uncollectable_total{generation="2"} 0.0 +# HELP python_gc_collections_total Number of times this generation was collected +# TYPE python_gc_collections_total counter +python_gc_collections_total{generation="0"} 4631.0 +python_gc_collections_total{generation="1"} 420.0 +python_gc_collections_total{generation="2"} 12.0 +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="12",patchlevel="3",version="3.12.3"} 1.0 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 3.2837738496e+10 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 2.370977792e+09 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.79210782535e+09 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 457.6 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 43.0 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 524288.0 +# HELP vllm:estimated_flops_per_gpu_total Estimated number of floating point operations per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_flops_per_gpu_total counter +vllm:estimated_flops_per_gpu_total{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:estimated_flops_per_gpu_created Estimated number of floating point operations per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_flops_per_gpu_created gauge +vllm:estimated_flops_per_gpu_created{engine="0",model_name="juggernaut"} 1.7921081022034154e+09 +# HELP vllm:estimated_read_bytes_per_gpu_total Estimated number of bytes read from memory per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_read_bytes_per_gpu_total counter +vllm:estimated_read_bytes_per_gpu_total{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:estimated_read_bytes_per_gpu_created Estimated number of bytes read from memory per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_read_bytes_per_gpu_created gauge +vllm:estimated_read_bytes_per_gpu_created{engine="0",model_name="juggernaut"} 1.792108102203435e+09 +# HELP vllm:estimated_write_bytes_per_gpu_total Estimated number of bytes written to memory per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_write_bytes_per_gpu_total counter +vllm:estimated_write_bytes_per_gpu_total{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:estimated_write_bytes_per_gpu_created Estimated number of bytes written to memory per GPU (for Model Flops Utilization calculations). +# TYPE vllm:estimated_write_bytes_per_gpu_created gauge +vllm:estimated_write_bytes_per_gpu_created{engine="0",model_name="juggernaut"} 1.7921081022034492e+09 +# HELP vllm:num_requests_running Number of requests in model execution batches. +# TYPE vllm:num_requests_running gauge +vllm:num_requests_running{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:num_requests_waiting Number of requests waiting to be processed. +# TYPE vllm:num_requests_waiting gauge +vllm:num_requests_waiting{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:num_requests_waiting_by_reason Number of waiting requests by reason. Reason labels: 'capacity' = waiting for scheduling capacity; 'deferred' = deferred by transient constraints (LoRA budget, KV transfer, blocked status). Sum of all reasons equals vllm:num_requests_waiting. +# TYPE vllm:num_requests_waiting_by_reason gauge +vllm:num_requests_waiting_by_reason{engine="0",model_name="juggernaut",reason="capacity"} 0.0 +vllm:num_requests_waiting_by_reason{engine="0",model_name="juggernaut",reason="deferred"} 0.0 +# HELP vllm:engine_sleep_state Engine sleep state; awake = 0 means engine is sleeping; awake = 1 means engine is awake; weights_offloaded = 1 means sleep level 1; discard_all = 1 means sleep level 2. +# TYPE vllm:engine_sleep_state gauge +vllm:engine_sleep_state{engine="0",model_name="juggernaut",sleep_state="awake"} 1.0 +vllm:engine_sleep_state{engine="0",model_name="juggernaut",sleep_state="weights_offloaded"} 0.0 +vllm:engine_sleep_state{engine="0",model_name="juggernaut",sleep_state="discard_all"} 0.0 +# HELP vllm:kv_cache_usage_perc KV-cache usage. 1 means 100 percent usage. +# TYPE vllm:kv_cache_usage_perc gauge +vllm:kv_cache_usage_perc{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:prefix_cache_queries_total Prefix cache queries, in terms of number of queried tokens. +# TYPE vllm:prefix_cache_queries_total counter +vllm:prefix_cache_queries_total{engine="0",model_name="juggernaut"} 1.8389909e+07 +# HELP vllm:prefix_cache_queries_created Prefix cache queries, in terms of number of queried tokens. +# TYPE vllm:prefix_cache_queries_created gauge +vllm:prefix_cache_queries_created{engine="0",model_name="juggernaut"} 1.7921081022037191e+09 +# HELP vllm:prefix_cache_hits_total Prefix cache hits, in terms of number of cached tokens. +# TYPE vllm:prefix_cache_hits_total counter +vllm:prefix_cache_hits_total{engine="0",model_name="juggernaut"} 1.210496e+07 +# HELP vllm:prefix_cache_hits_created Prefix cache hits, in terms of number of cached tokens. +# TYPE vllm:prefix_cache_hits_created gauge +vllm:prefix_cache_hits_created{engine="0",model_name="juggernaut"} 1.792108102203731e+09 +# HELP vllm:external_prefix_cache_queries_total External prefix cache queries from KV connector cross-instance cache sharing, in terms of number of queried tokens. +# TYPE vllm:external_prefix_cache_queries_total counter +vllm:external_prefix_cache_queries_total{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:external_prefix_cache_queries_created External prefix cache queries from KV connector cross-instance cache sharing, in terms of number of queried tokens. +# TYPE vllm:external_prefix_cache_queries_created gauge +vllm:external_prefix_cache_queries_created{engine="0",model_name="juggernaut"} 1.7921081022037423e+09 +# HELP vllm:external_prefix_cache_hits_total External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens. +# TYPE vllm:external_prefix_cache_hits_total counter +vllm:external_prefix_cache_hits_total{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:external_prefix_cache_hits_created External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens. +# TYPE vllm:external_prefix_cache_hits_created gauge +vllm:external_prefix_cache_hits_created{engine="0",model_name="juggernaut"} 1.792108102203755e+09 +# HELP vllm:mm_cache_queries_total Multi-modal cache queries, in terms of number of queried items. +# TYPE vllm:mm_cache_queries_total counter +vllm:mm_cache_queries_total{engine="0",model_name="juggernaut"} 8.0 +# HELP vllm:mm_cache_queries_created Multi-modal cache queries, in terms of number of queried items. +# TYPE vllm:mm_cache_queries_created gauge +vllm:mm_cache_queries_created{engine="0",model_name="juggernaut"} 1.7921081022037659e+09 +# HELP vllm:mm_cache_hits_total Multi-modal cache hits, in terms of number of cached items. +# TYPE vllm:mm_cache_hits_total counter +vllm:mm_cache_hits_total{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:mm_cache_hits_created Multi-modal cache hits, in terms of number of cached items. +# TYPE vllm:mm_cache_hits_created gauge +vllm:mm_cache_hits_created{engine="0",model_name="juggernaut"} 1.7921081022037778e+09 +# HELP vllm:num_preemptions_total Cumulative number of preemption from the engine. +# TYPE vllm:num_preemptions_total counter +vllm:num_preemptions_total{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:num_preemptions_created Cumulative number of preemption from the engine. +# TYPE vllm:num_preemptions_created gauge +vllm:num_preemptions_created{engine="0",model_name="juggernaut"} 1.7921081022037895e+09 +# HELP vllm:prompt_tokens_total Number of prefill tokens processed. +# TYPE vllm:prompt_tokens_total counter +vllm:prompt_tokens_total{engine="0",model_name="juggernaut"} 1.8389909e+07 +# HELP vllm:prompt_tokens_created Number of prefill tokens processed. +# TYPE vllm:prompt_tokens_created gauge +vllm:prompt_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022038019e+09 +# HELP vllm:prompt_tokens_by_source_total Number of prompt tokens by source. +# TYPE vllm:prompt_tokens_by_source_total counter +vllm:prompt_tokens_by_source_total{engine="0",model_name="juggernaut",source="local_compute"} 6.284949e+06 +vllm:prompt_tokens_by_source_total{engine="0",model_name="juggernaut",source="local_cache_hit"} 1.210496e+07 +vllm:prompt_tokens_by_source_total{engine="0",model_name="juggernaut",source="external_kv_transfer"} 0.0 +# HELP vllm:prompt_tokens_by_source_created Number of prompt tokens by source. +# TYPE vllm:prompt_tokens_by_source_created gauge +vllm:prompt_tokens_by_source_created{engine="0",model_name="juggernaut",source="local_compute"} 1.7921081022038164e+09 +vllm:prompt_tokens_by_source_created{engine="0",model_name="juggernaut",source="local_cache_hit"} 1.7921081022038229e+09 +vllm:prompt_tokens_by_source_created{engine="0",model_name="juggernaut",source="external_kv_transfer"} 1.7921081022038274e+09 +# HELP vllm:prompt_tokens_cached_total Number of cached prompt tokens (local + external). +# TYPE vllm:prompt_tokens_cached_total counter +vllm:prompt_tokens_cached_total{engine="0",model_name="juggernaut"} 1.210496e+07 +# HELP vllm:prompt_tokens_cached_created Number of cached prompt tokens (local + external). +# TYPE vllm:prompt_tokens_cached_created gauge +vllm:prompt_tokens_cached_created{engine="0",model_name="juggernaut"} 1.7921081022038393e+09 +# HELP vllm:generation_tokens_total Number of generation tokens processed. +# TYPE vllm:generation_tokens_total counter +vllm:generation_tokens_total{engine="0",model_name="juggernaut"} 611163.0 +# HELP vllm:generation_tokens_created Number of generation tokens processed. +# TYPE vllm:generation_tokens_created gauge +vllm:generation_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022038517e+09 +# HELP vllm:request_success_total Count of successfully processed requests. +# TYPE vllm:request_success_total counter +vllm:request_success_total{engine="0",finished_reason="stop",model_name="juggernaut"} 3488.0 +vllm:request_success_total{engine="0",finished_reason="length",model_name="juggernaut"} 0.0 +vllm:request_success_total{engine="0",finished_reason="abort",model_name="juggernaut"} 0.0 +vllm:request_success_total{engine="0",finished_reason="error",model_name="juggernaut"} 0.0 +vllm:request_success_total{engine="0",finished_reason="repetition",model_name="juggernaut"} 0.0 +# HELP vllm:request_success_created Count of successfully processed requests. +# TYPE vllm:request_success_created gauge +vllm:request_success_created{engine="0",finished_reason="stop",model_name="juggernaut"} 1.792108102203872e+09 +vllm:request_success_created{engine="0",finished_reason="length",model_name="juggernaut"} 1.7921081022038789e+09 +vllm:request_success_created{engine="0",finished_reason="abort",model_name="juggernaut"} 1.7921081022038846e+09 +vllm:request_success_created{engine="0",finished_reason="error",model_name="juggernaut"} 1.7921081022038896e+09 +vllm:request_success_created{engine="0",finished_reason="repetition",model_name="juggernaut"} 1.7921081022038949e+09 +# HELP vllm:request_prompt_tokens Number of prefill tokens processed. +# TYPE vllm:request_prompt_tokens histogram +vllm:request_prompt_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0 +vllm:request_prompt_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 5.0 +vllm:request_prompt_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 5.0 +vllm:request_prompt_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 30.0 +vllm:request_prompt_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 133.0 +vllm:request_prompt_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 612.0 +vllm:request_prompt_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 1135.0 +vllm:request_prompt_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 2002.0 +vllm:request_prompt_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3126.0 +vllm:request_prompt_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3227.0 +vllm:request_prompt_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3290.0 +vllm:request_prompt_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3383.0 +vllm:request_prompt_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3482.0 +vllm:request_prompt_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_prompt_tokens_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_prompt_tokens_sum{engine="0",model_name="juggernaut"} 1.8352277e+07 +# HELP vllm:request_prompt_tokens_created Number of prefill tokens processed. +# TYPE vllm:request_prompt_tokens_created gauge +vllm:request_prompt_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022039344e+09 +# HELP vllm:request_generation_tokens Number of generation tokens processed. +# TYPE vllm:request_generation_tokens histogram +vllm:request_generation_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0 +vllm:request_generation_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0 +vllm:request_generation_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0 +vllm:request_generation_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0 +vllm:request_generation_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 6.0 +vllm:request_generation_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 45.0 +vllm:request_generation_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 819.0 +vllm:request_generation_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 3206.0 +vllm:request_generation_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 3379.0 +vllm:request_generation_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 3423.0 +vllm:request_generation_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 3464.0 +vllm:request_generation_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3486.0 +vllm:request_generation_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3488.0 +vllm:request_generation_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3488.0 +vllm:request_generation_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3488.0 +vllm:request_generation_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3488.0 +vllm:request_generation_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_generation_tokens_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_generation_tokens_sum{engine="0",model_name="juggernaut"} 610140.0 +# HELP vllm:request_generation_tokens_created Number of generation tokens processed. +# TYPE vllm:request_generation_tokens_created gauge +vllm:request_generation_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022039804e+09 +# HELP vllm:iteration_tokens_total Histogram of number of tokens per engine_step. +# TYPE vllm:iteration_tokens_total histogram +vllm:iteration_tokens_total_bucket{engine="0",le="1.0",model_name="juggernaut"} 317139.0 +vllm:iteration_tokens_total_bucket{engine="0",le="8.0",model_name="juggernaut"} 461626.0 +vllm:iteration_tokens_total_bucket{engine="0",le="16.0",model_name="juggernaut"} 461631.0 +vllm:iteration_tokens_total_bucket{engine="0",le="32.0",model_name="juggernaut"} 461631.0 +vllm:iteration_tokens_total_bucket{engine="0",le="64.0",model_name="juggernaut"} 461632.0 +vllm:iteration_tokens_total_bucket{engine="0",le="128.0",model_name="juggernaut"} 461692.0 +vllm:iteration_tokens_total_bucket{engine="0",le="256.0",model_name="juggernaut"} 461852.0 +vllm:iteration_tokens_total_bucket{engine="0",le="512.0",model_name="juggernaut"} 462266.0 +vllm:iteration_tokens_total_bucket{engine="0",le="1024.0",model_name="juggernaut"} 462849.0 +vllm:iteration_tokens_total_bucket{engine="0",le="2048.0",model_name="juggernaut"} 463840.0 +vllm:iteration_tokens_total_bucket{engine="0",le="4096.0",model_name="juggernaut"} 464991.0 +vllm:iteration_tokens_total_bucket{engine="0",le="8192.0",model_name="juggernaut"} 465088.0 +vllm:iteration_tokens_total_bucket{engine="0",le="16384.0",model_name="juggernaut"} 465097.0 +vllm:iteration_tokens_total_bucket{engine="0",le="+Inf",model_name="juggernaut"} 465099.0 +vllm:iteration_tokens_total_count{engine="0",model_name="juggernaut"} 465099.0 +vllm:iteration_tokens_total_sum{engine="0",model_name="juggernaut"} 6.896112e+06 +# HELP vllm:iteration_tokens_total_created Histogram of number of tokens per engine_step. +# TYPE vllm:iteration_tokens_total_created gauge +vllm:iteration_tokens_total_created{engine="0",model_name="juggernaut"} 1.7921081022040136e+09 +# HELP vllm:request_max_num_generation_tokens Histogram of maximum number of requested generation tokens. +# TYPE vllm:request_max_num_generation_tokens histogram +vllm:request_max_num_generation_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 6.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 45.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 819.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 3206.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 3379.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 3423.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 3464.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3486.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3488.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3488.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3488.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3488.0 +vllm:request_max_num_generation_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_max_num_generation_tokens_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_max_num_generation_tokens_sum{engine="0",model_name="juggernaut"} 610140.0 +# HELP vllm:request_max_num_generation_tokens_created Histogram of maximum number of requested generation tokens. +# TYPE vllm:request_max_num_generation_tokens_created gauge +vllm:request_max_num_generation_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022040458e+09 +# HELP vllm:request_params_n Histogram of the n request parameter. +# TYPE vllm:request_params_n histogram +vllm:request_params_n_bucket{engine="0",le="1.0",model_name="juggernaut"} 3488.0 +vllm:request_params_n_bucket{engine="0",le="2.0",model_name="juggernaut"} 3488.0 +vllm:request_params_n_bucket{engine="0",le="5.0",model_name="juggernaut"} 3488.0 +vllm:request_params_n_bucket{engine="0",le="10.0",model_name="juggernaut"} 3488.0 +vllm:request_params_n_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0 +vllm:request_params_n_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_params_n_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_params_n_sum{engine="0",model_name="juggernaut"} 3488.0 +# HELP vllm:request_params_n_created Histogram of the n request parameter. +# TYPE vllm:request_params_n_created gauge +vllm:request_params_n_created{engine="0",model_name="juggernaut"} 1.7921081022040787e+09 +# HELP vllm:request_params_max_tokens Histogram of the max_tokens request parameter. +# TYPE vllm:request_params_max_tokens histogram +vllm:request_params_max_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 0.0 +vllm:request_params_max_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 1.0 +vllm:request_params_max_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 1.0 +vllm:request_params_max_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3139.0 +vllm:request_params_max_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3140.0 +vllm:request_params_max_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3183.0 +vllm:request_params_max_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3488.0 +vllm:request_params_max_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3488.0 +vllm:request_params_max_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_params_max_tokens_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_params_max_tokens_sum{engine="0",model_name="juggernaut"} 1.7011909e+07 +# HELP vllm:request_params_max_tokens_created Histogram of the max_tokens request parameter. +# TYPE vllm:request_params_max_tokens_created gauge +vllm:request_params_max_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022041035e+09 +# HELP vllm:time_to_first_token_seconds Histogram of time to first token in seconds. +# TYPE vllm:time_to_first_token_seconds histogram +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.001",model_name="juggernaut"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.005",model_name="juggernaut"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.01",model_name="juggernaut"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.02",model_name="juggernaut"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.04",model_name="juggernaut"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.06",model_name="juggernaut"} 0.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.08",model_name="juggernaut"} 4.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.1",model_name="juggernaut"} 4.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.25",model_name="juggernaut"} 592.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 1312.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="0.75",model_name="juggernaut"} 1924.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 2266.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 3433.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 3485.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="7.5",model_name="juggernaut"} 3488.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3489.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3490.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3490.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="80.0",model_name="juggernaut"} 3490.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="160.0",model_name="juggernaut"} 3490.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="640.0",model_name="juggernaut"} 3490.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="2560.0",model_name="juggernaut"} 3490.0 +vllm:time_to_first_token_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3490.0 +vllm:time_to_first_token_seconds_count{engine="0",model_name="juggernaut"} 3490.0 +vllm:time_to_first_token_seconds_sum{engine="0",model_name="juggernaut"} 2904.5298397541046 +# HELP vllm:time_to_first_token_seconds_created Histogram of time to first token in seconds. +# TYPE vllm:time_to_first_token_seconds_created gauge +vllm:time_to_first_token_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022041366e+09 +# HELP vllm:inter_token_latency_seconds Histogram of inter-token latency in seconds. +# TYPE vllm:inter_token_latency_seconds histogram +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.01",model_name="juggernaut"} 0.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.025",model_name="juggernaut"} 0.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.05",model_name="juggernaut"} 596060.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.075",model_name="juggernaut"} 605658.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.1",model_name="juggernaut"} 605758.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.15",model_name="juggernaut"} 606021.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.2",model_name="juggernaut"} 606268.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 606488.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.4",model_name="juggernaut"} 606669.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 606814.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="0.75",model_name="juggernaut"} 607341.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 607398.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="7.5",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="80.0",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_count{engine="0",model_name="juggernaut"} 607673.0 +vllm:inter_token_latency_seconds_sum{engine="0",model_name="juggernaut"} 22579.6616614901 +# HELP vllm:inter_token_latency_seconds_created Histogram of inter-token latency in seconds. +# TYPE vllm:inter_token_latency_seconds_created gauge +vllm:inter_token_latency_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022041779e+09 +# HELP vllm:request_time_per_output_token_seconds Histogram of time_per_output_token_seconds per request. +# TYPE vllm:request_time_per_output_token_seconds histogram +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.01",model_name="juggernaut"} 0.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.025",model_name="juggernaut"} 0.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.05",model_name="juggernaut"} 3383.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.075",model_name="juggernaut"} 3485.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.1",model_name="juggernaut"} 3487.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.15",model_name="juggernaut"} 3487.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.2",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.4",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="0.75",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="7.5",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="80.0",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_time_per_output_token_seconds_sum{engine="0",model_name="juggernaut"} 125.03691733873717 +# HELP vllm:request_time_per_output_token_seconds_created Histogram of time_per_output_token_seconds per request. +# TYPE vllm:request_time_per_output_token_seconds_created gauge +vllm:request_time_per_output_token_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022042146e+09 +# HELP vllm:e2e_request_latency_seconds Histogram of e2e request latency in seconds. +# TYPE vllm:e2e_request_latency_seconds histogram +vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 0.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 2.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 5.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 5.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 14.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 57.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 211.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 1389.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3241.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3339.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3369.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3404.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3418.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3436.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3449.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3480.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3486.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0 +vllm:e2e_request_latency_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:e2e_request_latency_seconds_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:e2e_request_latency_seconds_sum{engine="0",model_name="juggernaut"} 25442.03079867363 +# HELP vllm:e2e_request_latency_seconds_created Histogram of e2e request latency in seconds. +# TYPE vllm:e2e_request_latency_seconds_created gauge +vllm:e2e_request_latency_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022042491e+09 +# HELP vllm:request_queue_time_seconds Histogram of time spent in WAITING phase for request. +# TYPE vllm:request_queue_time_seconds histogram +vllm:request_queue_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 3485.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 3485.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 3485.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 3485.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 3487.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 3487.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 3487.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_queue_time_seconds_sum{engine="0",model_name="juggernaut"} 6.650666017842013 +# HELP vllm:request_queue_time_seconds_created Histogram of time spent in WAITING phase for request. +# TYPE vllm:request_queue_time_seconds_created gauge +vllm:request_queue_time_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022042854e+09 +# HELP vllm:request_inference_time_seconds Histogram of time spent in RUNNING phase for request. +# TYPE vllm:request_inference_time_seconds histogram +vllm:request_inference_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 0.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 2.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 6.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 6.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 14.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 61.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 214.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 1394.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3248.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3339.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3369.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3404.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3418.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3436.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3449.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3480.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3486.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0 +vllm:request_inference_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_inference_time_seconds_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_inference_time_seconds_sum{engine="0",model_name="juggernaut"} 25331.654430066672 +# HELP vllm:request_inference_time_seconds_created Histogram of time spent in RUNNING phase for request. +# TYPE vllm:request_inference_time_seconds_created gauge +vllm:request_inference_time_seconds_created{engine="0",model_name="juggernaut"} 1.79210810220433e+09 +# HELP vllm:request_prefill_time_seconds Histogram of time spent in PREFILL phase for request. +# TYPE vllm:request_prefill_time_seconds histogram +vllm:request_prefill_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 844.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 1341.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 2035.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 2314.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 3230.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 3384.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 3444.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 3484.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3487.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3487.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_time_seconds_sum{engine="0",model_name="juggernaut"} 2786.310553313582 +# HELP vllm:request_prefill_time_seconds_created Histogram of time spent in PREFILL phase for request. +# TYPE vllm:request_prefill_time_seconds_created gauge +vllm:request_prefill_time_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022043684e+09 +# HELP vllm:request_decode_time_seconds Histogram of time spent in DECODE phase for request. +# TYPE vllm:request_decode_time_seconds histogram +vllm:request_decode_time_seconds_bucket{engine="0",le="0.3",model_name="juggernaut"} 0.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="0.5",model_name="juggernaut"} 4.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="0.8",model_name="juggernaut"} 6.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="1.0",model_name="juggernaut"} 7.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="1.5",model_name="juggernaut"} 22.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="2.0",model_name="juggernaut"} 165.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="2.5",model_name="juggernaut"} 452.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="5.0",model_name="juggernaut"} 2092.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="10.0",model_name="juggernaut"} 3278.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="15.0",model_name="juggernaut"} 3354.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="20.0",model_name="juggernaut"} 3376.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="30.0",model_name="juggernaut"} 3406.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="40.0",model_name="juggernaut"} 3422.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="50.0",model_name="juggernaut"} 3438.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="60.0",model_name="juggernaut"} 3450.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="120.0",model_name="juggernaut"} 3481.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="240.0",model_name="juggernaut"} 3486.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="480.0",model_name="juggernaut"} 3488.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="960.0",model_name="juggernaut"} 3488.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="1920.0",model_name="juggernaut"} 3488.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="7680.0",model_name="juggernaut"} 3488.0 +vllm:request_decode_time_seconds_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_decode_time_seconds_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_decode_time_seconds_sum{engine="0",model_name="juggernaut"} 22545.34387675309 +# HELP vllm:request_decode_time_seconds_created Histogram of time spent in DECODE phase for request. +# TYPE vllm:request_decode_time_seconds_created gauge +vllm:request_decode_time_seconds_created{engine="0",model_name="juggernaut"} 1.7921081022044446e+09 +# HELP vllm:request_num_preemptions Histogram of the number of times a request was preempted. +# TYPE vllm:request_num_preemptions histogram +vllm:request_num_preemptions_bucket{engine="0",le="1.0",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_bucket{engine="0",le="2.0",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_bucket{engine="0",le="3.0",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_bucket{engine="0",le="4.0",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_bucket{engine="0",le="5.0",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_bucket{engine="0",le="10.0",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_bucket{engine="0",le="20.0",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_num_preemptions_sum{engine="0",model_name="juggernaut"} 0.0 +# HELP vllm:request_num_preemptions_created Histogram of the number of times a request was preempted. +# TYPE vllm:request_num_preemptions_created gauge +vllm:request_num_preemptions_created{engine="0",model_name="juggernaut"} 1.792108102204479e+09 +# HELP vllm:request_prefill_kv_computed_tokens Histogram of new KV tokens computed during prefill (excluding cached tokens). +# TYPE vllm:request_prefill_kv_computed_tokens histogram +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="1.0",model_name="juggernaut"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="2.0",model_name="juggernaut"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="5.0",model_name="juggernaut"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="10.0",model_name="juggernaut"} 0.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="20.0",model_name="juggernaut"} 5.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="50.0",model_name="juggernaut"} 6.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="100.0",model_name="juggernaut"} 32.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="200.0",model_name="juggernaut"} 136.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="500.0",model_name="juggernaut"} 628.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="1000.0",model_name="juggernaut"} 1206.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="2000.0",model_name="juggernaut"} 2210.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="5000.0",model_name="juggernaut"} 3423.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="10000.0",model_name="juggernaut"} 3485.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="20000.0",model_name="juggernaut"} 3486.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="50000.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="100000.0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_kv_computed_tokens_bucket{engine="0",le="+Inf",model_name="juggernaut"} 3488.0 +vllm:request_prefill_kv_computed_tokens_count{engine="0",model_name="juggernaut"} 3488.0 +vllm:request_prefill_kv_computed_tokens_sum{engine="0",model_name="juggernaut"} 6.267701e+06 +# HELP vllm:request_prefill_kv_computed_tokens_created Histogram of new KV tokens computed during prefill (excluding cached tokens). +# TYPE vllm:request_prefill_kv_computed_tokens_created gauge +vllm:request_prefill_kv_computed_tokens_created{engine="0",model_name="juggernaut"} 1.7921081022045038e+09 +# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig +# TYPE vllm:cache_config_info gauge +vllm:cache_config_info{_block_size_resolved="True",block_size="1568",cache_dtype="fp8",enable_mamba_fine_grained_prefix_cache="False",enable_prefix_caching="True",engine="0",gpu_memory_utilization="0.95",is_attention_free="False",kv_cache_dtype_skip_layers="[]",kv_cache_layout="None",kv_cache_max_concurrency="2.1333333333333333",kv_cache_memory_bytes="None",kv_cache_size_tokens="279620",kv_offloading_backend="native",kv_offloading_size="None",kv_sharing_fast_prefill="False",mamba_block_size="1568",mamba_cache_dtype="auto",mamba_cache_mode="align",mamba_page_size_padded="None",mamba_ssm_cache_dtype="float32",num_cpu_blocks="None",num_gpu_blocks="192",num_gpu_blocks_override="None",prefix_cache_retention_interval="0",prefix_caching_hash_algo="sha256",prefix_match_unit="None",replayssm_buffer_len="16",skip_page_size_padded="None",sliding_window="None",use_kda_recoverssm="False",use_replayssm="False",user_specified_block_size="False",user_specified_mamba_block_size="False"} 1.0 +# HELP http_requests_total Total number of requests by method, status and handler. +# TYPE http_requests_total counter +http_requests_total{handler="/v1/models",method="GET",status="2xx"} 236.0 +http_requests_total{handler="/v1/chat/completions",method="POST",status="2xx"} 3490.0 +http_requests_total{handler="/v1/chat/completions",method="POST",status="4xx"} 1.0 +# HELP http_requests_created Total number of requests by method, status and handler. +# TYPE http_requests_created gauge +http_requests_created{handler="/v1/models",method="GET",status="2xx"} 1.7921081971217349e+09 +http_requests_created{handler="/v1/chat/completions",method="POST",status="2xx"} 1.792108217381334e+09 +http_requests_created{handler="/v1/chat/completions",method="POST",status="4xx"} 1.7921641238703468e+09 +# HELP http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes summary +http_request_size_bytes_count{handler="/v1/models"} 236.0 +http_request_size_bytes_sum{handler="/v1/models"} 0.0 +http_request_size_bytes_count{handler="/v1/chat/completions"} 3491.0 +http_request_size_bytes_sum{handler="/v1/chat/completions"} 7.4767849e+07 +# HELP http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes_created gauge +http_request_size_bytes_created{handler="/v1/models"} 1.7921081971218035e+09 +http_request_size_bytes_created{handler="/v1/chat/completions"} 1.7921082173813493e+09 +# HELP http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes summary +http_response_size_bytes_count{handler="/v1/models"} 236.0 +http_response_size_bytes_sum{handler="/v1/models"} 113516.0 +http_response_size_bytes_count{handler="/v1/chat/completions"} 3491.0 +http_response_size_bytes_sum{handler="/v1/chat/completions"} 4.443412e+06 +# HELP http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes_created gauge +http_response_size_bytes_created{handler="/v1/models"} 1.7921081971218395e+09 +http_response_size_bytes_created{handler="/v1/chat/completions"} 1.7921082173813689e+09 +# HELP http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds histogram +http_request_duration_highr_seconds_bucket{le="0.01"} 236.0 +http_request_duration_highr_seconds_bucket{le="0.025"} 236.0 +http_request_duration_highr_seconds_bucket{le="0.05"} 236.0 +http_request_duration_highr_seconds_bucket{le="0.075"} 236.0 +http_request_duration_highr_seconds_bucket{le="0.1"} 236.0 +http_request_duration_highr_seconds_bucket{le="0.25"} 237.0 +http_request_duration_highr_seconds_bucket{le="0.5"} 239.0 +http_request_duration_highr_seconds_bucket{le="0.75"} 241.0 +http_request_duration_highr_seconds_bucket{le="1.0"} 242.0 +http_request_duration_highr_seconds_bucket{le="1.5"} 251.0 +http_request_duration_highr_seconds_bucket{le="2.0"} 294.0 +http_request_duration_highr_seconds_bucket{le="2.5"} 446.0 +http_request_duration_highr_seconds_bucket{le="3.0"} 678.0 +http_request_duration_highr_seconds_bucket{le="3.5"} 917.0 +http_request_duration_highr_seconds_bucket{le="4.0"} 1126.0 +http_request_duration_highr_seconds_bucket{le="4.5"} 1351.0 +http_request_duration_highr_seconds_bucket{le="5.0"} 1625.0 +http_request_duration_highr_seconds_bucket{le="7.5"} 3191.0 +http_request_duration_highr_seconds_bucket{le="10.0"} 3478.0 +http_request_duration_highr_seconds_bucket{le="30.0"} 3643.0 +http_request_duration_highr_seconds_bucket{le="60.0"} 3688.0 +http_request_duration_highr_seconds_bucket{le="+Inf"} 3727.0 +http_request_duration_highr_seconds_count 3727.0 +http_request_duration_highr_seconds_sum 25494.72712990234 +# HELP http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds_created gauge +http_request_duration_highr_seconds_created 1.7921081024104605e+09 +# HELP http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{handler="/v1/models",le="0.1",method="GET"} 236.0 +http_request_duration_seconds_bucket{handler="/v1/models",le="0.5",method="GET"} 236.0 +http_request_duration_seconds_bucket{handler="/v1/models",le="1.0",method="GET"} 236.0 +http_request_duration_seconds_bucket{handler="/v1/models",le="+Inf",method="GET"} 236.0 +http_request_duration_seconds_count{handler="/v1/models",method="GET"} 236.0 +http_request_duration_seconds_sum{handler="/v1/models",method="GET"} 0.09283922897884622 +http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="0.1",method="POST"} 0.0 +http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="0.5",method="POST"} 3.0 +http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="1.0",method="POST"} 6.0 +http_request_duration_seconds_bucket{handler="/v1/chat/completions",le="+Inf",method="POST"} 3491.0 +http_request_duration_seconds_count{handler="/v1/chat/completions",method="POST"} 3491.0 +http_request_duration_seconds_sum{handler="/v1/chat/completions",method="POST"} 25494.634290673363 +# HELP http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds_created gauge +http_request_duration_seconds_created{handler="/v1/models",method="GET"} 1.7921081971218753e+09 +http_request_duration_seconds_created{handler="/v1/chat/completions",method="POST"} 1.7921082173813894e+09 +# HELP vllm:tool_call_parser_invocations_total Total number of ToolParser invocations. Non-streaming increments once per choice; streaming increments once per delta. +# TYPE vllm:tool_call_parser_invocations_total counter +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="chat_completions"} 31000.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="responses"} 0.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="other"} 0.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="chat_completions"} 42734.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="responses"} 0.0 +vllm:tool_call_parser_invocations_total{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="other"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="chat_completions"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="responses"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="other"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="chat_completions"} 3139.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="responses"} 0.0 +vllm:tool_call_parser_invocations_total{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="other"} 0.0 +# HELP vllm:tool_call_parser_invocations_created Total number of ToolParser invocations. Non-streaming increments once per choice; streaming increments once per delta. +# TYPE vllm:tool_call_parser_invocations_created gauge +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="chat_completions"} 1.7921081024108987e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="responses"} 1.792108102410908e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="tool_call",request_type="other"} 1.7921081024109144e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="chat_completions"} 1.7921081024109201e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="responses"} 1.7921081024109259e+09 +vllm:tool_call_parser_invocations_created{mode="streaming",model_name="juggernaut",outcome="no_tool_call",request_type="other"} 1.792108102410931e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="chat_completions"} 1.7921081024109368e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="responses"} 1.7921081024109426e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="tool_call",request_type="other"} 1.7921081024109476e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="chat_completions"} 1.792108102410953e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="responses"} 1.7921081024109583e+09 +vllm:tool_call_parser_invocations_created{mode="non_streaming",model_name="juggernaut",outcome="no_tool_call",request_type="other"} 1.7921081024109638e+09 diff --git a/tests/test_agent.py b/tests/test_agent.py new file mode 100644 index 0000000..513d675 --- /dev/null +++ b/tests/test_agent.py @@ -0,0 +1,284 @@ +"""The host agent: stdlib-only, read-only by construction, and wrong on no box. + +Three things are asserted here: + * parsing of /proc and /sys files exactly as Linux writes them (fake roots, so the + weird cases are testable on any machine); + * the collector produces a complete, non-crashing snapshot against the *real* /proc of + whatever runs the tests, because that is the code that ships; + * the HTTP surface answers GETs, needs a token when it is exposed, and has no write path. +""" +from __future__ import annotations + +import importlib.util +import json +import re +import sys +import threading +import time +import urllib.error +import urllib.request +from http.server import ThreadingHTTPServer +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +AGENT_PATH = ROOT / "agent" / "host_agent.py" + +spec = importlib.util.spec_from_file_location("host_agent", AGENT_PATH) +assert spec and spec.loader +host_agent = importlib.util.module_from_spec(spec) +sys.modules["host_agent"] = host_agent +spec.loader.exec_module(host_agent) + +linux_only = pytest.mark.skipif(not sys.platform.startswith("linux"), + reason="reads /proc and /sys") + + +# ---- source-level helpers ------------------------------------------------- # + +@pytest.mark.parametrize("lspci,expected", [ + ("Advanced Micro Devices, Inc. [AMD/ATI] Navi 48 [Radeon AI PRO R9700]", + "Radeon AI PRO R9700"), + ("Advanced Micro Devices, Inc. [AMD/ATI] Strix Halo [Radeon Graphics / Radeon 8050S " + "Graphics / Radeon 8060S Graphics]", "Radeon 8060S Graphics"), + ("Advanced Micro Devices, Inc. [AMD/ATI] Phoenix1", "Phoenix1"), + ("00:02.0 VGA compatible controller: Intel Corporation UHD Graphics 620", + "UHD Graphics 620"), + ("NVIDIA GB200", "NVIDIA GB200"), + ("Vendor 1af1 Device 0001", "Vendor 1af1 Device 0001"), + (None, None), + ("", None), +]) +def test_gpu_model_is_shortened_for_legends_without_losing_identity(lspci, expected) -> None: + """`ATI` on a legend is worse than no name: it is true of every AMD GPU on the box.""" + assert host_agent.short_gpu_model(lspci) == expected + + +def test_gpu_model_is_capped_so_a_legend_cannot_blow_up() -> None: + long_name = "X" * 200 + assert len(host_agent.short_gpu_model(long_name)) <= 41 # 40 + ellipsis + + +# ---- /proc and /sys parsing (fake roots) ---------------------------------- # + +def write_file(root: Path, relative: str, text: str) -> None: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + +def test_read_number_takes_the_first_field(tmp_path: Path) -> None: + """/proc/uptime is `210036.18 9613750.86`; float() of the whole line is None.""" + write_file(tmp_path, "proc/uptime", "210036.18 9613750.86\n") + collector = host_agent.Collector(str(tmp_path)) + assert collector.read_number("proc/uptime") == 210036.18 + + +def test_read_number_handles_missing_and_junk_files(tmp_path: Path) -> None: + collector = host_agent.Collector(str(tmp_path)) + assert collector.read_number("proc/nope") is None + write_file(tmp_path, "proc/junk", "\n") + assert collector.read_number("proc/junk") is None + + +def test_core_counts_dedupe_socket_and_core(tmp_path: Path) -> None: + """One /proc/cpuinfo block per *thread*: counting 'core id' lines said '1 core'.""" + blocks = [] + for thread in range(8): # 2 sockets x 2 cores x 2 threads + socket, core = thread % 2, (thread // 2) % 2 + blocks.append(f"processor\t: {thread}\nphysical id\t: {socket}\ncore id\t\t: {core}\n") + write_file(tmp_path, "proc/cpuinfo", "\n".join(blocks)) + collector = host_agent.Collector(str(tmp_path)) + assert collector.core_counts() == (8, 4) + + +def test_core_counts_falls_back_to_smt_width_on_sysfs(tmp_path: Path) -> None: + """Some kernels/vms give no physical id/core id; cpu0's sibling list still says SMT=2.""" + write_file(tmp_path, "proc/cpuinfo", "processor\t: 0\nprocessor\t: 1\n") + write_file(tmp_path, "sys/devices/system/cpu/cpu0/topology/thread_siblings_list", "0,1\n") + collector = host_agent.Collector(str(tmp_path)) + logical, physical = collector.core_counts() + assert (logical, physical) == (2, 1) + + +def test_core_counts_survive_a_kernel_with_no_hints(tmp_path: Path) -> None: + collector = host_agent.Collector(str(tmp_path)) + assert collector.core_counts()[1] is None, "unknown must be None, never 1" + + +def test_cpu_modes_total_excludes_guest_ticks(tmp_path: Path) -> None: + """/proc/stat's idle/iowait are already inside 'total'; guest is inside user on Linux.""" + write_file(tmp_path, "proc/stat", + "cpu 100 10 20 500 5 3 2 1 0 0\nintr 1\nctxt 2\nprocesses 3\n") + collector = host_agent.Collector(str(tmp_path)) + sample = collector.cpu()["modes"] + assert sample["user"] == 100 and sample["idle"] == 500 + assert sample["total"] == pytest.approx(100 + 10 + 20 + 500 + 5 + 3 + 2 + 1) + + +# ---- the real collector --------------------------------------------------- # + +@linux_only +def test_snapshot_against_the_live_system_is_complete() -> None: + collector = host_agent.Collector("/") + snap = collector.snapshot() + for key in ("schema", "ts", "host", "cpu", "memory", "filesystems", "disk_io", + "network", "gpus", "temperatures", "processes", "errors"): + assert key in snap, f"{key} missing from a real snapshot" + assert snap["host"]["hostname"] + assert snap["cpu"]["modes"]["total"] > 0 + assert len(snap["cpu"]["cores"]) >= 1 + assert snap["memory"]["total_bytes"] > 0 + assert snap["filesystems"], "the box it runs on has at least one mounted filesystem" + assert snap["errors"] == [], f"degraded sections must be empty on a healthy box: {snap['errors']}" + + +def test_captured_fixture_has_the_shape_the_dashboard_depends_on(agent_snapshot) -> None: + """Ground truth from driveripper: if the agent's schema drifts, this fails first.""" + assert agent_snapshot["schema"] == host_agent.SCHEMA_VERSION + host = agent_snapshot["host"] + assert host["hostname"] == "driveripper" + assert (host["cpu_cores_physical"], host["cpu_cores_logical"]) == (24, 48) + assert host["uptime_seconds"] > 0 + gpu = agent_snapshot["gpus"][0] + assert gpu["model"] == "Radeon AI PRO R9700" + assert "model_full" in gpu and "Navi 48" in gpu["model_full"] + assert gpu["vram_total_bytes"] > 0 + # the dashboard renders n/a for anything a driver omits — it must never invent a 0 + assert gpu["busy_pct"] is None or 0.0 <= gpu["busy_pct"] <= 100.0 + + +@linux_only +def test_uptime_and_boot_ts_are_reported() -> None: + """Both are needed: uptime for the table, boot_ts to detect a counter-reset reboot.""" + info = host_agent.Collector("/").host_info() + assert info["uptime_seconds"] and info["uptime_seconds"] > 0 + assert info["boot_ts"] and info["boot_ts"] <= time.time() + assert abs((time.time() - info["boot_ts"]) - info["uptime_seconds"]) < 60 + assert info["cpu_cores_logical"] and info["cpu_cores_physical"] + + +@linux_only +def test_snapshot_is_repeatable_and_does_not_deadlock() -> None: + """snapshot() reads processes() internally; a non-reentrant lock used to hang it.""" + collector = host_agent.Collector("/") + first = collector.snapshot() + second = collector.snapshot(top=0) + assert first["processes"]["total"] > 0 + assert second["processes"]["top_cpu"] == [], "top=0 must skip the expensive scan" + + +@linux_only +def test_disk_io_comes_from_sysfs_not_proc_diskstats() -> None: + """/proc/diskstats field order has moved between kernels; /sys/block//stat has not.""" + devices = host_agent.Collector("/").disk_io()["devices"] + assert devices + for device in devices: + assert device["reads_total"] is None or device["reads_total"] >= 0 + assert device["size_bytes"] is None or device["size_bytes"] > 0 + + +# ---- HTTP surface --------------------------------------------------------- # + +@pytest.fixture +def agent_server(): + """Run the real handler on an ephemeral loopback port; yield (get, port).""" + servers: list[ThreadingHTTPServer] = [] + + def start(token: str | None = None) -> tuple[str, int]: + host_agent.Handler.collector = host_agent.Collector("/") + host_agent.Handler.token = token + httpd = ThreadingHTTPServer(("127.0.0.1", 0), host_agent.Handler) + servers.append(httpd) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return f"http://127.0.0.1:{httpd.server_address[1]}", httpd.server_address[1] + + yield start + for httpd in servers: + httpd.shutdown() + httpd.server_close() + + +def _get(url: str, headers: dict[str, str] | None = None) -> tuple[int, dict]: + request = urllib.request.Request(url, headers=headers or {}) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as error: + body = error.read() + try: + return error.code, json.loads(body) + except json.JSONDecodeError: # pragma: no cover - handler always sends JSON + return error.code, {"raw": body.decode(errors="replace")} + + +def test_health_info_schema_snapshot_all_answer(agent_server) -> None: + base, _port = agent_server() + for path in ("/health", "/info", "/schema", "/snapshot"): + status, body = _get(base + path) + assert status == 200, path + assert body["ok"] is True, path + + +def test_snapshot_query_param_controls_the_process_scan(agent_server) -> None: + base, _port = agent_server() + _status, body = _get(base + "/snapshot?top=0") + assert body["processes"]["top_cpu"] == [] + _status, full = _get(base + "/snapshot?top=3") + assert len(full["processes"]["top_cpu"]) <= 3 + + +def test_unknown_path_is_404_not_a_snapshot(agent_server) -> None: + base, _port = agent_server() + status, body = _get(base + "/etc/shadow") + assert status == 404 + assert body["ok"] is False + + +def test_token_is_required_when_configured_but_health_stays_open(agent_server) -> None: + base, _port = agent_server(token="s3cret") + assert _get(base + "/snapshot")[0] == 401 + assert _get(base + "/snapshot", {"Authorization": "Bearer wrong"})[0] == 401 + assert _get(base + "/snapshot", {"Authorization": "Bearer s3cret"})[0] == 200 + # /health stays open so a monitoring probe needs no secret + assert _get(base + "/health")[0] == 200 + + +def test_agent_has_no_write_path(agent_server) -> None: + """No do_POST/do_PUT exists: read-only is structural, not a policy toggle.""" + base, _port = agent_server() + request = urllib.request.Request(base + "/snapshot", data=b"{}", method="POST") + with pytest.raises(urllib.error.HTTPError) as excinfo: + urllib.request.urlopen(request, timeout=5) + assert excinfo.value.code in (405, 501) + + +def test_refuses_to_bind_off_loopback_without_a_token() -> None: + """0.0.0.0 with no token would hand host inventory to the whole LAN.""" + assert host_agent.main(["--host", "0.0.0.0", "--port", "0"]) == 2 + + +def test_once_mode_prints_json_and_exits(capsys) -> None: + assert host_agent.main(["--once"]) == 0 + printed = capsys.readouterr().out + assert json.loads(printed)["ok"] is True + + +def test_agent_is_dependency_free() -> None: + """The agent must run on a stock distro python: stdlib imports, and few of them. + + Two checks, because they fail for different reasons: anything outside the stdlib means a + `pip install` on every monitored host (deployment pain), while a growing footprint means + the agent can break on an older interpreter than the dashboard's own Python. + """ + text = AGENT_PATH.read_text() + imports = re.findall(r"^(?:import|from)\s+([a-zA-Z0-9_.]+)", text, re.M) + # indented (function-local) imports count too — subprocess is loaded lazily on purpose + imports += re.findall(r"^\s+(?:import|from)\s+([a-zA-Z0-9_.]+)", text, re.M) + top = {name.split(".")[0] for name in imports} + third_party = top - set(sys.stdlib_module_names) - {"host_agent"} + assert not third_party, f"agent grew a non-stdlib import: {third_party}" + assert top <= {"__future__", "argparse", "collections", "dataclasses", "http", "json", + "os", "pathlib", "re", "socket", "subprocess", "threading", "time", + "typing"}, f"agent import footprint grew: {sorted(top)}" diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..0875c95 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,346 @@ +"""HTTP contract: what the pages may assume, and what must never leak. + +The behaviours here are the ones a screenshot cannot show: an unconfigured source answering +404 `configured: false` instead of dialling a hostname that does not exist; a bearer token +that stops at the backend; the single write endpoint refusing on every gate; no external +asset the network can fail to deliver. +""" +from __future__ import annotations + +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient + +from dashboards.app import create_app +from dashboards.config import Settings, Target +from dashboards.store import Event, Sample, SampleStore + +# refused immediately, and never dials out of the test environment +DEAD_URL = "http://127.0.0.1:1" + + +def build_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, simulate: bool = False, + **overrides: Any) -> TestClient: + """A TestClient whose lifespan runs, against a private DB and no real network.""" + if simulate: + monkeypatch.setenv("NUT_SIMULATE", "1") + else: + monkeypatch.delenv("NUT_SIMULATE", raising=False) + db = str(tmp_path / "api.sqlite3") + settings = Settings(db_path=db, **overrides) + store = SampleStore(db) + app = create_app(settings, store) + client = TestClient(app) + client.__enter__() + return client + + +@pytest.fixture +def client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + with build_client(tmp_path, monkeypatch) as test_client: + yield test_client + + +@pytest.fixture +def ups_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + """NUT on the labelled simulator, actions allowed — the dev configuration.""" + with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1", + nut_allow_actions=True) as c: + yield c + + +def wait_for_payload(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]: + deadline = time.time() + timeout + while time.time() < deadline: + response = client.get(path) + if response.status_code == 200: + return response.json() + time.sleep(0.1) + pytest.fail(f"{path} never became available") + + +# ---- unconfigured: say so, and do not dial -------------------------------- # + +def test_config_reports_every_source_as_off(client) -> None: + body = client.get("/api/config").json() + assert body["system"]["enabled"] is False + assert body["system"]["targets"] == [] + assert body["llm"]["gufo"]["enabled"] is False + assert body["llm"]["vllm"]["enabled"] is False + assert body["ups"]["enabled"] is False + assert body["ups"]["actions_enabled"] is False + assert body["ranges"], "the page builds its window buttons from this list" + + +def test_health_with_nothing_configured_is_not_a_failure(client) -> None: + """Nothing configured is 'empty', not 'broken' — and the page must not go red.""" + body = client.get("/api/health").json() + assert body["status"] in ("ok", "empty") + assert body["ok"] is True + assert body["sources"] == [] + + +@pytest.mark.parametrize("path", ["/api/system/snapshot", "/api/llm/snapshot", + "/api/ups/snapshot", "/api/ups/commands"]) +def test_unconfigured_source_is_404_configured_false(client, path) -> None: + response = client.get(path) + assert response.status_code == 404 + assert response.json()["configured"] is False + assert "detail" in response.json() + + +def test_summary_still_answers_with_no_sources(client) -> None: + body = client.get("/api/summary").json() + assert body["system"]["enabled"] is False + assert body["system"]["fleet"] is None + assert body["llm"]["enabled"] is False + assert body["ups"]["enabled"] is False + + +def test_unknown_route_is_json_not_html(client) -> None: + response = client.get("/api/nope") + assert response.status_code == 404 + assert response.headers["content-type"].startswith("application/json") + assert "no such route" in response.json()["detail"] + + +# ---- tokens --------------------------------------------------------------- # + +def test_agent_tokens_never_reach_the_browser(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NUT_SIMULATE", "") + targets = [Target(name="driveripper", url="http://driveripper:9120", + headers={"Authorization": "Bearer SEKRIT-TOKEN"})] + db = str(tmp_path / "tokens.sqlite3") + store = SampleStore(db) + app = create_app(Settings(db_path=db, host_targets=tuple(targets)), store) + with TestClient(app) as client: + body = client.get("/api/config") + assert "SEKRIT-TOKEN" not in body.text + target = body.json()["system"]["targets"][0] + assert target["name"] == "driveripper" + assert "SEKRIT" not in target["url"] and "@" not in target["url"] + # and a host that never answered stays in the list, marked, with the fleet card intact + snapshot = wait_for_state(client, "/api/system/snapshot") + assert snapshot["hosts"][0]["name"] == "driveripper" + assert snapshot["hosts"][0].get("unreachable") is True + assert snapshot["fleet"]["hosts_total"] == 1 + assert snapshot["fleet"]["hosts_live"] == 0 + assert snapshot["fleet"]["memory_total_bytes"] is None + store.close() + + +def wait_for_state(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]: + deadline = time.time() + timeout + while time.time() < deadline: + response = client.get(path) + if response.status_code == 200: + return response.json() + time.sleep(0.1) + pytest.fail(f"{path} never answered") + + +def test_target_problems_are_surfaced_not_swallowed(client) -> None: + body = client.get("/api/config").json() + assert body["system"]["target_problems"] == [] + + +# ---- pages ---------------------------------------------------------------- # + +@pytest.mark.parametrize("path,title", [ + ("/", "overview"), ("/system", "system"), ("/llm", "llm"), ("/ups", "ups"), +]) +def test_pages_serve_and_declare_themselves(client, path, title) -> None: + response = client.get(path) + assert response.status_code == 200 + assert "text/html" in response.headers["content-type"] + assert title in response.text.lower() + + +def test_no_page_loads_anything_from_the_internet(client) -> None: + """No CDN, no webfonts: this dashboard has to work with no route to the internet.""" + for path in ("/", "/system", "/llm", "/ups"): + html = client.get(path).text + for fragment in ("//cdn.", "https://fonts.", "//unpkg.com", "//cdnjs"): + assert fragment not in html, f"{path} references {fragment}" + + +def test_static_assets_are_served(client) -> None: + for asset in ("/static/dash.css", "/static/app.js", "/static/charts.js", + "/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js", + "/static/page-index.js"): + assert client.get(asset).status_code == 200, asset + + +def test_api_responses_are_not_cacheable(client) -> None: + for path in ("/api/config", "/api/health", "/api/summary"): + assert "no-store" in client.get(path).headers.get("cache-control", "") + + +# ---- history -------------------------------------------------------------- # + +def test_history_returns_seeded_samples(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + db = str(tmp_path / "hist.sqlite3") + store = SampleStore(db) + now = time.time() + for i in range(30): + store.add("host", "driveripper", "cpu.usage_pct", 10.0 + i, ts=now - 300 + i) + app = create_app(Settings(db_path=db, host_targets=(Target("driveripper", DEAD_URL),)), store) + with TestClient(app) as client: + body = client.get("/api/history", params={"source": "host", "host": "driveripper", + "metric": "cpu.usage_pct", "range": "15m"}) + assert response_ok(body) + points = body.json()["points"] + assert len(points) >= 25 + assert points[-1]["value"] == pytest.approx(39.0) + store.close() + + +def test_history_rejects_an_unknown_range(client) -> None: + response = client.get("/api/history", params={"source": "host", "host": "h", + "metric": "m", "range": "banana"}) + assert response.status_code == 400 + + +def test_history_requires_the_series_coordinates(client) -> None: + assert client.get("/api/history").status_code == 422 + + +def test_metrics_endpoint_lists_what_actually_has_history(client) -> None: + body = client.get("/api/metrics").json() + assert body["metrics"] == [] + + +# ---- events --------------------------------------------------------------- # + +def test_events_are_returned_in_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + db = str(tmp_path / "events.sqlite3") + store = SampleStore(db) + store.log_event(Event(ts=time.time(), source="host", host="deskwork", kind="down", + severity="error", message="deskwork is not answering")) + app = create_app(Settings(db_path=db), store) + with TestClient(app) as client: + events = client.get("/api/events").json()["events"] + assert events[0]["host"] == "deskwork" + assert events[0]["severity"] == "error" + store.close() + + +# ---- the one write endpoint ------------------------------------------------ # + +def test_command_endpoint_is_post_only(ups_client) -> None: + assert ups_client.post("/api/ups/command", json={}).status_code in (400, 403, 404) + assert ups_client.put("/api/ups/command", json={}).status_code == 405 + assert ups_client.delete("/api/ups/command").status_code == 405 + + +def test_commands_are_offered_when_actions_are_enabled(ups_client) -> None: + body = wait_for_payload(ups_client, "/api/ups/snapshot") + assert body["simulated"] is True, "the simulator must label itself" + assert body["actions_enabled"] is True + ups = body["ups"][0] + enabled = [item for item in ups["commands"] if item["enabled"]] + assert enabled, "with NUT_ALLOW_ACTIONS=1 and a simulator, safe commands are sendable" + assert all(item["risk"] in ("safe", "caution", "dangerous") for item in ups["commands"]) + + +def test_a_safe_command_actually_reaches_the_driver(ups_client) -> None: + body = wait_for_payload(ups_client, "/api/ups/snapshot") + name = body["ups"][0]["name"] + response = ups_client.post("/api/ups/command", + json={"ups": name, "command": "test.panel.start"}) + assert response.status_code == 200, response.text + payload = response.json() + assert payload["ok"] is True and payload["simulated"] is True + events = ups_client.get("/api/events").json()["events"] + assert any("test.panel.start" in event["message"] or "Panel test" in event["message"] + for event in events), "every sent command must leave an audit trail" + + +def test_a_dangerous_command_needs_the_confirmation_word(ups_client) -> None: + name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"] + deep = "test.battery.start.deep" + refusal = ups_client.post("/api/ups/command", json={"ups": name, "command": deep}) + assert refusal.status_code == 422 + assert "DANGER" in refusal.json()["detail"] + assert ups_client.post("/api/ups/command", + json={"ups": name, "command": deep, "confirm": "DANGER"}).status_code == 200 + + +def test_unknown_commands_are_refused_even_with_actions_on(ups_client) -> None: + name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"] + for command in ("load.format", "rm -rf /", "INSTCMD x y"): + response = ups_client.post("/api/ups/command", json={"ups": name, "command": command}) + assert response.status_code == 400, command + + +def test_commands_for_an_unknown_ups_are_404(ups_client) -> None: + wait_for_payload(ups_client, "/api/ups/snapshot") + response = ups_client.post("/api/ups/command", + json={"ups": "not-real", "command": "beeper.mute"}) + assert response.status_code == 404 + + +def test_missing_fields_are_a_400(ups_client) -> None: + assert ups_client.post("/api/ups/command", json={"ups": ""}).status_code == 400 + + +def test_actions_flag_off_blocks_every_command(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1") as client: + body = wait_for_payload(client, "/api/ups/snapshot") + name = body["ups"][0]["name"] + assert body["actions_enabled"] is False + response = client.post("/api/ups/command", + json={"ups": name, "command": "beeper.mute"}) + assert response.status_code == 403 + assert "NUT_ALLOW_ACTIONS" in response.json()["detail"] + + +def test_read_only_beats_the_actions_flag(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1", + nut_allow_actions=True, read_only=True) as client: + body = wait_for_payload(client, "/api/ups/snapshot") + name = body["ups"][0]["name"] + response = client.post("/api/ups/command", + json={"ups": name, "command": "beeper.mute"}) + assert response.status_code == 403 + commands = wait_for_payload(client, "/api/ups/commands") + assert commands["read_only"] is True + assert all(item["enabled"] is False for ups in commands["ups"] + for item in ups["commands"]) + + +def test_ups_history_answers_for_simulated_ups(ups_client) -> None: + wait_for_payload(ups_client, "/api/ups/snapshot") + time.sleep(1.2) + body = ups_client.get("/api/ups/history", params={"metrics": "ups.charge_pct", + "range": "1h"}).json() + assert body["ups"], "the UPS page needs a name to chart before any history exists" + + +def response_ok(response) -> bool: + assert response.status_code == 200, response.text + return True + + +def test_samples_written_by_the_poller_are_queryable(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + """The write path (poller -> store) and the read path (/api/history) must agree on names.""" + db = str(tmp_path / "roundtrip.sqlite3") + store = SampleStore(db) + store.add_many([Sample("llm", "gufo", "llm.decode_tokens_per_second", 40.0, time.time() - 60), + Sample("llm", "gufo", "llm.decode_tokens_per_second", 45.0, time.time() - 30)]) + app = create_app(Settings(db_path=db), store) + with TestClient(app) as client: + metrics = client.get("/api/metrics", params={"source": "llm"}).json()["metrics"] + assert "llm.decode_tokens_per_second" in metrics + points = client.get("/api/history", params={"source": "llm", "host": "gufo", + "metric": "llm.decode_tokens_per_second", + "range": "15m"}).json()["points"] + assert [p["value"] for p in points] == [40.0, 45.0] + store.close() diff --git a/tests/test_collectors.py b/tests/test_collectors.py new file mode 100644 index 0000000..fd93afd --- /dev/null +++ b/tests/test_collectors.py @@ -0,0 +1,443 @@ +"""Collectors: the fetch half of every poll, exercised with a fake transport. + +`httpx.MockTransport` means these run at the speed of the parser and still cover what the +live hosts do badly: a 401 from a rotated agent token, a half-degraded agent snapshot, a +gufo server that only answers `/v1/slots`, an engine that counts nothing yet. + +NUT is covered here for the *monitor* bookkeeping (payload shape, command gating, status +transitions); the socket protocol itself is in test_nut.py. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, ClassVar + +import httpx +import pytest + +from dashboards.collectors import ( + NutMonitor, + UpstreamError, + fetch_engine, + fetch_host_snapshot, +) +from dashboards.config import Target +from dashboards.rates import RateEstimator +from dashboards.sources import nut +from dashboards.sources.host import HostCollector + +FIXTURES = Path(__file__).parent / "fixtures" + + +def client_for(handler: Any) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.MockTransport(handler), + base_url="http://agent.test") + + +# --------------------------------------------------------------------------- # +# host agent +# --------------------------------------------------------------------------- # +def agent_body() -> dict[str, Any]: + return json.loads((FIXTURES / "agent_driveripper.json").read_text()) + + +def advance_cpu(body: dict[str, Any], *, total: float, idle: float) -> None: + """Move the jiffy counters the way a real /proc/stat does between polls.""" + modes = body["cpu"]["modes"] + for mode, value in modes.items(): + modes[mode] = value + (total if mode == "total" else idle if mode == "idle" else 0) + + +async def test_a_real_agent_snapshot_parses() -> None: + body = agent_body() + calls: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/snapshot" + calls.append(request.url.path) + return httpx.Response(200, json=body) + + target = Target("driveripper", "http://agent.test") + collector = HostCollector(RateEstimator()) + async with client_for(handler) as client: + first = (await fetch_host_snapshot(client, target, collector)).payload + # CPU usage is Δjiffies/Δt: one scrape has no window, so it is n/a and not 0 % + assert first["cpu"]["usage_pct"] is None + # the counters have to actually move — total 0 delta means "cannot divide", not idle + same = (await fetch_host_snapshot(client, target, collector)).payload + assert same["cpu"]["usage_pct"] is None, "no counter movement must read as n/a" + advance_cpu(body, total=1000, idle=700) + result = await fetch_host_snapshot(client, target, collector) + payload = result.payload + assert payload["cpu"]["usage_pct"] == pytest.approx(30.0, abs=1.0) + assert payload["name"] == "driveripper" + assert payload["cpu"]["usage_pct"] is not None + assert payload["memory"]["total_bytes"] > 0 + # real captures have real GPUs; the legend wants the short name, the tooltip the full one + assert payload["gpus"], "driveripper's capture must yield at least one GPU" + assert payload["gpus"][0]["model"] and "model_full" in payload["gpus"][0] + assert any(s.metric == "cpu.usage_pct" for s in result.samples) + + +async def test_target_headers_are_sent_to_the_agent() -> None: + seen: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["authorization"] = request.headers.get("authorization", "") + return httpx.Response(200, json=agent_body()) + + target = Target("gateway", "http://agent.test", headers={"Authorization": "Bearer tok"}) + async with client_for(handler) as client: + await fetch_host_snapshot(client, target, HostCollector(RateEstimator())) + assert seen["authorization"] == "Bearer tok" + + +@pytest.mark.parametrize("status,fragment", [ + (401, "token"), + (403, "HTTP 403"), + (500, "HTTP 500"), +]) +async def test_agent_rejections_say_which_one_it_was(status, fragment) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(status, text="nope") + + async with client_for(handler) as client: + with pytest.raises(UpstreamError) as excinfo: + await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + assert fragment in excinfo.value.message + assert excinfo.value.status == 502 + + +async def test_a_refused_connection_is_an_upstream_error_not_a_traceback() -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection refused") + + async with client_for(handler) as client: + with pytest.raises(UpstreamError, match="unreachable"): + await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + + +async def test_a_slow_agent_is_a_gateway_timeout() -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("too slow") + + async with client_for(handler) as client: + with pytest.raises(UpstreamError) as excinfo: + await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + assert excinfo.value.status == 504 + + +async def test_html_instead_of_json_is_reported_as_such() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="502 Bad Gateway") + + async with client_for(handler) as client: + with pytest.raises(UpstreamError, match="non-JSON"): + await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + + +async def test_an_agent_answering_not_ok_is_not_silently_empty() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"}) + + async with client_for(handler) as client: + with pytest.raises(UpstreamError, match="procfs unreadable"): + await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + + +async def test_a_degraded_agent_still_renders_and_says_what_broke() -> None: + body = agent_body() + body["errors"] = {"temperatures": "no hwmon sensors"} + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=body) + + async with client_for(handler) as client: + result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + assert result.payload["degraded"] == {"temperatures": "no hwmon sensors"} + assert result.payload["memory"]["total_bytes"] > 0 + + +# --------------------------------------------------------------------------- # +# LLM engines +# --------------------------------------------------------------------------- # +GUFO = (FIXTURES / "metrics_gufo.txt").read_text() +VLLM = (FIXTURES / "metrics_vllm.txt").read_text() + + +def bump(text: str, series: str, amount: float) -> str: + lines = [] + for line in text.splitlines(): + if line.startswith(series + " "): + head, _, value = line.rpartition(" ") + line = f"{head} {float(value) + amount}" + lines.append(line) + return "\n".join(lines) + "\n" + + +async def test_first_gufo_poll_has_no_measured_rate_yet() -> None: + """One scrape cannot produce Δ/Δt. The cell is n/a, not a stale single-request gauge.""" + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/metrics": + return httpx.Response(200, text=GUFO) + return httpx.Response(404, text="nope") + + async with client_for(handler) as client: + result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(), + with_slots=True) + decode = result.payload["metrics"]["decode_tokens_per_second"] + assert decode["value"] is None + assert decode["source"] is None + assert result.payload["sessions"]["active"] == 0.0 + assert "no /slots answer" in result.payload["sessions"]["detail"] + + +async def test_second_gufo_poll_measures_throughput_over_a_window() -> None: + state = {"text": GUFO} + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/metrics": + return httpx.Response(200, text=state["text"]) + if request.url.path == "/slots": + return httpx.Response(200, json=[{"index": 0, "n_ctx": 8192, "predicted": 12}]) + return httpx.Response(404) + + rates = RateEstimator(span=10.0) + async with client_for(handler) as client: + await fetch_engine(client, "gufo", "http://engine.test", "", rates) + state["text"] = bump(GUFO, "llamacpp:tokens_predicted_total", 40.0) + result = await fetch_engine(client, "gufo", "http://engine.test", "", rates, + with_slots=True) + decode = result.payload["metrics"]["decode_tokens_per_second"] + assert decode["source"].startswith("computed:Δllamacpp:") + assert decode["derived"] is True + assert decode["window"] > 0 + assert decode["value"] == pytest.approx(40.0 / decode["window"], rel=0.05) + assert result.payload["sessions"]["source"] == "gufo /slots" + assert result.payload["sessions"]["list"][0]["index"] == 0 + + +async def test_gufo_slots_alias_is_used_when_plain_slots_is_missing() -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/metrics": + return httpx.Response(200, text=GUFO) + if request.url.path == "/v1/slots": + return httpx.Response(200, json={"slots": [{"index": 1}]}) + return httpx.Response(404) + + async with client_for(handler) as client: + result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(), + with_slots=True) + assert result.payload["sessions"]["list"] == [{"index": 1}] + + +async def test_vllm_metrics_parse_and_carry_the_api_key() -> None: + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.headers.get("authorization", "")) + return httpx.Response(200, text=VLLM) + + async with client_for(handler) as client: + result = await fetch_engine(client, "vllm", "http://engine.test", "SECRET", + RateEstimator()) + assert seen == ["Bearer SECRET"] + metrics = result.payload["metrics"] + assert metrics["num_requests_running"]["value"] is not None + # vLLM publishes no per-session list: the page is told, rather than shown an empty table + assert "counts, not sessions" in result.payload["sessions"]["detail"] + assert any(s.metric.startswith("llm.") for s in result.samples) + + +@pytest.mark.parametrize("status", [404, 500, 503]) +async def test_engine_http_errors_name_the_status(status) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(status, text="") + + async with client_for(handler) as client: + with pytest.raises(UpstreamError, match=f"HTTP {status}"): + await fetch_engine(client, "vllm", "http://engine.test", "", RateEstimator()) + + +async def test_engine_timeout_is_504() -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("engine busy") + + async with client_for(handler) as client: + with pytest.raises(UpstreamError) as excinfo: + await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator()) + assert excinfo.value.status == 504 + + +# --------------------------------------------------------------------------- # +# NUT monitor +# --------------------------------------------------------------------------- # +class FakeSimulator: + """Stands in for NutSimulator: hands back readings we control.""" + + def __init__(self, readings: list[nut.UpsReading]) -> None: + self.readings = readings + + async def read(self) -> list[nut.UpsReading]: + return list(self.readings) + + +def reading(status: str, **vars: str) -> nut.UpsReading: + item = nut.UpsReading(name="cyberpower", description="CP1350PFCLCD", ts=1_700_000_000.0) + item.vars = {"ups.status": status, "battery.charge": "92", "ups.load": "35", + "battery.runtime": "1800", **vars} + item.commands = ["beeper.mute", "test.panel.start", "test.battery.start.deep"] + return item + + +class SettingsStub: + nut_host, nut_port, nut_timeout = "ups.test", 3493, 5.0 + nut_user = nut_password = "" + nut_interval = 5.0 + nut_enabled = True + actions_allowed = True + read_only = False + + +async def test_monitor_payload_labels_the_simulator() -> None: + monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")])) + result = await monitor.collect() + payload = result.payload + assert payload["simulated"] is True + assert payload["server"] is None, "a simulator must not claim a server address" + assert payload["actions_enabled"] is True + card = payload["ups"][0] + assert card["name"] == "cyberpower" + assert card["status"] == "OL" + assert card["numbers"]["charge_pct"] == 92.0 + # a driver that reports no battery date, temperature, or self-test result is `unknown`, + # never a silent "healthy" — and never a failure either + assert card["health"]["overall"] == "unknown", card["health"] + assert card["health"]["counts"]["fail"] == 0 + assert card["health"]["counts"]["warn"] == 0 + # commands are offered with their risk, and the dangerous one is present but marked + risks = {item["command"]: item for item in card["commands"]} + assert risks["beeper.mute"]["enabled"] is True + assert risks["test.battery.start.deep"]["risk"] == "dangerous" + + +async def test_a_fully_report_ups_is_called_healthy() -> None: + full = reading("OL", **{"battery.temperature": "27.4", "battery.mfr.date": "2024-06-01", + "battery.test.result": "OK", "input.voltage": "230"}) + full.vars["battery.charge"] = "100" + card = (await NutMonitor(SettingsStub(), simulator=FakeSimulator([full])).collect()).payload["ups"][0] + assert card["health"]["overall"] == "healthy", card["health"] + assert card["health"]["counts"]["unknown"] == 0 + + +async def test_monitor_sends_history_samples_for_every_ups_metric() -> None: + monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OB")])) + result = await monitor.collect() + names = {sample.metric for sample in result.samples} + assert "ups.charge_pct" in names + assert "ups.load_pct" in names + assert all(sample.source == "ups" for sample in result.samples) + + +async def test_status_changes_become_one_event_each() -> None: + monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")])) + assert (await monitor.collect()).events == [] + monitor.simulator = FakeSimulator([reading("OL")]) + assert (await monitor.collect()).events == [], "an unchanged status is not news" + monitor.simulator = FakeSimulator([reading("OB")]) + events = (await monitor.collect()).events + assert len(events) == 1 + assert events[0].kind == "status" + assert "OL → OB" in events[0].message + assert events[0].severity == "warn", "on battery is a warning, not a failure" + monitor.simulator = FakeSimulator([reading("OL FUB")]) + events = (await monitor.collect()).events + assert events[0].severity == "error", "FUB (the UPS decided to power off) is a failure" + + +async def test_a_self_test_result_is_reported_when_it_changes() -> None: + monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")])) + await monitor.collect() + # the variable is `ups.test.result` (drivers also use battery.test.result) — see VAR_MAP + monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "INPROGRESS"})]) + assert (await monitor.collect()).events == [], "first sighting is not news" + monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "FAIL"})]) + events = (await monitor.collect()).events + assert [event.kind for event in events] == ["test"] + assert events[0].severity == "warn" + monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "OK"})]) + assert (await monitor.collect()).events[0].severity == "info" + + +async def test_gating_is_applied_where_the_commands_are_built() -> None: + class ReadOnly(SettingsStub): + actions_allowed = False + + monitor = NutMonitor(ReadOnly(), simulator=FakeSimulator([reading("OL")])) + card = (await monitor.collect()).payload["ups"][0] + assert all(item["enabled"] is False for item in card["commands"]) + assert all(item["reason"] for item in card["commands"]), "every refusal must explain itself" + assert set(card["advertised_commands"]) == {"beeper.mute", "test.panel.start", + "test.battery.start.deep"}, ( + "the driver still advertises them — the refusal is ours, and must say so") + + +class FakeNutClient: + """The real upsd branch without a socket: vars work, the optional lists do not.""" + + instances: ClassVar[list[FakeNutClient]] = [] + + def __init__(self, host: str, port: int, **kwargs: Any) -> None: + self.host, self.port, self.kwargs = host, port, kwargs + self.entered = False + FakeNutClient.instances.append(self) + + async def __aenter__(self) -> FakeNutClient: + self.entered = True + return self + + async def __aexit__(self, *exc: Any) -> None: + self.entered = False + + async def list_ups(self) -> list[dict[str, str]]: + return [{"name": "cyberpower", "description": "CP1350PFCLCD"}] + + async def list_vars(self, ups: str) -> dict[str, str]: + return {"ups.status": "OL", "battery.charge": "100", "battery.temperature": "301.1"} + + async def list_cmds(self, ups: str) -> list[str]: + raise nut.NutError("this driver exports no commands") + + async def list_rw_vars(self, ups: str) -> list[str]: + raise nut.NutError("LIST RW denied") + + +async def test_real_upsd_branch_survives_drivers_that_refuse_the_optional_lists( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(nut, "NutClient", FakeNutClient) + monitor = NutMonitor(SettingsStub()) + result = await monitor.collect() + client = FakeNutClient.instances[-1] + assert client.entered is False, "the connection must not outlive one poll" + assert client.kwargs["timeout"] == 5.0 + card = result.payload["ups"][0] + assert card["commands"] == [] and card["advertised_commands"] == [] + assert card["numbers"]["charge_pct"] == 100.0 + # a driver reporting Kelvin is normalised, not charted as 301 °C + assert card["numbers"]["battery_temperature"] == pytest.approx(301.1 - 273.15, abs=0.01) + samples = {sample.metric: sample.value for sample in result.samples} + assert samples["ups.battery_temperature_c"] == pytest.approx(27.95, abs=0.01) + + +async def test_a_real_server_names_itself_in_the_payload(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(nut, "NutClient", FakeNutClient) + payload = (await NutMonitor(SettingsStub()).collect()).payload + assert payload["simulated"] is False + assert payload["server"] == "ups.test:3493" diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..952320a --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,181 @@ +"""Configuration: env-only, no defaults that point at somebody else's machine.""" +from __future__ import annotations + +import importlib +import os +from dataclasses import FrozenInstanceError +from typing import Any + +import pytest + +from dashboards import config +from dashboards.config import Settings, parse_targets + + +def test_parses_whitespace_and_comma_separated_targets() -> None: + targets, problems = parse_targets("a=http://h1:9120 b=http://h2:9120,c=http://h3:9120") + assert [t.name for t in targets] == ["a", "b", "c"] + assert [t.url for t in targets] == ["http://h1:9120", "http://h2:9120", "http://h3:9120"] + assert problems == [] + + +def test_trailing_slash_is_stripped_so_urls_do_not_double_up() -> None: + targets, _ = parse_targets("a=http://h1:9120/") + assert targets[0].url == "http://h1:9120" + + +def test_one_bad_entry_does_not_sink_the_others() -> None: + targets, problems = parse_targets("good=http://h:1 broken-no-equals another=ftp://h:1") + assert [t.name for t in targets] == ["good"] + assert len(problems) == 2 + assert any("not name=url" in p for p in problems) + assert any("http://" in p for p in problems) + + +def test_empty_input_yields_no_targets_and_no_noise() -> None: + assert parse_targets("") == ([], []) + assert parse_targets(" \n ") == ([], []) + + +def test_token_in_url_becomes_a_bearer_header_and_stays_out_of_the_display_url() -> None: + targets, problems = parse_targets("driveripper=http://SEKRIT@driveripper:9120") + assert problems == [] + target = targets[0] + assert target.headers == {"Authorization": "Bearer SEKRIT"} + assert target.url == "http://driveripper:9120", "the token must not stay in the URL" + assert target.display_url == "http://driveripper:9120" + assert "SEKRIT" not in target.display_url + + +def test_target_names_are_restricted_because_they_are_db_keys() -> None: + """Names key the history DB and appear in URLs, so they are validated, not guessed. + + Whitespace separates entries, which means "Bad Name=http://h" is two entries — both + reported, because the operator needs to see exactly what was rejected. + """ + targets, problems = parse_targets("Bad=http://h:1 /=http://h:1 ok-1_2=http://h:1") + assert [t.name for t in targets] == ["ok-1_2"] + assert len(problems) == 2 + assert any("Bad" in p for p in problems) + + _targets, problems = parse_targets("Bad Name=http://h:1") + assert len(problems) == 2, "a space splits the entry, so both halves are rejected" + + +def test_settings_default_to_no_sources_configured() -> None: + settings = Settings() + assert settings.host_targets == () + assert settings.gufo_url == "" and settings.vllm_url == "" + assert settings.nut_host == "" + assert settings.nut_enabled is False + assert settings.any_source_enabled is False + + +def test_actions_need_the_flag_and_are_dead_under_read_only() -> None: + assert Settings(nut_allow_actions=True, read_only=False).actions_allowed is True + assert Settings(nut_allow_actions=True, read_only=True).actions_allowed is False + assert Settings(nut_allow_actions=False, read_only=False).actions_allowed is False + + +def test_nut_host_accepts_a_nut_scheme_prefix() -> None: + assert Settings(nut_host="nut://ups-box:3493").nut_enabled is True + + +def test_load_reads_host_targets_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HOST_TARGETS", "deskwork=http://127.0.0.1:9122") + settings = config.load() + assert [t.name for t in settings.host_targets] == ["deskwork"] + + +def test_env_driven_scalars_are_read_at_import(monkeypatch: pytest.MonkeyPatch) -> None: + """The dataclass defaults read os.environ, so a reload must pick up a changed env.""" + monkeypatch.setenv("DASH_PORT", "8123") + monkeypatch.setenv("RETAIN_RAW_HOURS", "6") + monkeypatch.setenv("NUT_ALLOW_ACTIONS", "1") + reloaded = importlib.reload(config) + try: + assert reloaded.SETTINGS.port == 8123 + assert reloaded.SETTINGS.raw_keep_hours == 6 + assert reloaded.SETTINGS.actions_allowed is True + finally: + monkeypatch.delenv("DASH_PORT") + monkeypatch.delenv("RETAIN_RAW_HOURS") + monkeypatch.delenv("NUT_ALLOW_ACTIONS") + importlib.reload(config) + + +def test_flags_accept_the_usual_spellings(monkeypatch: pytest.MonkeyPatch) -> None: + for truthy in ("1", "true", "TRUE", "yes", "on"): + monkeypatch.setenv("DASH_READ_ONLY", truthy) + assert config._flag("DASH_READ_ONLY") is True, truthy + for falsey in ("0", "false", "", "no"): + monkeypatch.setenv("DASH_READ_ONLY", falsey) + assert config._flag("DASH_READ_ONLY") is False, falsey + + +def test_numeric_env_must_be_a_number_or_the_default_wins(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HOST_INTERVAL_SECONDS", "not-a-number") + assert config._num("HOST_INTERVAL_SECONDS", 10.0) == 10.0 + monkeypatch.setenv("HOST_INTERVAL_SECONDS", "2.5") + assert config._num("HOST_INTERVAL_SECONDS", 10.0) == 2.5 + + +def test_settings_is_immutable_so_nobody_mutates_shared_state() -> None: + settings: Any = Settings() + with pytest.raises(FrozenInstanceError): + settings.port = 1 # type: ignore[misc] + + +# ---- the .env convenience -------------------------------------------------- # + +def test_no_env_file_is_the_normal_case(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: + monkeypatch.setenv("DASH_ENV_FILE", str(tmp_path / "absent.env")) + assert config.load_env_file() is None + + +def test_env_file_seeds_values_a_deployed_env_does_not_set(monkeypatch: pytest.MonkeyPatch, + tmp_path: Any) -> None: + env = tmp_path / ".env" + env.write_text("DASH_PORT=9999\nNUT_HOST=ups.internal\n") + monkeypatch.setenv("DASH_ENV_FILE", str(env)) + monkeypatch.delenv("DASH_PORT", raising=False) + monkeypatch.delenv("NUT_HOST", raising=False) + assert config.load_env_file() == env + # Settings reads its defaults when the module is imported, so the reload is what a real + # process gets for free and what the test has to ask for explicitly + reloaded = importlib.reload(config) + assert reloaded.SETTINGS.port == 9999 + assert reloaded.SETTINGS.nut_host == "ups.internal" + + +def test_a_real_environment_variable_beats_the_file(monkeypatch: pytest.MonkeyPatch, + tmp_path: Any) -> None: + """The container/quadlet env is the source of truth; the file is for laptops.""" + env = tmp_path / ".env" + env.write_text("DASH_PORT=9999\n") + monkeypatch.setenv("DASH_ENV_FILE", str(env)) + monkeypatch.setenv("DASH_PORT", "7777") + reloaded = importlib.reload(config) + assert os.environ["DASH_PORT"] == "7777" + assert reloaded.SETTINGS.port == 7777 + + +def test_env_file_expands_other_variables(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: + """`.env.example` shows `DASH_DB=${HOME}/...` — the file has to mean what it shows.""" + env = tmp_path / ".env" + env.write_text("MY_BASE=/srv/dash\nDASH_DB=${MY_BASE}/dashboards.sqlite3\n") + monkeypatch.setenv("DASH_ENV_FILE", str(env)) + monkeypatch.delenv("MY_BASE", raising=False) + monkeypatch.delenv("DASH_DB", raising=False) + reloaded = importlib.reload(config) + assert reloaded.SETTINGS.db_path == "/srv/dash/dashboards.sqlite3" + + +def test_the_example_file_is_never_loaded(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: + example = tmp_path / ".env.example" + example.write_text("NUT_HOST=should-not-load\n") + monkeypatch.setenv("DASH_ENV_FILE", str(example)) + monkeypatch.delenv("NUT_HOST", raising=False) + reloaded = importlib.reload(config) + assert reloaded.load_env_file() is None + assert reloaded.SETTINGS.nut_host == "" diff --git a/tests/test_host_source.py b/tests/test_host_source.py new file mode 100644 index 0000000..5bc7b3e --- /dev/null +++ b/tests/test_host_source.py @@ -0,0 +1,242 @@ +"""Agent snapshot -> dashboard payload: rates, null-safety, and honest fleet totals. + +The rule this file defends hardest is *null in, null out*. A missing sensor rendered as 0 +is a dashboard that says "the GPU is idle" when it means "the driver said nothing", and on +a machine you are debugging at 2am that difference is the whole point. +""" +from __future__ import annotations + +import copy +import math +from typing import Any + +import pytest + +from dashboards.rates import RateEstimator +from dashboards.sources.host import HostCollector, fleet_summary, parse_host_snapshot + +TICKS_PER_SECOND = 100.0 + + +def make_collector() -> HostCollector: + return HostCollector(RateEstimator(span=60.0)) + + +def advance(snapshot: dict[str, Any], dt: float = 10.0, *, total_ticks: float = 1000.0, + idle_ticks: float = 900.0, read_bytes: float = 0.0, write_bytes: float = 0.0, + rx_bytes: float = 0.0, tx_bytes: float = 0.0) -> dict[str, Any]: + """A second scrape `dt` later with counters moved by known amounts.""" + snap = copy.deepcopy(snapshot) + snap["ts"] = float(snapshot["ts"]) + dt + modes = snap["cpu"]["modes"] + idle_parts = {"idle": idle_ticks * 0.9, "iowait": idle_ticks * 0.1} + for key, value in modes.items(): + if key == "total": + modes["total"] = value + total_ticks + elif key in idle_parts: + modes[key] = value + idle_parts[key] + elif key != "guest" and key != "guest_nice": + modes[key] = value + (total_ticks - idle_ticks) / 5 + for device in snap["disk_io"]["devices"]: + device["read_bytes_total"] += read_bytes + device["write_bytes_total"] += write_bytes + device["reads_total"] += read_bytes / 4096 + device["writes_total"] += write_bytes / 4096 + for iface in snap["network"]["interfaces"]: + iface["rx_bytes_total"] += rx_bytes + iface["tx_bytes_total"] += tx_bytes + return snap + + +def parse_twice(agent_snapshot) -> tuple[dict[str, Any], dict[str, Any]]: + collector = make_collector() + first = parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0) + second_snap = advance(agent_snapshot, 10.0, read_bytes=4_096_000, write_bytes=2_048_000, + rx_bytes=100_000, tx_bytes=50_000) + second = parse_host_snapshot("driveripper", second_snap, collector, now=1_010.0) + return first.payload, second.payload + + +def test_first_scrape_has_no_rates_but_reports_instantaneous_values(agent_snapshot) -> None: + collector = make_collector() + payload = parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0).payload + assert payload["cpu"]["usage_pct"] is None, "one sample cannot make a rate" + assert payload["cpu"]["load1"] is not None, "load is a gauge, available immediately" + assert payload["memory"]["used_pct"] is not None + assert payload["disks"]["read_bytes_per_s"] is None + assert payload["network"]["rx_bytes_per_s"] is None + assert payload["window_seconds"] is None + + +def test_second_scrape_measures_the_window_it_claims(agent_snapshot) -> None: + """Two scrapes are enough — a dashboard that needs three looks to say anything about + CPU is a dashboard whose first minute of history is a hole.""" + _first, payload = parse_twice(agent_snapshot) + assert payload["window_seconds"] == 10.0 + usage = payload["cpu"]["usage_pct"] + assert usage is not None + # 1000 ticks total, 900 idle over 10s => exactly 10% busy, whatever the host was doing + assert usage == 10.0 + + +def test_iowait_counts_as_idle_but_is_still_reported(agent_snapshot) -> None: + _first, payload = parse_twice(agent_snapshot) + modes = payload["cpu"]["modes_pct"] + assert modes["iowait"] is not None and modes["iowait"] > 0 + assert sum(v for v in modes.values() if v is not None) <= 100.0001 + + +def test_per_core_series_is_one_point_per_logical_cpu(agent_snapshot) -> None: + collector = make_collector() + parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0) + payload = parse_host_snapshot("driveripper", advance(agent_snapshot, 10.0), collector, + now=1_010.0).payload + cores = payload["cpu"]["core_usage_pct"] + assert len(cores) == payload["info"]["cpu_cores_logical"] == 48 + assert all(value is None or 0.0 <= value <= 100.0 for value in cores) + + +def test_disk_and_network_rates_are_bytes_per_second_over_the_window(agent_snapshot) -> None: + _first, payload = parse_twice(agent_snapshot) + devices = payload["disks"]["devices"] + assert devices, "captured host has block devices" + # the fixture scrape added the same amount to every device/interface, so the per-device + # figure is delta/dt and the headline total is the sum of what the payload shows + assert devices[0]["read_bytes_per_s"] == 4_096_000 / 10.0 + assert payload["disks"]["read_bytes_per_s"] == sum( + d["read_bytes_per_s"] or 0.0 for d in devices) + assert payload["disks"]["write_bytes_per_s"] == sum( + d["write_bytes_per_s"] or 0.0 for d in devices) + interfaces = payload["network"]["interfaces"] + assert payload["network"]["rx_bytes_per_s"] == 100_000 / 10.0 * len(interfaces) + assert payload["network"]["tx_bytes_per_s"] == 50_000 / 10.0 * len(interfaces) + + +def test_idle_machine_reports_zero_not_none_once_measured(agent_snapshot) -> None: + """0 and n/a are different claims: 'nothing happened' vs 'we could not tell'.""" + collector = make_collector() + parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0) + quiet = advance(agent_snapshot, 10.0, total_ticks=1_000.0, idle_ticks=1_000.0) + payload = parse_host_snapshot("driveripper", quiet, collector, now=1_010.0).payload + assert payload["cpu"]["usage_pct"] == 0.0 + assert payload["disks"]["read_bytes_per_s"] == 0.0 + assert payload["network"]["rx_bytes_per_s"] == 0.0 + + +def test_missing_sensors_stay_missing(agent_snapshot) -> None: + snap = copy.deepcopy(agent_snapshot) + snap["temperatures"] = [] # the agent reports a flat sensor list + snap["cpu_temperature_celsius"] = None # ...plus its own pick for "the CPU temp" + snap["cpu"]["freq_mhz"] = None + for gpu in snap["gpus"]: + gpu["power_watts"] = None + gpu["temps"] = {} + collector = make_collector() + payload = parse_host_snapshot("driveripper", snap, collector, now=1_000.0).payload + assert payload["cpu"]["temp_c"] is None + assert payload["cpu"]["freq_mhz"] is None + assert payload["temperatures"]["max_c"] is None + assert payload["temperatures"]["sensors"] == [] + for gpu in payload["gpus"]: + assert gpu["power_watts"] is None + assert gpu["temps"] == {} + assert gpu["temp_c"] is None + + +def test_no_sample_is_ever_written_as_zero_for_a_missing_value(agent_snapshot) -> None: + snap = copy.deepcopy(agent_snapshot) + snap["cpu"]["freq_mhz"] = None + snap["gpus"] = [] + collector = make_collector() + result = parse_host_snapshot("driveripper", snap, collector, now=1_000.0) + for sample in result.samples: + assert sample.value is not None + assert math.isfinite(sample.value), f"{sample.metric} is {sample.value}" + assert not any(s.metric == "cpu.freq_mhz" for s in result.samples) + + +def test_gpu_pool_percent_accounts_for_vram_and_gtt(agent_snapshot) -> None: + """On an APU the model lives in GTT; reporting only vram says '32 MB used' on a 128 GB box.""" + collector = make_collector() + payload = parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0).payload + gpu = payload["gpus"][0] + assert gpu["model_full"] and gpu["model"] + assert gpu["vram_total_bytes"] > 0 + assert gpu["mem_used_pct"] is not None and 0.0 <= gpu["mem_used_pct"] <= 100.0 + assert "gtt_total_bytes" in gpu + + +def test_reboot_is_reported_as_an_event(agent_snapshot) -> None: + collector = make_collector() + parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0) + snap = advance(agent_snapshot, 10.0) + snap["host"]["boot_ts"] = float(agent_snapshot["host"]["boot_ts"]) + 5_000 + result = parse_host_snapshot("driveripper", snap, collector, now=1_010.0) + reboots = [e for e in result.events if e.kind == "reboot"] + assert len(reboots) == 1 + assert reboots[0].severity == "warn" + assert "rebooted" in reboots[0].message + + +def test_agent_errors_travel_to_the_ui(agent_snapshot) -> None: + snap = copy.deepcopy(agent_snapshot) + snap["errors"] = ["disk_io: permission denied reading /sys/block/sdb"] + payload = parse_host_snapshot("driveripper", snap, make_collector(), now=1_000.0).payload + assert payload["agent"]["errors"], "a partial snapshot must be labelled partial" + + +def test_counter_reset_from_a_reboot_does_not_chart_a_spike(agent_snapshot) -> None: + collector = make_collector() + parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0) + snap = copy.deepcopy(agent_snapshot) + snap["cpu"]["modes"]["total"] = 1.0 # counter restarted from zero + payload = parse_host_snapshot("driveripper", snap, collector, now=1_010.0).payload + assert payload["cpu"]["usage_pct"] is None + + +def test_fleet_summary_ignores_hosts_that_are_not_answering(agent_snapshot) -> None: + payload = parse_host_snapshot("driveripper", agent_snapshot, make_collector(), + now=1_000.0).payload + live = copy.deepcopy(payload) + live["cpu"]["usage_pct"] = 40.0 + live["info"]["cpu_cores_logical"] = 48 + dead = copy.deepcopy(payload) + dead["name"] = "deskwork" + dead["cpu"]["usage_pct"] = None + fleet = fleet_summary([live, dead]) + assert fleet["hosts_total"] == 2 + assert fleet["hosts_live"] == 1 + assert fleet["cpu_usage_pct_avg"] == 40.0, "a dead host must not drag the average to 0" + + +def test_fleet_summary_weights_cpu_by_cores(agent_snapshot) -> None: + base = parse_host_snapshot("driveripper", agent_snapshot, make_collector(), + now=1_000.0).payload + big = copy.deepcopy(base) + big["cpu"]["usage_pct"] = 10.0 + big["info"]["cpu_cores_logical"] = 100 + small = copy.deepcopy(base) + small["name"] = "gateway" + small["cpu"]["usage_pct"] = 90.0 + small["info"]["cpu_cores_logical"] = 2 + fleet = fleet_summary([big, small]) + assert fleet["cpu_usage_pct_avg"] == pytest.approx((10 * 100 + 90 * 2) / 102) + assert fleet["cpu_cores"] == 102 + + +def test_fleet_summary_worst_disk_and_memory_totals(agent_snapshot) -> None: + base = parse_host_snapshot("driveripper", agent_snapshot, make_collector(), + now=1_000.0).payload + fleet = fleet_summary([base, base]) + assert fleet["disk_worst_used_pct"] == pytest.approx(base["disks"]["worst_used_pct"]) + assert fleet["memory_total_bytes"] == pytest.approx(2 * base["memory"]["total_bytes"]) + assert fleet["memory_used_pct"] == pytest.approx( + base["memory"]["used_bytes"] / base["memory"]["total_bytes"] * 100) + + +def test_fleet_summary_of_nothing_is_empty_not_zero() -> None: + fleet = fleet_summary([]) + assert fleet["hosts_total"] == 0 + assert fleet["cpu_usage_pct_avg"] is None + assert fleet["memory_used_bytes"] is None + diff --git a/tests/test_llm_source.py b/tests/test_llm_source.py new file mode 100644 index 0000000..55534f1 --- /dev/null +++ b/tests/test_llm_source.py @@ -0,0 +1,267 @@ +"""LLM engine parsing: real captures, provenance on every number, absent never 0. + +The engines do not export the names people expect, so these tests are pinned to captures of +the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentation. +""" +from __future__ import annotations + +import re + +import pytest + +from dashboards.rates import RateEstimator +from dashboards.sources.llm import ( + engine_result, + fleet_totals, + parse_gufo_metrics, + parse_vllm_metrics, + series_names, +) + + +def bump_metric(text: str, name: str, delta: float) -> str: + """Return the exposition with one counter increased — a scrape `dt` later.""" + pattern = re.compile(rf"^({re.escape(name)}(?:\{{[^}}]*\}})?) (\S+)$", re.M) + + def replace(match: re.Match[str]) -> str: + return f"{match.group(1)} {float(match.group(2)) + delta}" + + bumped, count = pattern.subn(replace, text) + assert count, f"{name} not present in the capture" + return bumped + + +# ---- gufo / llama.cpp ----------------------------------------------------- # + +def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None: + metrics = parse_gufo_metrics(metrics_text("gufo")) + assert metrics["tokens_generated_total"]["value"] == 877_336.0 + assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0 + assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0 + assert metrics["requests_processing"]["value"] == 0.0 + assert metrics["device_lost_total"]["value"] == 0.0 + assert metrics["kv_cache_usage_ratio"]["value"] == 0.0 + + +def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None: + metrics = parse_gufo_metrics(metrics_text("gufo")) + accept = metrics["spec_accept_rate"]["value"] + assert accept == pytest.approx(578_514 / 789_028) + assert str(metrics["spec_accept_rate"]["source"]).startswith("computed") + + +def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None: + """gufo's prompt counter excludes cache hits; the lifetime total must not.""" + metrics = parse_gufo_metrics(metrics_text("gufo")) + total = metrics["tokens_total"]["value"] + assert total == 1_350_666 + 124_529_105 + 877_336 + + +def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None: + """llamacpp:predicted_tokens_seconds is ONE request's last pass — never the headline.""" + text = metrics_text("gufo") + rates = RateEstimator(span=10) + now = 1_000.0 + first = parse_gufo_metrics(text, rates, now) + assert "decode_tokens_per_second" not in first or first["decode_tokens_per_second"]["value"] is None + + text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0) + metrics = parse_gufo_metrics(text, rates, now + 10.0) + decode = metrics["decode_tokens_per_second"] + assert decode["value"] == pytest.approx(40.0) + assert decode["window"] == pytest.approx(10.0) + assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"] + # the native gauge is still there, labelled, because it answers a different question + assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597) + + +def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None: + text = metrics_text("gufo") + rates = RateEstimator(span=10) + parse_gufo_metrics(text, rates, 1_000.0) + text = bump_metric(text, "llamacpp:prompt_tokens_total", 1_000.0) + metrics = parse_gufo_metrics(text, rates, 1_010.0) + assert metrics["prompt_tokens_per_second"]["value"] == pytest.approx(100.0) + + +def test_gufo_counter_reset_produces_n_a_not_a_spike(metrics_text) -> None: + text = metrics_text("gufo") + rates = RateEstimator(span=60) + parse_gufo_metrics(text, rates, 1_000.0) + restarted = bump_metric(text, "llamacpp:tokens_predicted_total", -800_000.0) + metrics = parse_gufo_metrics(restarted, rates, 1_010.0) + assert metrics.get("decode_tokens_per_second", {}).get("value") is None + + +def test_absent_gufo_metrics_are_none_with_no_source() -> None: + metrics = parse_gufo_metrics("# nothing here\n") + assert metrics["requests_processing"]["value"] is None + assert metrics["requests_processing"]["source"] is None + assert metrics.get("spec_accept_rate", {}).get("value") is None + + +# ---- vLLM ----------------------------------------------------------------- # + +def test_vllm_counters_and_gauges(metrics_text) -> None: + metrics = parse_vllm_metrics(metrics_text("vllm")) + assert metrics["prompt_tokens_total"]["value"] == pytest.approx(1.8389909e7) + assert metrics["generation_tokens_total"]["value"] == pytest.approx(611_163.0) + usage = metrics["gpu_cache_usage"]["value"] + assert usage is None or 0.0 <= usage <= 1.0, "a ratio summed over label sets would exceed 1" + assert metrics["num_requests_running"]["value"] is not None + assert metrics["num_preemptions_total"]["value"] is not None + + +def test_vllm_latency_histograms(metrics_text) -> None: + metrics = parse_vllm_metrics(metrics_text("vllm")) + ttft = metrics["time_to_first_token_seconds"] + assert ttft and ttft["count"] > 0 and ttft["mean"] > 0 + assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"] + assert metrics["time_per_output_token_seconds"]["mean"] > 0 + + +def test_vllm_per_request_speed_is_the_reciprocal_of_tpot(metrics_text) -> None: + metrics = parse_vllm_metrics(metrics_text("vllm")) + tpot = metrics["time_per_output_token_seconds"]["mean"] + per_request = metrics["per_request_decode_tokens_per_second"] + assert per_request["value"] == pytest.approx(1.0 / tpot) + assert per_request["derived"] is True + + +def test_vllm_prefix_cache_hit_ratio(metrics_text) -> None: + metrics = parse_vllm_metrics(metrics_text("vllm")) + ratio = metrics["prefix_cache_hit_ratio"]["value"] + assert ratio is not None and 0.0 <= ratio <= 1.0 + assert "hits" in metrics["prefix_cache_hit_ratio"]["source"] + + +def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) -> None: + text = metrics_text("vllm") + rates = RateEstimator(span=10) + parse_vllm_metrics(text, rates, 2_000.0) + text = bump_metric(text, "vllm:generation_tokens_total", 250.0) + metrics = parse_vllm_metrics(text, rates, 2_010.0) + assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0) + + +def test_vllm_waiting_requests_are_broken_down_by_reason(metrics_text) -> None: + metrics = parse_vllm_metrics(metrics_text("vllm")) + reasons = metrics["waiting_by_reason"]["value"] + assert isinstance(reasons, dict) + + +def test_vllm_empty_exposition_is_all_none() -> None: + metrics = parse_vllm_metrics("") + assert metrics["num_requests_running"]["value"] is None + assert metrics["time_to_first_token_seconds"] is None + assert metrics["per_request_decode_tokens_per_second"]["value"] is None + + +# ---- Result assembly ------------------------------------------------------ # + +def test_engine_result_payload_and_samples(metrics_text) -> None: + rates = RateEstimator(span=10) + result = engine_result("gufo", "http://gufo.internal:8731", metrics_text("gufo"), rates, + now=3_000.0) + payload = result.payload + assert payload["engine"] == "gufo" + assert payload["upstream"] == "http://gufo.internal:8731" + assert payload["series_count"] > 10 + assert payload["unparsed"] == 0 + assert payload["sessions"]["detail"], "a missing /slots must be explained, not blank" + for sample in result.samples: + assert sample.source == "llm" and sample.host == "gufo" + assert sample.metric.startswith("llm.") + assert sample.value == sample.value and sample.value is not None # no NaN + + +def test_engine_result_charts_a_measured_rate_after_the_second_poll(metrics_text) -> None: + rates = RateEstimator(span=10) + engine_result("gufo", "http://g:1", metrics_text("gufo"), rates, now=3_000.0) + bumped = bump_metric(metrics_text("gufo"), "llamacpp:tokens_predicted_total", 100.0) + result = engine_result("gufo", "http://g:1", bumped, rates, now=3_010.0) + metrics = result.payload["metrics"] + assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(10.0) + assert any(s.metric == "llm.decode_tokens_per_second" for s in result.samples) + + +def test_sessions_pass_through_slots_for_gufo() -> None: + slots = [{"id": 0, "idle": False, "n_prompt": 12, "n_predict": 40}] + result = engine_result("gufo", "http://g:1", "", RateEstimator(), slots=slots, now=1.0) + sessions = result.payload["sessions"] + assert sessions["list"] == slots + assert sessions["source"] == "gufo /slots" + + +def test_sessions_say_vllm_has_no_per_session_view() -> None: + result = engine_result("vllm", "http://v:1", "", RateEstimator(), now=1.0) + sessions = result.payload["sessions"] + assert sessions["list"] == [] + assert "vLLM" in sessions["detail"] + + +# ---- fleet totals --------------------------------------------------------- # + +def _engine(name: str, **values) -> dict: + metrics = {key: {"value": value} for key, value in values.items()} + return {"engine": name, "metrics": metrics} + + +def test_fleet_totals_sum_only_engines_that_answered() -> None: + engines = [ + _engine("gufo", decode_tokens_per_second=40.0, prompt_tokens_per_second=10.0, + kv_cache_usage_ratio=0.25, requests_processing=1.0, requests_deferred=0.0, + tokens_total=100.0), + _engine("vllm", decode_tokens_per_second=10.0, prompt_tokens_per_second=5.0, + gpu_cache_usage=0.75, num_requests_running=2.0, num_requests_waiting=3.0, + tokens_total=200.0), + {}, # an engine whose poll failed + ] + totals = fleet_totals(engines) + assert totals["engines_total"] == 3 + assert totals["engines_live"] == 2 + assert totals["decode_tokens_per_second"] == 50.0 + assert totals["prompt_tokens_per_second"] == 15.0 + assert totals["requests_active"] == 3.0 + assert totals["requests_waiting"] == 3.0 + assert totals["kv_cache_usage_max"] == 0.75, "cache pressure is a max, never a sum" + assert totals["tokens_total"] == 300.0 + + +def test_fleet_totals_report_the_gufo_vllm_seam() -> None: + totals = fleet_totals([_engine("gufo"), _engine("vllm")]) + assert totals["seams"], "the two engines count prompt tokens differently; say so" + + +def test_fleet_totals_of_nothing_is_none_not_zero() -> None: + totals = fleet_totals([]) + assert totals["decode_tokens_per_second"] is None + assert totals["tokens_total"] is None + assert totals["engines_live"] == 0 + + +def test_series_names_are_the_parse_catalog() -> None: + assert "tokens_generated_total" in series_names("gufo") + assert "generation_tokens_total" in series_names("vllm") + assert "generation_tokens_total" not in series_names("gufo") + + +def test_gufo_a_measured_zero_is_not_replaced_by_the_gauge(metrics_text) -> None: + """`measured or gauge` made an idle engine advertise its last request's speed.""" + text = metrics_text("gufo") + rates = RateEstimator(span=10) + parse_gufo_metrics(text, rates, 1_000.0) + metrics = parse_gufo_metrics(text, rates, 1_010.0) # counters unchanged: truly idle + decode = metrics["decode_tokens_per_second"] + assert decode["value"] == 0.0 + assert "predicted_tokens_seconds" not in str(decode["source"]), "native gauge must not leak in" + + +def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None: + text = metrics_text("vllm") + rates = RateEstimator(span=10) + parse_vllm_metrics(text, rates, 2_000.0) + metrics = parse_vllm_metrics(text, rates, 2_010.0) + decode = metrics["decode_tokens_per_second"] + assert decode["value"] == 0.0 + assert "running/tpot" not in str(decode["source"]) diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..fc5033b --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,85 @@ +"""Startup wiring: `python -m dashboards` / the `dashboards` console script. + +Two things are worth pinning: the reload path must hand uvicorn an import string (an +app object cannot be reloaded), and debugpy must stay completely inert unless asked. +""" +from __future__ import annotations + +import sys +from types import ModuleType +from typing import Any + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from dashboards import __main__ as entry +from dashboards import debug as debug_module +from dashboards.config import Settings + + +class FakeUvicorn(ModuleType): + def __init__(self) -> None: + super().__init__("uvicorn") + self.calls: list[tuple[Any, dict[str, Any]]] = [] + + def run(self, target: Any, **kwargs: Any) -> None: + self.calls.append((target, kwargs)) + + +@pytest.fixture +def fake_uvicorn(monkeypatch: pytest.MonkeyPatch) -> FakeUvicorn: + module = FakeUvicorn() + monkeypatch.setitem(sys.modules, "uvicorn", module) + return module + + +def test_main_serves_the_assembled_app(fake_uvicorn: FakeUvicorn, tmp_path: Any, + monkeypatch: pytest.MonkeyPatch) -> None: + # Settings is frozen on purpose, so the module attribute is replaced, not poked. The DB + # path is private: a test must not open the developer's running database. + monkeypatch.setattr(entry, "SETTINGS", + Settings(host="0.0.0.0", port=8090, db_path=str(tmp_path / "main.sqlite3"))) + monkeypatch.delenv("DASH_RELOAD", raising=False) + entry.main() + (target, kwargs), = fake_uvicorn.calls + assert kwargs == {"host": "0.0.0.0", "port": 8090} + # FastAPI keeps included routers in a private wrapper, so the honest check is a request + assert isinstance(target, FastAPI) + with TestClient(target) as client: + assert client.get("/api/health").status_code == 200 + assert client.get("/ups").status_code == 200 + + +def test_reload_mode_passes_an_import_string(fake_uvicorn: FakeUvicorn, tmp_path: Any, + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(entry, "SETTINGS", Settings(db_path=str(tmp_path / "reload.sqlite3"))) + # uvicorn cannot reload an object it was handed, only a module:app string + monkeypatch.setenv("DASH_RELOAD", "1") + entry.main() + (target, kwargs), = fake_uvicorn.calls + assert target == "dashboards.app:app" + assert kwargs["reload"] is True + + +def test_debugpy_is_inert_by_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DEBUGPY", raising=False) + assert debug_module.maybe_listen() is False + + +def test_debugpy_listens_when_asked(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, int]] = [] + fake = ModuleType("debugpy") + fake.listen = lambda addr: calls.append(addr) # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "debugpy", fake) + monkeypatch.setenv("DEBUGPY", "yes") + monkeypatch.setenv("DEBUGPY_PORT", "5699") + assert debug_module.maybe_listen() is True + assert calls == [("0.0.0.0", 5699)] + + +def test_debugpy_missing_is_a_message_not_a_crash(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DEBUGPY", "1") + # a None entry makes `import debugpy` raise ImportError, as it does in a prod venv + monkeypatch.setitem(sys.modules, "debugpy", None) # type: ignore[arg-type] + assert debug_module.maybe_listen() is False diff --git a/tests/test_nut.py b/tests/test_nut.py new file mode 100644 index 0000000..4c71107 --- /dev/null +++ b/tests/test_nut.py @@ -0,0 +1,486 @@ +"""NUT: the wire protocol, the health rules, and the gates around the only write path. + +The protocol tests run against a fake upsd over a real socket — this client speaks a +line protocol with quoting rules, and a mock that agrees with my memory of it proves +nothing. The health rules and the command gates are pure functions and are asserted +directly, because those are what decide whether a battery test gets sent. +""" +from __future__ import annotations + +import asyncio + +import pytest +import pytest_asyncio + +from dashboards.sources import nut +from dashboards.sources.nut import NutClient, NutError, UpsReading, normalise_celsius +from dashboards.sources.nut_sim import NutSimulator + +VAR_ROWS = { + "battery.charge": "100", + "battery.runtime": "3720", + "battery.voltage": "27.3", + "battery.temperature": "300.4", # kelvin, as several drivers report it + "battery.date": "2021/04/01", + "battery.type": "Pb", + "input.voltage": "121.8", + "input.voltage.nominal": "120", + "input.transfer.low": "92", + "input.transfer.high": "139", + "output.voltage": "120.9", + "ups.load": "34", + "ups.realpower.nominal": "900", + "ups.load.high": "1500", + "ups.status": "OL CHRG", + "ups.mfr": "Cyber Power Systems", + "ups.model": "PR1500LCDRT2U", + "battery.test.result": "PASSED", + "battery.test.date": "2026-09-01", +} + +CMDS = ["beeper.mute", "test.panel.start", "test.battery.start.quick", + "test.battery.start.deep", "load.off"] + + +class FakeUpsd: + """Just enough of upsd's protocol to exercise the client, over a real socket.""" + + def __init__(self, *, require_auth: bool = False, close_early: bool = False, + hang: bool = False) -> None: + self.require_auth = require_auth + self.close_early = close_early + self.hang = hang + self.received: list[str] = [] + self.authenticated = False + self._server: asyncio.AbstractServer | None = None + + async def start(self) -> int: + self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0) + await self._server.start_serving() + return int(self._server.sockets[0].getsockname()[1]) + + async def stop(self) -> None: + if self._server: + self._server.close() + await self._server.wait_closed() + + async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + if self.close_early: + writer.close() + return + while True: + raw = await reader.readline() + if not raw: + break + line = raw.decode().strip() + self.received.append(line) + verb, _, rest = line.partition(" ") + await self._respond(writer, verb, rest) + + async def _respond(self, writer: asyncio.StreamWriter, verb: str, rest: str) -> None: + def send(text: str) -> None: + writer.write((text + "\n").encode()) + + if verb == "LIST": + kind, _, target = rest.partition(" ") + if kind == "UPS": + send("BEGIN LIST UPS") + send('UPS "cyberpower" "Cyber Power Systems PR1500LCDRT2U"') + send("END LIST UPS") + elif kind == "VAR": + send("BEGIN LIST VAR") + for key, value in VAR_ROWS.items(): + send(f'VAR "{_strip(target)}" "{key}" "{value}"') + send("END LIST VAR") + elif kind == "CMD": + send("BEGIN LIST CMD") + for command in CMDS: + send(f'CMD "{_strip(target)}" "{command}"') + send("END LIST CMD") + elif kind == "RW": + send("BEGIN LIST RW") + send(f'RW "{_strip(target)}" "battery.energysave" "0"') + send("END LIST RW") + else: + send("ERR UNKNOWN-COMMAND") + elif verb == "USER": + self.user = rest + send("OK") + elif verb == "PASS": + self.authenticated = not self.require_auth or getattr(self, "user", "") == "monitor" + send("OK" if self.authenticated else "ERR ACCESS-DENIED") + elif verb == "INSTCMD": + if self.require_auth and not self.authenticated: + send("ERR ACCESS-DENIED") + elif self.hang: + await asyncio.sleep(30) + else: + send("OK Send shorthand command successful") + elif verb == "LOGOUT": + send("OK Goodbye") + else: + send("ERR UNKNOWN-COMMAND") + + +def _strip(value: str) -> str: + return value.strip().strip('"') + + +@pytest_asyncio.fixture +async def upsd(): + server = FakeUpsd() + port = await server.start() + yield server, port + await server.stop() + + +async def test_list_ups_parses_names_and_descriptions(upsd) -> None: + _server, port = upsd + async with NutClient("127.0.0.1", port) as client: + ups_list = await client.list_ups() + assert ups_list[0]["name"] == "cyberpower" + assert ups_list[0]["description"] == "Cyber Power Systems PR1500LCDRT2U" + + +async def test_list_vars_keeps_spaces_and_quotes(upsd) -> None: + _server, port = upsd + async with NutClient("127.0.0.1", port) as client: + variables = await client.list_vars("cyberpower") + assert variables["ups.mfr"] == "Cyber Power Systems" + assert variables["battery.charge"] == "100" + assert variables["ups.status"] == "OL CHRG" + + +async def test_list_cmds_returns_what_the_driver_offers(upsd) -> None: + _server, port = upsd + async with NutClient("127.0.0.1", port) as client: + commands = await client.list_cmds("cyberpower") + assert commands == CMDS + + +async def test_list_rw_vars_names_the_variables_not_the_ups(upsd) -> None: + """`RW `: reading field 1 returns the UPS name as a writable variable.""" + _server, port = upsd + async with NutClient("127.0.0.1", port) as client: + rw = await client.list_rw_vars("cyberpower") + assert rw == ["battery.energysave"] + + +async def test_instcmd_without_credentials_refuses_locally(upsd) -> None: + """No socket write: a dashboard must not even try when it knows it cannot authorise.""" + _server, port = upsd + async with NutClient("127.0.0.1", port) as client: + with pytest.raises(NutError) as excinfo: + await client.instcmd("cyberpower", "test.panel.start") + assert excinfo.value.code == "ACCESS-DENIED" + assert "INSTCMD" not in _server.received or not any("INSTCMD" in line for line in _server.received) + + +async def test_instcmd_with_credentials_succeeds() -> None: + server = FakeUpsd(require_auth=True) + port = await server.start() + try: + async with NutClient("127.0.0.1", port, username="monitor", password="pw") as client: + assert await client.login() is True + reply = await client.instcmd("cyberpower", "test.panel.start") + assert "OK" in reply + assert any(line.startswith("INSTCMD") for line in server.received) + finally: + await server.stop() + + +async def test_upsd_rejection_carries_the_error_code() -> None: + server = FakeUpsd(require_auth=True) + port = await server.start() + try: + async with NutClient("127.0.0.1", port, username="nobody", password="pw") as client: + with pytest.raises(NutError) as excinfo: + await client.login() + assert excinfo.value.code == "ACCESS-DENIED" + assert "upsd.users" in str(excinfo.value) or "refused" in str(excinfo.value) or str(excinfo.value) + finally: + await server.stop() + + +async def test_connection_refused_is_reported_as_unreachable() -> None: + """A closed port must read as 'upsd is not answering', never as an empty UPS list.""" + client = NutClient("127.0.0.1", 1, timeout=1.0) + with pytest.raises(NutError) as excinfo: + await client.connect() + assert excinfo.value.code == "UNREACHABLE" + + +async def test_server_closing_early_is_reported() -> None: + server = FakeUpsd(close_early=True) + port = await server.start() + try: + async with NutClient("127.0.0.1", port, timeout=2.0) as client: + with pytest.raises(NutError) as excinfo: + await client.list_ups() + assert excinfo.value.code == "CLOSED" + finally: + await server.stop() + + +async def test_a_silent_upsd_times_out_instead_of_hanging() -> None: + server = FakeUpsd(hang=True) + port = await server.start() + try: + async with NutClient("127.0.0.1", port, timeout=0.3) as client: + with pytest.raises(NutError) as excinfo: + await client._send("INSTCMD cyberpower beeper.mute") + await client._read_line() + assert excinfo.value.code == "TIMEOUT" + finally: + await server.stop() + + +# ---- reading semantics ---------------------------------------------------- # + +def reading(**overrides: str) -> UpsReading: + values = dict(VAR_ROWS) + values.update(overrides) + return UpsReading(name="cyberpower", description="Cyber Power PR1500LCDRT2U", + vars=values, commands=list(CMDS)) + + +def test_status_flags_and_state_chips() -> None: + ups = reading() + assert ups.status_flags == ["OL", "CHRG"] + labels = {state["label"] for state in ups.states()} + assert "online" in labels and "charging" in labels + assert all(state["severity"] in ("ok", "warn", "fail", "info") for state in ups.states()) + + +def test_unknown_status_flag_is_shown_not_dropped() -> None: + ups = reading(**{"ups.status": "OL OFF"}) + ids = {state["id"] for state in ups.states()} + assert "OFF" in ids, "an unrecognised-in-combination flag must still be shown, not swallowed" + + +def test_on_battery_is_derived_from_the_flag_not_from_charge() -> None: + assert reading().on_battery is False + assert reading(**{"ups.status": "OB"}).on_battery is True + + +def test_alarms_are_split_and_cleaned() -> None: + ups = reading(**{"ups.alarm": "BATT.VERIFYFAIL, OVERLOAD "}) + assert ups.alarms == ["BATT.VERIFYFAIL", "OVERLOAD"] + + +def test_number_returns_none_for_absent_and_non_numeric() -> None: + ups = reading() + assert ups.number("battery.charge") == 100.0 + assert ups.number("does.not.exist") is None + assert ups.number("ups.status") is None + + +@pytest.mark.parametrize("raw,expected", [ + (None, None), (27.4, 27.4), (300.4, 27.25), (150.0, 150.0), (151.0, -122.15), +]) +def test_kelvin_is_detected_because_the_driver_cannot_be_trusted(raw, expected) -> None: + result = normalise_celsius(raw) + assert result == (None if expected is None else pytest.approx(expected, abs=0.01)) + + +# ---- health assessment ---------------------------------------------------- # + +def test_a_healthy_ups_reports_every_check_as_ok() -> None: + health = nut.assess(reading(**{"battery.temperature": "27.4", "battery.mfr.date": "2026-01-01", + "ups.status": "OL"}), now=1_767_225_600.0) + assert health["overall"] == "healthy" + assert health["counts"]["fail"] == 0 and health["counts"]["warn"] == 0 + assert any(check["name"] == "Power path" for check in health["checks"]) + + +def test_on_battery_with_little_runtime_is_a_failure() -> None: + health = nut.assess(reading(**{"ups.status": "OB", "battery.charge": "18", + "battery.runtime": "90", + "battery.temperature": "27.4"}), now=1_767_225_600.0) + assert health["overall"] == "at risk" + runtime = next(c for c in health["checks"] if c["name"] == "Runtime estimate") + assert runtime["status"] in ("warn", "fail") + + +def test_old_battery_is_warned_about_with_the_actual_age() -> None: + health = nut.assess(reading(**{"battery.temperature": "27.4", "battery.mfr.date": "2019-01-01"}), + now=1_767_225_600.0) + age = next(c for c in health["checks"] if c["name"] == "Battery age") + assert age["status"] == "warn" + assert "2019" in age["detail"] or "7" in age["detail"] + + +def test_a_kelvin_temperature_is_not_reported_as_a_melting_battery() -> None: + health = nut.assess(reading(), now=1_767_225_600.0) # 300.4 in the fixture + temp = next(c for c in health["checks"] if c["name"] == "Battery temperature") + assert temp["status"] == "ok" + assert "kelvin" in temp["detail"] + assert 20 < temp["value"] < 35 + + +def test_a_hot_battery_fails() -> None: + health = nut.assess(reading(**{"battery.temperature": "52"}), now=1_767_225_600.0) + temp = next(c for c in health["checks"] if c["name"] == "Battery temperature") + assert temp["status"] == "fail" + + +def test_a_failed_self_test_is_surfaced() -> None: + health = nut.assess(reading(**{"battery.test.result": "FAILED", "battery.temperature": "27.4"}), + now=1_767_225_600.0) + check = next(c for c in health["checks"] if c["name"] == "Last self-test") + assert check["status"] in ("warn", "fail") + + +def test_missing_test_history_is_unknown_not_ok() -> None: + values = {key: value for key, value in VAR_ROWS.items() + if not key.startswith("battery.test")} + values["battery.temperature"] = "27.4" + health = nut.assess(nut.UpsReading(name="cyberpower", vars=values, commands=[]), + now=1_767_225_600.0) + check = next(c for c in health["checks"] if c["name"] == "Last self-test") + assert check["status"] == "unknown" + + +def test_alarms_drive_the_overall_verdict() -> None: + health = nut.assess(reading(**{"ups.alarm": "BATT.FAIL", "battery.temperature": "27.4"}), + now=1_767_225_600.0) + assert health["overall"] in ("degraded", "at risk") + + +# ---- command gating ------------------------------------------------------- # + +def base_commands() -> list[str]: + return list(CMDS) + + +def by_name(items: list[dict], command: str) -> dict: + return next(item for item in items if item["command"] == command) + + +def test_commands_are_disabled_when_actions_are_off() -> None: + items = nut.available_commands(base_commands(), allow_actions=False, read_only=False, has_credentials=True, + on_battery=False) + deep = by_name(items, "test.battery.start.deep") + assert deep["enabled"] is False + assert "NUT_ALLOW_ACTIONS" in deep["reason"] + + +def test_commands_are_disabled_without_credentials_that_have_actions() -> None: + items = nut.available_commands(base_commands(), allow_actions=True, read_only=False, has_credentials=False, + on_battery=False) + assert by_name(items, "beeper.mute")["enabled"] is False + assert "upsd.users" in by_name(items, "beeper.mute")["reason"] + + +def test_dangerous_commands_are_blocked_while_on_battery() -> None: + """A deep test while already on battery can end the runtime early — that is the whole + thing the UPS is protecting against, so the UI must not offer it.""" + items = nut.available_commands(base_commands(), allow_actions=True, read_only=False, has_credentials=True, + on_battery=True) + deep = by_name(items, "test.battery.start.deep") + assert deep["enabled"] is False + assert "battery" in deep["reason"].lower() + safe = by_name(items, "beeper.mute") + assert safe["enabled"] is True + + +def test_read_only_overrides_everything_else() -> None: + items = nut.available_commands(base_commands(), allow_actions=True, read_only=True, + has_credentials=True, on_battery=False) + assert items and all(item["enabled"] is False for item in items) + assert any("DASH_READ_ONLY" in (item["reason"] or "") for item in items) + + +def test_risk_levels_are_labelled_per_command() -> None: + items = nut.available_commands(base_commands(), allow_actions=True, read_only=False, has_credentials=True, + on_battery=False) + assert by_name(items, "beeper.mute")["risk"] == nut.RISK_SAFE + assert by_name(items, "test.battery.start.deep")["risk"] == nut.RISK_DANGEROUS + assert by_name(items, "load.off")["risk"] == nut.RISK_DANGEROUS + + +def test_commands_the_driver_does_not_advertise_are_never_offered() -> None: + items = nut.available_commands(["beeper.mute"], allow_actions=True, read_only=False, has_credentials=True, + on_battery=False) + assert [item["command"] for item in items] == ["beeper.mute"] + + +def test_commands_outside_the_risk_catalog_are_never_offered() -> None: + """A driver can advertise anything; the dashboard only sends what it has a risk label for.""" + items = nut.available_commands(["beeper.mute", "something.new"], allow_actions=True, + read_only=False, has_credentials=True, on_battery=False) + assert [item["command"] for item in items] == ["beeper.mute"] + + +def test_command_advice_catalog_is_complete_for_the_ui() -> None: + advice = nut.command_advice("test.battery.start.deep") + assert advice and advice["risk"] == nut.RISK_DANGEROUS + assert advice["label"] and advice["help"] + assert nut.command_advice("not.a.command") is None + + +# ---- payload & simulator -------------------------------------------------- # + +def test_payload_carries_numbers_health_and_what_the_driver_offers() -> None: + payload = nut.payload(reading(), now=1_767_225_600.0) + assert payload["name"] == "cyberpower" + assert payload["identity"]["model"] == "PR1500LCDRT2U" + assert payload["numbers"]["charge_pct"] == 100.0 + assert payload["numbers"]["battery_temperature"] == pytest.approx(27.25, abs=0.01) + assert payload["health"]["overall"] in ("healthy", "degraded", "at risk") + assert payload["advertised_commands"] == CMDS + assert payload["on_battery"] is False + # the raw dump keeps the original kelvin string: normalising for charts must not + # destroy the value the operator would compare against `upsc` + assert payload["extra"] or payload["status"] == "OL CHRG" + + +def test_history_samples_only_include_real_numbers() -> None: + values = {key: value for key, value in VAR_ROWS.items() if key != "battery.runtime"} + samples = nut.history_samples(nut.UpsReading(name="cyberpower", vars=values, commands=[])) + assert "ups.charge_pct" in samples + assert "ups.battery_runtime_seconds" not in samples + assert all(isinstance(value, float) and value == value for value in samples.values()) + + +def test_history_samples_normalise_the_temperature() -> None: + samples = nut.history_samples(reading()) + assert samples["ups.battery_temperature_c"] == pytest.approx(27.25, abs=0.01) + + +async def test_simulator_reads_like_upsd_and_is_labelled() -> None: + readings = await NutSimulator().read() + assert readings, "the simulator must present at least one UPS" + for ups in readings: + assert "(simulated)" in ups.description, "simulated data must be identifiable" + assert ups.vars["battery.charge"] + assert "test.battery.start.deep" in ups.commands + + +async def test_simulator_commands_change_the_model_like_a_real_driver() -> None: + sim = NutSimulator() + name = (await sim.read())[0].name + assert "OK" in sim.command(name, "test.battery.start.quick") + after = (await sim.read())[0] + assert after.on_battery is True, "a quick test puts the load on battery" + assert "OK" in sim.command(name, "test.battery.stop") + assert (await sim.read())[0].on_battery is False + + +async def test_simulator_discharges_while_on_battery() -> None: + """A simulated UPS that never drains cannot exercise a runtime or health path.""" + sim = NutSimulator() + name = (await sim.read())[0].name + before = float((await sim.read())[0].vars["battery.charge"]) + sim.command(name, "test.battery.start.deep") + ups = next(item for item in sim.ups if item.name == name) + ups.on_battery = True + for _ in range(30): + ups.step(60.0) # half an hour of model time on battery + assert ups.charge < before + + +async def test_simulator_command_for_unknown_ups_raises() -> None: + sim = NutSimulator() + with pytest.raises(KeyError): + sim.command("nope", "beeper.mute") diff --git a/tests/test_poller.py b/tests/test_poller.py new file mode 100644 index 0000000..71e8526 --- /dev/null +++ b/tests/test_poller.py @@ -0,0 +1,217 @@ +"""The poll loop: isolation, last-good-value, and events that mark transitions only. + +These three behaviours are why the dashboard can be trusted when something is broken, so +they are asserted directly rather than through HTTP: + +* a source that throws must not take the loop, the hub, or its neighbours down; +* a failed poll keeps the previous payload and is marked stale — never zeroed; +* ok→fail and fail→ok each write exactly one event; 500 more failures write nothing; +* maintenance failing is logged, because a silent rollup failure means every long-window + chart is quietly empty (that bug is what the logging was added for). +""" +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import Iterator +from typing import Any + +import pytest + +from dashboards.poller import Hub, Job, Result +from dashboards.rates import RateEstimator +from dashboards.store import Event, Sample, SampleStore + + +@pytest.fixture +def store(tmp_path: Any) -> Iterator[SampleStore]: + instance = SampleStore(tmp_path / "poller.sqlite3") + try: + yield instance + finally: + instance.close() + + +def ok_result(value: float = 1.0) -> Result: + return Result(payload={"value": value}, + samples=[Sample("host", "gw", "cpu.usage_pct", value, time.time())]) + + +async def failing() -> Result: + raise ConnectionError("agent is down") + + +async def test_a_successful_poll_stores_payload_samples_and_health(store) -> None: + job = Job("host:gw", "system", "gw", 0.5, lambda: asyncio.sleep(0, ok_result(7.0))) + outcome = await job.run_once(store) + assert outcome is True, job.state.error + assert job.state.ok is True and job.state.error is None + assert job.state.payload == {"value": 7.0} + assert job.state.successes == 1 and job.state.samples_written == 1 + assert job.state.latency_ms is not None + assert store.recent_events(limit=5) == [], "a first successful poll is not a recovery" + + +async def test_a_failing_poll_keeps_the_last_good_payload(store) -> None: + """The UI shows `stale 12s` against real numbers; it must not show zeros or n/a.""" + job = Job("host:gw", "system", "gw", 0.5, lambda: asyncio.sleep(0, ok_result(7.0))) + await job.run_once(store) + job.fn = failing + assert await job.run_once(store) is False + assert job.state.ok is False + assert job.state.payload == {"value": 7.0}, "a dead host must not blank its last values" + assert job.state.error == "ConnectionError: agent is down" + assert job.state.as_json()["age_seconds"] is not None + + +async def test_transitions_write_one_event_each_and_spam_writes_none(store) -> None: + job = Job("host:gw", "system", "gw", 0.5, lambda: asyncio.sleep(0, ok_result()), + grace_failures=1) + await job.run_once(store) + job.fn = failing + for _ in range(5): + await job.run_once(store) + events = store.recent_events(limit=50) + down = [event for event in events if event.kind == "unreachable"] + assert len(down) == 1, "five consecutive failures must not log five events" + assert down[0].host == "gw", "events carry the display label, not the internal job key" + assert down[0].severity == "error" + + job.fn = lambda: asyncio.sleep(0, ok_result()) + await job.run_once(store) + events = store.recent_events(limit=50) + up = [event for event in events if event.kind == "reachable"] + assert len(up) == 1 + assert up[0].message == "gw is answering again after 5 failures" + + +async def test_one_failure_reads_as_a_failure_not_failures(store) -> None: + job = Job("host:gw", "system", "gw", 0.5, lambda: asyncio.sleep(0, ok_result())) + await job.run_once(store) + job.fn = failing + await job.run_once(store) + job.fn = lambda: asyncio.sleep(0, ok_result()) + await job.run_once(store) + up = [event for event in store.recent_events(limit=50) if event.kind == "reachable"] + assert up[0].message.endswith("after 1 failure") + + +async def test_grace_allows_a_blip_without_an_event(store) -> None: + """One failure inside the grace window must not page anybody.""" + job = Job("host:gw", "system", "gw", 0.5, failing, grace_failures=2) + await job.run_once(store) + assert store.recent_events(limit=10) == [] + await job.run_once(store) + events = store.recent_events(limit=10) + assert [event.kind for event in events] == ["unreachable"] + assert events[0].severity == "warn", "never answered is a config problem, not an outage" + assert "never answered" in events[0].message + for _ in range(4): + await job.run_once(store) + assert len(store.recent_events(limit=10)) == 1, "one event per outage, not per poll" + + +async def test_a_slow_poll_never_overlaps_itself(store) -> None: + calls = 0 + gate = asyncio.Event() + + async def slow() -> Result: + nonlocal calls + calls += 1 + await gate.wait() + return ok_result() + + job = Job("host:gw", "system", "gw", 0.5, slow) + first = asyncio.create_task(job.run_once(store)) + await asyncio.sleep(0.01) + assert await job.run_once(store) is False, "an overlapping poll double-counts rates" + gate.set() + assert await first is True + assert calls == 1 + + +async def test_a_counter_that_resets_is_reported_not_silently_negative() -> None: + """The reset path is what ships: an agent restart must not read as negative throughput.""" + rates = RateEstimator(span=10.0) + first, _ = rates.rate("cpu.total", 1000.0, now=100.0) + assert first is None, "one sample cannot produce a rate" + second, window = rates.rate("cpu.total", 2000.0, now=110.0) + assert second == pytest.approx(100.0) and window == pytest.approx(10.0) + after, _ = rates.rate("cpu.total", 5.0, now=120.0) # agent restarted + assert after is None, "a reset counter must read n/a, never as negative throughput" + + +async def test_the_hub_runs_every_job_and_stops_cleanly(store) -> None: + counts = {"a": 0, "b": 0} + + async def good(key: str) -> Result: + counts[key] += 1 + return ok_result() + + hub = Hub(store) + hub.add(Job("host:a", "system", "a", 0.5, lambda: good("a"))) + hub.add(Job("host:b", "system", "b", 0.5, failing)) + hub.add(Job("llm:off", "llm", "off", 0.5, failing, enabled=False)) + hub.start() + try: + await asyncio.sleep(1.2) + finally: + await hub.stop() + assert counts["a"] >= 2, "a neighbour that throws must not slow the healthy one" + states = {state["key"]: state for state in hub.states()} + assert states["host:a"]["ok"] is True + assert states["host:b"]["ok"] is False + assert states["llm:off"]["configured"] is False, "not configured is not the same as broken" + assert hub.jobs["host:b"].state.failures >= 1 + + +async def test_states_can_be_filtered_to_one_dashboard(store) -> None: + hub = Hub(store) + hub.add(Job("host:a", "system", "a", 0.5, lambda: asyncio.sleep(0, ok_result()))) + hub.add(Job("nut", "ups", "NUT server", 0.5, lambda: asyncio.sleep(0, ok_result()))) + assert [s["key"] for s in hub.states("ups")] == ["nut"] + + +async def test_maintenance_failure_is_logged_not_swallowed(store, monkeypatch, + caplog: pytest.LogCaptureFixture) -> None: + """A broken rollup means every chart past the raw window is empty. Retrying quietly + turned a data-loss bug into nothing at all — this is the regression that added the log.""" + def boom() -> dict[str, int]: + raise RuntimeError("no such table: sample_rollup") + + monkeypatch.setattr(store, "maintain", boom) + hub = Hub(store) + task = asyncio.create_task(hub._maintain_loop(interval=0.02)) + try: + with caplog.at_level(logging.ERROR, logger="dashboards.poller"): + await asyncio.sleep(0.15) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert "store maintenance failed" in caplog.text + assert "no such table" in caplog.text, "the log line must carry the real reason" + + +async def test_maintenance_runs_from_the_hub_and_records_its_pass(store) -> None: + hub = Hub(store) + calls = [] + store.maintain = lambda: calls.append(1) or {"to_minute": 0, "to_hour": 0, # type: ignore[method-assign] + "pruned": 0} + task = asyncio.create_task(hub._maintain_loop(interval=0.02)) + try: + await asyncio.sleep(0.1) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert calls, "the maintenance loop must actually call maintain" + + +async def test_events_written_by_a_job_land_in_the_store(store) -> None: + event = Event(1_700_000_000.0, "ups", "cyberpower", "status", "warn", "OL → OB") + job = Job("nut", "ups", "NUT server", 0.5, + lambda: asyncio.sleep(0, Result(payload={}, events=[event]))) + await job.run_once(store) + assert [e.message for e in store.recent_events(limit=5)] == ["OL → OB"] diff --git a/tests/test_prom.py b/tests/test_prom.py new file mode 100644 index 0000000..0bb16a8 --- /dev/null +++ b/tests/test_prom.py @@ -0,0 +1,112 @@ +"""The hand-rolled Prometheus parser, tested against real engine expositions. + +This parser exists so the app has no Prometheus dependency for the two upstreams that +already speak the format. That makes it load-bearing on text we do not control: llama.cpp +emits names with colons, vLLM emits label sets and histograms, and both are free to change +them. Fixtures are captured verbatim (scripts/capture-fixtures.sh --with-llm). +""" +from __future__ import annotations + +import math + +from dashboards.prom import Prom + + +def test_parses_counters_and_gauges_from_a_real_capture(metrics_text) -> None: + prom = Prom(metrics_text("gufo")) + assert prom.scalar(["llamacpp:tokens_predicted_total"]) == 877_336.0 + assert prom.scalar(["llamacpp:kv_cache_usage_ratio"]) == 0.0 + assert prom.parse_errors == 0 + + +def test_help_and_type_comments_are_not_samples(metrics_text) -> None: + prom = Prom(metrics_text("gufo")) + assert not any(name.startswith("#") for name in prom.series) + assert "# HELP llamacpp:prompt_tokens_total" not in prom.series + + +def test_labels_are_parsed_and_filterable(metrics_text) -> None: + prom = Prom(metrics_text("vllm")) + running = prom.values(["vllm:num_requests_running"]) + assert running, "vLLM exposes num_requests_running" + by_model = prom.by_label(["vllm:generation_tokens_total"], "model_name") + assert by_model and all(model for model in by_model) + + +def test_aggregations(metrics_text) -> None: + prom = Prom(metrics_text("vllm")) + total = prom.scalar(["vllm:prompt_tokens_total"], agg="sum") + assert total is not None and total > 1e6 + # ratios must never be summed across label sets — a 3-way sum would read as 300% used + usage = prom.scalar(["vllm:kv_cache_usage_perc"], agg="max") + assert usage is not None and 0.0 <= usage <= 1.0 + + +def test_missing_series_is_none_not_zero(metrics_text) -> None: + prom = Prom(metrics_text("gufo")) + assert prom.scalar(["vllm:does_not_exist"]) is None + assert prom.values(["vllm:does_not_exist"]) == [] + assert prom.by_label(["nope"], "model_name") == {} + + +def test_histogram_mean_and_p95(metrics_text) -> None: + hist = Prom(metrics_text("vllm")).histogram(["vllm:time_to_first_token_seconds"]) + assert hist is not None + assert hist["count"] and hist["count"] > 0 + assert hist["mean"] and hist["mean"] > 0 + assert hist["p95"] is not None + assert hist["p95"] >= hist["mean"], "p95 below the mean means the interpolation is wrong" + assert math.isfinite(hist["p95"]) + + +def test_histogram_resolves_through_suffixes(metrics_text) -> None: + prom = Prom(metrics_text("vllm")) + # the bare name never appears in the exposition; only _bucket/_sum/_count do + assert "vllm:time_to_first_token_seconds" not in prom.series + assert prom.resolve_hist(["vllm:time_to_first_token_seconds"]) == "vllm:time_to_first_token_seconds" + assert prom.resolve(["vllm:time_to_first_token_seconds"]) is None + + +def test_resolve_returns_the_first_present_name() -> None: + prom = Prom('a_total 5\nb_total 7\n') + assert prom.resolve(["missing_total", "b_total", "a_total"]) == "b_total" + assert prom.resolve(["nope"]) is None + + +def test_label_match_narrows_the_series() -> None: + prom = Prom('x{engine="0"} 1\nx{engine="1"} 2\n') + assert prom.scalar(["x"], agg="sum") == 3 + assert prom.scalar(["x"], label_match={"engine": "1"}) == 2 + assert prom.scalar(["x"], label_match={"engine": "9"}) is None + + +def test_malformed_lines_are_counted_not_fatal() -> None: + prom = Prom("good_metric 1\nthis is not prometheus\nalso_bad{unclosed 2\n") + assert prom.scalar(["good_metric"]) == 1.0 + assert prom.parse_errors >= 1 + + +def test_empty_exposition_is_harmless() -> None: + prom = Prom("") + assert prom.series == {} + assert prom.scalar(["anything"]) is None + assert prom.histogram(["anything"]) is None + + +def test_quantile_ignores_non_finite_values() -> None: + prom = Prom('m_sum 10\ncount 2\nm{le="1"} NaN\n') + assert prom.scalar(["m"], label_match={"le": "1"}) is None + + +def test_multiple_histogram_label_sets_do_not_invent_a_p95(metrics_text) -> None: + """Mixing per-DP-rank buckets into one cumulative curve would fabricate a percentile.""" + two = ( + 'h_sum{dp="0"} 10\nh_sum{dp="1"} 10\n' + 'h_count{dp="0"} 2\nh_count{dp="1"} 2\n' + 'h_bucket{dp="0",le="1"} 1\nh_bucket{dp="0",le="+Inf"} 2\n' + 'h_bucket{dp="1",le="1"} 1\nh_bucket{dp="1",le="+Inf"} 2\n' + ) + hist = Prom(two).histogram(["h"]) + assert hist is not None + assert hist["mean"] == 5.0 + assert hist["p95"] is None, "p95 across mixed label sets is a made-up number" diff --git a/tests/test_rates.py b/tests/test_rates.py new file mode 100644 index 0000000..ac32e72 --- /dev/null +++ b/tests/test_rates.py @@ -0,0 +1,95 @@ +"""Rate estimation over a labelled window. + +Every throughput number in this dashboard is Δcounter/Δt. If the estimator returns a +plausible-looking number for a counter that restarted, or a silent 0 for a scrape that +arrived late, the UI cannot tell the difference — so the None cases matter more here than +the happy path. +""" +from __future__ import annotations + +import math + +import pytest + +from dashboards.rates import RateEstimator + + +def test_first_sample_has_no_rate() -> None: + r = RateEstimator(span=10) + assert r.rate("cpu", 100.0, now=1_000.0) == (None, None) + + +def test_second_sample_gives_rate_and_window() -> None: + r = RateEstimator(span=10) + r.rate("cpu", 100.0, now=1_000.0) + rate, window = r.rate("cpu", 300.0, now=1_010.0) + assert rate == 20.0 + assert window == 10.0 + + +def test_window_is_labelled_with_the_span_actually_used() -> None: + r = RateEstimator(span=10) + r.rate("x", 0.0, now=0.0) + r.rate("x", 10.0, now=2.0) # inside the span + rate, window = r.rate("x", 30.0, now=11.0) + # base is the sample 11s ago (outside the span) -> falls back to the previous sample + assert window == 9.0 + assert rate == pytest.approx(20.0 / 9.0) + + +def test_none_value_does_not_poison_the_window() -> None: + r = RateEstimator(span=10) + r.rate("x", 10.0, now=0.0) + assert r.rate("x", None, now=1.0) == (None, None) + rate, _window = r.rate("x", 30.0, now=2.0) + assert rate == 10.0, "the None scrape must not become the baseline" + + +def test_nan_and_inf_are_treated_as_missing() -> None: + r = RateEstimator(span=10) + r.rate("x", 1.0, now=0.0) + for bad in (float("nan"), float("inf")): + assert r.rate("x", bad, now=1.0) == (None, None) + + +def test_counter_reset_yields_none_once_then_recovers() -> None: + """A restart must not report a huge negative — or, worse, a huge positive — rate.""" + r = RateEstimator(span=60) + r.rate("tokens", 5_000.0, now=0.0) + rate, _ = r.rate("tokens", 10.0, now=10.0) # process restarted + assert rate is None + rate, _ = r.rate("tokens", 60.0, now=20.0) + assert rate == 5.0, "the window re-seeds from the post-restart counter" + + +def test_flat_counter_reports_zero_not_none() -> None: + r = RateEstimator(span=10) + r.rate("idle", 5.0, now=0.0) + rate, window = r.rate("idle", 5.0, now=5.0) + assert rate == 0.0 and window == 5.0 + + +def test_keys_are_independent() -> None: + r = RateEstimator(span=10) + r.rate("a", 0.0, now=0.0) + r.rate("b", 100.0, now=0.0) + assert r.rate("a", 10.0, now=1.0)[0] == 10.0 + assert r.rate("b", 100.0, now=1.0)[0] == 0.0 + + +def test_forget_drops_a_removed_target() -> None: + r = RateEstimator(span=10) + for key in ("driveripper:cpu.total", "driveripper:disk.read", "deskwork:cpu.total"): + r.rate(key, 1.0, now=0.0) + assert r.forget("driveripper:") == 2 + assert r.keys() == ["deskwork:cpu.total"] + # and the surviving target is untouched + assert r.rate("deskwork:cpu.total", 11.0, now=1.0)[0] == 10.0 + + +def test_history_is_trimmed_to_max_samples() -> None: + r = RateEstimator(span=1e6, max_samples=8) + for i in range(50): + r.rate("x", float(i), now=float(i)) + assert len(r._samples["x"]) <= 8 # bounded memory is the contract + assert all(math.isfinite(v) for _, v in r._samples["x"]) diff --git a/tests/test_store.py b/tests/test_store.py new file mode 100644 index 0000000..ec4391d --- /dev/null +++ b/tests/test_store.py @@ -0,0 +1,254 @@ +"""History store: rollups, retention, and the honesty of an empty chart. + +The rollup queries are plain SQL with positional bindings, and a mismatch there used to +raise on every maintenance tick while the poller swallowed it — so every window past the +raw retention was silently empty. These tests fail loudly if that ever happens again. +""" +from __future__ import annotations + +import time +from typing import Any + +import pytest + +from dashboards.store import ( + HOUR, + MINUTE, + RES_HOUR, + RES_MIN, + RES_RAW, + Event, + Sample, + SampleStore, + samples_from_flat, + select_window, +) + + +def add_series(db, source: str, host: str, metric: str, values, *, start: float, step: float, + res: str = RES_RAW) -> None: + for i, value in enumerate(values): + # the private handle is the point: writing at an explicit resolution is what + # the resolution-choice tests need + db._db.execute( + "INSERT INTO sample (source, host, metric, res, ts, value) VALUES (?,?,?,?,?,?)", + (source, host, metric, res, start + i * step, float(value)), + ) + db._db.commit() + + +def test_store_starts_in_wal_mode(db) -> None: + """Readers poll while the writer commits; without WAL the API would hit locked errors.""" + assert db._db.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal" + + +def test_add_and_query_roundtrip(db) -> None: + now = time.time() + db.add("host", "driveripper", "cpu.usage_pct", 42.5, ts=now) + points = db.query("host", "driveripper", "cpu.usage_pct", start=now - 60) + assert points == [(now, 42.5)] + + +def test_query_is_bounded_and_ascending(db) -> None: + base = 1_700_000_000.0 + add_series(db, "host", "h", "m", [3, 1, 2], start=base, step=10) + assert db.query("host", "h", "m", start=base - 1) == [(base, 3.0), (base + 10, 1.0), (base + 20, 2.0)] + assert len(db.query("host", "h", "m", start=base - 1, limit=2)) == 2 + + +def test_query_tail_keeps_the_newest_rows(db) -> None: + base = 1_700_000_000.0 + add_series(db, "host", "h", "m", range(100), start=base, step=10) + head = db.query("host", "h", "m", start=base - 1, limit=3) + tail = db.query("host", "h", "m", start=base - 1, limit=3, tail=True) + assert [v for _, v in head] == [0.0, 1.0, 2.0] + assert [v for _, v in tail] == [97.0, 98.0, 99.0] + assert tail == sorted(tail), "tail must still come back oldest-first" + + +def test_samples_from_flat_drops_unknown_and_none() -> None: + payload: dict[str, Any] = {"a": 1.0, "b": None, "c": "nope"} + samples = samples_from_flat("host", "gw", payload, ts=5.0) + assert [(s.metric, s.value) for s in samples] == [("a", 1.0)] + + +def test_maintain_rolls_raw_into_minutes(db) -> None: + """Regression: the rollup statement had 7 placeholders and 6 bindings.""" + now = time.time() + # two full, aligned minute buckets — see the hour test for why alignment matters + minute = int(now // MINUTE) * MINUTE + add_series(db, "host", "h", "cpu.usage_pct", [10, 20, 30, 40], start=minute - 4 * MINUTE, + step=1) + add_series(db, "host", "h", "cpu.usage_pct", [1, 1, 1, 1], start=minute - 3 * MINUTE, step=1) + counts = db.maintain(now=now) + assert counts["to_minute"] > 0 + minutes = db.query("host", "h", "cpu.usage_pct", res=RES_MIN, start=now - 3600) + assert minutes, "no 1m rollup was written" + assert [round(value) for _ts, value in minutes] == [25, 1] + + +def test_maintain_is_idempotent(db) -> None: + """Rollups run on a timer: running twice must not double-count or duplicate rows.""" + now = time.time() + minute = int(now // MINUTE) * MINUTE + add_series(db, "host", "h", "m", [10, 20], start=minute - 4 * MINUTE, step=1) + first = db.maintain(now=now) + rows = db.query("host", "h", "m", res=RES_MIN, start=now - 3600) + count = db.count() + second = db.maintain(now=now) + assert rows == db.query("host", "h", "m", res=RES_MIN, start=now - 3600) + assert first["to_minute"] == second["to_minute"] + assert db.count() == count, "re-running maintenance duplicated samples" + + +def test_maintain_rolls_minutes_into_hours(db) -> None: + now = time.time() + # buckets are aligned to the epoch (`ts/3600*3600`), not to "hours from now", so the + # fixtures start on a boundary or the two hours mix into a third, averaged bucket + hour = int(now // HOUR) * HOUR + # two *complete* hours of minute buckets: the current and previous hour are still + # filling in, and maintenance must not touch them (see the partial-bucket test) + add_series(db, "host", "h", "m", [1] * 60, start=hour - 5 * HOUR, step=60, res=RES_MIN) + add_series(db, "host", "h", "m", [3] * 60, start=hour - 4 * HOUR, step=60, res=RES_MIN) + counts = db.maintain(now=now) + assert counts["to_hour"] > 0 + hours = db.query("host", "h", "m", res=RES_HOUR, start=now - 86_400) + assert sorted(round(v) for _, v in hours) == [1.0, 3.0] + + + +def test_maintain_prunes_expired_raw(db) -> None: + now = time.time() + old = now - (db.raw_keep_seconds + 7200) + add_series(db, "host", "h", "m", [1, 2], start=old, step=60) + add_series(db, "host", "h", "fresh", [1], start=now - 60, step=1) + counts = db.maintain(now=now) + assert counts["pruned"] >= 2 + assert db.query("host", "h", "m", start=0) == [] + + +def test_maintain_records_timestamp_in_stats(db) -> None: + db.maintain() + stats = db.stats() + assert stats["last_maintain"] is not None + assert "samples_by_res" in stats and "path" in stats + + +def test_pick_res_prefers_the_finest_resolution_that_fits(db) -> None: + now = time.time() + add_series(db, "host", "h", "m", list(range(500)), start=now - 3600, step=1) + assert db.pick_res("host", "h", "m", 3600, 600) == RES_RAW + add_series(db, "host", "h", "m", list(range(4000)), start=now - 7200, step=1) + assert db.pick_res("host", "h", "m", 7200, 600) == RES_RAW, "raw still fits" + add_series(db, "host", "h", "m", list(range(120)), start=now - 7200, step=60, + res=RES_MIN) + # 4500 raw points blow the 600 budget, so the minute rollup takes over + assert db.pick_res("host", "h", "m", 7200, 600) == RES_MIN + + +def test_pick_res_never_picks_an_empty_resolution(db) -> None: + """A fresh install has raw only: choosing 1h because it 'fits' would chart nothing.""" + now = time.time() + add_series(db, "host", "h", "m", [1] * 2000, start=now - 7200, step=1) + assert db.pick_res("host", "h", "m", 86_400, 600) == RES_RAW + + +def test_history_falls_back_to_raw_when_rollups_lag(db) -> None: + now = time.time() + add_series(db, "host", "h", "m", list(range(1500)), start=now - 1500, step=1) + points = db.history("host", "h", "m", span=3600, max_points=600) + assert len(points) > 0, "empty chart while raw data exists" + assert len(points) <= 600 + + +def test_history_always_keeps_the_newest_point(db) -> None: + now = time.time() + values = list(range(2000)) + add_series(db, "host", "h", "m", values, start=now - 2000, step=1) + points = db.history("host", "h", "m", span=3600, max_points=50) + assert points[-1][1] == pytest.approx(values[-1]) + + +def test_series_names_and_hosts(db) -> None: + db.add("host", "a", "cpu.usage_pct", 1.0) + db.add("host", "a", "mem.used_pct", 2.0) + db.add("host", "b", "cpu.usage_pct", 3.0) + db.add("llm", "gufo", "llm.decode_tokens_per_second", 4.0) + assert db.series_names("host", "a") == ["cpu.usage_pct", "mem.used_pct"] + assert db.hosts("host") == ["a", "b"] + assert db.series_names("llm") == ["llm.decode_tokens_per_second"] + + +def test_delete_series_cleans_up(db) -> None: + db.add("host", "a", "gone.metric", 1.0) + db.add("host", "a", "keep.metric", 2.0) + assert db.delete_series("host", "a", "gone.metric") == 1 + assert db.series_names("host", "a") == ["keep.metric"] + + +def test_events_roundtrip_and_filter(db) -> None: + now = time.time() + db.log_event(Event(ts=now, source="host", host="a", kind="down", severity="error", + message="a is not answering", data={"fails": 3})) + db.log_event(Event(ts=now + 1, source="nut", host="cyberpower", kind="state", + severity="warn", message="OL → OB")) + everything = db.recent_events(10) + assert [e.host for e in everything] == ["cyberpower", "a"] + assert everything[-1].data == {"fails": 3} + assert [e.source for e in db.recent_events(10, source="nut")] == ["nut"] + assert len(db.recent_events(1)) == 1 + # the API hands out plain JSON, so the boundary has to be serialisable + assert everything[0].as_json()["message"] == "OL → OB" + + +def test_event_as_json_is_serialisable(db) -> None: + event = Event(ts=1.0, source="host", host="a", kind="up", severity="info", message="ok") + payload = event.as_json() + assert payload["severity"] == "info" and payload["data"] is None + + +def test_count_filters_by_resolution(db) -> None: + now = time.time() + db.add("host", "h", "m", 1.0, ts=now) + add_series(db, "host", "h", "m", [2.0], start=now, step=1, res=RES_MIN) + assert db.count() == 2 + assert db.count(res=RES_RAW) == 1 + assert db.count(res=RES_HOUR) == 0 + + +def test_add_many_reports_inserted_rows(db) -> None: + now = time.time() + rows = [Sample("host", "h", f"m{i}", float(i), now) for i in range(5)] + assert db.add_many(rows) == 5 + assert db.count() == 5 + + +def test_store_reopens_without_losing_history(tmp_path) -> None: + path = tmp_path / "reopen.sqlite3" + first = SampleStore(path) + now = time.time() + first.add("host", "h", "m", 7.0, ts=now) + first.close() + second = SampleStore(path) + try: + assert second.query("host", "h", "m", start=now - 5) == [(now, 7.0)] + finally: + second.close() + + +def test_select_window_maps_range_onto_presets() -> None: + assert select_window(60, [300, 3600, 86_400]) == 300 + assert select_window(3600, [300, 3600, 86_400]) == 3600 + assert select_window(86_401, [300, 3600, 86_400]) == 86_400 + + +def test_rollup_does_not_average_a_partial_minute(db) -> None: + """A bucket is only rolled once its whole window is in the past, or it gets rewritten.""" + now = time.time() + add_series(db, "host", "h", "m", [1, 1, 1], start=now - 5, step=1) # current minute + db.maintain(now=now) + assert db.query("host", "h", "m", res=RES_MIN, start=now - 600) == [] + + +def test_hour_constant_is_used_in_retention(db) -> None: + assert db.raw_keep_seconds == pytest.approx(48 * HOUR) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..b0e308b --- /dev/null +++ b/uv.lock @@ -0,0 +1,869 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/55/d1eaf3e73781174340a00dc1ba2aee8a65f82fadb18e2797b192b6b3925b/coverage-7.16.2.tar.gz", hash = "sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa", size = 971999, upload-time = "2026-09-27T12:29:01.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/f6/8eb4f220ef24f84fb27d852d4f9bf83e0c73ec1a4a08dd9a87e3f4529739/coverage-7.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9", size = 223993, upload-time = "2026-09-27T12:26:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/d4bbaf0c154e0b0c2b5264890dbf6ef098dcb50ec8f2469be9490d191660/coverage-7.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b", size = 224368, upload-time = "2026-09-27T12:26:41.762Z" }, + { url = "https://files.pythonhosted.org/packages/7f/48/fc1e88fd571ec5cb38150b7f89f7696ca1bdf9920e01432febb69774cc85/coverage-7.16.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204", size = 255358, upload-time = "2026-09-27T12:26:43.442Z" }, + { url = "https://files.pythonhosted.org/packages/1d/56/6785397d07c29c8e70fbb9a07e97d062b43c21ffc5f12385917847f09f63/coverage-7.16.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5", size = 257955, upload-time = "2026-09-27T12:26:45.725Z" }, + { url = "https://files.pythonhosted.org/packages/27/3b/c8cdd07721e5f99abd81cea970d971997f99bf158c0b85f51bd284179c8b/coverage-7.16.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7", size = 259183, upload-time = "2026-09-27T12:26:47.208Z" }, + { url = "https://files.pythonhosted.org/packages/9b/11/606b192fe43d32574ec6238549d48de588fdcc18485682a5ec0a8ac357f2/coverage-7.16.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541", size = 261325, upload-time = "2026-09-27T12:26:49.084Z" }, + { url = "https://files.pythonhosted.org/packages/67/90/eea481f8b0305ceeb33f081a5f47e298391dbd1b589de0c4b3b3aa50d3f2/coverage-7.16.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc", size = 255531, upload-time = "2026-09-27T12:26:50.509Z" }, + { url = "https://files.pythonhosted.org/packages/6b/be/dedbf9aea1457b120c27ac10b8fc2a357f37fa2b54c3e7286d42980a0a2a/coverage-7.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967", size = 257329, upload-time = "2026-09-27T12:26:52.005Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cb/b25c19d5bb2bd0f2e4e27fe8e2ffcae80c7a91ae181c0dc749ed60e9b1a4/coverage-7.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7", size = 255294, upload-time = "2026-09-27T12:26:53.634Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a2/892c5c5f4ad44b7b2ca009aee705191f3f268f15052244f2f9e3539b2e35/coverage-7.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23", size = 259444, upload-time = "2026-09-27T12:26:55.243Z" }, + { url = "https://files.pythonhosted.org/packages/ed/99/a562537deba0a3e370182ae71c149be796c39d8087365f17a09188f27145/coverage-7.16.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8", size = 255111, upload-time = "2026-09-27T12:26:56.851Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/854ec68641a9b3362ff068a32dfa41637299761617ef253791dbade6fc76/coverage-7.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4", size = 256882, upload-time = "2026-09-27T12:26:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/db/0d/748e4518b0ac0f9ff2687c248a6e5f8c0737306e709372632a2556f84443/coverage-7.16.2-cp313-cp313-win32.whl", hash = "sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865", size = 226045, upload-time = "2026-09-27T12:26:59.983Z" }, + { url = "https://files.pythonhosted.org/packages/31/fa/6e46edba66a183fe4d99d4bb52c173287e9b8dddabe0888d24cb8210e580/coverage-7.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6", size = 226581, upload-time = "2026-09-27T12:27:01.494Z" }, + { url = "https://files.pythonhosted.org/packages/1b/d9/9ef6845367600b336ff75d000444a0d32497d6972c833141bd39356abf68/coverage-7.16.2-cp313-cp313-win_arm64.whl", hash = "sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79", size = 226409, upload-time = "2026-09-27T12:27:03.116Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/577fc0803dab4155dcf808faffbdd7b159256781c0874a8586e17b81b149/coverage-7.16.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4ee546b9e4872ffa194bf07ac87bfa1202ebb824d0795dc1ef22f175545ca90a", size = 224138, upload-time = "2026-09-27T12:27:05.141Z" }, + { url = "https://files.pythonhosted.org/packages/75/9e/e3785ba3ecba2bd11efc74bfe2801ca4b78c4480b15a375648d809a59da3/coverage-7.16.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a2fac6895eb299a2e52d7bbb8fb3903502b9da8d3f5309ceb16ec40c646b58ee", size = 224445, upload-time = "2026-09-27T12:27:06.805Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d0/963ff22d3fd27117da3b8cc442f5bdc91196f783321e1a8ff0ec43476772/coverage-7.16.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:57ff3783f99d75a1e81dd56a9737eb5665e6736a5d93258ba596b6dcad8fd05b", size = 256112, upload-time = "2026-09-27T12:27:08.43Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d4/a306940c81c6ae759e82fff27d20b7fdc6896e422b821f51313cce212b6c/coverage-7.16.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f37886699cb9abd29958247d718628d5bc6f39e623dff66a09e546c42a7e03", size = 258727, upload-time = "2026-09-27T12:27:09.927Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a3/d3d99d93b02517087aa05bc0cf2d04d372956b849e5443e059079901429b/coverage-7.16.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0fd7a86fdda7cb6d616d178654bd0ad6bc0f3f33c2e478aa598500a1a9e34eda", size = 259909, upload-time = "2026-09-27T12:27:11.55Z" }, + { url = "https://files.pythonhosted.org/packages/08/44/39dd599181726758dd185ae4dc0c0ab3aeabf7ca70e68e145060feeaaa16/coverage-7.16.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ac0f3b379c94acc2f7dce5f5f0b24d44fa1cc6a509717ef83dfee07450c2117c", size = 262481, upload-time = "2026-09-27T12:27:13.17Z" }, + { url = "https://files.pythonhosted.org/packages/99/e8/91ee43f6ded411460c359d7e1aebde4d6fd8f00a2e5394182d9d212eb23c/coverage-7.16.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7d0732c83746bc24123c581a85d9dd96b70ddb538c9076020aa1a041790361e9", size = 256012, upload-time = "2026-09-27T12:27:14.91Z" }, + { url = "https://files.pythonhosted.org/packages/11/8c/e9499ddc33197bd7eabcb1118ca81756fc874457b324e2b479a4804b2ad2/coverage-7.16.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b451c68218c150f616bc9649783ec8de76a59792c759b43aa0c9c0466a465e4", size = 257898, upload-time = "2026-09-27T12:27:16.588Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6e/c081cb5991a0afba99f9c4ad6c74a5fce9513a38ddc64e3e6680c6fed9af/coverage-7.16.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a56ac4fa5a75c7e182e8f62600cfb4aff43c5ed7356a034f3557659c3bec1d90", size = 255938, upload-time = "2026-09-27T12:27:18.19Z" }, + { url = "https://files.pythonhosted.org/packages/b2/42/1c3d819e8f9b6eb01c2fe90874d67a8882adb9507e0bbb09361ed131ea89/coverage-7.16.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4cc4f73aa3fabc36e32046d6cd2971405948d8a903636508a3d3b2f9128b3a95", size = 260368, upload-time = "2026-09-27T12:27:19.903Z" }, + { url = "https://files.pythonhosted.org/packages/19/4f/d70eac07901fd587b6ab05e659b52afe13959992aa5113bf6cce059cc572/coverage-7.16.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:723dcdab91357159b722935b500ee8abc0a66c8c432e1e9fabf4cc7598952de8", size = 255620, upload-time = "2026-09-27T12:27:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/34/5e/6d87af88317d3d9a9b18a9ca1bc1673eb516917f296e579d0d4a55cb3490/coverage-7.16.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5397e21a90dde0e9c6896b77ded8f0be26b66f8b22b33aed41f6043ed95d55e6", size = 257521, upload-time = "2026-09-27T12:27:23.358Z" }, + { url = "https://files.pythonhosted.org/packages/79/bb/90c2641170d2fa1a6757b3f8450ba2740197317b0ddd749e9604b914e886/coverage-7.16.2-cp314-cp314-win32.whl", hash = "sha256:848893e1d361448c113dc2f0913503522a6f7be231d0e38333d2a22d9698a011", size = 226248, upload-time = "2026-09-27T12:27:25.153Z" }, + { url = "https://files.pythonhosted.org/packages/30/08/d8d0478bb02c8eb0ae20a496fc80c40fcf4d3450bd184300d682ba2d28a6/coverage-7.16.2-cp314-cp314-win_amd64.whl", hash = "sha256:5a27b731c171e43dc8b5f32b76a5051dde2ec9b9366c87028f08a7088ebc2c7b", size = 226732, upload-time = "2026-09-27T12:27:26.907Z" }, + { url = "https://files.pythonhosted.org/packages/32/3f/0001da22155b0a8ce063ec0f7e64ecbe17b373f306e7a74435f6d6accb72/coverage-7.16.2-cp314-cp314-win_arm64.whl", hash = "sha256:1c569a9fd25505f1cd6bea90588818f90373ce90e2632e2cacf19ddbd6e14fdb", size = 226645, upload-time = "2026-09-27T12:27:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/d7/85/6d8813aff9b8b8586691a9d33c43c5604f7227622574da7cdc3d91a86861/coverage-7.16.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d93db87adb6b1c1b408dce4763314b55d76a9f589e96783a84ac9e7689e48bdf", size = 224871, upload-time = "2026-09-27T12:27:30.32Z" }, + { url = "https://files.pythonhosted.org/packages/5c/70/444f3a4981ac2cda40fdcf4cc9b56a4e1a33c222abeb33e51ed3e3eb2a6b/coverage-7.16.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:aa62c85046473959c13ba9edca9dc90a77d5c1095b1ba313556314d77fe5b036", size = 225123, upload-time = "2026-09-27T12:27:32.33Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c1/980681cd7b33eb66ac835044116ef0a92e11fcc7bdd866cc89d10b1130b9/coverage-7.16.2-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db76506aa5416081f3e8974ae0f7965c58ada0bb0ef7339ac86099588dbb20d3", size = 265049, upload-time = "2026-09-27T12:27:34.085Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e3/87679875c33bb2191f0f05544a1cc9adcc940fe0c35443a10f2df753dde5/coverage-7.16.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a0f2285329dac10ab08f79cb11f5692c497018e6c7c511f95e6fd63a70b8f831", size = 267717, upload-time = "2026-09-27T12:27:36.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/64/5d372776d6eb523d4e93bafba2253f96984e3b18261c4cc56a50863c6d0d/coverage-7.16.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:382d3346d56b0eec1b793d53a4c88799c8053f516aa3a8d7c44315696954bacf", size = 270047, upload-time = "2026-09-27T12:27:37.96Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/44082ff0cbf9f97d0043f57970a71204097ec7ba606361a9fd2065393669/coverage-7.16.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:648352b94507179d82637292e7ae8802508d95f78e2f00a705a50b6c48011681", size = 271360, upload-time = "2026-09-27T12:27:39.766Z" }, + { url = "https://files.pythonhosted.org/packages/b8/17/9a215efe25b5e0ecc87c89dbe525c4a87d14d87c8c0c7316ef140a5f6f3e/coverage-7.16.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fb2bde05838fffae1a1bf75e5d411a6cac3e4e9bb97e6640fed8cd47888b33f0", size = 264736, upload-time = "2026-09-27T12:27:42.072Z" }, + { url = "https://files.pythonhosted.org/packages/a2/da/7f0a31af8e448107d4d32844bd684757f51ea907bc0c68c8fd537b2123ff/coverage-7.16.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6a75180829efb8ae62b4aded25be6ddca1c888d138d2d82e21d93bfbd88f41cb", size = 267600, upload-time = "2026-09-27T12:27:43.85Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a4/3bfecbd3366b775bacdcb3330394d356cf384b5d8f5b2146ac4b14b252b5/coverage-7.16.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:99704f73721e23859112072d522076e11c31744fc96b5652e5dd2018aa4359f7", size = 264767, upload-time = "2026-09-27T12:27:45.768Z" }, + { url = "https://files.pythonhosted.org/packages/b8/3f/5d62163732d87e4a0c4710a0eab30f0fd6a2d480112abe2029f014fe8c9d/coverage-7.16.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:29309ccc86b7f33df7db12813c299f215bbbc470ed6292d0bedd63ffae1ebf64", size = 269026, upload-time = "2026-09-27T12:27:47.787Z" }, + { url = "https://files.pythonhosted.org/packages/49/4d/8e4579f225426535085a9be371cc75e3b026d058d679b80affbdfb4c3ef0/coverage-7.16.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:30c1b65d529e46569899fadca59e4a87c1faf2886923f1307ba61e654d4f3c20", size = 264147, upload-time = "2026-09-27T12:27:49.681Z" }, + { url = "https://files.pythonhosted.org/packages/d1/36/ef1f77e2c3f7bb03c2b13b9a2006f88700fdd75535ef158d70049f425c1c/coverage-7.16.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dcf4bc2aab4e16b1c4c0c2005918f23a7dd5d7821ddae82caed9e3342dc2fcce", size = 266376, upload-time = "2026-09-27T12:27:51.551Z" }, + { url = "https://files.pythonhosted.org/packages/be/79/0cb2bf4428830dec971c718c2c841a039c084415c99e67281f5a72841aab/coverage-7.16.2-cp314-cp314t-win32.whl", hash = "sha256:a9cd3de0a5bfe7b0e21ee10e1a14e3d61bf52efc88217ab1d95d6ace6970bd46", size = 226585, upload-time = "2026-09-27T12:27:53.945Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f9/da17121c16667fd84998e972200ae226a41540f6ea4795776c6d99e8976f/coverage-7.16.2-cp314-cp314t-win_amd64.whl", hash = "sha256:611a44e5229a59d7483ce830160e1a0e85f700562c7a5651c7c63fb8f4eb528c", size = 227378, upload-time = "2026-09-27T12:27:55.778Z" }, + { url = "https://files.pythonhosted.org/packages/74/89/01179c62d1b7e6e33bd5001566b02d7f778cf33d3ec1e81e94ca170c517f/coverage-7.16.2-cp314-cp314t-win_arm64.whl", hash = "sha256:22957cef43ce038641de78ba995de7568d2d6a37c6ddbf7fa0fd7d1ae2344d91", size = 227064, upload-time = "2026-09-27T12:27:57.496Z" }, + { url = "https://files.pythonhosted.org/packages/4c/57/52935003c3f627ba6e5203d7179aad32448c10899663a30336aba8e81a2c/coverage-7.16.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:414c26dfdb96aac2d570a54e03008f001e32eb2d413705365503648c6bd361d8", size = 224140, upload-time = "2026-09-27T12:27:59.343Z" }, + { url = "https://files.pythonhosted.org/packages/31/38/df472520f3e626524d7e2fc9d6da0afe7895a2f1489d36b48af8ca40bb41/coverage-7.16.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:00d3eb96e9988c45f50cccd1f1496571ac5c1f91386ac02c4d55516eeda19a24", size = 224449, upload-time = "2026-09-27T12:28:01.299Z" }, + { url = "https://files.pythonhosted.org/packages/0c/aa/3be084d5b82e63ccdad4ed751e4acbae294673573e30481d29f8b7402eec/coverage-7.16.2-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4dbbd1155ca46e6e0b6b89d204428c56ef6a459af21333f365d135a2820e5a09", size = 256060, upload-time = "2026-09-27T12:28:03.185Z" }, + { url = "https://files.pythonhosted.org/packages/de/29/48fca82a7ebf7ff7b2e35019cc9537e7f65e4d2aa1215cc5a8792c989251/coverage-7.16.2-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8fc15cc8d0d06e873c00ef18e1372d605f9aaf3de27d8c24e50782e75bc8b843", size = 259106, upload-time = "2026-09-27T12:28:05.15Z" }, + { url = "https://files.pythonhosted.org/packages/06/3d/b2d5986f2dd53fe201aa1be2e4ab204fa1aed5101e67c0dbbb419b850aee/coverage-7.16.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c6afdd69218202bc1758c9a14b86b8cf1084f37ed2ca143e567a103772b16d1", size = 260674, upload-time = "2026-09-27T12:28:06.868Z" }, + { url = "https://files.pythonhosted.org/packages/ce/7e/b50160be3506ead12e6480d14279af7f0f17627694300a2d1fd2c42d2ff5/coverage-7.16.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aba5c63b7afdc749cc9eae943d5b868cba2b261a176378fa1c5a30bc8bc89982", size = 263139, upload-time = "2026-09-27T12:28:08.771Z" }, + { url = "https://files.pythonhosted.org/packages/14/5e/7c805ac9a32606de1399bd7e9bd375aa2f973dc61b12680d9e6403c2e891/coverage-7.16.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9174f0af24e5eff248b9dbfe76ec5275a3d19d37edbc2810543f12cf97347a34", size = 256591, upload-time = "2026-09-27T12:28:10.842Z" }, + { url = "https://files.pythonhosted.org/packages/ab/9e/76f1ed129a2daf658a3ea17122824cf2e3b91fea0460d8d3664fc5a61018/coverage-7.16.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:80e9fdb4c3d926b6ba721d4bf7435bdb869c3527ae7803290361d0ab73db13b6", size = 258708, upload-time = "2026-09-27T12:28:12.962Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b7/8d62e75f48b527619239a65294f842d4b7fd02a0839d43ae1de80184e2df/coverage-7.16.2-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:7b3bce4a0d05401d70b7d0d5ca783e686bc9d30e81dbd7d980d532609bf809e4", size = 256562, upload-time = "2026-09-27T12:28:14.934Z" }, + { url = "https://files.pythonhosted.org/packages/b8/8d/0a15f95c3afb78e947c52644786ba4bc9de259905687dd720d5e6fae2e76/coverage-7.16.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:44f21e407b278efdfc1ee5e481e00518bd1d500310a30a5fbf2bcbedfef4aaf0", size = 261077, upload-time = "2026-09-27T12:28:17.215Z" }, + { url = "https://files.pythonhosted.org/packages/25/00/88389987305a47d732866c07c8a500000ab574df9505e3114ac69c8d027f/coverage-7.16.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:59c3926585e1cd1f2190f4b2ac9014de1bbeaf0d5d0587b0dc6b0aa90d17896a", size = 256034, upload-time = "2026-09-27T12:28:19.08Z" }, + { url = "https://files.pythonhosted.org/packages/92/02/34d079d4952ad461bde037d353f9a6e037a7edc45fe0f9ee8781ff73f028/coverage-7.16.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:066429634299e14dd2d511e1e85f8f9cecc500781f6b41907c0dd6f1baea7e63", size = 258034, upload-time = "2026-09-27T12:28:21.242Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d8/3e59a62879285b464ec1b10fd824fbc1af9ce66e842cd39974f80a0becc4/coverage-7.16.2-cp315-cp315-win32.whl", hash = "sha256:893ea9cf86cb8d2546812ac93d973aaf2ee1fb45110a873b014214fd23e3725e", size = 226252, upload-time = "2026-09-27T12:28:23.102Z" }, + { url = "https://files.pythonhosted.org/packages/f4/e1/128026e1b2836e9ad6b219207ba9edf1c5e0088a7869e23088aee7fbbe7a/coverage-7.16.2-cp315-cp315-win_amd64.whl", hash = "sha256:01c6908bc613b420c26c818fe948e1b97dfd041a53c98b01c63bd8321f5c9aae", size = 226728, upload-time = "2026-09-27T12:28:25.21Z" }, + { url = "https://files.pythonhosted.org/packages/a8/f4/c9fa8e7cf525ca7748ac52b0ee89331d13fe09808e45c679830708782e90/coverage-7.16.2-cp315-cp315-win_arm64.whl", hash = "sha256:967d72c835d7a8cf0af99ec813a2d06e3db6df706402f1fe85b31b437645f495", size = 226648, upload-time = "2026-09-27T12:28:27.136Z" }, + { url = "https://files.pythonhosted.org/packages/a2/13/e96b045447a856666f36f9c653e2a80bdaa732aaaf72412b19aa2c26a473/coverage-7.16.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:98d9c97f51b334b0adce7b964442a9af33c1a00c6ac856984cc5dc8d18f81c75", size = 224858, upload-time = "2026-09-27T12:28:29.169Z" }, + { url = "https://files.pythonhosted.org/packages/23/90/087f6ad1bd3df059632ca3407a4e6552ed1053ee35354de0a771acf35423/coverage-7.16.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:3e861f1071dcc2fec1e88bef0920f6b1eaa66a143555b4f8ab79ba2b0f30ef55", size = 225139, upload-time = "2026-09-27T12:28:31.131Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8e/285dcef0184358044e7cbcd810a1bdc9566bc620f54702d605477155df4a/coverage-7.16.2-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fb9d92ecfe2d5b494367c67f7446f8b75b68d8d0c8cf3bc3e6997478be25d9e2", size = 265090, upload-time = "2026-09-27T12:28:33.04Z" }, + { url = "https://files.pythonhosted.org/packages/06/b2/cc83f3a6e5789a4e89059c69555bc641c2efcde568405a1c06fc702951ab/coverage-7.16.2-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb57acff4a74246ae513c142d4b36e18c389c3aed8661914a53f7cd0071031b2", size = 268254, upload-time = "2026-09-27T12:28:35.135Z" }, + { url = "https://files.pythonhosted.org/packages/ac/41/f548c19530f5d66ac6e3c92bbcbc49da7261de3a458b9f3e54a3efb1a0b2/coverage-7.16.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:444889f7f66b74e4455c0a97e0e166dd41177f1dca8c0239a47cff25e05ba7e1", size = 270695, upload-time = "2026-09-27T12:28:36.959Z" }, + { url = "https://files.pythonhosted.org/packages/94/61/4dc27cf82ef96434d2874110ad0cc10ea4621025705dc5049862bd3bd181/coverage-7.16.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a740ea6f083c6db7b926534d159508f80ba275ab35e722522de0d18d0f56e55f", size = 271853, upload-time = "2026-09-27T12:28:38.821Z" }, + { url = "https://files.pythonhosted.org/packages/38/29/bf8072b1b8bd5f2de8b21460a404460b1a2b97e80a9464c78ec0271f6199/coverage-7.16.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8e209591f7c41ae4a9171335cf6156afda0b21de73b02f73f5aa95b2d5fbb08d", size = 265644, upload-time = "2026-09-27T12:28:40.815Z" }, + { url = "https://files.pythonhosted.org/packages/7c/2f/0aecb8721be5cdeb8afd9d6d9f6b463f074e4d8d37f00f4c42442522709f/coverage-7.16.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:396bb16e04ce04efbb3df91456ae4e3da918e69ecdf67fb711b0a0fdf35ccce0", size = 268530, upload-time = "2026-09-27T12:28:42.725Z" }, + { url = "https://files.pythonhosted.org/packages/ab/0b/92b4b7628268ee711249958e68fc0328779bd3d9a7ab4715379465aedb84/coverage-7.16.2-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9cdf19874e0d247f32f03609200370343c3c7aa260b191d8c2bb251d36198283", size = 265154, upload-time = "2026-09-27T12:28:44.684Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/41c95c1ab29b3dcd357cd1227181d1c98185632aca41ce670ce671b23a43/coverage-7.16.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:fd3d72233eb8b48acc94fa57d44e2d32ce8e7abed02882ccb6d855ccc4ed33ec", size = 269799, upload-time = "2026-09-27T12:28:46.672Z" }, + { url = "https://files.pythonhosted.org/packages/80/07/ebeb259aa5362b033a137b86d7274ff4b109d59be8cc9913889b783bf75a/coverage-7.16.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:bb4ffe96aa663cee727659db5a2afeb38c95f8677b747d447b90d6d4874ea2c5", size = 265196, upload-time = "2026-09-27T12:28:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/b2/18/8437620f90d023680a072eee02f968055f3658bbfb7d386d0ea34cfb7f30/coverage-7.16.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dba2edfb054f6d4a08df9d1637c39a5aa3865bca6617c13c86be21e45658a59c", size = 267266, upload-time = "2026-09-27T12:28:51.361Z" }, + { url = "https://files.pythonhosted.org/packages/28/6c/f08e8ee4293e6434035424180bef4d45e028e8ecc006c61bf9453e74405e/coverage-7.16.2-cp315-cp315t-win32.whl", hash = "sha256:251aed777c47c77aba047096d4542889db089227655711dfc2b9c54ef0e15e35", size = 226583, upload-time = "2026-09-27T12:28:53.33Z" }, + { url = "https://files.pythonhosted.org/packages/f7/fd/3f939c2847f4a72c20cff8b1ac33da78ea91a2d38d9b43336e60db719103/coverage-7.16.2-cp315-cp315t-win_amd64.whl", hash = "sha256:2aca0bdfa9e91621d5b09d815357bf63def4fc0e9cb66da67bf2cf93f3b1a6f5", size = 227374, upload-time = "2026-09-27T12:28:55.158Z" }, + { url = "https://files.pythonhosted.org/packages/5a/35/b98cdc354c952402132e675a87f2cc3227fb68f959c84aaa491fbe15933d/coverage-7.16.2-cp315-cp315t-win_arm64.whl", hash = "sha256:b88841e654f09732804809e435b3e005a929ffd9998b872b7b213957b8759cb8", size = 227064, upload-time = "2026-09-27T12:28:57.075Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0c/7a64e1ac90541a8edf50daef0914848011fb057a5bf55284a4811e21939a/coverage-7.16.2-py3-none-any.whl", hash = "sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f", size = 215754, upload-time = "2026-09-27T12:28:59.075Z" }, +] + +[[package]] +name = "dashboards" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "fastapi" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[package.dev-dependencies] +dev = [ + { name = "debugpy" }, + { name = "pyright" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "fastapi", specifier = ">=0.143.3" }, + { name = "httpx", specifier = ">=0.28.1" }, + { name = "pydantic", specifier = ">=2.14.1" }, + { name = "python-dotenv", specifier = ">=1.2.4" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.54.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "debugpy", specifier = ">=1.8.22" }, + { name = "pyright", specifier = ">=1.1.414" }, + { name = "pytest", specifier = ">=9.1.1" }, + { name = "pytest-asyncio", specifier = ">=1.4.0" }, + { name = "pytest-cov", specifier = ">=7.1.0" }, + { name = "ruff", specifier = ">=0.17.1" }, +] + +[[package]] +name = "debugpy" +version = "1.8.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/9d/3cb6693342acf96802dba89934a5b8da43c201d764ebd1f2c0514a3d42bd/debugpy-1.8.22.tar.gz", hash = "sha256:e489c7268e1c7b41e13b438d9c533d2a7af73fb59bf8cd30fead8286c1c39c4e", size = 1710877, upload-time = "2026-09-15T21:44:02.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/f4/fee6d40f5865aa84e1100c9df1a0f2cf8991776b61f11f433ff277df4415/debugpy-1.8.22-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:c21dd7e1ec22556bb41dc3bf6b86c603e440ec889c4acb27d7206354b2106c54", size = 2493531, upload-time = "2026-09-15T21:44:25.835Z" }, + { url = "https://files.pythonhosted.org/packages/bc/4f/8df91270afbf61fc5ab202951f28451194a7eb093704b26da11cb0ecd308/debugpy-1.8.22-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:c1ffb9953708b648ecf6acd2e5ad2c002bf68785ed618197dd4463a2a7226f39", size = 3994351, upload-time = "2026-09-15T21:44:27.31Z" }, + { url = "https://files.pythonhosted.org/packages/4f/11/2e34a4f21d6155b0c58003fd6fca9819df7d6fe7d7acd83b146b648a57ee/debugpy-1.8.22-cp313-cp313-win32.whl", hash = "sha256:ba810a66b437e3c43ca0ce0892404f010338286263ca8e5108442d76a9485337", size = 5359071, upload-time = "2026-09-15T21:44:29.064Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f5/c2b3e8260388d7b76bc4e5a3a29648a0ca47dca6a2721d6549ff2d944eee/debugpy-1.8.22-cp313-cp313-win_amd64.whl", hash = "sha256:f49b1d6cecf326b63c06d7f517e4a6642777f758c58cb799e7bc5a6dd74721c2", size = 5400362, upload-time = "2026-09-15T21:44:30.857Z" }, + { url = "https://files.pythonhosted.org/packages/ea/95/d97240bb68b86db8d0a07602be068aae9e50f17063d696a4c6f619975311/debugpy-1.8.22-cp314-cp314-macosx_15_0_universal2.whl", hash = "sha256:e6744ac1850c73c2ba7b29a126cf9ab74efd1831190720e8fee17ef5389c8f5d", size = 2493148, upload-time = "2026-09-15T21:44:32.518Z" }, + { url = "https://files.pythonhosted.org/packages/70/9e/994fe1fb23dd4a5a8d059a0dc95da4b0821907200a61c2d5a4fdc21b6cf8/debugpy-1.8.22-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:0c1104233340196e5cbf5514e7dfdbb98968e730cfd7fdd6f3d72a082be838a9", size = 3961025, upload-time = "2026-09-15T21:44:34.387Z" }, + { url = "https://files.pythonhosted.org/packages/37/e0/66319918883cf6dd6dea3805210c3cafd9e65dfaa6f8b852482cff10a026/debugpy-1.8.22-cp314-cp314-win32.whl", hash = "sha256:feea785c7bbeb8cfd5b01a63f9061c899dc468c8be81671141a29305774fa294", size = 5359408, upload-time = "2026-09-15T21:44:36.128Z" }, + { url = "https://files.pythonhosted.org/packages/94/53/5b4d91f7fffb9393ac907133201d1da0c070d57e463ee8e2ec1db3cb63b6/debugpy-1.8.22-cp314-cp314-win_amd64.whl", hash = "sha256:d593a330297332ec435f3965c448b6d500e03f33675cf2063178bc76377a788e", size = 5397429, upload-time = "2026-09-15T21:44:38.429Z" }, + { url = "https://files.pythonhosted.org/packages/24/71/580df4aed96c9b23c9cbb049907d8ac1347050afdb8f450df20649d51578/debugpy-1.8.22-cp315-cp315-macosx_15_0_universal2.whl", hash = "sha256:a9eca6ab09a61534e923064400081e016f7e1d8430cc6e8b7a9cecd2f59e2b07", size = 2493380, upload-time = "2026-09-15T21:44:40.074Z" }, + { url = "https://files.pythonhosted.org/packages/b0/10/a323945b0966e560c40fe48470d2847bfc7692a8f550d7086e9a28f5b623/debugpy-1.8.22-cp315-cp315-manylinux_2_34_x86_64.whl", hash = "sha256:b7dbde1fb822d100802d505b2aed6d0813b1a0a2015d495cb8798b2215d5d1e5", size = 3975456, upload-time = "2026-09-15T21:44:41.623Z" }, + { url = "https://files.pythonhosted.org/packages/ef/f9/4383635281999c3eaa44bf327de523ba42240b35eb87c1130adac4a829c6/debugpy-1.8.22-cp315-cp315-win32.whl", hash = "sha256:225d063f81708c2546999e7edfac0198b2d5c2f144797dc64858f867948633e0", size = 5359469, upload-time = "2026-09-15T21:44:43.234Z" }, + { url = "https://files.pythonhosted.org/packages/7b/14/4fe9f46a9c825c8cbc6dc4aebc2a885b0bceb37ce2a37699c170482e7a40/debugpy-1.8.22-cp315-cp315-win_amd64.whl", hash = "sha256:b17a4896520f1c6da09ce76ec8215df0b85b0b6f3617526a4c65c2315340875a", size = 5397869, upload-time = "2026-09-15T21:44:44.783Z" }, + { url = "https://files.pythonhosted.org/packages/56/3d/c7dc9f35bc9e22cdb53679f844bee40fd5cff871862f60595869a433400d/debugpy-1.8.22-py2.py3-none-any.whl", hash = "sha256:a9e9d3550e15ca479c59333e90845029190531f0cacfedab3b815a57bd913947", size = 5374857, upload-time = "2026-09-15T21:44:54.454Z" }, +] + +[[package]] +name = "fastapi" +version = "0.143.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/d5/b0aeb0b303650ac87b0411de4aaf3f901b06042dcc3fb00f30b97b7ff585/fastapi-0.143.3.tar.gz", hash = "sha256:b9ab567ee57afb4454004ce04c889f8f0160ac47946b99795887cd6e88484848", size = 469834, upload-time = "2026-10-16T14:14:13.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/1a/03d246befcf307219bb2161c97d8c5d314f84125bf7dd245730a28589828/fastapi-0.143.3-py3-none-any.whl", hash = "sha256:75e8bd68e2f52f77d29b2879749c257d64de2d85282d25270caf2bfccce5efe5", size = 144849, upload-time = "2026-10-16T14:14:11.763Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httptools" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/ec/deed52912ab7ca6c0b12859330c571c60c61d7267b341b28951fcbf13694/httptools-0.9.0.tar.gz", hash = "sha256:d484ebb7e3a3f3597b0f645fbd1b85633674ca808c1f5ba11c2caf7c66f5c8b6", size = 282523, upload-time = "2026-10-09T19:57:04.301Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/04/223994f8589750d2a36ceb43203e739cf75bd9e12c226680d73567766908/httptools-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fb995082fe41ec410b33c48b54fb1d44abb8a6ee762c31e8c42519e8c3a30a9", size = 117115, upload-time = "2026-10-09T19:54:53.356Z" }, + { url = "https://files.pythonhosted.org/packages/31/d8/b4407836e567a862ce79d78a628d785db99aba52e63496d68c60eed0d475/httptools-0.9.0-cp313-cp313-macosx_11_0_x86_64.whl", hash = "sha256:b9cd15cb7cf0d5cc41f649fd789aae12c56c3b83eff593f8e095c1d4555ad5c3", size = 113225, upload-time = "2026-10-09T19:54:54.81Z" }, + { url = "https://files.pythonhosted.org/packages/79/f6/0caa51b077492a7306bdbd9dfb907a2246985f0aed1fe2d086255921848b/httptools-0.9.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:088de1738e1af624466a01c35d652dbe6fb825be887c76d68aa850621d81db88", size = 520112, upload-time = "2026-10-09T19:54:56.3Z" }, + { url = "https://files.pythonhosted.org/packages/fa/da/7a47b7c2106bb10e6d4c04a139d045257a4f93c672fae6f0b9e92b1f7bc2/httptools-0.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b1ac7f1bc6c0dbf90684b77571a51a21b2463909fd916ce0ac9bfc4d566dc75", size = 516079, upload-time = "2026-10-09T19:54:57.938Z" }, + { url = "https://files.pythonhosted.org/packages/0f/4d/417b42d2663acf4f5aeb2718dc894ec2be4e3dcfd8caa2d3bf9ee2dce511/httptools-0.9.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9430f65db521db7962ad951571d446171213686f96c998a54dc18ed574821e2", size = 535040, upload-time = "2026-10-09T19:54:59.769Z" }, + { url = "https://files.pythonhosted.org/packages/cb/de/8df4c09a33ddaf50f697719f20201cf93631ef4b50cec05e42acf179a7c1/httptools-0.9.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:52fe0176682a25b15370f23f5b0f1366a84771df89144fb0cd979cb72a94b5ca", size = 462799, upload-time = "2026-10-09T19:55:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/e8/90/1bfe91e3fca29c541d85d7ba8ed92a406d4dd13608c281baf7ec75369fec/httptools-0.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:757e3f79cb865a7db94e0db5f4d0ed3284a69e39d53568f433982ea13c60cac1", size = 497596, upload-time = "2026-10-09T19:55:03.201Z" }, + { url = "https://files.pythonhosted.org/packages/b0/af/2bbd5af0dd7a0e0c3b63bfefafd87a07041eb13d7cd710fbf30708b70773/httptools-0.9.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6ff5f0ed70783dcb9562dbd20edca51c3d4d277f128223709e3da6b75986d1d4", size = 517110, upload-time = "2026-10-09T19:55:05.011Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7a/9f165817c3e27df9098f3d50a675417d8721253f1073434f48a3f9d9a6c2/httptools-0.9.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0f537e5e8152e8d9cae82804024790cb973061abd3b7ef8f66f46e2b5c7bb51", size = 459198, upload-time = "2026-10-09T19:55:06.985Z" }, + { url = "https://files.pythonhosted.org/packages/93/20/b93279e334946c359d39aaf405241c6fd60f9e60da709bc4156731a4413c/httptools-0.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a7f1df31829c258158be01bb04eb668c4fba7df1ddf2262131a972962e651b6", size = 502996, upload-time = "2026-10-09T19:55:08.733Z" }, + { url = "https://files.pythonhosted.org/packages/86/c9/ac3657943d40c5a9949b72565ee03151e480fb18c062c7c13c0c0276df6f/httptools-0.9.0-cp313-cp313-win32.whl", hash = "sha256:714bf348f468532d86bed670837e7d5ddff3834dd7f5d3c08066da400c86f088", size = 85878, upload-time = "2026-10-09T19:55:10.275Z" }, + { url = "https://files.pythonhosted.org/packages/74/69/d23079cd4bc16d11e49c3f51c2540c018736f26701a2a73183cae9255a1c/httptools-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:805b0f2618e5d4c3e28f45b731eb1a0539691ae4a2f97b4ce014de0bf96a1ff5", size = 91549, upload-time = "2026-10-09T19:55:11.701Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ed/5ff678a774b721f054c095f04d84fc536e7369ea4f4c9af3813a518d95b6/httptools-0.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:bfdabac0c6d3d6a5be8c2a100a001c92c14a39bbafd5999545a675c493626e64", size = 88043, upload-time = "2026-10-09T19:55:13.046Z" }, + { url = "https://files.pythonhosted.org/packages/31/39/0965023968452245ece67b161adbf7c5652f8d0697ac69312f9d21849411/httptools-0.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a4050a651e1f2faf05eb028ce9f2168abbcee9e24b209f5c1f2eb96d8c569e4", size = 118142, upload-time = "2026-10-09T19:55:14.491Z" }, + { url = "https://files.pythonhosted.org/packages/31/39/a6ec662d81059e505e953af709797038e83e489014df721e506f4fd0d3c5/httptools-0.9.0-cp314-cp314-macosx_11_0_x86_64.whl", hash = "sha256:130635fea6e611a6b2026120037965ddb88b3dafd11bb64e264b101a70a76630", size = 113943, upload-time = "2026-10-09T19:55:15.887Z" }, + { url = "https://files.pythonhosted.org/packages/72/04/4ecb7251a6c55bef61b157bb93fd44678943c35702a5966e4d5ebda2d450/httptools-0.9.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:18d800aaa2d6bff7d889df810d1b19a5fde72b1f6c0ca96e8d9f28a692fe5460", size = 514397, upload-time = "2026-10-09T19:55:17.48Z" }, + { url = "https://files.pythonhosted.org/packages/31/5a/0c26c98ee06f0f39608de715e7ca868baec942171a77feace5a0ba548ca6/httptools-0.9.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0e45def4d9ce7073e2226535572442d9d6efb4047c7a5fd8960807e877ce70a", size = 514383, upload-time = "2026-10-09T19:55:19.221Z" }, + { url = "https://files.pythonhosted.org/packages/d4/6c/0f85d4f1f579c49aea6e4946dd304e9f33a680382b5117970ab887885bc7/httptools-0.9.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f6da814aeecbc6cb8872d6d3e85ed16e8ab1653f9557cea8658725ce212348a", size = 534993, upload-time = "2026-10-09T19:55:20.992Z" }, + { url = "https://files.pythonhosted.org/packages/3b/32/97a836533b7bc9e269fc6d075c2d27669ca9786bf43f229158b9b4b15021/httptools-0.9.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8e1e037bb57dbc549c6fe20370b763ea74bdb09413cdcf857e4f14d9e4e2fb13", size = 461494, upload-time = "2026-10-09T19:55:22.785Z" }, + { url = "https://files.pythonhosted.org/packages/67/cf/a2d5e8dc3bad9b0b966bb546170234b4614275346cccbc01f6cdb6fce3b3/httptools-0.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:cd3e55223a77d6e08d5730ebacb4930ecca5d2ce7c57e7ba10833be7e52903f1", size = 495859, upload-time = "2026-10-09T19:55:24.9Z" }, + { url = "https://files.pythonhosted.org/packages/bd/d9/7472c4ca2aa1cfe6d0f9923380784b034cb77addc88589f2e5c92fd3b4df/httptools-0.9.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:beb2c8a34cc90fb4d862b7284eafdb322030d6a8b2ee5eb6a744f84205beedc3", size = 516303, upload-time = "2026-10-09T19:55:26.84Z" }, + { url = "https://files.pythonhosted.org/packages/c1/dd/f9be002ba859714cc306fe86204b7cb12bac091be66a7e23d7bb25d259bb/httptools-0.9.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0cc339a807c156d840b54f8bf050ba0fc265eb81692c24bca8535b52fbd797c6", size = 458188, upload-time = "2026-10-09T19:55:28.571Z" }, + { url = "https://files.pythonhosted.org/packages/89/7a/ed8bb5344071afd12c87e57e8839fa65abc3895b92a5d065be79ecacb919/httptools-0.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b6ee42112d785a913dd63ec0335435a3dddbea5040c151252db815b0095cf066", size = 498356, upload-time = "2026-10-09T19:55:30.301Z" }, + { url = "https://files.pythonhosted.org/packages/04/8d/3f1390c901d4a266ad9d5b988c47c4883e322e6f6cc021c592b9a050fb19/httptools-0.9.0-cp314-cp314-win32.whl", hash = "sha256:d1e329a1866981efe0201d05a374617f6c6cf14434a501d78ab22793d1ab1fa6", size = 88479, upload-time = "2026-10-09T19:55:32.071Z" }, + { url = "https://files.pythonhosted.org/packages/99/05/7de70a4eea3b52d31a95fe64eb5775ccdead01e4913e4741b4424e9ef180/httptools-0.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:edd5aa045fa3cc57143db018dd32ce7962bd5b525d05230709015d7e570100aa", size = 94809, upload-time = "2026-10-09T19:55:33.423Z" }, + { url = "https://files.pythonhosted.org/packages/e8/79/7f6c354a8f8f74381fd473f365d2db3cd976ee8d1422b8dd7455dfc52b62/httptools-0.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:6ff0145b34610e57c9fae20df4e133c8d54266447387de6fcc0bdabfe4db4569", size = 91508, upload-time = "2026-10-09T19:55:34.764Z" }, + { url = "https://files.pythonhosted.org/packages/94/0c/f9e8148ca684b41b4b5d0ced0860530b9a9bcb7c38bf727d83dcbfea42d0/httptools-0.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:80eae881cfb69383303e9a4d7961a478025b89c24f38f2e69b30c516fa0d57f2", size = 123684, upload-time = "2026-10-09T19:55:36.445Z" }, + { url = "https://files.pythonhosted.org/packages/3d/54/3c1d910e8f0bc9ee0ba7867b687e3272c8ae4a7da2df2fbf1b2bce77f0f9/httptools-0.9.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:b2ab3aad55d75d0b8df8d8a1b5920baaec9b161112cd5e95984848b4d2cd3dfe", size = 118378, upload-time = "2026-10-09T19:55:37.851Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/3b9694880da927ae69b5629b8847cfe73d14584be2aa974a92ed2675b7da/httptools-0.9.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db735a23ecb0f0450d2b24e0a05fb00a8a35c9db172919c4d3e023e7c7ee4c9b", size = 591295, upload-time = "2026-10-09T19:55:39.501Z" }, + { url = "https://files.pythonhosted.org/packages/3c/89/1ff2835b6adf5c08a477d3a199e72b71e7f26df55ceaaed7d7364d745a1d/httptools-0.9.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:995b52f7c260ac7023640221f27472303968753cb6fc6fce1ddfb0e9db59a398", size = 603709, upload-time = "2026-10-09T19:55:41.404Z" }, + { url = "https://files.pythonhosted.org/packages/24/40/4f59a0d9dca6d60002e7cb5dbf1441b558ced5a65b5b4131d57cbbd7c806/httptools-0.9.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3af4e45ff455fce5511fdf2653c1ce428ef09c56fe37a83eb4d924c2d474f31e", size = 607379, upload-time = "2026-10-09T19:55:43.119Z" }, + { url = "https://files.pythonhosted.org/packages/bf/19/381d444a3ba704cd5c67eb4617ae7a08e920a8239c688f23ba0de07a270b/httptools-0.9.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ce8e723b4637034b76f5382a30a6b725518c332273e8d62a6c7d46e90837c947", size = 530031, upload-time = "2026-10-09T19:55:44.85Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c5/c9ba7758bf266240f598934510af4a800edafd9c8eb1fcf15feac0427063/httptools-0.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:465bc1526debf53a3be92022a16ca0c38f891ea3b5c1587af4f52e44020f8a07", size = 575129, upload-time = "2026-10-09T19:55:46.536Z" }, + { url = "https://files.pythonhosted.org/packages/db/87/c17f3a53616a3849681f7c8e913ce966487b95038504bbb035c38f5f2fbe/httptools-0.9.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:8463b34ebde3f000627e9dbd8a545f995ad49fbf7ff9dd5abc0cd507da98a603", size = 582996, upload-time = "2026-10-09T19:55:48.545Z" }, + { url = "https://files.pythonhosted.org/packages/88/e3/cb33ba1348ddfa5853f96021f4c38674ac383b92c944492cf7638bd6bfd0/httptools-0.9.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f9489c1d87160c126f73b004742fe8654fa1ce37ed89e9e01330a1c10aaecde4", size = 526150, upload-time = "2026-10-09T19:55:50.261Z" }, + { url = "https://files.pythonhosted.org/packages/e9/00/af0e2f33ba5be60803a492ad377e798714d0c970e76015e313849b351ef7/httptools-0.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:06bfe7fad972a417269d8a5fc53b87e4eca970354abf5e9e24336fd06d64292e", size = 571731, upload-time = "2026-10-09T19:55:52.422Z" }, + { url = "https://files.pythonhosted.org/packages/b6/35/e67e9c9dd3da036ebfcbd273eec44bd39213f952d638858b09b9f3ecaf3f/httptools-0.9.0-cp314-cp314t-win32.whl", hash = "sha256:c42424213c28804f8d0e20f5692106cfb57bf72e1dbc4092b8481fb2f9e4c707", size = 94622, upload-time = "2026-10-09T19:55:53.982Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5c/af620c73de59b5f3d431ae778c7412d30bba7bf56ca8b4140107a8ac0e54/httptools-0.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bb1533541c729ad422f870a780d8b4af924f9817d45b5f580390418cda72eaa2", size = 101934, upload-time = "2026-10-09T19:55:55.417Z" }, + { url = "https://files.pythonhosted.org/packages/90/90/fc6019b5179d13007c6c3039346ea2696cf2e94369d6ca96e57f23b01989/httptools-0.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6f9549ca354a1d6d6167c458a1f1b12147726b968f02dd64b6a5801dba91ae0f", size = 97211, upload-time = "2026-10-09T19:55:56.878Z" }, + { url = "https://files.pythonhosted.org/packages/d2/77/e226b16a2f291f2a4ce25a24a3297e98749d80b8a713b8f3b11d8a82e904/httptools-0.9.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:d3906b5c549ff2ad2473cb711e1fc65d76715c2726a402108fbf55eab6c6b49d", size = 117817, upload-time = "2026-10-09T19:55:58.295Z" }, + { url = "https://files.pythonhosted.org/packages/ff/08/050ad8985ec34064e4401e6e5aeca7238685bc218eaff20025f7c04b0723/httptools-0.9.0-cp315-cp315-macosx_11_0_x86_64.whl", hash = "sha256:cb2bb3ac0af7fdab2311b895c9eb95442b45deb14cc949b9e65545e74aa0be69", size = 113669, upload-time = "2026-10-09T19:55:59.915Z" }, + { url = "https://files.pythonhosted.org/packages/52/0f/af812488a4963ce59d97b73a00c72bba49f5eebca1a13ab6f114372b5e82/httptools-0.9.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:63d38e9a9a10a20fb57593742e63c6b1e78dd7f6ef5472de8e0b1e4cf4f3db26", size = 515633, upload-time = "2026-10-09T19:56:01.529Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/73c987b84e0d02fa6c4109c7ce6ea00518d0aa3005fb92b75553ffd5ddf8/httptools-0.9.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eae4e9c7a0785a1a715de0a74fb822ab40084c060f444f18f075d05e322aa7ef", size = 516049, upload-time = "2026-10-09T19:56:03.327Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f9/74cc01fba5a0ea05501eb39eddba4baa00c10e4d1caebdb78f23eaacafe5/httptools-0.9.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0adc974916efe1fbf89d0363a86dcb2c746727643e362ff398de1a4b50b6bc77", size = 534832, upload-time = "2026-10-09T19:56:05.068Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a2/a7bb90643c059e8136c2a5fdfb0d7e1a18b2c5c4f1a78f2de14b1303184d/httptools-0.9.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:050f84b7ec46a6efe0e5f521cf8729e3397c1cef4384f62ed8d5d68ca0045776", size = 466489, upload-time = "2026-10-09T19:56:06.757Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/bb3f18e05cbad9628e7f1254176c475e05ac79c72697ec7c144fc2cc877f/httptools-0.9.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9b4da5789d7cf576c7e81f0088c632f6ee3786d87d17f08e90e703c22ce15633", size = 497146, upload-time = "2026-10-09T19:56:08.641Z" }, + { url = "https://files.pythonhosted.org/packages/25/e6/90e2433d7a947bec66a5ad22e948626a26672ff62aa3ebf949899f687a3e/httptools-0.9.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:f78f7ae1c2e5aabf29583fc0d302d8081a663776f84578025662eb6f5d63a921", size = 516153, upload-time = "2026-10-09T19:56:10.415Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c7/86373edd9d800eb723b8b68d3fce0e31d3e3211f9d7b0eaf8c3deadfada0/httptools-0.9.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:b2cc6991f16f6d666d48e4b57318104e7b29109e32e2f6b86e9d44c4e6a27f4e", size = 463141, upload-time = "2026-10-09T19:56:12.406Z" }, + { url = "https://files.pythonhosted.org/packages/65/46/8dc41d9ebf78fa56f609f251ed8ac5a9f66513b0ce712040bd7ada7b19cc/httptools-0.9.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:dbc9fd1521e573045d71b6afab7398439c5cc259e8cb9d416fe62d485c4899c6", size = 499125, upload-time = "2026-10-09T19:56:14.109Z" }, + { url = "https://files.pythonhosted.org/packages/7a/41/38db94fda8b266dcde50722a4fcef825b189380a220e02c682518bc1b430/httptools-0.9.0-cp315-cp315-win32.whl", hash = "sha256:34266cec8c1d4e3e91fcca7efe38971d6bdda64a7944f2a46ab576da15173680", size = 88370, upload-time = "2026-10-09T19:56:15.873Z" }, + { url = "https://files.pythonhosted.org/packages/4a/cd/347f12eb16e20972dcdacbca907f2c52d72a36542199a5bf3ca342c92098/httptools-0.9.0-cp315-cp315-win_amd64.whl", hash = "sha256:b5a3f5f70967a1aa2bc47fec42a1e19d2fb38c61700e3ee62b63a4af4f4fd001", size = 94617, upload-time = "2026-10-09T19:56:17.257Z" }, + { url = "https://files.pythonhosted.org/packages/f3/08/086ba2f53989d504a05f4669b03673a04fc72554bc37d4696c3c6132be75/httptools-0.9.0-cp315-cp315-win_arm64.whl", hash = "sha256:e0acbd474d0af4afacc6e66c4273f8a19e25f8af4379fc816388095ea6b01371", size = 91358, upload-time = "2026-10-09T19:56:18.641Z" }, + { url = "https://files.pythonhosted.org/packages/3e/3a/9ba59ec76d45bf8eb7ad3a18f2c6e9074fa4ce5cbbd3900fffb8d840f9e7/httptools-0.9.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:02bc5b3dcb6394b9d825fd62a7bfa0b2943063a3c89abc4492ad45e334a20eb5", size = 123063, upload-time = "2026-10-09T19:56:20.023Z" }, + { url = "https://files.pythonhosted.org/packages/18/2d/49eb389bda75a8ef0d04bf025dfb8412a3646637051c8a88bdeea700e343/httptools-0.9.0-cp315-cp315t-macosx_11_0_x86_64.whl", hash = "sha256:fc1a4f9d18d32a6e0a0a0a382986a60a2126f5144dd08715be7adb8df18e8a46", size = 117328, upload-time = "2026-10-09T19:56:21.439Z" }, + { url = "https://files.pythonhosted.org/packages/a0/6b/2d6439378fd3d1f9c06272b35d61f4519e2d9bf9967611df069fa6c23044/httptools-0.9.0-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:df3867518b205be3648e2fbd522bf380c851b5c2500588047505afdd786b6669", size = 588680, upload-time = "2026-10-09T19:56:23.056Z" }, + { url = "https://files.pythonhosted.org/packages/08/65/3fb50e861bbb6103ca58fd88b4127d346fc909eb9f06d250455033a3f698/httptools-0.9.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26e1d9629f3bf70d23f0d22238152aec51c837a7c9e384cb74f356fdccad7eb3", size = 600031, upload-time = "2026-10-09T19:56:25.216Z" }, + { url = "https://files.pythonhosted.org/packages/90/9b/40d33d4098fde007845804b1c923ddf5a27fd48aca1c8080bdbdac6c16fa/httptools-0.9.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:050f7ab098121873c8f13e35857f97ab60a76185c8302bde9a384939bb7c3b96", size = 604407, upload-time = "2026-10-09T19:56:27.04Z" }, + { url = "https://files.pythonhosted.org/packages/17/37/472afc9000aca3c7dd61a9b8ac6f3e2765900e3614f8d7f13e772c9c5438/httptools-0.9.0-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8d90d10e9b6594c28f27896a68fab97fd784c43804e9fe419dab8e8dcfcf4b02", size = 529459, upload-time = "2026-10-09T19:56:28.944Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/9956910fb1d181578249cd2cc966c0c46ad3c558b43ac2b79af50f94589f/httptools-0.9.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b928ab0ecaa664e8caecc529dcb8bc881b6b35bb2b74bf9a39ae25f982ee8812", size = 570845, upload-time = "2026-10-09T19:56:30.602Z" }, + { url = "https://files.pythonhosted.org/packages/30/8c/d1c160a3cc2c18e41a6f763c3aad979530dfb295039449312b8814e19753/httptools-0.9.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:2319858018eedd0c0b2f950a620413c0a9d1352607be4267eb28209eca8b1e3f", size = 579194, upload-time = "2026-10-09T19:56:32.353Z" }, + { url = "https://files.pythonhosted.org/packages/90/3c/3f7cc49925928a8c82f4141d504b8b8c2901c4b35cb88800211828312561/httptools-0.9.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:931f45f84e15daafec5f82cc92e6710569e1f50933f3253d206eab4132bec678", size = 524950, upload-time = "2026-10-09T19:56:34.103Z" }, + { url = "https://files.pythonhosted.org/packages/19/98/8e2154e99b8e8818fad3e6c5dd7cf21c050f6314b1bd8072e8dc29f49eb5/httptools-0.9.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f67db0ba2bedafec15b8e5330d40da1e1c7921559fa715af021252bfef81a6f8", size = 568603, upload-time = "2026-10-09T19:56:35.876Z" }, + { url = "https://files.pythonhosted.org/packages/79/a3/86fe9fef3a1bfab5db62262f8880c294cbf8a8d94cffe2a2aa8b4aeed40c/httptools-0.9.0-cp315-cp315t-win32.whl", hash = "sha256:2095207b75a83c9e947346da9c127fb7e4fb29f41589df2643764f06b750989c", size = 94042, upload-time = "2026-10-09T19:56:37.441Z" }, + { url = "https://files.pythonhosted.org/packages/54/4d/f2d88782251467325a62ec4ad704249bb1b09c21aacb997181a9f4421f30/httptools-0.9.0-cp315-cp315t-win_amd64.whl", hash = "sha256:bca180cbe84e4fba7807eb408a8655295f697928512324517e30a091ede522a8", size = 100837, upload-time = "2026-10-09T19:56:38.831Z" }, + { url = "https://files.pythonhosted.org/packages/00/4b/5e96c4e0d171f959a0064971c3fced9cea5a19e5fab7a8e7d57aceb80506/httptools-0.9.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a4d8c2c7e73ba5967be74d7c3a5ff81fde815ee1b48d9c5c0f14de8463a847b", size = 95947, upload-time = "2026-10-09T19:56:40.562Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/105de02c1322cfada6d9710d9146ef8026419d433c9d08359a2d35805811/nodeenv-1.11.0.tar.gz", hash = "sha256:3ce8fe5b71d16e8af7039ca65257354100bc772965d6bc549070649e53b1b146", size = 99301, upload-time = "2026-09-26T11:29:21.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/c8/12811c9b48fde162bb72b6f2e78fada9a247a09d7bb5be2050a5d099c77b/nodeenv-1.11.0-py2.py3-none-any.whl", hash = "sha256:edaa16e6c14d7cf395d75d4bbd5a26390f4dc06501a33b4e76282b02cc688a25", size = 34277, upload-time = "2026-09-26T11:29:19.933Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.45.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pydantic" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/0b/8e10b2e693af8ec54346a14caf36221334775975a747c9ceaa3f8371d96d/pydantic-2.14.1.tar.gz", hash = "sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26", size = 900362, upload-time = "2026-10-11T18:37:55.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ea/a56b9fe5066f3537b7882f77e9c5dfb26d8c2eefdaed9b5fc73d57b4dc22/pydantic-2.14.1-py3-none-any.whl", hash = "sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454", size = 488106, upload-time = "2026-10-11T18:37:53.437Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.50.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/24/af4ec4be49fbc810f35b0bdcacc3d433b56bbfa469fd3bfd4ae116cd9bf1/pydantic_core-2.50.1.tar.gz", hash = "sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09", size = 513515, upload-time = "2026-10-11T18:35:44.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/57/0e237d7091d2cd44a35d243b7344227f90440166860469cd036afc23a04d/pydantic_core-2.50.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:d5e062c01286d861fd6a1c4ff6e063547b3e713067f2df033c0ff97ac2ca006b", size = 2222774, upload-time = "2026-10-11T18:32:43.103Z" }, + { url = "https://files.pythonhosted.org/packages/f7/b9/c720e56858d4e1539503297ed37063e0c08e0f3541c41b777f6a800f75fa/pydantic_core-2.50.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0c003c3b7f49debb893d2d85ae099ac5959c9839e2f330fadb1fcdf7a6594482", size = 2024738, upload-time = "2026-10-11T18:32:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b8/fbfc25875219cc060e613170ff10e850c6d8924beb4910da57c2ee3ba1d2/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:409e0ea40ec30d9158f33574fd758e689f6045a0f2596701828c27816ca9687d", size = 2047049, upload-time = "2026-10-11T18:32:46.164Z" }, + { url = "https://files.pythonhosted.org/packages/d7/43/34210124d504c553688f2f04b48500b131237528ac545b445e0d6d30e0d5/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:131059670f1d2444269b8585cb888963994871932447c08b39ac6a51fcfef658", size = 2173415, upload-time = "2026-10-11T18:32:47.722Z" }, + { url = "https://files.pythonhosted.org/packages/65/cf/6e178e8fdc11da5965bef980983bf46326a42f436871dd51ba0a57f39df1/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6dbcbee53bf17196a7f745aa9bf5a9603953a1e365b1f020be3207c676a3e7c4", size = 2361719, upload-time = "2026-10-11T18:32:49.217Z" }, + { url = "https://files.pythonhosted.org/packages/11/14/bd5169d356aa91bf777e28ba0b281c192d286d03c2812c9c9db023a2f00e/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:325c23f3e35cfbf0fe3486fa5f7260d1e45885173002d30a28ca019994124255", size = 2439803, upload-time = "2026-10-11T18:32:51.025Z" }, + { url = "https://files.pythonhosted.org/packages/1c/bc/d79d000e5203ebef39af839f2ce77a777fcad6e08ad26af9a2fcd114ffc8/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17e722e156d0444ecaefbe640bdb60928752bf2013e2b7a11cdb099aaae19bec", size = 2158402, upload-time = "2026-10-11T18:32:52.714Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/4902cb5fd599422c130bd3124ab31ee8b771199bd56662702eed07d3fec7/pydantic_core-2.50.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:aa8224f10880d9bf1b5993988ba153d42a8b4f3f4f511f93b1f09c93ff613c72", size = 2229755, upload-time = "2026-10-11T18:32:54.126Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f5/c1481f8669f6060d89110c9b1374173fc8767ca276b84f4870bd8acd5e3f/pydantic_core-2.50.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:41bc8237121bd8dc8d888dfd6279fc166ffc88c1f1bf3a8bf00869680533ca4c", size = 2242303, upload-time = "2026-10-11T18:32:55.641Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/77fc3c7653ba9b6e42049e17e96b25388187d4a7c274ab1cb07f58ac8419/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:45c6266d071c241f2a168d45bf8c54344f0effce35e7e6b73afdec11f3687568", size = 2278345, upload-time = "2026-10-11T18:32:57.38Z" }, + { url = "https://files.pythonhosted.org/packages/95/9b/0579c5d12e7f2b16b27e6782427987341fcce07b0725e02ddb0b74add0c4/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:1deeacb112d14d3f4fcb16b165f7dbaf76c70ba6e82f37ba042bdab51970a0b8", size = 2449947, upload-time = "2026-10-11T18:32:58.896Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ac/1b677db91eba54cc5922f4de6edc46ba45c2bd712dfdc0130382d158e214/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1c96fd793b73d1b92e65570132505498fe7b21eaef73cdf74e67e5dfba7ac9e4", size = 2476889, upload-time = "2026-10-11T18:33:00.665Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2a/3a9f6624ee3ea9ccba5249dde11418bb4c35780a7a92608f8768bd3fea39/pydantic_core-2.50.1-cp313-cp313-win32.whl", hash = "sha256:06ead20d39ffd6f2f6f2a8f8a6de67ff8bb1b4f14a8a30e058502514ee2ac685", size = 2101118, upload-time = "2026-10-11T18:33:02.329Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1f/323f78ddd9d9938aac420c1abb4e8ba799fc8bdab0acc67c0593b837c979/pydantic_core-2.50.1-cp313-cp313-win_amd64.whl", hash = "sha256:7816e98acc08119dc0f340ab167048ecc54126316330c1f0caf7c6756c88e28f", size = 2170840, upload-time = "2026-10-11T18:33:03.919Z" }, + { url = "https://files.pythonhosted.org/packages/cb/09/8497c52a739ae425c3ac2f7f56414cbc711c67d374346174f40fe2062644/pydantic_core-2.50.1-cp313-cp313-win_arm64.whl", hash = "sha256:c17799a62c142d61b8a3c51752a7cbc87fe2ad4ccfab10e628a77b405075c662", size = 2161633, upload-time = "2026-10-11T18:33:05.518Z" }, + { url = "https://files.pythonhosted.org/packages/49/33/28b96e81677153715e3eafb9f26663a80841e859fde282a380359b0d3fa1/pydantic_core-2.50.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1cf41f1ae3fa155cf167a72689ad044bcc1e3c97e064123677149bdfb5dafc4a", size = 2224428, upload-time = "2026-10-11T18:33:07.153Z" }, + { url = "https://files.pythonhosted.org/packages/94/40/15c06410c9b7b8da5805d27b64e09bd3f900e986728106db2689dbc51513/pydantic_core-2.50.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4df197990c15b5a37c5a277d131d9f2c67de6133f2e5dafd80d9bba4b99f46f9", size = 2023740, upload-time = "2026-10-11T18:33:08.763Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4e/5eb629f6efc2a27e421d789dd4bfacbb5f6d09c80c28b13d1e87d73163d5/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0036473f5583e6a60e50b8b21651511564277a3f05cc5dab8cf579f552cd5f6c", size = 2049435, upload-time = "2026-10-11T18:33:10.366Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8e/f195aebec49ad12318876ac2368c197a7939f5d52f8e156d2238ef8a4588/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:992c3514ec891fa7858099183e4d64e6bd5a5d4ff452fae29df22faa77a006bb", size = 2172975, upload-time = "2026-10-11T18:33:12.368Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f5/7ad9fb83010cd5ea0948409db105676ed779c4e709e2d3d89b9fe2558794/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:739dc730e6be3bd5ec2f4ab5cfc7eb047cc45fc1497b3bafec74ff2ed07df597", size = 2364197, upload-time = "2026-10-11T18:33:14.244Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/bf0aab3e78301d202b82a0322ec968cd703b11fc0623fde9a28708936f62/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32fad3a91e51b6d2039c572db04a5a873260b399f6bd62c3552671fa7a4a2899", size = 2444259, upload-time = "2026-10-11T18:33:15.79Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/38f6d6564fae57d9b12e5676dfa947e0e7d5c46dbeaa7eb3352261d6d299/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42b54c2c90ad348b5e3a85e03e715d572c1fde357ef104cdfe3b03b697a404ea", size = 2159857, upload-time = "2026-10-11T18:33:17.51Z" }, + { url = "https://files.pythonhosted.org/packages/65/a0/fb0a3ca10f139dcf765b2d312d10cf0f65c57c59993229d00ad12b14ceff/pydantic_core-2.50.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:2df1ff41884de2bc4b307bafd7c40a691094fad2ff8e767e5b45a319257bcf4e", size = 2233954, upload-time = "2026-10-11T18:33:19.591Z" }, + { url = "https://files.pythonhosted.org/packages/8c/6a/e63842252702aa4ec6e6b7178ca85e541a77459076592c23ebe2d9840b33/pydantic_core-2.50.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe90228920fd8ff2be62622b6bb8a2b11acd65046d50c6b130614b5879605a20", size = 2243876, upload-time = "2026-10-11T18:33:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/39/25/5991cf8318b37e0dfab47b87541619a1df8501a793cba0d978846cba37a7/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:844b869f118e22a41a091bdcedda8a71bc1b0f62c38d1a0c3211cece47e1d8fc", size = 2277908, upload-time = "2026-10-11T18:33:23.324Z" }, + { url = "https://files.pythonhosted.org/packages/a7/3e/3ee8baaa6cc25a6961c69168cf9ff0f002d56f4e0d541ad6724c18fb61f3/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:2eb75304506894a281d346220a4f7481a1b8729577c5ed2a05395991966a8396", size = 2449285, upload-time = "2026-10-11T18:33:24.942Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/acbec6deac13fe697a80c275a9b6661db62c4d323bac8a47347b1f39c7cd/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6b20a4bffabdad0db2927ac034ae3b8a681b1f7a0182f3e60b479ad2fde21ebb", size = 2479013, upload-time = "2026-10-11T18:33:26.925Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/21a3f237b264389f6219d053ee78fc1b5c2fd402c1b1cb2b6cb8b85f9834/pydantic_core-2.50.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:99ba9bc2b8062ea0c326a990f7f00e6530c23579de66dd246e72c4cafef950a5", size = 1484400, upload-time = "2026-10-11T18:33:28.693Z" }, + { url = "https://files.pythonhosted.org/packages/09/ce/077a6d262d12ef09108377ac0717f029f420d773ace63cafd6143af75568/pydantic_core-2.50.1-cp314-cp314-win32.whl", hash = "sha256:cf356f70551d40374eaffb1aa63f1eb6d2006681cbd7a9faea173ce0f4dd7cd2", size = 2103222, upload-time = "2026-10-11T18:33:30.326Z" }, + { url = "https://files.pythonhosted.org/packages/14/4c/350a2415209c43d670eb71d3c040f332c04a30583d39e311b05c7ac15762/pydantic_core-2.50.1-cp314-cp314-win_amd64.whl", hash = "sha256:d32f3acc081cc3923386d88f422cde8892335e95f034e0104bb4cf9310d9915f", size = 2171586, upload-time = "2026-10-11T18:33:32.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/92/9bea6ca96580a0902fed366f064f0829e404f41889b34543279a1162b888/pydantic_core-2.50.1-cp314-cp314-win_arm64.whl", hash = "sha256:bed5163e03b98bc1fa2eb05d74c63d9c5c95d8ed6254985481640fbf5e237dea", size = 2155773, upload-time = "2026-10-11T18:33:33.896Z" }, + { url = "https://files.pythonhosted.org/packages/86/8c/f121f073cf32bdd5cba7e6230b1ce0ac845b0cf43e44dd597d95af272db2/pydantic_core-2.50.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:9572c1369e9c9da2d64a7b7992c786d90ff295abc93964cfe3125e4290768070", size = 2234017, upload-time = "2026-10-11T18:33:35.588Z" }, + { url = "https://files.pythonhosted.org/packages/a4/69/2bb2bcd6146cffe98d760a46c42ae71efd5151d9b2f9c9bf6619a3b32083/pydantic_core-2.50.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2005207aafe1231315718bf6ed5d064a7300fb4772754af35ee72fc68159492e", size = 2001016, upload-time = "2026-10-11T18:33:37.57Z" }, + { url = "https://files.pythonhosted.org/packages/20/b3/fbf854c7d07ec114260c26e9e2071a4381740f9ae09641dbfcbdf2a18c45/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64f6047f62a6c5ae08d0a6afb035667aa2d97c3d20d69762e034c5ea144d92a5", size = 2023015, upload-time = "2026-10-11T18:33:39.433Z" }, + { url = "https://files.pythonhosted.org/packages/65/20/6de55b2f92cdb614b745c6e9ced639fc4fd7e1e77604825a88bdece6fcbd/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ef800dd7d85bcdadf4c3076e4c94e43939493558a3b69a1ea830c706d4617bb", size = 2192543, upload-time = "2026-10-11T18:33:41.381Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d9/19e91c94bd5c405945ce2f15526808aea37e162c253160de8ed7bf70b406/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0135bcdcaa0f23573f286e4cb5e0fd2962700964ed13df085b85f2b97aeab9e", size = 2388704, upload-time = "2026-10-11T18:33:43.167Z" }, + { url = "https://files.pythonhosted.org/packages/b6/bc/2e24c8415eae1a25ee5a5484946a917123bad2e9d01689a759236928175a/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b3a6f334c6a2345ca15318ff894502a90012536404b37c844a976c76c846e0b", size = 2464118, upload-time = "2026-10-11T18:33:44.856Z" }, + { url = "https://files.pythonhosted.org/packages/bd/1f/945b8053cb061c64e102bcaf7bfb9ed740c0bd4349f9bb978a7d4ddff4ab/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06e01fbbfdb9be777b316a71b6c49efaf4a08b615d0a98d678cda3023f79d019", size = 2150142, upload-time = "2026-10-11T18:33:46.661Z" }, + { url = "https://files.pythonhosted.org/packages/2b/78/96a3e50bf0d64aaae781107eb9335b7529d05a85793ed6e7241c4cd1d931/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:a29a061fec0b4e2d714f277e70a3a18125ecff803f2fea6eade2f2e53711d112", size = 2245577, upload-time = "2026-10-11T18:33:48.574Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5d/a6038a0322232758a6ebfa709f4ca14a60bf5b93940cd0b345056a557da4/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f5187624823423e1d1b82b1072ac41dc837389e18d3d0572cc19bbee46cd550a", size = 2263728, upload-time = "2026-10-11T18:33:50.527Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4b/76ded3333a457a9344c4f2d63f2d82d0101654b05178fa4ddfe7d3627674/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3e46a9eb0a0901dd6275e6b06ac3a464885ef350ec4121fe486869de8053e4bb", size = 2293630, upload-time = "2026-10-11T18:33:52.362Z" }, + { url = "https://files.pythonhosted.org/packages/a9/00/9eca378335c9c1b72bc779bf6e4c2a82ce784f14ec48a0e87102c9870e05/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:756d669f04e62ec4148ecfe22be6a4484d9b1181a6ef32e205ebfd200540858b", size = 2468048, upload-time = "2026-10-11T18:33:54.118Z" }, + { url = "https://files.pythonhosted.org/packages/35/ca/e3832e9cf93651251de43c8c5c029ed680a3c1a0a1b17ee26c81bed08cbd/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:c516cc5367ca3448995d42cb994bf3f4c9002d2a7c22eac9622551269ad1b807", size = 2512159, upload-time = "2026-10-11T18:33:55.99Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f2/773469b5a10a39116a2cb17edaf6d722f017dd8da07161b371465311c542/pydantic_core-2.50.1-cp314-cp314t-win32.whl", hash = "sha256:9d1bed94af6a63835461f3cf7502058eb166c58c4778e11d0f433cfb1bd69e19", size = 2098406, upload-time = "2026-10-11T18:33:58.043Z" }, + { url = "https://files.pythonhosted.org/packages/ee/42/0bb74f8f25204b259b11ab7c12dc7f180893b6bd706118b385147fb6efd5/pydantic_core-2.50.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c8dce1f1e0e5358b682a6ad3fa5e31b31d4560997b8e61417e9217c8d60f8a0c", size = 2148481, upload-time = "2026-10-11T18:33:59.913Z" }, + { url = "https://files.pythonhosted.org/packages/47/0d/d801646c9679a4e630e15cf521d4108b93854b42a6e6e02391bd4c6b1095/pydantic_core-2.50.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ceff0acc940be2715bd6ad17b24c0e5304abf44f6efd0f81ee8499e640f9dc86", size = 2174972, upload-time = "2026-10-11T18:34:01.875Z" }, + { url = "https://files.pythonhosted.org/packages/b2/84/23984b763d8862a02a13d27a44b6e8169428fd85ecfed88f54c29108604d/pydantic_core-2.50.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:8a6791afa2245e6c6b180122d105941644f5bd410bb18623b408808cc41a3102", size = 2224599, upload-time = "2026-10-11T18:34:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/4d/90/a63cf8586abc1d1a3f6d9b18f0224789ebf003851f092ebda3c7c863fd32/pydantic_core-2.50.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:84f34323a61a365b4e9295de6028474754829aaddd59c7bf1a040e7487ef8f3c", size = 2024191, upload-time = "2026-10-11T18:34:05.901Z" }, + { url = "https://files.pythonhosted.org/packages/a6/6a/f34bff9808ffb4907fbf5f5040457d253cacf2da7fce9efbc99ca1b1a44d/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23edad659e8dbd8ca7e4e877fe6c81573abbdf215bd25a68b53e1272f58b80c7", size = 2048252, upload-time = "2026-10-11T18:34:07.757Z" }, + { url = "https://files.pythonhosted.org/packages/b2/54/13f419bf1eb59852003818e25935aaf175687d978f4c4bca70e08fe40a3b/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a5fce22f1e87d181e924e12da7d81cfe031fb3881a5ddf26ad28f141756ca43", size = 2173376, upload-time = "2026-10-11T18:34:09.592Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6c/b5a34d24cd0c81669d8f8339d74e6815abcf2f8fb48ab4b49b84c09be1d5/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c73622ef819328873b53109ee4f77ceb598bffedd02daf916102be3228866b78", size = 2364559, upload-time = "2026-10-11T18:34:11.534Z" }, + { url = "https://files.pythonhosted.org/packages/eb/8d/d64d6216a8df365082665927ff923f183056f9049fee08e9777c9ac05296/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce8c25ca38cc0e3d7753ba180808de2c0c8cb24eae0df64491e40921454e9831", size = 2444697, upload-time = "2026-10-11T18:34:13.535Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0d/1b1149f60a00ea21ba5f70e28acbd40feb4598af414f80f53c921fac07c9/pydantic_core-2.50.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7689580e72a642ab5ec64d5f55b2e33636fa43b4ebe63c0c2c965ef307c7d1aa", size = 2159961, upload-time = "2026-10-11T18:34:15.56Z" }, + { url = "https://files.pythonhosted.org/packages/1e/35/f236549299dcc78e71e945495d7ec67e78d20844d0999c60601685003031/pydantic_core-2.50.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:d5c0e32fdbce7f1e8ef4d11f655694bf5f4175c757a9f1dc2be09b8864e5bcf5", size = 2234581, upload-time = "2026-10-11T18:34:17.502Z" }, + { url = "https://files.pythonhosted.org/packages/6a/85/26901a490522b7f75ef9bb9a7afb73e5bb550f0e1cb5b99a0069183e2eb9/pydantic_core-2.50.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40f523349960fa30f3ea51404308ff50f9997a90df639590f47a057c1f32b415", size = 2244090, upload-time = "2026-10-11T18:34:19.402Z" }, + { url = "https://files.pythonhosted.org/packages/ca/2c/481bcfc70ceeb77a778ca6e5b705fe592cb64735c108157f81b7dd9c280e/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:d4193206b6587047437f6f11d7e776df23e1c1e23af2a54d9347275614791e10", size = 2277948, upload-time = "2026-10-11T18:34:21.317Z" }, + { url = "https://files.pythonhosted.org/packages/24/eb/f1e09333faa7ba447cde967310f758430cc7c5e987bdab5228816c70030d/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_armv7l.whl", hash = "sha256:84bc765b282a9d5b7fe0348b8648904f25a6a04b2139da52b1dd30c8ac3a2c8f", size = 2449688, upload-time = "2026-10-11T18:34:23.321Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b3/036bde636db8f76d92996e81aefc75678ab5cec4a07eea1ad0c72a893fc3/pydantic_core-2.50.1-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:ed1e728b39a383c81035b2459cfcb35d99dfb01f7d6ebe3a913bc1cc5b81e459", size = 2479286, upload-time = "2026-10-11T18:34:25.214Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/07f018294ee18d144afeb6df1a47d9e92960f014be45c5ed13519fa5af95/pydantic_core-2.50.1-cp315-cp315-win32.whl", hash = "sha256:bc94f474417604bd383d2cd445d071b07dd55fedceed3ce33407bf1fcc107290", size = 2103683, upload-time = "2026-10-11T18:34:27.42Z" }, + { url = "https://files.pythonhosted.org/packages/5f/98/f9bd7e1f9b6709f155acb9ef826d9c3884fe925f811fd8e55b9b52280bad/pydantic_core-2.50.1-cp315-cp315-win_amd64.whl", hash = "sha256:983a662de2571cb2502fc8ff47b6770b03d025d2eb314c92f77b3f07c74720ed", size = 2171681, upload-time = "2026-10-11T18:34:29.508Z" }, + { url = "https://files.pythonhosted.org/packages/10/87/4bb3e1e7f385c076ab5af4d6dd0571b22042cd8207eb810d5b9fef15ae31/pydantic_core-2.50.1-cp315-cp315-win_arm64.whl", hash = "sha256:94845ff54dc5193f228cab81b2662a04bfbb892e95bdc15edf7399000ce57d54", size = 2156251, upload-time = "2026-10-11T18:34:31.455Z" }, + { url = "https://files.pythonhosted.org/packages/47/47/83643225b08f2aef6c8cc4bbe6e3f79c5e139c4364a6e450d0f399d87774/pydantic_core-2.50.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:4a53d13cdfbedbfa87f08b83c1a0a5efcc767d785a4b41934fa9cb672670493a", size = 2234363, upload-time = "2026-10-11T18:34:33.65Z" }, + { url = "https://files.pythonhosted.org/packages/5e/66/127ca649ba2f2462039dc1e694c6c01e00a3689f023a617364d3507e6d97/pydantic_core-2.50.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:efbecf43d321f7b9281441f1f213f7c21c66988b0e06c2730ba13ed47a46bb08", size = 2000972, upload-time = "2026-10-11T18:34:36.037Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4f/e421e0a5d653b1203b090b2e48745976988d7338a647940704b5b9c2b399/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc1f08f68dac9f9e83845a8039880aba2ab553eb9b2259c3243a313182c253fe", size = 2022925, upload-time = "2026-10-11T18:34:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/ea5568e2491e1a71100f15ae8c2d01ef52db42a184a2d4e716bc79e5eb8f/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5dfe41f232befddb9c4377f6cfc702b51595e2d78ed082672adf8758d2c4619f", size = 2192867, upload-time = "2026-10-11T18:34:39.921Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5e/3b8c3a35acbe219909ada5defad5d7d9fed845fb2b37bd5ec518f453c119/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adc06d218a1cadfd2ec4628424d7d79ce4eba69c2965e7e7b55106f0da5208c8", size = 2388687, upload-time = "2026-10-11T18:34:42.184Z" }, + { url = "https://files.pythonhosted.org/packages/b6/aa/7889b4e515f91a2e8c0ae6b5081fec0feb30c4398d1434e14793c60f173a/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2cf91809d0721ab81592ba67bea7694821679c10b1a2e3c3460082b286c1918a", size = 2464415, upload-time = "2026-10-11T18:34:44.384Z" }, + { url = "https://files.pythonhosted.org/packages/08/78/93449e628eb8a6fdcce3eff9043081179d1bc7ce6f1bff32dc5006f41e00/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23923ab9292c40da026330b1ecf4dc2618c8e86e0422e5d1fbf50d94d64ca4f8", size = 2150823, upload-time = "2026-10-11T18:34:46.392Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f1/72c5bc129fceb0d00f05dc1e67f518c1728de1928c55f81fc13d7690de39/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:f3377c8c2b3ce898423c5e5dd94c7982e30aa7717a7e6ab2470b9de364963709", size = 2245806, upload-time = "2026-10-11T18:34:48.805Z" }, + { url = "https://files.pythonhosted.org/packages/28/2a/922a0e78f3aa6ab837b59f88190233fb8dad546c17995fde9bb3ed3b9b49/pydantic_core-2.50.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:455a773617b5913bf5c20d0692e5787b119e52c4d40ea644ca31f5758fd31be2", size = 2264160, upload-time = "2026-10-11T18:34:50.876Z" }, + { url = "https://files.pythonhosted.org/packages/52/a8/0f1449e3e1b20941c9372faa18e7b6a092e30cdfa841902473f7989532ac/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_aarch64.whl", hash = "sha256:1a9006395dece0e32e704c315eff8a00bede494f6108546cfc5539c89fef4f9a", size = 2294031, upload-time = "2026-10-11T18:34:52.876Z" }, + { url = "https://files.pythonhosted.org/packages/d1/d0/1031f492857de70355fb16524bbb03efce5fd34c93ed4a1ec60be07e0d4b/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_armv7l.whl", hash = "sha256:d2d82aa62521c55ddfb000ae70f88cdd8de974078f6024e821dfe5addd0c818f", size = 2468423, upload-time = "2026-10-11T18:34:54.995Z" }, + { url = "https://files.pythonhosted.org/packages/46/52/269ffffa645b8e47906395a39cf9db1151ae9fa4bcd7f47b960bc31baf37/pydantic_core-2.50.1-cp315-cp315t-musllinux_1_1_x86_64.whl", hash = "sha256:009634b83993777ddcd69cad0ffcace43dabde692109528e35f0fde91e386a8b", size = 2512598, upload-time = "2026-10-11T18:34:57.149Z" }, + { url = "https://files.pythonhosted.org/packages/31/5c/e47e28281f20326ff6f3c31d626f0a83e615d2d94ba41bb0ad6ec237184a/pydantic_core-2.50.1-cp315-cp315t-win32.whl", hash = "sha256:3fde4fdc6487a58d944ca87cf5adc95d5f266e872c19599f5f4c0a8a1b1f9f9f", size = 2098535, upload-time = "2026-10-11T18:34:59.313Z" }, + { url = "https://files.pythonhosted.org/packages/03/ad/759e181e69c1b472c60b2049e5f61d5da1deefdd5ce1df4bb2ebcf771f25/pydantic_core-2.50.1-cp315-cp315t-win_amd64.whl", hash = "sha256:1c8632d4ac04e6f91128fca584b3a8a507d81604c24eeaaad00d4be42765c32b", size = 2148645, upload-time = "2026-10-11T18:35:01.571Z" }, + { url = "https://files.pythonhosted.org/packages/6d/56/8a702c27e5be9f47e5f19d8669227424290e4c024e7c370279cbaf244b4e/pydantic_core-2.50.1-cp315-cp315t-win_arm64.whl", hash = "sha256:c3ede305158e75510be50869b319550ab072008c13d64d4ab1e094fb286b6f44", size = 2175297, upload-time = "2026-10-11T18:35:04.079Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pyright" +version = "1.1.414" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/1b/244c7b710031ada80f27e579ec20d28a2285dfc318fed0339866b1047f12/pyright-1.1.414.tar.gz", hash = "sha256:523c0a97c60da6333234955c277730c9cf4f5bd6d5399e7b7d2b0fc5d3599524", size = 4154638, upload-time = "2026-09-10T12:26:53.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/ba/18b6e682ead424ad24bcc134339ae5d1b931cd9ae260540592a058a91279/pyright-1.1.414-py3-none-any.whl", hash = "sha256:2a6b4b3298c9eec174c5ed83bd338de6eee82df2992f3e1930e6199d381be36f", size = 6225049, upload-time = "2026-09-10T12:26:51.427Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/26/2fbeedb218a787a5eea551c7532cac4e009f83d689dd2faa0d0353473f86/python_dotenv-1.2.4.tar.gz", hash = "sha256:f0d53e69935a851c0dcc78f3ab7aaccd8cabef0b92382b576b824212902873c0", size = 60824, upload-time = "2026-10-01T05:36:10Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/d1/38f3a3405989a89ac18390803e70c6ad7c7760da4f9b83cbeca0c44a0c72/python_dotenv-1.2.4-py3-none-any.whl", hash = "sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc", size = 23266, upload-time = "2026-10-01T05:36:08.633Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e7/e6d5749983b020e6cb1055abec0a6f361a8e94f0184cdcf56d4760995f02/ruff-0.17.1.tar.gz", hash = "sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f", size = 5323349, upload-time = "2026-10-15T15:23:18.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/78/90c46ca6250709f7c37e397b60d06061adee8d0f7d13959ed6a8e49b14fd/ruff-0.17.1-py3-none-linux_armv6l.whl", hash = "sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a", size = 10147528, upload-time = "2026-10-15T15:22:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/a2e28bc35b083ae51fa07dfba5280a667c8b01ecaee7802fc8849dcd5c46/ruff-0.17.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb", size = 10225974, upload-time = "2026-10-15T15:22:36.328Z" }, + { url = "https://files.pythonhosted.org/packages/8e/71/7a9029a81c8206ee01a98f6cfeeb56e6c095af86857581a37045c46d7e94/ruff-0.17.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74", size = 9270361, upload-time = "2026-10-15T15:22:38.651Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/e075e9c917cb0575297880c96768304e881240171c870cbd8d9b7b54bec2/ruff-0.17.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978", size = 9938300, upload-time = "2026-10-15T15:22:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/87/27/0b4bb926a49de2383f161272b4d863571a4c7ecdbf595a6bc58813a13367/ruff-0.17.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640", size = 9948464, upload-time = "2026-10-15T15:22:43.681Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/518d57e0e02aecb8ea4acac6b57b51a53173d25fa734952bfdd7d531e88e/ruff-0.17.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1", size = 10638639, upload-time = "2026-10-15T15:22:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/01/82/d89a4e498ade59e6cee52d67f8da569019372212b87468c1fdeae884601c/ruff-0.17.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05", size = 11337238, upload-time = "2026-10-15T15:22:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7c/3b0fdef9352656317c977086fe1d802902406e4f2a0d8578da90c0e6d1c5/ruff-0.17.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785", size = 10896327, upload-time = "2026-10-15T15:22:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/5566290219708ce2928f3a29c4f843a1185b341ec6111c4631a33c5ceced/ruff-0.17.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189", size = 10350811, upload-time = "2026-10-15T15:22:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/ca8462dabb3c413990d49703ba306c30c862195a75448d080fa8fcce9775/ruff-0.17.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467", size = 10569994, upload-time = "2026-10-15T15:22:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ad/dd1864c1f1db17c8cfb297ce70b352388e6977620f66f7182c52855407df/ruff-0.17.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab", size = 10059672, upload-time = "2026-10-15T15:23:00.084Z" }, + { url = "https://files.pythonhosted.org/packages/65/45/3f56cf1d2168c2165cdd5ac6f9668aee66a4cfdd4caf2b2cc5ccb2b1d568/ruff-0.17.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca", size = 9947803, upload-time = "2026-10-15T15:23:03.132Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4b/5457cb2be4c1d1ea5e919558131a0f197a4a454a0530d9e30820e8206e4c/ruff-0.17.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4", size = 10351807, upload-time = "2026-10-15T15:23:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/49/fb/db15b4f8fa24a1e36b99e1401f42ef6df16ee299a48670d5d262b771e7d7/ruff-0.17.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a", size = 10765716, upload-time = "2026-10-15T15:23:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e8/1e186578076aff7a80735f625005fde678412ab609cd323ed182eb7d6b87/ruff-0.17.1-py3-none-win32.whl", hash = "sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a", size = 9895093, upload-time = "2026-10-15T15:23:10.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2e/da3d9589bae8e8b58f3a48bd5bc0b980232fb14a83bf02c3be7660f87c1b/ruff-0.17.1-py3-none-win_amd64.whl", hash = "sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7", size = 10097955, upload-time = "2026-10-15T15:23:13.309Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/0f2a4dea06a5e12c1191ee3d96b1e1233797d0d5e18f97a2b909a95a9836/ruff-0.17.1-py3-none-win_arm64.whl", hash = "sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4", size = 9853542, upload-time = "2026-10-15T15:23:16.05Z" }, +] + +[[package]] +name = "starlette" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/0c/6efb252d091ecccd7d62048ae11f0ea35cd75a4fbaeea5e30f9c3bf91d10/starlette-1.8.0.tar.gz", hash = "sha256:1565dc0b35d5737a271ed1e0e04e949f4e81198799f216d2667b0a0fb9cf9522", size = 2730457, upload-time = "2026-10-13T07:54:39.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/b0/5742e4ac7af5eb58ec3470a537a49d7aa507e5539413e504b3a65ef50ba8/starlette-1.8.0-py3-none-any.whl", hash = "sha256:dfdd6b29c26483288088d990eee59631dedadd66ce20d203402a7ca8e3c4656f", size = 79612, upload-time = "2026-10-13T07:54:38.019Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/42/02c739ce85fb2ee8d99212c61417da8140c6b87e9d97c430bea520d76044/uvloop-0.23.0.tar.gz", hash = "sha256:28d160f51ab4da3b187063652e643dea6831072add4adc1e6d62afbe73b6be27", size = 2559185, upload-time = "2026-10-01T03:17:04.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/83/eb980d64e6dd5da46d4dc35755fa6afd6b5b47141437cf89615f1117c5a6/uvloop-0.23.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2dcff2d69be43e6559e5dad2c5a7a2dbfb60e05a77311b6c4b7a4a8123d86c65", size = 1412726, upload-time = "2026-10-01T03:15:52.49Z" }, + { url = "https://files.pythonhosted.org/packages/04/c1/02a725e7698134c647904bdee6589e2be14a0e7fc9942c74f86e2b90d48b/uvloop-0.23.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:19c64108b507cd0bc140e400e3396bacebd9d504956aa7726272bf6de7d9aabb", size = 779071, upload-time = "2026-10-01T03:15:54.02Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1d/cde53c79e8c01884ad1cdca8e407e086d523362cfe4139e2c2a8dde27304/uvloop-0.23.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1748321e3c59a14a75404b1ae8d5a8d81c4e201803ea0e14c1b6fd84421024b5", size = 4395323, upload-time = "2026-10-01T03:15:55.549Z" }, + { url = "https://files.pythonhosted.org/packages/98/54/b12915bebbf99d7ae0796211e7f5977b95f069830dca45dc1a346d84125d/uvloop-0.23.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2cba180d6451822763eda8364f342435a873bcfb3849cbd82fdeca248ca65eb", size = 4480449, upload-time = "2026-10-01T03:15:57.362Z" }, + { url = "https://files.pythonhosted.org/packages/f7/8e/da6de68c31549a052a105fc76f5a9a204f6df22cb0909440aa4dbb06f9a2/uvloop-0.23.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dc61e4f9e37b507069dc7e659ae28bca7adcb04c993c3508214315d12c63f848", size = 4219177, upload-time = "2026-10-01T03:15:59.351Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c3/1b53c6a89dc9c9d5cb75eb9a0b891ad69b32e1421ad3aa01617a9cbdcc78/uvloop-0.23.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7337b06a9f9ed9ea3049f04b76f65819db9b19bb832ee598e97b388eadf25e5f", size = 4346132, upload-time = "2026-10-01T03:16:01.064Z" }, + { url = "https://files.pythonhosted.org/packages/4e/a4/00e85345871c59c834a23c136c1771205856028ecc8ba940b3951178e59b/uvloop-0.23.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b90397a50ad6332ed3e459c648ac20d182cce24a557354363ad85fc9ea4a17cd", size = 1421363, upload-time = "2026-10-01T03:16:02.599Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a9/e5f0f3cfde30af3ec32eba8ec07bccdba2b5116afbd1ecc53edfeb0a0790/uvloop-0.23.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:be53e1d5f83de43dc175c87612ecc128d444b38e5c56cb3f807f5a73d6887476", size = 785177, upload-time = "2026-10-01T03:16:04.018Z" }, + { url = "https://files.pythonhosted.org/packages/9e/79/9ddf78f8cd75a15c14a09a57f59c587b8cd9d82802c5c8368b9c3ebefa0b/uvloop-0.23.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b3cbc4f96ddfa1fb88a78a69dd851369825b7816d9702eee8c4461505ba172e", size = 4381060, upload-time = "2026-10-01T03:16:05.642Z" }, + { url = "https://files.pythonhosted.org/packages/1e/20/57d63c44d32326878fcad5c63854afc9deb394ed95673c1b1a429178c79d/uvloop-0.23.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31e0cf90bc8fd88784f6802cdba968a51fb1aec1cc3feec74d862b2d371d1330", size = 4418891, upload-time = "2026-10-01T03:16:07.326Z" }, + { url = "https://files.pythonhosted.org/packages/12/c5/0795abecda2cc3dfe41033f880a32a9ff103be4e6b177ac736833c153a0e/uvloop-0.23.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa8ed556fcc87a4091cf61587ef172fa104323dc89ecc085a618ba7ff8629a8f", size = 4214811, upload-time = "2026-10-01T03:16:09.13Z" }, + { url = "https://files.pythonhosted.org/packages/20/18/9010dacd5221eec1bd79a4a83ac68f3db6a42d7bb657f7b640c4838ca6b6/uvloop-0.23.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3fbfe82829d8e381426a289b87e59e585278728361db9ce975b88b51f64f410", size = 4294876, upload-time = "2026-10-01T03:16:10.875Z" }, + { url = "https://files.pythonhosted.org/packages/b1/08/f6384a03c771d00067cba4f542a69b2fc1a982e9fd78b357c2f788678d72/uvloop-0.23.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:7e35c9bc977760981693e1a7a51493b58ee5a501f9ebb1e547565ee40b6c6208", size = 1494811, upload-time = "2026-10-01T03:16:12.399Z" }, + { url = "https://files.pythonhosted.org/packages/ac/01/756a4fb24a449f313cf4a153eb0c6210b49cfe5539255ec9fb1e17d2c4ef/uvloop-0.23.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5bb9be71d9ee39b4359b832f9569518ec9bc08704194034e79e4958e6bc4d46d", size = 819396, upload-time = "2026-10-01T03:16:14.094Z" }, + { url = "https://files.pythonhosted.org/packages/3e/45/e314b0c600b14f53dad3a3c2d7a922a249a88225fd727652b53e1854b9dd/uvloop-0.23.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e84575f11873c109cf3962ad0bdf679094466184125f4cadcc41a73febff41f", size = 4734966, upload-time = "2026-10-01T03:16:15.815Z" }, + { url = "https://files.pythonhosted.org/packages/66/0d/8686a7f0b1b2d55ebd770ba21f8e0e4ffa0cde5ab738f43ffb8264499052/uvloop-0.23.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbbdb8fcd5e7062e546eec1ac78c28bb21ae7df54c18f8e4b06e15a18d661a49", size = 4584963, upload-time = "2026-10-01T03:16:18.198Z" }, + { url = "https://files.pythonhosted.org/packages/78/b2/034a2d47e435ac02357c42956246887167bdc0357bdd6ad31c5f6d94497b/uvloop-0.23.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:76345f51367fb1f23e08605c6efb18374f669be5b223658fbab6b17627950507", size = 4421388, upload-time = "2026-10-01T03:16:19.953Z" }, + { url = "https://files.pythonhosted.org/packages/f0/77/131f4b583e6b4b715c404a66b51c812d701db20f25c9018b188a2b00062c/uvloop-0.23.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c7ef4701a96553514b2688e342ef1bf2beae6cfd172d89a76c768292aabf405", size = 4402414, upload-time = "2026-10-01T03:16:21.716Z" }, + { url = "https://files.pythonhosted.org/packages/58/3d/ee11f4718ea1280595c67ed25c83d4c92115dc100bbdfd192d3ed9339168/uvloop-0.23.0-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:f1341c6abcee1c31277cfe28d34e46196f2143ec3d755e6efe7452126e1f626d", size = 1418095, upload-time = "2026-10-01T03:16:23.241Z" }, + { url = "https://files.pythonhosted.org/packages/f8/0c/7ca516a0671418517d79a09d3ff2ccbb44af94c75711afa6e4cf58aa6f65/uvloop-0.23.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:e095f9e105af76593b4c183bb0bcbdae64bd913a59ec595732dc108b48730ab5", size = 784837, upload-time = "2026-10-01T03:16:24.666Z" }, + { url = "https://files.pythonhosted.org/packages/35/95/75d4e28e596d505b7ae11de517646b4ca3d369fb8537ba755410380da11a/uvloop-0.23.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f673d835bdb1a60229cc3609a113fd2c9ce3f4a3c75ad4eaed111180c00199d2", size = 4380276, upload-time = "2026-10-01T03:16:26.389Z" }, + { url = "https://files.pythonhosted.org/packages/10/99/68daf827ad62efaf4667d1f3fda127046d42161178396bdd93aab3684082/uvloop-0.23.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3f23f403a273900d57de6ee5ca0614c650f7f58563065dad1a4744498960e53", size = 4451496, upload-time = "2026-10-01T03:16:28.364Z" }, + { url = "https://files.pythonhosted.org/packages/71/69/f67e696ee688f426a96f99099bae26fec14a1d0fa75dccdd6518ee267c0c/uvloop-0.23.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:cbe8d03d4efcccdb7fcedecbaa1e1fa02913eaf3a74cb933634a6bc6d2ea9e2a", size = 4212541, upload-time = "2026-10-01T03:16:30.014Z" }, + { url = "https://files.pythonhosted.org/packages/f1/6a/c8c436a9d7453297b4be70bdf6a9f9fc9400da45e0059ddf7b28ab63f4c7/uvloop-0.23.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:4f1798f56c6f4ba5ac11fa2869e5717926e4470d97a1dd42b4f59219d43b5027", size = 4319377, upload-time = "2026-10-01T03:16:31.705Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2c/8fc15a03489299aab8a6212dfe0f137dc39836f915c87f7fd9d9ddd814de/uvloop-0.23.0-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:098a85e1393ef5202767b7e5fb41a32cd8bd81e6ee4af364c179801c4aa3f6d4", size = 1493428, upload-time = "2026-10-01T03:16:33.859Z" }, + { url = "https://files.pythonhosted.org/packages/b7/7c/05e4a210790229607f71460fcb2ed4a2c7bc72668d8a928ce577c22e38f8/uvloop-0.23.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:5a2bbad3a63007f7e9524d4903ba04fee252557c2acd86f9a3d4f91786695254", size = 818115, upload-time = "2026-10-01T03:16:35.45Z" }, + { url = "https://files.pythonhosted.org/packages/65/14/a40b11c6c024213803b13955664a15754c72f64c873a33d986b26ec9ff5b/uvloop-0.23.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a08875543bbd4519faf30497506c9cda8a48470467ffdf967c7313c7a5981a8", size = 4734149, upload-time = "2026-10-01T03:16:37.025Z" }, + { url = "https://files.pythonhosted.org/packages/9f/83/f421a077712c1e87603bfec62744c3cd3a2f4b47378025db3d740df9af0d/uvloop-0.23.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:12634f15e6625f78b3f2922f91404c4d7173487eba11746764153f556e9852dc", size = 4661763, upload-time = "2026-10-01T03:16:38.719Z" }, + { url = "https://files.pythonhosted.org/packages/f5/62/25dcaa6b7e7b48f82ce633854ce96597ab768f9650931f4f86c572de392c/uvloop-0.23.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:378188efbb1524f2219d05246a3e1e5907217848d2882144dff59585f1b81d55", size = 4421324, upload-time = "2026-10-01T03:16:40.488Z" }, + { url = "https://files.pythonhosted.org/packages/05/46/04628239b43dcef703af314202a3307d6060918e2d76aa86c5b1188f5551/uvloop-0.23.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4b8e207c67d207a8608fec57e116511030af3495dc0109b8c333cf9cb412b16f", size = 4462501, upload-time = "2026-10-01T03:16:42.359Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/68/e6aa0b77d217b31f8f486ec0cdfe5e00e6e38dc0be657e7d85819b9faf0a/watchfiles-1.3.0.tar.gz", hash = "sha256:99aee4a07847c06820765fd7b1b49ceac4f3f711ccb7d104655a33231de1c207", size = 108119, upload-time = "2026-09-21T09:08:55.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/fa/c0b840d5d8bafe640925408ce11095948c8945bb61be1e67d0ab629b872a/watchfiles-1.3.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:000b9688fc8133037a8b075c8ebf98f32844ff8964dda61e85c1db547dafc441", size = 399703, upload-time = "2026-09-21T09:08:13.518Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/894799b485fe9473ad10422a0d9668e53fb78e3a2b5cc6061159572844ad/watchfiles-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:bbc1198edfdc90fda0600f825aa94150f428dfcbf8138746f55998e0e660d64c", size = 397647, upload-time = "2026-09-21T09:08:15.16Z" }, + { url = "https://files.pythonhosted.org/packages/5a/61/f277cd5ead05f5d37b98b41fbdd0c943ee0c04d49738f62e902cb291cf43/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af9c464d410e8c44b58ddf1cbec7c3a02c1cb2c6c80df9c74659ba2fbda0ab5", size = 454994, upload-time = "2026-09-21T09:08:16.587Z" }, + { url = "https://files.pythonhosted.org/packages/46/66/ea4c01382975e53b1ac1b0a0ea1b353416b8127fb6e04f5c1c313119aab6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb94e40b9a0db19636b5e3fed0a9803452ecc7381df13b385272abb779ee83ca", size = 460105, upload-time = "2026-09-21T09:08:17.961Z" }, + { url = "https://files.pythonhosted.org/packages/31/c3/1a501b597817096c9831c255f6270c124ce0e2faeb74124be24dcc93b86d/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7398930a2b76b9dc67a4e6b8bfc7baf30946892a86f417a3920c07bdd6febc30", size = 493761, upload-time = "2026-09-21T09:08:19.327Z" }, + { url = "https://files.pythonhosted.org/packages/5a/16/77352d05e152c7786a016e9f1decdb4507e8aa5bcf3343c72142a0413d14/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:025b108f2d5cc2799cb941f79380df3e8b22bfb589c218e801767fe78d598c95", size = 577500, upload-time = "2026-09-21T09:08:20.723Z" }, + { url = "https://files.pythonhosted.org/packages/95/fe/81001c55b24466fe9911135e50269fd3a29f43e446627a92adfab54553b6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9ef3cd403d756a304a0a08a112163c63da7ec72c8c9d083c98818291ff8f29", size = 469209, upload-time = "2026-09-21T09:08:22.346Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cf/e3ad894ed5909a7c44b0707e89bc25d19a3d78b3a7aec846119443b6ae56/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5768b49e426fd5b550b012c866db347cdf15c398ef98dc557b6e6b72fa74cd1", size = 458353, upload-time = "2026-09-21T09:08:23.7Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f5/2f106829e6e2bc91ad74614730f6df484202b9a4d20d927dbf2f08d37d44/watchfiles-1.3.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:d36c72fcdc08f143d87316a4c4c3b064e340da2c7383d39803950740ed418935", size = 468707, upload-time = "2026-09-21T09:08:25.167Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/2741ab776b978669c1f3bba1e573f1a184f727ab8b4d3a6d9c61090800ed/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c9865240e065a2247f4b5248528ae3d01e0e8ae071462c250c884dc16e3ee3", size = 632510, upload-time = "2026-09-21T09:08:26.532Z" }, + { url = "https://files.pythonhosted.org/packages/48/64/db4a4684275fe9b696ee5118c194705c1423b3d42181c3f8ee70194d5c73/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:380f513d26cc2e598266b88d66456e07d67ef4be8f0f435a1154d9c44b43d509", size = 661723, upload-time = "2026-09-21T09:08:28.004Z" }, + { url = "https://files.pythonhosted.org/packages/52/80/95ddb24f6a2f595c031c24432ab7b8aa7d6d1ada0c6d63a552166e06e8d1/watchfiles-1.3.0-cp310-abi3-win32.whl", hash = "sha256:509d9f74d2bec5c1f4cd868ddcfe0c9601f7ea53066dfcb19d8ede742c4de661", size = 277761, upload-time = "2026-09-21T09:08:29.335Z" }, + { url = "https://files.pythonhosted.org/packages/48/a1/1c4a1b3c8030cd03206232612be8a6e8799093490c220b0fcb935d02b75a/watchfiles-1.3.0-cp310-abi3-win_amd64.whl", hash = "sha256:1acabde19b67e673274e89a04d613b7fb1f122e2c4b8ec9a581e59c149264b61", size = 290643, upload-time = "2026-09-21T09:08:30.45Z" }, + { url = "https://files.pythonhosted.org/packages/82/e0/d2db577c632d26d08078c4390dd870b8555c4e558df25d22270f0166c71f/watchfiles-1.3.0-cp310-abi3-win_arm64.whl", hash = "sha256:d978cc1dd7ba44f5590d7de74f7a9c8be7262d9e469c2c0efb8bbfc2574321dc", size = 285510, upload-time = "2026-09-21T09:08:31.561Z" }, + { url = "https://files.pythonhosted.org/packages/4f/55/a122743607939429ab6a7272672f49303aa02bbd34c3f06676765a3861de/watchfiles-1.3.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e0204f90677b7fb8d3a824279a71a01166bea9f6e53923b8056592d87b33b0b6", size = 395978, upload-time = "2026-09-21T09:08:32.747Z" }, + { url = "https://files.pythonhosted.org/packages/3e/9d/0ee1b0ea42acfe72afa2484f0222e2386dd09992e9034a81e63cc3cdab0a/watchfiles-1.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7c1125e99f84934a92996ae343501285617373acd66f16b59fd5654b27ad8d80", size = 387997, upload-time = "2026-09-21T09:08:33.953Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f2/4319c6b683e8bd28465dfc53c750617a2a07f072eb2e3ab0182ccf812229/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a85e243c87109d9f48a2e0d10d6abdee8768cc114848b32281ec83d7f81d86ef", size = 448067, upload-time = "2026-09-21T09:08:35.104Z" }, + { url = "https://files.pythonhosted.org/packages/b7/71/9f37ac192a51a24de3a2d7fe9aac1e99b2b2668300674ff7a9f85d95beb1/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd538c71766c59c732e2c5cedc39323c99c11897b99f93eac773438e54af206a", size = 452881, upload-time = "2026-09-21T09:08:36.238Z" }, + { url = "https://files.pythonhosted.org/packages/34/07/1224667c32a4a96876ddd642da69733cbfe6a15897db0865aec5a882cebc/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da213822ec9082b62cadf3008e07a8c690ec7a3c89f6dc5e55a8cf07b56a1b26", size = 484802, upload-time = "2026-09-21T09:08:37.717Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4d/7f29cd87e1f826df86fc0418d0b8f370ca0311c29c618406dc492aad9951/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4593392e87669836670f87d62fc754059eaeb0e6157af44060a46558cebe5a3", size = 567522, upload-time = "2026-09-21T09:08:38.94Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ac/3aa1f84abd4dbbfbae99a817dd931f77b439261c911189cad4c30fdfd470/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26b81bc515a0f03f66a69a6fea86eea54c8eca4173c46dd7222569f79dd2f977", size = 461997, upload-time = "2026-09-21T09:08:40.142Z" }, + { url = "https://files.pythonhosted.org/packages/82/1c/27c66764d5b0dbb70a36770a32eca217eba739c28b24e8583a035087a3d7/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca39661934749df580d89f3dde266f2af7c6968f4c21c72ce2dc387d299e6aa7", size = 451856, upload-time = "2026-09-21T09:08:41.305Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b3/a45827084d953c1d0dd3d75abf85dcac26a0f5d115ad2fe0b7e9e931573b/watchfiles-1.3.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:04c5500faa725d69a99b0f63750e80795e455bb951849e9c98a811b4324997c4", size = 459889, upload-time = "2026-09-21T09:08:42.58Z" }, + { url = "https://files.pythonhosted.org/packages/7a/3d/07740d23ce2bb23419bb24055c435a9abe5fe03efdaea1ba0720b831aab6/watchfiles-1.3.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:ec3cd4bb181a7b6329134204c2277c7144ab5f62cce9c5335d6f901ee7ce8d6a", size = 624482, upload-time = "2026-09-21T09:08:44.064Z" }, + { url = "https://files.pythonhosted.org/packages/91/46/9540f8123eaec86df989f26d7704cad0448d9fb7271763a4f8f9fe23617d/watchfiles-1.3.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:e5871d4a7f788a9f64fb06f5793389d29d171e5c32d37c3ebfd520b017cc7694", size = 653724, upload-time = "2026-09-21T09:08:45.257Z" }, + { url = "https://files.pythonhosted.org/packages/25/9d/b9deb52f1e4528ffe44338c5ebab95c32ed79203082da661de50288e130a/watchfiles-1.3.0-cp314-cp314t-win32.whl", hash = "sha256:81c989d7267bfb7b676cd32c83bec97914e396870983cbf493e29415037bff5e", size = 270863, upload-time = "2026-09-21T09:08:46.518Z" }, + { url = "https://files.pythonhosted.org/packages/59/59/8b1881afccbafda06248be4185e28034dd73c80c85e4da24f2cb0c1321eb/watchfiles-1.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d754e049006e81a9be49dc78ba7802081cee426e968cbc63ee19e164fa2d0e39", size = 286879, upload-time = "2026-09-21T09:08:47.874Z" }, + { url = "https://files.pythonhosted.org/packages/46/dd/41c12ae9745d5d72f709333287679d71cbba68ad7678fed872207bba94f4/watchfiles-1.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:a5b631db08fd3032db57ec6a92d1777cd60f88803287e4e941a4858c53aab1e0", size = 276924, upload-time = "2026-09-21T09:08:48.976Z" }, +] + +[[package]] +name = "websockets" +version = "17.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/89/3f825ab71c242fffb62ea8fe638741c290f62f8d7aadf8125ff897747af3/websockets-17.2.tar.gz", hash = "sha256:36c2fb94c990cc2545143b12690e2de6c16300f9dbe5b4f33fa300cf57dc8792", size = 188355, upload-time = "2026-10-03T14:56:53.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/54/a935a32dbc2e7365b1b59eb74b5ab7515456f02370fdca4c4efc3574e96f/websockets-17.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:b24b83fbb34b2d8de06cf0f0d4bd7737344ef854482a614826d4356c0c3f0c12", size = 217752, upload-time = "2026-10-03T14:53:54.59Z" }, + { url = "https://files.pythonhosted.org/packages/cd/95/cb8881851abe2662730e6c61cc521b4c96513fdf9103a44f169afce2eba8/websockets-17.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8a829db795e3f87053904493d184b185c8eb1f497c852f434168ec856aa6f997", size = 215436, upload-time = "2026-10-03T14:53:56.034Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1e/621bb93f35ab7d337be98f1958294437527e2a1797089b5e734ddc5eec5f/websockets-17.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cf8811d285acc91216368df7fb55cc8c9bf6fcd90eea42429c7186c7385a12b9", size = 215690, upload-time = "2026-10-03T14:53:57.587Z" }, + { url = "https://files.pythonhosted.org/packages/62/4a/49d0c983c082676d5d413b28e6ba5ae1d174c00268467bf78d9fe986a2d2/websockets-17.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:89c4898da776193577279173dcf9860487590611d7320d379435a145881b048d", size = 225080, upload-time = "2026-10-03T14:53:59.081Z" }, + { url = "https://files.pythonhosted.org/packages/04/13/95a45eb410019772002d8f53d81396dad4120f7df39ca9962f86f5d7cd01/websockets-17.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d87091c4347daadbcc0833b65812ff38d7350c67339625d4e4a512cf38e3e8ef", size = 225361, upload-time = "2026-10-03T14:54:00.61Z" }, + { url = "https://files.pythonhosted.org/packages/f8/fe/0f0eda80bb441f54becdaf793eb20ee080926f8d2356388377cf262187e5/websockets-17.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1110fbfd530c447380e6e6db88b7e43ffe33d54178f5b0ff0aaa5a280301e668", size = 226602, upload-time = "2026-10-03T14:54:02.098Z" }, + { url = "https://files.pythonhosted.org/packages/5c/36/067fc09d8e6f154abde7c2f747c52cc442a02c5eb14816f5c39cb9f8bcc6/websockets-17.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:83abd8beab056aa77a116364811f8fc262dffbcc7abea48de0c85ccbfc6f1428", size = 228035, upload-time = "2026-10-03T14:54:03.545Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a2/939bade7a396b4c381aebbf3941969f124d0f98d56753f81cd256f3fc4d6/websockets-17.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:876da8ca5520d65b5d0f2ca6b4e7a00d35bb90ccda35cb2ce3cda4b6c711e84a", size = 227227, upload-time = "2026-10-03T14:54:05.045Z" }, + { url = "https://files.pythonhosted.org/packages/e5/8a/37b1033e21709dd7fa39239ea4d9cd7f348ad5bcba94eb47253878576f8a/websockets-17.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8462395df8f224d2daa3d80db3ae4450d9d4b7243c8483ac79a82862f1599dd6", size = 225985, upload-time = "2026-10-03T14:54:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/a0/3a/0d89539900b06d86366facb7558198046de125ab8c371d9248d6262da70d/websockets-17.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6e9a04e69456015e6ae5e0d486d995137fd435794442122b00ce5f9526ea3ba8", size = 223226, upload-time = "2026-10-03T14:54:08.583Z" }, + { url = "https://files.pythonhosted.org/packages/31/9a/bfc5633e3d538d0a71cfbe7a5fee56c712e16c2dbd0ce17c83196a2a96a9/websockets-17.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a2321bcb73758c44c8076509024d02c15ee484fe77ce04edea4bf4d257492cc", size = 226042, upload-time = "2026-10-03T14:54:10.254Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/cbaf1786d8e3aeafe9d76951fc01139ec353b92555580336f23669382a55/websockets-17.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8be4a87b3baca380ec3c7b1643b2dd268ac9d42c5097c0e8dc9a49342faf4774", size = 224639, upload-time = "2026-10-03T14:54:11.911Z" }, + { url = "https://files.pythonhosted.org/packages/80/49/175faa5bd169486f835602ac0ae6303318aa65693b79cdc72c5ee53b148d/websockets-17.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:eb7b737ce8d18c8a08beb68f751572b7bf6a18093ecd1406ca1256b50592552e", size = 225407, upload-time = "2026-10-03T14:54:13.489Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d1/3662f612456cfb2dcc128c8e596f0a55fb7b695025e2ebe8ba2abb355c3b/websockets-17.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d6605630c2808b33f362d6d08582e79821f77ed2bd3f49f9d467ea70defea06d", size = 226513, upload-time = "2026-10-03T14:54:15.046Z" }, + { url = "https://files.pythonhosted.org/packages/73/6b/07af5177a49e30156b0922556fa93624a920a2b17d3e63bf4ad94668112c/websockets-17.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:dd9252828073fd0d69e7667af4275a1b17c18d0833b1ab7f59db272f194a6b9a", size = 224072, upload-time = "2026-10-03T14:54:16.574Z" }, + { url = "https://files.pythonhosted.org/packages/eb/34/d18054ff4d8314524164f8b8efec2cb17627287e099f122c28ed6fa598e0/websockets-17.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:06c7386128a9d85de4e1960114604f3031c084d2f4eee8db382637f1634cbab1", size = 225022, upload-time = "2026-10-03T14:54:18.143Z" }, + { url = "https://files.pythonhosted.org/packages/e9/12/75433caa3e9fa3e51d7751dc6bad24a86addf76cbfb51e52b11d037ba7fd/websockets-17.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:98f2d03df74977fd252831c997c388cd6c3f691a8a9d022b266d3cbd9849838f", size = 225303, upload-time = "2026-10-03T14:54:19.679Z" }, + { url = "https://files.pythonhosted.org/packages/6f/de/23e21c002aa2786ac9807c0876faa3b2576493b29ca3386287b0db46f021/websockets-17.2-cp313-cp313-win32.whl", hash = "sha256:5b43a1f7e4853ce08c3f6d3bf69799ee5b46548bfb71792a8158f7e45d66b547", size = 218219, upload-time = "2026-10-03T14:54:21.232Z" }, + { url = "https://files.pythonhosted.org/packages/13/eb/960411c0c574535d629c16e96a2b4e5353dbe4109df8ecea859e1b5245ee/websockets-17.2-cp313-cp313-win_amd64.whl", hash = "sha256:27c7a59b5352a8f741b422820adfe89dfe47c8f2d84fb32111e76111edaa0e83", size = 218531, upload-time = "2026-10-03T14:54:23.025Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1a/3ac07bb52378952eff1d52d04a7ee6e82ce84e3da319a52a4739cd9c78f5/websockets-17.2-cp313-cp313-win_arm64.whl", hash = "sha256:533b7c82bb1eafbeb921dfe131c9f88e55451ddc328d84bde1c9340ba72d2808", size = 218466, upload-time = "2026-10-03T14:54:24.857Z" }, + { url = "https://files.pythonhosted.org/packages/8b/74/6bc991a28ac983600e65de408ebd1b1413d554ed0468ae5c831bc52dded6/websockets-17.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ecb748910e9ba4624ebe2057791df51dcbffb48c37108ab94a3c593472023c9e", size = 217791, upload-time = "2026-10-03T14:54:26.381Z" }, + { url = "https://files.pythonhosted.org/packages/cb/2f/158e99426be6e71d09520bae53f29294fbb614b2fc5fbf8867b1d08395a7/websockets-17.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2ab9af5cb7265899e659f079eb71691375a1025b6d5fbd3caa495dd08f70833a", size = 215486, upload-time = "2026-10-03T14:54:27.962Z" }, + { url = "https://files.pythonhosted.org/packages/5c/09/1abf942723c0001d9c2fca1551907dade6304517b982b0bf10bba107fa81/websockets-17.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:06e46da092bca3a52e98f0458c66b247993ce501a07cd09c858be3296511ab7d", size = 215699, upload-time = "2026-10-03T14:54:29.523Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1d/1ade03963ef497c47e6bad79e24370827b2fe6145fa8f58070ff2b7dcbac/websockets-17.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fcce735ffd72ac4056db05325d9f0232382b74826f0196eb6a15ca903abdaa0f", size = 225081, upload-time = "2026-10-03T14:54:31.278Z" }, + { url = "https://files.pythonhosted.org/packages/9f/fd/47b8a0361c49da939b976a07b27a72a9f893d01dfcf4d2a28b53419ce1ef/websockets-17.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:42cbca10f82a8b2fb1536e8a0830ca6ceeb6bb3d8d64b766e0795369135654a8", size = 225430, upload-time = "2026-10-03T14:54:32.917Z" }, + { url = "https://files.pythonhosted.org/packages/f0/26/f4d4c76264ee037c5556ab5f50fcba302746dabf7528955534e4dda9965e/websockets-17.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c63ff5a21f26bd0e6a8464b53fadbe174825c8718ac14180df45665eaacdb6af", size = 226676, upload-time = "2026-10-03T14:54:34.833Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/c8b1c981322a050c4babfd327ffc9880f9c3834f5b15d2574e37eeb8768c/websockets-17.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:63f543463601c1558b755f8dd7618b6ec3dd0934dda051d3b7030d8c76e54de2", size = 228048, upload-time = "2026-10-03T14:54:36.424Z" }, + { url = "https://files.pythonhosted.org/packages/f0/5a/1cb29ddb23e6bc27ffd1c5316cd3616360d1ba0c3854eaa134ee3207bd28/websockets-17.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4c32eb565ad9ce8a6444248e5b7a19dbb86a81c811fe5fcc2fba7a735aed5163", size = 227281, upload-time = "2026-10-03T14:54:38.01Z" }, + { url = "https://files.pythonhosted.org/packages/ba/64/135274572dc0c845fc1111e2b932c807c395daac75d6eae6cfa148d8a208/websockets-17.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5d459bbb6c22f26dcebea56924a362aba50d453b9867912862c970434fcf0d94", size = 226025, upload-time = "2026-10-03T14:54:39.613Z" }, + { url = "https://files.pythonhosted.org/packages/58/75/f1e386aec3124489411caf5138cdd5a2bc43d3fd4a681c69adcf5f6272a5/websockets-17.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f19ca1a21871f024e38faf4107b433047df27558dff1b72a1dac31481e2c1fe5", size = 223277, upload-time = "2026-10-03T14:54:41.165Z" }, + { url = "https://files.pythonhosted.org/packages/60/eb/24733a0f568c2eb99e60f9faa620a98fb228c06a01e7e2f348b33290ed9c/websockets-17.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c76b4bcbf0f713194591673fc86a42820e14da6bbd1bb445d3d002cc4d1e4521", size = 226148, upload-time = "2026-10-03T14:54:42.779Z" }, + { url = "https://files.pythonhosted.org/packages/55/6d/ea66a30af74f5983cae31ebb9ef78b178b366a12856a414e1472225c4a34/websockets-17.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:30201a7f69833b015556c72feb69ea501b645986fd0b90dab13f589e995ff428", size = 224615, upload-time = "2026-10-03T14:54:44.41Z" }, + { url = "https://files.pythonhosted.org/packages/87/80/c6f2228ad89774429d270179375ebddb657119215f52d1df7c680d65cad7/websockets-17.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0c8600aec354cc259f1691b0b42816f04a9886a953f82cb227246df76057f97a", size = 225398, upload-time = "2026-10-03T14:54:46.063Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4a/3d8da19732ad468d4be7f1e3ac298078b60bdda55edde6589bef84a5eb7e/websockets-17.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:307fc22ea496be8542d67b82ae8c867a978dfd19ac35573d4f15943fd9277dfe", size = 226571, upload-time = "2026-10-03T14:54:47.672Z" }, + { url = "https://files.pythonhosted.org/packages/58/22/1231657122d9cc24791bb90af13cc2f4e84cf0d3a454cb37e3abfdcb2fd9/websockets-17.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:9c88697fa943bd4ef67cc919a17d81de6581846f52bfa8c6f64a916098986556", size = 224125, upload-time = "2026-10-03T14:54:49.537Z" }, + { url = "https://files.pythonhosted.org/packages/1a/04/350ca2445da758bc42cdb4218b44d4ce0d5a9c1d5e4cc4a58d64348ad9da/websockets-17.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f7eac84d4969da82166d5e90d9c38d2f416fe24f9708a7013569b193745b9a31", size = 225081, upload-time = "2026-10-03T14:54:51.075Z" }, + { url = "https://files.pythonhosted.org/packages/da/c4/dec952b0df3a5d918ed2a545abb0c25ae519c3bc2d9aba3b7c46abae8f05/websockets-17.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:313f6703023d53baabab6d6c5c37cf637b2c4fee255acf2ed5e92ad69e28f1b7", size = 225376, upload-time = "2026-10-03T14:54:52.675Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b4/198a260afbcc086ff4979774e51834ed7fb5b95f9ef305e0c4924630b857/websockets-17.2-cp314-cp314-win32.whl", hash = "sha256:08d90cf344bdb971ba3a826b78d4da9bfd56cc6a97a604d9b88cbd40bfa6c735", size = 217760, upload-time = "2026-10-03T14:54:54.247Z" }, + { url = "https://files.pythonhosted.org/packages/e5/9e/0523f8bc2f7aaddf39562d4fa01b4d38fa61b23d980917a16d2dd19c8dac/websockets-17.2-cp314-cp314-win_amd64.whl", hash = "sha256:dac93bf7a9beb215be3282b8441173cd50806c41c007b8be9bb24e03c60ad563", size = 218104, upload-time = "2026-10-03T14:54:55.845Z" }, + { url = "https://files.pythonhosted.org/packages/55/17/7b8bb4cb64a199e7082f1f9be784d657842fefc327ac777d6c1493504804/websockets-17.2-cp314-cp314-win_arm64.whl", hash = "sha256:2ab742249f953d148a9ba696c8b9944361e8cb92e8bc61ba2dd53a178403afd3", size = 217989, upload-time = "2026-10-03T14:54:57.376Z" }, + { url = "https://files.pythonhosted.org/packages/ee/76/f54ed054b6e860f1e0bbc7019542a048352d41231fdff6d904b379f881c7/websockets-17.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a69ce25be5f1330ee1c74eb6fabbbceaa96b384beedd2627cecded7546490c40", size = 218125, upload-time = "2026-10-03T14:54:58.943Z" }, + { url = "https://files.pythonhosted.org/packages/e6/4c/0f3375cea66a125ae01d21fb9c537aae955ef499bfe7e2b2376a34362f2a/websockets-17.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:8e24b878cf54843a63985d90480f163ca7f692689fbcbe9cdbd8165521083a8b", size = 215658, upload-time = "2026-10-03T14:55:00.674Z" }, + { url = "https://files.pythonhosted.org/packages/0c/05/7c871a67bfb4b61adc1fe13583db97803f87dfeca644fe6ef51df7bb276d/websockets-17.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f33c7908a6885dcae9f462a4a8347b637053b4ff2b96beb4c23fba1cf7818e5f", size = 215858, upload-time = "2026-10-03T14:55:02.379Z" }, + { url = "https://files.pythonhosted.org/packages/41/8e/59df4d9cd357e902d1c74b13c3c0c3841c8df6e4b1b3d131bf26a23fdcb1/websockets-17.2-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c796a1bb3e4015249639849f30e8e680df8a431b45d417ba8acf843d2451d95f", size = 225443, upload-time = "2026-10-03T14:55:03.966Z" }, + { url = "https://files.pythonhosted.org/packages/5c/64/5e486a3a44e041203c62eccf1fc89c7f8824e21104a7b82b182e5b21c228/websockets-17.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:983bcdc898662f6ba9d6a025c30d29946ff0986d9ad60d400af0da3671f7cbf3", size = 225726, upload-time = "2026-10-03T14:55:05.797Z" }, + { url = "https://files.pythonhosted.org/packages/f0/98/b6eb53121c91fbe8b6897aba06861ce60f9ab58faffc6bca5750cbc21681/websockets-17.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:35e0f088ddfd9d9bc5019e27ff3767411779e92b59db5bb1507f2731a5b61158", size = 226895, upload-time = "2026-10-03T14:55:07.626Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/8c091321b99c91eb3eaec9acbd940e69308b4e465b5605c430af0cf7d3a5/websockets-17.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:19e2511412ad3393191de652513bc7a0ca3c93af143b32d96d46e59fbbddf1d4", size = 229040, upload-time = "2026-10-03T14:55:09.321Z" }, + { url = "https://files.pythonhosted.org/packages/1a/96/3a92f944305b7de42fcb7530b9fa69607b4b4ce993c36a9f2330dbc318ba/websockets-17.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb5e2bf969ac99a6ae3c71208a5eb05cfde973192540ffa6e1068b57fb78c4f8", size = 227469, upload-time = "2026-10-03T14:55:10.935Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a9/624f6d75ba326c22d03698b34c0ada984f1d76196322a62f6c22903b831d/websockets-17.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:691780fca2be3dec512cb603cb91060271968cb4af86b51d07c57445c5754a37", size = 226202, upload-time = "2026-10-03T14:55:12.536Z" }, + { url = "https://files.pythonhosted.org/packages/47/af/1e6e8c625aeb268830af2c4227fe05e8db59f4f4debe1dadfd0ada214895/websockets-17.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2d39c19b1ba6a6791050383fd69efdd3b63533e2254693d0263879cd5f5921ba", size = 223743, upload-time = "2026-10-03T14:55:14.164Z" }, + { url = "https://files.pythonhosted.org/packages/dd/81/33c5280f4f6f81637c93ae065c6a594dfe35935622af135a5f7c3768bf22/websockets-17.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e48ac2b302986c6f55cf61e8e36b4dd97d0132c5078a713a697a940934ba422e", size = 226492, upload-time = "2026-10-03T14:55:15.796Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/7aa9fc36e67caccbcfee2c48f4ada41e9da512d41523c024d039f0f22ba3/websockets-17.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:e136197f1262620ef2e507afc3ea759c1ae7d221886da20eec5f4c9f2618c2aa", size = 224940, upload-time = "2026-10-03T14:55:17.661Z" }, + { url = "https://files.pythonhosted.org/packages/3f/8c/457aff7081a63d1261608bb4d7b0b0f9dfe780697a2a334671745742850b/websockets-17.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3eb44019a2b0b3b91bac95998f1e4e5589730421170e060fe654a2b7be727dc7", size = 225835, upload-time = "2026-10-03T14:55:19.607Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c3/7a13a3b3050db2c36772ded49f8d48f99eb080948e9f6f762e7529925ab5/websockets-17.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e5855e574804398859c5fbaf4fc7882b96278b7f6572a3d889627e6eb6cfca59", size = 226848, upload-time = "2026-10-03T14:55:21.274Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3e/d5b2c1e473b1031a4a0ec0e10de69df5b981ab4a10aa482bb45c18dd43f5/websockets-17.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5dc29815520c329f5662f6eb3ebadecf0d4f8c82dfa416d4d6efbf8f39245559", size = 224541, upload-time = "2026-10-03T14:55:22.874Z" }, + { url = "https://files.pythonhosted.org/packages/79/5d/bb81976cc1aa546afb51395ce42913521e9dea062bb34a61308cfff30726/websockets-17.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:d1a4f9462da6496b6cb79bbb09c60d17f7e63e8a1df136797b3afabec9560e4d", size = 225315, upload-time = "2026-10-03T14:55:24.443Z" }, + { url = "https://files.pythonhosted.org/packages/f4/6b/314962d5440c61b4c107914599c13ceeecc6bdb6e2e73a5f7e566a7d1f26/websockets-17.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9496bff5541086478264678bac73c0a75b2fde94fdf6568893bca1f7c6d50d18", size = 225747, upload-time = "2026-10-03T14:55:26.033Z" }, + { url = "https://files.pythonhosted.org/packages/98/fc/9eb64b34a3a4458eb08f3f24bde01508f72a00790330723c158ebb965048/websockets-17.2-cp314-cp314t-win32.whl", hash = "sha256:e1e3bc8090a7eae79fdf634b63bdbfa3c93999991023c37c6fd3b469fc8ff5dc", size = 217891, upload-time = "2026-10-03T14:55:27.681Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ed/3a4e2a09b0822d6e525cbc6e44a4885669bad5b22ab9c64fa2444bc15325/websockets-17.2-cp314-cp314t-win_amd64.whl", hash = "sha256:65a89a5bde227bfe908016f35b5bd347970cd1e5b0360f389502eba1c7fde6e0", size = 218229, upload-time = "2026-10-03T14:55:29.314Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/cffb75ee746dd060984c3c3e2eac7f875a866225a30dfa53e2cd18232565/websockets-17.2-cp314-cp314t-win_arm64.whl", hash = "sha256:1c27339934109dfaca83f18ab2c23db06714e9d5deca2c8e37e8f492ab90d20b", size = 218146, upload-time = "2026-10-03T14:55:31.001Z" }, + { url = "https://files.pythonhosted.org/packages/12/e9/10a9b1633b63594054c87b97af048628cea2b21b5089a52a9fc1e0af60a3/websockets-17.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:a7c4bb26de6ef496d24822aee4f6a305d97cd33d21a2b85f290292d69ba1c25e", size = 217719, upload-time = "2026-10-03T14:55:32.674Z" }, + { url = "https://files.pythonhosted.org/packages/0c/00/ff4020fe0886dac7199a16ce2805c7afd7b981bd2e81d3fa18dff5d9863a/websockets-17.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c08da1f15040bd1e1a6074bd4518a6ef20e67b1594ecfb0aa75e5b45f87e6d6d", size = 215448, upload-time = "2026-10-03T14:55:34.338Z" }, + { url = "https://files.pythonhosted.org/packages/66/06/bc7b944f81514378b2c2ab96c17df19e871cd33b9be0f1f6dfc975457e5e/websockets-17.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:3117abfd32b183bdb6194df9317766d32c6517f3d1c0aa8c62d5c6ccfda0b4a8", size = 215674, upload-time = "2026-10-03T14:55:35.918Z" }, + { url = "https://files.pythonhosted.org/packages/a8/da/2b2b76faa2f10c4813e3872c9577fd13a798f5918b1785b86ff7d635eb2a/websockets-17.2-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a046227daa7f191e843d26b911c1146233e9a33d249e0c954dcb3ac7c398710e", size = 225119, upload-time = "2026-10-03T14:55:37.777Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d4/22cbe288c0d5cef7620503be92c0098d82220353fc7e188034a19c517240/websockets-17.2-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2901bdf24f20bc884124b3e88c61f7ece260c20c81e610f2196007395264a4aa", size = 225549, upload-time = "2026-10-03T14:55:39.364Z" }, + { url = "https://files.pythonhosted.org/packages/4c/0a/504b0d3063679f2c60430c3539482d42a4cb8bd1a76646baf742030a93cc/websockets-17.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f60e39adfecf998488166aca8ff24ab1ac406c9ecbecbcf9b3bcfc43cb1ec9a1", size = 226717, upload-time = "2026-10-03T14:55:40.942Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ea/5da9309cc55c2665a6eebc22c369d9918c0d77258c61e92058e6b08d5ff1/websockets-17.2-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d4df62fd8448a85c752bbea1803cb3a2785e6fc8352009ab64ad7447af079b3c", size = 228413, upload-time = "2026-10-03T14:55:42.54Z" }, + { url = "https://files.pythonhosted.org/packages/a6/74/5a24df72aa5500f311105687af864c27f1f9da910e968e97818c6149e6b0/websockets-17.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c8eea55fdfa9ba65c6981eea38bd20c800bce2f092a2803d82de764ecf0f071a", size = 227196, upload-time = "2026-10-03T14:55:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ee/ca32cc1ed892dc4ac30a922e8f648048233fbdb8b0bce7048860ec4c60ec/websockets-17.2-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3f0def1279644acaa9bc861d4234af3f82ea9cee7e460dffac5cb63e691501e9", size = 226092, upload-time = "2026-10-03T14:55:45.842Z" }, + { url = "https://files.pythonhosted.org/packages/7d/0c/12d4a73324aa9798d5165d20c088f9dba66c75c871960e5d921ec66694e4/websockets-17.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fb78fb4158c12f77a934a003006784108a27a6553cfc0c6f10483c9c02e94f48", size = 223486, upload-time = "2026-10-03T14:55:47.45Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a4/7fe15da5abb8f0f61e6a357593f7f2ed55724825b7db0ffe72b5c5fad68d/websockets-17.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:f8969ad228115ad8869b5fed801f899e52ab8ad376fdb165ba4760a277c8258a", size = 226200, upload-time = "2026-10-03T14:55:49.126Z" }, + { url = "https://files.pythonhosted.org/packages/08/b9/4cd3a311f96a2eea0ed458bc01fe2cce42f9cd50aa9e64315dfc855d63a9/websockets-17.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:4a49ca342efc0800e6ae94ed5c9cbdcb319308f75e73c21181e4c24d6710e8dd", size = 224862, upload-time = "2026-10-03T14:55:50.674Z" }, + { url = "https://files.pythonhosted.org/packages/41/b5/22caa3460f75e42bfcc74028870b556d22847ea9a9034aa03986f07f16a9/websockets-17.2-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:06fa3ce9c3154826c33d4395b225b2994aa64f1f3bcd8be8ed932019175d9268", size = 225391, upload-time = "2026-10-03T14:55:52.393Z" }, + { url = "https://files.pythonhosted.org/packages/95/be/8d28f92092076abf1ddfb3206b0ce956120a22e7c3105f6a3029d727deae/websockets-17.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:50644d8715be7e0ec0682f9d7744b63008e199c5e1618a48fa153756a332235f", size = 226545, upload-time = "2026-10-03T14:55:54.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7b/ff943fa383e540fe17f066cc10a3eeedef26e50fd45aae2bdc6746d6f95a/websockets-17.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:60deca33e584c09e91f70f8b55a0b1de7d671d6a63f051d154920f48bed717c7", size = 224352, upload-time = "2026-10-03T14:55:55.856Z" }, + { url = "https://files.pythonhosted.org/packages/e9/df/1e6c3e06c473c9fd833a5c1620b15e2c3b37647b91b7d41871d20bc098de/websockets-17.2-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:b5f79366a8d8dbb981d53ba800bb54a95454595ab8a4548c2b95501b32a08326", size = 225255, upload-time = "2026-10-03T14:55:57.497Z" }, + { url = "https://files.pythonhosted.org/packages/db/f8/d8a4f988f7cbb568d8bd69da4632c5b6010aa9cd9366f285e23b73b678d9/websockets-17.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f2bbf3f28d0b63157577c8b774b9136f076afa6797e1a52a2ecd477f23cad3a8", size = 225513, upload-time = "2026-10-03T14:55:59.338Z" }, + { url = "https://files.pythonhosted.org/packages/75/e0/920357165b2797a2530fc9e271d79a9b5fee2b750b154c990c740f767af3/websockets-17.2-cp315-cp315-win32.whl", hash = "sha256:74836317b7010b579522bb52426f1e225608b042c9e78cbe2493522bebb8a318", size = 217722, upload-time = "2026-10-03T14:56:01.307Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/25bdca25bbc329ffb330ef33993397d6556a871e40a0d196e757699ea3f7/websockets-17.2-cp315-cp315-win_amd64.whl", hash = "sha256:aaead3d926e9ab4124ada727d20cd62d396649917822df4f771d1f07f1079b40", size = 218017, upload-time = "2026-10-03T14:56:02.914Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cb/ea30a552bbcd1c75f0d14bfce6c884ee36187030b85b74a242aacc02406e/websockets-17.2-cp315-cp315-win_arm64.whl", hash = "sha256:40960554e60eb60c3eec4ff9e42a80f84f8cd3ca9bc80a5481a61f1e64d807c9", size = 217929, upload-time = "2026-10-03T14:56:04.604Z" }, + { url = "https://files.pythonhosted.org/packages/4a/01/477664c619af8aa3c908d482e2a95e13ceed9d78f21d15902013c3bc6c28/websockets-17.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:9a2a60a7f0ea5f239efb6391d2b28630a640d82dad63e3bee47cf2c623c4495d", size = 218029, upload-time = "2026-10-03T14:56:06.336Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a9/b0be62ff1c0e2bc966da56b36d3d820c7e2ad3c0c4a4ac414fc7335b214f/websockets-17.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:cca2fcb72c007103740fa4fc3df19fdb1a318c641c69f3b0cc47ed63a889336e", size = 215607, upload-time = "2026-10-03T14:56:08.035Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2b/a6738530de0437a31c1b168e4096ecf790aafaf561f33a009886c7d8042e/websockets-17.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b789356bc4e2e6c20ba52817f92c3fed74e24657654237ecd536c54843b80c6c", size = 215817, upload-time = "2026-10-03T14:56:09.852Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c2/2fc44ddc419cbb09ee1708af3e78d8a4b018db01fc7e4f91bd730e2f8d9e/websockets-17.2-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:222fb626fa15701a850eccc778be17312142b2f6a0e16aea80770b7459adb784", size = 225979, upload-time = "2026-10-03T14:56:11.85Z" }, + { url = "https://files.pythonhosted.org/packages/2e/91/a215b14caa7ea65bc36db81609108899c259503300d1560dae9c70a135e7/websockets-17.2-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4497e87c34a2d21cbec1227858fec3af8e514dd70c47625557a122fcebc081dc", size = 226250, upload-time = "2026-10-03T14:56:13.548Z" }, + { url = "https://files.pythonhosted.org/packages/65/b9/9406a18e9edf558ed504d2a7679371d0f8107e4ef526c80b154ea4ec9752/websockets-17.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6281c171557ce0e408e19d9a223f22d915117ac38a5a7f32ed83809e7492316c", size = 227579, upload-time = "2026-10-03T14:56:15.143Z" }, + { url = "https://files.pythonhosted.org/packages/fe/45/a73af119244f46f5130005d7ab63f1c75890c890141a0ca2adc9d97d4671/websockets-17.2-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:08d97098644728bd1895caa7ecf3090b8e563d70809870d2adb33a107bd061d0", size = 229205, upload-time = "2026-10-03T14:56:17.086Z" }, + { url = "https://files.pythonhosted.org/packages/c1/92/ccd8e2e921d134a56f1ed4642d276500d9e33b3dc4d6deb63d614b3e53a6/websockets-17.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1fdb8d5a1660307dc6d36d0b7fc725213cbd7f80800904dc4896aa3208b89121", size = 228011, upload-time = "2026-10-03T14:56:18.716Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ef/7d71105d19a7aaab5ff87b9c712f6c1dda44e72ea56aa0e7b777f2fc274b/websockets-17.2-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18b0a46e5e9b315e2b54ce8c3bafdeef0e1388ca363114fa868e6aab2dc58512", size = 226892, upload-time = "2026-10-03T14:56:20.412Z" }, + { url = "https://files.pythonhosted.org/packages/56/f7/87012d628b21e66e699440f39bfa7cc55fae7f52b2c532ab62184a589624/websockets-17.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f115d5d804a2163dd89245710049078b0e726a58c1f44a1f86c2c6e79055d76", size = 224241, upload-time = "2026-10-03T14:56:22.257Z" }, + { url = "https://files.pythonhosted.org/packages/55/f5/495371068b27ee5f7c435187f9dafd62402f195e2c76063bdd4653da1565/websockets-17.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:1d829946a2e7630f92f9d7b45b62f3abe9f393cc2dea6a35edb3988f865e75f2", size = 227076, upload-time = "2026-10-03T14:56:23.909Z" }, + { url = "https://files.pythonhosted.org/packages/18/18/3dce3cc6099be5e044e0fd5d0e0c9931c8e3387511cdec8014a345f619e5/websockets-17.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:6c274fc1572edf7c197094a0eb1887d45fdc95254bc80597dc7599550486c06a", size = 225727, upload-time = "2026-10-03T14:56:25.689Z" }, + { url = "https://files.pythonhosted.org/packages/47/30/57d0c7aaf8d4473926fa8829b8136483f561388d1e747ae71c9f2a83d5fd/websockets-17.2-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:4173a4b8a025ae44313d9d9b4ecf31e886c7b7faf45386d51a8ca4ff2dcf3f2a", size = 226225, upload-time = "2026-10-03T14:56:27.246Z" }, + { url = "https://files.pythonhosted.org/packages/0c/9f/9dce1203756756c00b407b9a6b13a7500fcd38f2634d4daa3f65575814ec/websockets-17.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:d8cfe9522ad69b6abb26b413ed1deca43cb915cefc588433d557cb3ae1c783e2", size = 227333, upload-time = "2026-10-03T14:56:28.811Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2f/d3b6b876678ebb03017b7afd7111fe44d54b93f036a80ebb4b481dd1ab74/websockets-17.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:908d81d88bb16141613a6275059b5114656d5c2f0b5400b421d54fe6f1943507", size = 225082, upload-time = "2026-10-03T14:56:30.578Z" }, + { url = "https://files.pythonhosted.org/packages/32/b0/a69b573a5e56d2e7a5dcbb447466f442380cf81515e1cb1220cd626c8042/websockets-17.2-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:c6590e1eb624ff6b15b872421bc9a10bc6d2057635d69c6cd244ac3f928f85c6", size = 225945, upload-time = "2026-10-03T14:56:32.32Z" }, + { url = "https://files.pythonhosted.org/packages/70/be/a72911dc8e33f74c196012366ce4d99b1a803894a377a1ed0c8e66df9caa/websockets-17.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:61040f6f7da5a279d2f77496c69d51132aba75f701c52bded400d4c639277b18", size = 226241, upload-time = "2026-10-03T14:56:34.142Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a9/02a68c1d8e5572918e0962d3aad881078f73ede43abd9b1336e4efaa8909/websockets-17.2-cp315-cp315t-win32.whl", hash = "sha256:f90bad2839c185a1edf8ee22a257cfc8a39e0e337a0490ab185dfa76ef04d1bd", size = 217847, upload-time = "2026-10-03T14:56:36.204Z" }, + { url = "https://files.pythonhosted.org/packages/2b/bf/3d7c33b8d5e7712a60e0149c017ed50394ec5e8cf72e5cb6a1ffaf11a42d/websockets-17.2-cp315-cp315t-win_amd64.whl", hash = "sha256:315551f4ccedbbf9fd4f7e8bf037a5948c976ade0e919ba5d8f581d465f6f725", size = 218169, upload-time = "2026-10-03T14:56:37.79Z" }, + { url = "https://files.pythonhosted.org/packages/27/57/ab34cc6460c5322e6932750fa5c6c64be89e6ee4e2707d13c4e9d3312b25/websockets-17.2-cp315-cp315t-win_arm64.whl", hash = "sha256:0a6220bdf8d5f11af71251a599092d89ac1d6bfac691c7f5951c5b07953947a0", size = 218089, upload-time = "2026-10-03T14:56:39.427Z" }, + { url = "https://files.pythonhosted.org/packages/8a/58/835cd51934d6780fa586f275b5d9901eead6d81569b4343b3767cdbaae4c/websockets-17.2-py3-none-any.whl", hash = "sha256:6aa59f0ef92e796b2db6f5f26550c4713c0e4036899fadf02f55e2ed4db0b7ae", size = 211883, upload-time = "2026-10-03T14:56:51.898Z" }, +]