Add homelab dashboards: system, LLM, and UPS on one domain
Three dashboards served by one FastAPI app with a SQLite history store
(raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana,
no CDN, no build step. Nothing is fetched from the internet at runtime.
* system: a stdlib-only agent (agent/host_agent.py) reports /proc and
/sys counters; the dashboard turns them into rates over a labelled
window, so every number says how it was measured.
* llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house
exposition parser. Measured delta/elapsed throughput beats a native
per-request gauge, and each value carries its provenance.
* ups: a NUT wire-protocol client plus health checks that name what
they looked at. Commands are gated by NUT_ALLOW_ACTIONS,
DASH_READ_ONLY and a DANGER confirmation, re-checked server-side.
Missing values render as n/a rather than 0, and an unconfigured source
answers 404 {"configured": false} without dialling anything, so "not
deployed here" never looks like an outage.
253 tests at 94% coverage against captured real fixtures; ruff and
pyright clean.
LLM-Generated: true
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
"""Homelab dashboards: system resources, LLM engines, and UPS battery health.
|
||||
|
||||
One FastAPI app, one domain, one SQLite file. See AGENTS.md for the rules and
|
||||
.agent(s)/PLAN.md for the locked decisions.
|
||||
"""
|
||||
|
||||
from .config import SETTINGS
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__all__ = ["SETTINGS", "__version__"]
|
||||
@@ -0,0 +1,28 @@
|
||||
"""`python -m dashboards` — the same server as `uv run uvicorn dashboards.app:app`.
|
||||
|
||||
DASH_PORT=8090 DEBUGPY=1 python -m dashboards
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from .app import create_app
|
||||
from .config import SETTINGS
|
||||
from .debug import maybe_listen
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import uvicorn # noqa: PLC0415 - only the entry point needs uvicorn
|
||||
|
||||
maybe_listen()
|
||||
reload = os.environ.get("DASH_RELOAD", "").lower() in ("1", "true", "yes", "on")
|
||||
if reload:
|
||||
uvicorn.run("dashboards.app:app", host=SETTINGS.host, port=SETTINGS.port, reload=True)
|
||||
else:
|
||||
# pass the object: create_app() is cheap and avoids a second import of config
|
||||
uvicorn.run(create_app(), host=SETTINGS.host, port=SETTINGS.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,399 @@
|
||||
"""HTTP API. GET everywhere except one endpoint: `POST /api/ups/command`.
|
||||
|
||||
Everything a browser can see is derived from the in-memory payloads (fast, no DB on the
|
||||
hot path) plus SQLite for anything older than a few minutes. Two rules from llm_monitor that
|
||||
are worth restating here because they are easy to break by accident:
|
||||
|
||||
* `/api/config` never leaks a token or password — target URLs are passed through
|
||||
`Target.display_url`, which strips userinfo.
|
||||
* an unconfigured source answers 404 `{"configured": false}` without dialling anything, so
|
||||
the page can tell "not deployed here" apart from "deployed and broken".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request
|
||||
|
||||
from .collectors import NutMonitor
|
||||
from .config import Settings
|
||||
from .poller import Hub
|
||||
from .sources import nut
|
||||
from .sources.host import fleet_summary
|
||||
from .sources.llm import fleet_totals
|
||||
from .store import Event, SampleStore
|
||||
|
||||
NO_STORE = {"cache-control": "no-store, max-age=0"}
|
||||
RANGE_SECONDS = {"5m": 300, "15m": 900, "30m": 1800, "1h": 3600, "3h": 10_800,
|
||||
"6h": 21_600, "12h": 43_200, "24h": 86_400, "7d": 604_800,
|
||||
"30d": 2_592_000, "90d": 7_776_000}
|
||||
|
||||
|
||||
@dataclass
|
||||
class AppContext:
|
||||
"""Everything the routes need, assembled once at startup and passed around."""
|
||||
|
||||
settings: Settings
|
||||
store: SampleStore
|
||||
hub: Hub
|
||||
nut_monitor: NutMonitor | None = None
|
||||
started_at: float = 0.0
|
||||
|
||||
|
||||
def get_context(request: Request) -> AppContext:
|
||||
"""The context built during lifespan. FastAPI hands it to every route."""
|
||||
return request.app.state.ctx # type: ignore[no-any-return]
|
||||
|
||||
|
||||
def parse_range(value: str) -> float:
|
||||
"""`1h` / `7d` / `45m` / raw seconds. Unknown values are a 400, not a silent default:
|
||||
a chart that quietly shows the wrong window is worse than an error."""
|
||||
raw = (value or "1h").strip().lower()
|
||||
if raw in RANGE_SECONDS:
|
||||
return float(RANGE_SECONDS[raw])
|
||||
if raw[-1] in ("s", "m", "h", "d") and raw[:-1].isdigit():
|
||||
multiplier = {"s": 1, "m": 60, "h": 3600, "d": 86_400}[raw[-1]]
|
||||
return float(int(raw[:-1]) * multiplier)
|
||||
if raw.isdigit():
|
||||
return float(raw)
|
||||
raise HTTPException(400, f"unknown range {value!r}; try 5m 1h 24h 7d 30d")
|
||||
|
||||
|
||||
def _history(store: SampleStore, source: str, host: str, metric: str, span: float,
|
||||
points: int) -> dict[str, Any]:
|
||||
series = store.history(source, host, metric, span=span, max_points=points)
|
||||
return {
|
||||
"source": source,
|
||||
"host": host,
|
||||
"metric": metric,
|
||||
"range_seconds": span,
|
||||
"points": [{"ts": ts, "value": value} for ts, value in series],
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Common routes
|
||||
# --------------------------------------------------------------------------- #
|
||||
def common_router() -> APIRouter:
|
||||
router = APIRouter(prefix="/api", tags=["meta"])
|
||||
|
||||
@router.get("/config")
|
||||
async def config(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
settings = ctx.settings
|
||||
return {
|
||||
"app": {"name": "dashboards", "version": "0.1.0",
|
||||
"started_at": ctx.started_at, "read_only": settings.read_only},
|
||||
"system": {
|
||||
"enabled": bool(settings.host_targets),
|
||||
"interval_seconds": settings.host_interval,
|
||||
"poll_ms": settings.poll_ms_system,
|
||||
"targets": [{"name": t.name, "url": t.display_url} for t in settings.host_targets],
|
||||
"target_problems": list(settings.host_target_problems),
|
||||
},
|
||||
"llm": {
|
||||
"gufo": {"enabled": bool(settings.gufo_url), "base_url": settings.gufo_url,
|
||||
"auth": bool(settings.gufo_api_key)},
|
||||
"vllm": {"enabled": bool(settings.vllm_url), "base_url": settings.vllm_url,
|
||||
"auth": bool(settings.vllm_api_key)},
|
||||
"interval_seconds": settings.llm_interval,
|
||||
"poll_ms": settings.poll_ms_llm,
|
||||
"rate_window_seconds": settings.rate_window,
|
||||
},
|
||||
"ups": {
|
||||
"enabled": settings.nut_enabled,
|
||||
"server": f"{settings.nut_host}:{settings.nut_port}" if settings.nut_enabled else None,
|
||||
"auth": bool(settings.nut_user),
|
||||
"interval_seconds": settings.nut_interval,
|
||||
"poll_ms": settings.poll_ms_ups,
|
||||
"actions_enabled": settings.actions_allowed,
|
||||
"simulated": bool(ctx.nut_monitor and ctx.nut_monitor.simulated),
|
||||
},
|
||||
"ranges": sorted(RANGE_SECONDS, key=lambda key: RANGE_SECONDS[key]),
|
||||
}
|
||||
|
||||
@router.get("/health")
|
||||
async def health(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
states = ctx.hub.states()
|
||||
enabled = [s for s in states if s["configured"]]
|
||||
healthy = [s for s in enabled if s["ok"]]
|
||||
status = "ok" if enabled and len(healthy) == len(enabled) else (
|
||||
"degraded" if healthy else ("empty" if not enabled else "down"))
|
||||
# a fresh install with nothing configured is empty, not broken — `ok` answers "is the
|
||||
# app working", the banner on the page answers "is there anything to look at"
|
||||
return {"ok": status in ("ok", "empty"), "status": status, "sources": states,
|
||||
"db": ctx.store.stats(), "uptime_seconds": time.time() - ctx.started_at}
|
||||
|
||||
@router.get("/events")
|
||||
def events(limit: int = Query(60, ge=1, le=500), source: str | None = None,
|
||||
since: float | None = None,
|
||||
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
"""Event log (transitions, UPS state changes, actions taken).
|
||||
|
||||
Served by a sync handler on purpose: SQLite reads belong in Starlette's
|
||||
threadpool, not on the event loop the collectors share.
|
||||
"""
|
||||
rows = ctx.store.recent_events(limit, source=source, since=since)
|
||||
return {"events": [row.as_json() for row in rows]}
|
||||
|
||||
@router.get("/history")
|
||||
def history(source: str = Query(...), host: str = Query(...), metric: str = Query(...),
|
||||
range: str = Query("1h"), points: int = Query(600, ge=10, le=4000),
|
||||
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
return _history(ctx.store, source, host, metric, parse_range(range), points)
|
||||
|
||||
@router.get("/metrics")
|
||||
def metrics(source: str | None = None, host: str | None = None,
|
||||
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
"""Which series actually have history — the honest answer to "why is this chart empty"."""
|
||||
return {"metrics": ctx.store.series_names(source, host)}
|
||||
|
||||
@router.get("/summary")
|
||||
async def summary(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
"""One call for the landing page: is each dashboard alive, and what stands out."""
|
||||
hosts = [job.state.payload for job in ctx.hub.jobs.values()
|
||||
if job.dashboard == "system" and job.state.payload]
|
||||
engines = [job.state.payload for job in ctx.hub.jobs.values()
|
||||
if job.dashboard == "llm" and job.state.payload]
|
||||
ups_payload = None
|
||||
nut_job = ctx.hub.jobs.get("nut")
|
||||
if nut_job and nut_job.state.payload:
|
||||
ups_payload = nut_job.state.payload
|
||||
return {
|
||||
"ts": time.time(),
|
||||
"system": {"enabled": bool(ctx.settings.host_targets),
|
||||
"fleet": fleet_summary(hosts) if hosts else None,
|
||||
"hosts": [{"name": h["name"], "os": (h.get("info") or {}).get("os_pretty"),
|
||||
"cpu_usage_pct": h["cpu"]["usage_pct"],
|
||||
"load1": h["cpu"].get("load1"),
|
||||
"cores": (h.get("info") or {}).get("cpu_cores_logical"),
|
||||
"mem_used_pct": h["memory"]["used_pct"],
|
||||
"disk_worst_used_pct": h["disks"]["worst_used_pct"],
|
||||
"gpu_busy_pct": [g.get("busy_pct") for g in h.get("gpus") or []],
|
||||
"temp_c": h["cpu"].get("temp_c"),
|
||||
"uptime_seconds": (h.get("info") or {}).get("uptime_seconds")}
|
||||
for h in hosts],
|
||||
"states": ctx.hub.states("system")},
|
||||
"llm": {"enabled": bool(ctx.settings.gufo_url or ctx.settings.vllm_url),
|
||||
"fleet": fleet_totals(engines) if engines else None,
|
||||
"engines": [{"engine": e["engine"],
|
||||
"decode_tokens_per_second": e["metrics"].get(
|
||||
"decode_tokens_per_second", {}).get("value"),
|
||||
"active": e["sessions"].get("active")}
|
||||
for e in engines],
|
||||
"states": ctx.hub.states("llm")},
|
||||
"ups": {"enabled": ctx.settings.nut_enabled,
|
||||
"simulated": bool(ups_payload and ups_payload.get("simulated")),
|
||||
"ups": [{"name": card["name"], "overall": card["health"]["overall"],
|
||||
"status": card["status"],
|
||||
"charge_pct": card["numbers"].get("charge_pct"),
|
||||
"load_pct": card["numbers"].get("load_pct"),
|
||||
"runtime_seconds": card["numbers"].get("battery_runtime")}
|
||||
for card in (ups_payload or {}).get("ups", [])],
|
||||
"states": ctx.hub.states("ups")},
|
||||
"recent_events": [event.as_json() for event in ctx.store.recent_events(12)],
|
||||
}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# System dashboard
|
||||
# --------------------------------------------------------------------------- #
|
||||
def system_router() -> APIRouter:
|
||||
router = APIRouter(prefix="/api/system", tags=["system"])
|
||||
|
||||
@router.get("/snapshot")
|
||||
async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
if not ctx.settings.host_targets:
|
||||
raise HTTPException(404, detail={"configured": False,
|
||||
"detail": "no hosts configured — set HOST_TARGETS"})
|
||||
hosts = []
|
||||
for job in ctx.hub.jobs.values():
|
||||
if job.dashboard != "system":
|
||||
continue
|
||||
payload = job.state.payload
|
||||
if payload is None:
|
||||
# keep an unreachable host in the list: absent from the list and present-but-
|
||||
# red are different statements, and only one of them is a config problem
|
||||
payload = {"name": job.label, "ts": job.state.last_attempt, "unreachable": True}
|
||||
hosts.append(payload)
|
||||
hosts.sort(key=lambda item: item.get("name") or "")
|
||||
return {"ts": time.time(), "hosts": hosts, "fleet": fleet_summary(hosts),
|
||||
"states": ctx.hub.states("system"),
|
||||
"window_seconds": ctx.settings.host_interval}
|
||||
|
||||
@router.get("/history")
|
||||
def history(metrics: str = Query("cpu.usage_pct"), hosts: str = Query(""),
|
||||
range: str = Query("1h"), points: int = Query(600, ge=10, le=4000),
|
||||
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
"""One request for a whole chart: N hosts x M metrics.
|
||||
|
||||
`hosts` is comma-separated, or empty for every configured target. Charting three
|
||||
hosts should cost one round trip, not three, because the browser polls this.
|
||||
"""
|
||||
span = parse_range(range)
|
||||
wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()]
|
||||
wanted_hosts = [item.strip() for item in hosts.split(",") if item.strip()] or [
|
||||
target.name for target in ctx.settings.host_targets]
|
||||
series = [_history(ctx.store, "host", host, metric, span, points)
|
||||
for host in wanted_hosts for metric in wanted_metrics]
|
||||
return {"hosts": wanted_hosts, "range_seconds": span, "series": series}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# LLM dashboard
|
||||
# --------------------------------------------------------------------------- #
|
||||
def llm_router() -> APIRouter:
|
||||
router = APIRouter(prefix="/api/llm", tags=["llm"])
|
||||
|
||||
@router.get("/snapshot")
|
||||
async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
if not (ctx.settings.gufo_url or ctx.settings.vllm_url):
|
||||
raise HTTPException(404, detail={"configured": False,
|
||||
"detail": "no engines configured — set "
|
||||
"GUFO_BASE_URL / VLLM_BASE_URL"})
|
||||
engines = []
|
||||
for job in ctx.hub.jobs.values():
|
||||
if job.dashboard != "llm":
|
||||
continue
|
||||
payload = job.state.payload or {
|
||||
"engine": job.label, "metrics": {}, "sessions": {"active": None, "list": []},
|
||||
"unreachable": True}
|
||||
engines.append(payload)
|
||||
return {"ts": time.time(), "engines": engines, "fleet": fleet_totals(engines),
|
||||
"states": ctx.hub.states("llm")}
|
||||
|
||||
@router.get("/history")
|
||||
def history(metrics: str = Query("decode_tokens_per_second"), engines: str = Query(""),
|
||||
range: str = Query("1h"), points: int = Query(600, ge=10, le=4000),
|
||||
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
span = parse_range(range)
|
||||
wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()]
|
||||
wanted_engines = [item.strip() for item in engines.split(",") if item.strip()] or [
|
||||
name for name, url in (("gufo", ctx.settings.gufo_url), ("vllm", ctx.settings.vllm_url)) if url]
|
||||
return {"engines": wanted_engines, "range_seconds": span,
|
||||
"series": [_history(ctx.store, "llm", engine, f"llm.{metric}", span, points)
|
||||
for engine in wanted_engines for metric in wanted_metrics]}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# UPS dashboard
|
||||
# --------------------------------------------------------------------------- #
|
||||
def ups_router() -> APIRouter:
|
||||
router = APIRouter(prefix="/api/ups", tags=["ups"])
|
||||
|
||||
def require_monitor(ctx: AppContext) -> NutMonitor:
|
||||
if ctx.nut_monitor is None or not ctx.settings.nut_enabled:
|
||||
raise HTTPException(404, detail={"configured": False,
|
||||
"detail": "no NUT server configured — set NUT_HOST"})
|
||||
return ctx.nut_monitor
|
||||
|
||||
@router.get("/snapshot")
|
||||
async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
monitor = require_monitor(ctx)
|
||||
job = ctx.hub.jobs.get("nut")
|
||||
payload = (job.state.payload if job else None) or {
|
||||
"ts": time.time(), "ups": [], "simulated": monitor.simulated,
|
||||
"actions_enabled": ctx.settings.actions_allowed}
|
||||
events = ctx.store.recent_events(40, source="ups")
|
||||
return {**payload, "states": ctx.hub.states("ups"),
|
||||
"events": [event.as_json() for event in events]}
|
||||
|
||||
@router.get("/history")
|
||||
def history(metrics: str = Query("ups.charge_pct"), ups: str = Query(""),
|
||||
range: str = Query("24h"), points: int = Query(600, ge=10, le=4000),
|
||||
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
span = parse_range(range)
|
||||
wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()]
|
||||
job = ctx.hub.jobs.get("nut")
|
||||
wanted_ups = [item.strip() for item in ups.split(",") if item.strip()] or [
|
||||
card["name"] for card in ((job.state.payload if job else None) or {}).get("ups", [])]
|
||||
return {"ups": wanted_ups, "range_seconds": span,
|
||||
"series": [_history(ctx.store, "ups", name, metric, span, points)
|
||||
for name in wanted_ups for metric in wanted_metrics]}
|
||||
|
||||
@router.get("/commands")
|
||||
async def commands(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
|
||||
"""What may be sent right now, and why not when it may not."""
|
||||
require_monitor(ctx)
|
||||
job = ctx.hub.jobs.get("nut")
|
||||
payload = (job.state.payload if job else None) or {"ups": []}
|
||||
return {"actions_enabled": ctx.settings.actions_allowed,
|
||||
"read_only": ctx.settings.read_only,
|
||||
"risk_levels": [nut.RISK_SAFE, nut.RISK_CAUTION, nut.RISK_DANGEROUS],
|
||||
"catalog": [{"command": key, **value} for key, value in nut.COMMANDS.items()],
|
||||
"ups": [{"name": card["name"], "commands": card["commands"]}
|
||||
for card in payload.get("ups", [])]}
|
||||
|
||||
@router.post("/command")
|
||||
async def send_command(ctx: AppContext = Depends(get_context),
|
||||
payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
"""The one write endpoint in the application. Deliberately loud about it.
|
||||
|
||||
Gated on: NUT_ALLOW_ACTIONS, not DASH_READ_ONLY, credentials that carry `actions`
|
||||
in upsd.users, the command being advertised by the driver for that UPS, and — for
|
||||
anything that can cut power — the browser echoing the confirmation word back.
|
||||
"""
|
||||
monitor = require_monitor(ctx)
|
||||
ups_name = str(payload.get("ups") or "").strip()
|
||||
command = str(payload.get("command") or "").strip()
|
||||
confirm = str(payload.get("confirm") or "").strip()
|
||||
if not ups_name or not command:
|
||||
raise HTTPException(400, "both `ups` and `command` are required")
|
||||
if not ctx.settings.actions_allowed:
|
||||
raise HTTPException(403, "UPS actions are disabled — set NUT_ALLOW_ACTIONS=1 "
|
||||
"(and don't run with DASH_READ_ONLY=1)")
|
||||
|
||||
job = ctx.hub.jobs.get("nut")
|
||||
card = next((item for item in ((job.state.payload or {}).get("ups", []) if job else [])
|
||||
if item["name"] == ups_name), None)
|
||||
if card is None:
|
||||
raise HTTPException(404, f"unknown UPS {ups_name!r} — not in the last poll")
|
||||
advice = nut.command_advice(command)
|
||||
if advice is None:
|
||||
raise HTTPException(400, f"{command} is not a command this dashboard will send")
|
||||
if command not in card.get("advertised_commands", []):
|
||||
raise HTTPException(400, f"the driver for {ups_name} does not advertise {command}")
|
||||
if advice["risk"] == nut.RISK_DANGEROUS and confirm != "DANGER":
|
||||
raise HTTPException(422, f"{advice['label']} can cut power: send "
|
||||
f"confirm=\"DANGER\" to prove you read the warning")
|
||||
if any(item["command"] == command and not item["enabled"] for item in card["commands"]):
|
||||
blocked = next(item for item in card["commands"] if item["command"] == command)
|
||||
raise HTTPException(409, blocked["reason"] or f"{command} is not available")
|
||||
|
||||
try:
|
||||
if monitor.simulated:
|
||||
detail = monitor.simulator.command(ups_name, command) # type: ignore[union-attr]
|
||||
else:
|
||||
async with nut.NutClient(
|
||||
ctx.settings.nut_host, ctx.settings.nut_port,
|
||||
timeout=ctx.settings.nut_timeout,
|
||||
username=ctx.settings.nut_user, password=ctx.settings.nut_password,
|
||||
) as client:
|
||||
await client.login()
|
||||
detail = await client.instcmd(ups_name, command)
|
||||
except nut.NutError as exc:
|
||||
ctx.store.log_event(Event(time.time(), "ups", ups_name, "action", "error",
|
||||
f"{advice['label']} on {ups_name} failed: {exc}",
|
||||
{"command": command, "code": exc.code}))
|
||||
raise HTTPException(502, f"upsd refused {command}: {exc}") from exc
|
||||
|
||||
ctx.store.log_event(_action_event(ups_name, command, advice, detail))
|
||||
return {"ok": True, "ups": ups_name, "command": command, "detail": detail,
|
||||
"risk": advice["risk"], "simulated": monitor.simulated,
|
||||
"hint": "watch the state card — the driver answers over the next few polls"}
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _action_event(ups_name: str, command: str, advice: dict[str, str], detail: str) -> Event:
|
||||
severity = "warn" if advice["risk"] != nut.RISK_SAFE else "info"
|
||||
return Event(time.time(), "ups", ups_name, "action", severity,
|
||||
f"{advice['label']} sent to {ups_name}: {detail}", {"command": command})
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Application assembly: one FastAPI app, four pages, one SQLite file, N poll loops.
|
||||
|
||||
`create_app()` takes injectable settings and store so tests can build an app with an
|
||||
in-memory DB and no targets — the same code path production uses, without the network.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from .api import NO_STORE, AppContext, common_router, llm_router, system_router, ups_router
|
||||
from .collectors import NutMonitor, UpstreamError, host_jobs, llm_jobs, nut_job
|
||||
from .config import SETTINGS as DEFAULT_SETTINGS
|
||||
from .config import Settings
|
||||
from .poller import Hub
|
||||
from .rates import RateEstimator
|
||||
from .sources.host import HostCollector
|
||||
from .sources.nut_sim import NutSimulator
|
||||
from .store import SampleStore
|
||||
|
||||
WEB = Path(__file__).resolve().parent / "web"
|
||||
PAGES = {"/": "index.html", "/system": "system.html", "/llm": "llm.html", "/ups": "ups.html"}
|
||||
VERSION = "0.1.0"
|
||||
|
||||
|
||||
def build_context(settings: Settings, client: httpx.AsyncClient) -> tuple[
|
||||
AppContext, SampleStore, Callable[[], None]]:
|
||||
"""Create the store, the collectors, and the hub's jobs. Returns (ctx, store, start)."""
|
||||
store = SampleStore(
|
||||
settings.db_path,
|
||||
raw_keep_hours=settings.raw_keep_hours,
|
||||
minute_keep_days=settings.minute_keep_days,
|
||||
hour_keep_days=settings.hour_keep_days,
|
||||
)
|
||||
rates = RateEstimator(span=settings.rate_window)
|
||||
host_collector = HostCollector(rates)
|
||||
nut_monitor: NutMonitor | None = None
|
||||
if settings.nut_enabled:
|
||||
simulator = NutSimulator() if _truthy(os.environ.get("NUT_SIMULATE")) else None
|
||||
nut_monitor = NutMonitor(settings, simulator=simulator)
|
||||
|
||||
hub = Hub(store)
|
||||
for job in host_jobs(list(settings.host_targets), client, host_collector,
|
||||
settings.host_interval):
|
||||
hub.add(job)
|
||||
for job in llm_jobs(settings, client, rates):
|
||||
hub.add(job)
|
||||
if nut_monitor is not None:
|
||||
hub.add(nut_job(settings, nut_monitor, client))
|
||||
|
||||
ctx = AppContext(settings=settings, store=store, hub=hub, nut_monitor=nut_monitor,
|
||||
started_at=time.time())
|
||||
return ctx, store, hub.start
|
||||
|
||||
|
||||
def _truthy(value: str | None) -> bool:
|
||||
return (value or "").strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None, store: SampleStore | None = None) -> FastAPI:
|
||||
"""`store` is for tests: when supplied it is not created and not closed here."""
|
||||
settings = settings or DEFAULT_SETTINGS
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
owned_store: SampleStore | None = store
|
||||
ctx: AppContext | None = None
|
||||
client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(max(settings.host_timeout, settings.llm_timeout)),
|
||||
follow_redirects=False,
|
||||
limits=httpx.Limits(max_connections=32, max_keepalive_connections=16),
|
||||
)
|
||||
try:
|
||||
if store is not None:
|
||||
# injected: rebuild everything except the DB
|
||||
rates = RateEstimator(span=settings.rate_window)
|
||||
hub = Hub(store)
|
||||
nut_monitor = None
|
||||
if settings.nut_enabled:
|
||||
simulator = NutSimulator() if _truthy(os.environ.get("NUT_SIMULATE")) else None
|
||||
nut_monitor = NutMonitor(settings, simulator=simulator)
|
||||
for job in host_jobs(list(settings.host_targets), client,
|
||||
HostCollector(rates), settings.host_interval):
|
||||
hub.add(job)
|
||||
for job in llm_jobs(settings, client, rates):
|
||||
hub.add(job)
|
||||
if nut_monitor is not None:
|
||||
hub.add(nut_job(settings, nut_monitor, client))
|
||||
ctx = AppContext(settings=settings, store=store, hub=hub,
|
||||
nut_monitor=nut_monitor, started_at=time.time())
|
||||
start = hub.start
|
||||
else:
|
||||
ctx, owned_store, start = build_context(settings, client)
|
||||
app.state.ctx = ctx
|
||||
app.state.hub = ctx.hub
|
||||
start()
|
||||
yield
|
||||
finally:
|
||||
# `ctx` is None when startup itself failed; the client and DB still need closing
|
||||
if ctx is not None:
|
||||
await ctx.hub.stop()
|
||||
await client.aclose()
|
||||
if owned_store is not None and store is None:
|
||||
owned_store.close()
|
||||
|
||||
app = FastAPI(title="Homelab Dashboards", version=VERSION, lifespan=lifespan,
|
||||
docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.mount("/static", StaticFiles(directory=str(WEB / "static")), name="static")
|
||||
|
||||
for path, filename in PAGES.items():
|
||||
app.add_api_route(path, _page_handler(filename), methods=["GET"], name=f"page:{path}")
|
||||
|
||||
app.include_router(common_router())
|
||||
app.include_router(system_router())
|
||||
app.include_router(llm_router())
|
||||
app.include_router(ups_router())
|
||||
|
||||
@app.exception_handler(UpstreamError)
|
||||
async def upstream_error_handler(request: Request, exc: UpstreamError) -> JSONResponse:
|
||||
return JSONResponse({"ok": False, "detail": exc.message}, status_code=exc.status,
|
||||
headers=NO_STORE)
|
||||
|
||||
@app.middleware("http")
|
||||
async def no_store_for_api(request: Request, call_next):
|
||||
"""Every /api response is `no-store`, success or failure.
|
||||
|
||||
A dashboard is a polling app: a cached snapshot in a proxy or the browser makes a
|
||||
dead host look alive for as long as the TTL. Static assets keep their caching.
|
||||
"""
|
||||
response = await call_next(request)
|
||||
if request.url.path.startswith("/api"):
|
||||
response.headers["cache-control"] = "no-store, max-age=0"
|
||||
return response
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_error_handler(request: Request,
|
||||
exc: StarletteHTTPException) -> JSONResponse:
|
||||
"""One JSON error shape for every failure, deliberate or not.
|
||||
|
||||
A dict detail is a structured answer (`{"configured": false, ...}`) and is spread
|
||||
into the body so clients can branch on it. A status-specific 404 handler must NOT be
|
||||
registered: Starlette resolves handlers by status code first, so it would also swallow
|
||||
the deliberate `404 {"configured": false}` from an unconfigured source — the difference
|
||||
between "not deployed here" and "deployed and broken" on the page. Unmatched routes
|
||||
arrive here as Starlette's plain "Not Found" and get relabelled.
|
||||
"""
|
||||
body: dict[str, Any] = {"ok": False}
|
||||
detail: Any = exc.detail # Starlette types this as str | dict | list | None
|
||||
if isinstance(detail, dict):
|
||||
body.update(detail)
|
||||
elif exc.status_code == 404 and str(detail) in ("Not Found", "Not Found."):
|
||||
body["detail"] = f"no such route: {request.url.path}"
|
||||
else:
|
||||
body["detail"] = detail
|
||||
return JSONResponse(body, status_code=exc.status_code, headers=NO_STORE)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def _page_handler(filename: str) -> Callable[..., Awaitable[FileResponse]]:
|
||||
async def handler() -> FileResponse:
|
||||
return FileResponse(WEB / "pages" / filename, media_type="text/html", headers=NO_STORE)
|
||||
|
||||
return handler
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Collectors: the fetch half of each poll. Parse + store live in the Job wrapper.
|
||||
|
||||
Every collector returns a `Result`; every failure is an exception the Job turns into a
|
||||
red dot plus an event. Nothing here writes to any upstream — with one deliberate exception,
|
||||
`run_ups_command`, which is the app's only mutating path and is gated in three places.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from .config import Target
|
||||
from .poller import Job, Result
|
||||
from .rates import RateEstimator
|
||||
from .sources import llm as llm_source
|
||||
from .sources import nut
|
||||
from .sources.host import HostCollector, parse_host_snapshot
|
||||
from .store import Event, Sample
|
||||
|
||||
|
||||
class UpstreamError(Exception):
|
||||
"""An upstream answered badly or not at all; 502/504 in the API, red dot in the UI."""
|
||||
|
||||
def __init__(self, message: str, *, status: int = 502) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.status = status
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# System hosts
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def fetch_host_snapshot(client: httpx.AsyncClient, target: Target,
|
||||
host_collector: HostCollector) -> Result:
|
||||
url = f"{target.url}/snapshot"
|
||||
try:
|
||||
response = await client.get(url, headers=target.headers)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise UpstreamError(f"{target.name}: {url} timed out", status=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise UpstreamError(f"{target.name}: {url} unreachable ({type(exc).__name__})") from exc
|
||||
if response.status_code == 401:
|
||||
raise UpstreamError(f"{target.name}: agent rejected the token (401) — check the "
|
||||
f"userinfo token in HOST_TARGETS")
|
||||
if response.status_code >= 400:
|
||||
raise UpstreamError(f"{target.name}: {url} -> HTTP {response.status_code}")
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as exc:
|
||||
raise UpstreamError(f"{target.name}: agent returned non-JSON "
|
||||
f"({response.text[:80]!r})") from exc
|
||||
if not body.get("ok", False):
|
||||
raise UpstreamError(f"{target.name}: agent said not ok: {body.get('detail')}")
|
||||
result = parse_host_snapshot(target.name, body, host_collector)
|
||||
if body.get("errors"):
|
||||
# The agent degraded a section (a /proc read failed) but still answered. Say so
|
||||
# in the payload rather than presenting a partial snapshot as a complete one.
|
||||
result.payload["degraded"] = body["errors"]
|
||||
return result
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# LLM engines
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def fetch_engine(client: httpx.AsyncClient, engine: str, base_url: str,
|
||||
api_key: str, rates: RateEstimator, *,
|
||||
with_slots: bool = False) -> Result:
|
||||
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
|
||||
url = f"{base_url}/metrics"
|
||||
try:
|
||||
response = await client.get(url, headers=headers)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise UpstreamError(f"{engine}: {url} timed out", status=504) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise UpstreamError(f"{engine}: {url} unreachable ({type(exc).__name__})") from exc
|
||||
if response.status_code >= 400:
|
||||
raise UpstreamError(f"{engine}: {url} -> HTTP {response.status_code} "
|
||||
f"{response.reason_phrase}")
|
||||
text = response.text
|
||||
|
||||
slots: list[dict[str, Any]] | None = None
|
||||
if with_slots:
|
||||
# /slots is gufo's; /v1/slots is the documented alias, so a server that only
|
||||
# exposes the alias still gives us the per-session list.
|
||||
for path in ("/slots", "/v1/slots"):
|
||||
try:
|
||||
slots_response = await client.get(f"{base_url}{path}", headers=headers)
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
if slots_response.status_code == 200:
|
||||
try:
|
||||
data = slots_response.json()
|
||||
except ValueError:
|
||||
continue
|
||||
slots = data if isinstance(data, list) else data.get("slots", [])
|
||||
break
|
||||
return llm_source.engine_result(engine, base_url, text, rates, slots=slots)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# UPS / NUT
|
||||
# --------------------------------------------------------------------------- #
|
||||
@dataclass
|
||||
class NutState:
|
||||
"""Previous poll's per-UPS status, so transitions are events and not spam."""
|
||||
|
||||
status: dict[str, str] = field(default_factory=dict)
|
||||
test_results: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
async def read(self, client: httpx.AsyncClient | None, settings: Any) -> Result:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class NutMonitor:
|
||||
"""Polls upsd once and returns every UPS. Also owns the transition bookkeeping."""
|
||||
|
||||
def __init__(self, settings: Any, simulator: Any | None = None) -> None:
|
||||
self.settings = settings
|
||||
self.simulator = simulator
|
||||
self._status: dict[str, str] = {}
|
||||
self._test_results: dict[str, str] = {}
|
||||
self.simulated = simulator is not None
|
||||
|
||||
async def collect(self) -> Result:
|
||||
now = time.time()
|
||||
readings: list[nut.UpsReading] = []
|
||||
if self.simulator is not None:
|
||||
readings = await self.simulator.read()
|
||||
else:
|
||||
async with nut.NutClient(
|
||||
self.settings.nut_host, self.settings.nut_port,
|
||||
timeout=self.settings.nut_timeout,
|
||||
username=self.settings.nut_user or None,
|
||||
password=self.settings.nut_password or None,
|
||||
) as client:
|
||||
for entry in await client.list_ups():
|
||||
ups = entry["name"]
|
||||
reading = nut.UpsReading(name=ups, description=entry.get("description", ""),
|
||||
ts=now)
|
||||
reading.vars = await client.list_vars(ups)
|
||||
try:
|
||||
reading.commands = await client.list_cmds(ups)
|
||||
except nut.NutError:
|
||||
reading.commands = []
|
||||
try:
|
||||
reading.writable = await client.list_rw_vars(ups)
|
||||
except nut.NutError:
|
||||
reading.writable = []
|
||||
readings.append(reading)
|
||||
|
||||
samples = []
|
||||
events = []
|
||||
payloads = []
|
||||
for reading in readings:
|
||||
card = nut.payload(reading, now=now)
|
||||
card["commands"] = nut.available_commands(
|
||||
reading.commands,
|
||||
allow_actions=self.settings.actions_allowed,
|
||||
read_only=self.settings.read_only,
|
||||
# the simulator needs no credentials; refusing its commands would make the
|
||||
# dev buttons dead while claiming a permissions problem that does not exist
|
||||
has_credentials=bool(self.settings.nut_user and self.settings.nut_password) or self.simulated,
|
||||
on_battery=reading.on_battery,
|
||||
)
|
||||
payloads.append(card)
|
||||
samples.extend(
|
||||
Sample("ups", reading.name, metric, value, reading.ts)
|
||||
for metric, value in nut.history_samples(reading).items()
|
||||
)
|
||||
events.extend(self._transitions(reading, now))
|
||||
|
||||
payload = {
|
||||
"ts": now,
|
||||
"server": None if self.simulated else f"{self.settings.nut_host}:{self.settings.nut_port}",
|
||||
"simulated": self.simulated,
|
||||
"ups": payloads,
|
||||
"actions_enabled": self.settings.actions_allowed,
|
||||
}
|
||||
return Result(payload=payload, samples=samples, events=events)
|
||||
|
||||
def _transitions(self, reading: nut.UpsReading, now: float) -> list[Event]:
|
||||
"""Log a state change once, at the severity of the state we landed in."""
|
||||
out: list[Event] = []
|
||||
status = reading.get("status") or ""
|
||||
previous = self._status.get(reading.name)
|
||||
if previous is None:
|
||||
self._status[reading.name] = status
|
||||
return out
|
||||
if status != previous:
|
||||
severities = {s["severity"] for s in reading.states()}
|
||||
severity = ("error" if "fail" in severities else "warn" if "warn" in severities
|
||||
else "info")
|
||||
out.append(Event(now, "ups", reading.name, "status", severity,
|
||||
f"{reading.name}: {previous or '(no state)'} → {status or '(no state)'}",
|
||||
{"from": previous, "to": status,
|
||||
"states": reading.states(),
|
||||
"charge_pct": reading.number("charge_pct"),
|
||||
"runtime_seconds": reading.number("battery_runtime")}))
|
||||
self._status[reading.name] = status
|
||||
|
||||
result = reading.get("test_result")
|
||||
if result and self._test_results.get(reading.name) != result:
|
||||
first_seen = reading.name not in self._test_results
|
||||
self._test_results[reading.name] = result
|
||||
if not first_seen: # the first sighting is not news, a change is
|
||||
passed = result.strip().lower() in ("ok", "passed", "pass", "completed")
|
||||
out.append(Event(now, "ups", reading.name, "test",
|
||||
"info" if passed else "warn",
|
||||
f"{reading.name}: self-test reported {result}"))
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Job factories (what app.py wires into the Hub)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def host_jobs(targets: list[Target], client: httpx.AsyncClient, host_collector: HostCollector,
|
||||
interval: float) -> list[Job]:
|
||||
return [
|
||||
Job(f"host:{target.name}", "system", target.name, interval,
|
||||
lambda t=target: fetch_host_snapshot(client, t, host_collector))
|
||||
for target in targets
|
||||
]
|
||||
|
||||
|
||||
def llm_jobs(settings: Any, client: httpx.AsyncClient, rates: RateEstimator) -> list[Job]:
|
||||
jobs: list[Job] = []
|
||||
if settings.gufo_url:
|
||||
jobs.append(Job("llm:gufo", "llm", "gufo", settings.llm_interval,
|
||||
lambda: fetch_engine(client, "gufo", settings.gufo_url,
|
||||
settings.gufo_api_key, rates, with_slots=True)))
|
||||
if settings.vllm_url:
|
||||
jobs.append(Job("llm:vllm", "llm", "vLLM", settings.llm_interval,
|
||||
lambda: fetch_engine(client, "vllm", settings.vllm_url,
|
||||
settings.vllm_api_key, rates)))
|
||||
return jobs
|
||||
|
||||
|
||||
def nut_job(settings: Any, monitor: NutMonitor, client: httpx.AsyncClient) -> Job:
|
||||
return Job("nut", "ups", "NUT server", settings.nut_interval, monitor.collect,
|
||||
enabled=settings.nut_enabled)
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Environment-driven configuration. Read once at import; restart to pick up changes.
|
||||
|
||||
There is no config file format and no CLI flags on purpose: deployment is a container with
|
||||
env vars (see Containerfile + the deployment repo's quadlet env template). A `.env` file is
|
||||
only a convenience for the dev loop — it seeds the environment and never overrides a real
|
||||
variable. Every value has a safe default EXCEPT the sources, which default to *unset* — an
|
||||
unset source means "this deployment does not run it", which must render as "not configured",
|
||||
never as a red panel for a server the operator never installed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load_env_file() -> Path | None:
|
||||
"""Seed the environment from `DASH_ENV_FILE` (default: `.env` at the repo root).
|
||||
|
||||
`override=False`, so a variable the container, systemd unit, or shell already set always
|
||||
beats the file. `.env.example` is documentation and is never loaded, and a missing file is
|
||||
the normal case in production — not a warning.
|
||||
"""
|
||||
raw = os.environ.get("DASH_ENV_FILE", "").strip()
|
||||
path = Path(raw).expanduser() if raw else ROOT / ".env"
|
||||
if not path.is_file() or path.name == ".env.example":
|
||||
return None
|
||||
from dotenv import load_dotenv # noqa: PLC0415 - only needed when a file exists
|
||||
|
||||
load_dotenv(path, override=False)
|
||||
return path
|
||||
|
||||
|
||||
# This runs here, at the top of the module, because `Settings` below reads the environment
|
||||
# in its field defaults — which are evaluated once, when the class body executes. Loading the
|
||||
# file from `load()` instead would seed the environment after those defaults had already been
|
||||
# taken, and the file would silently do nothing.
|
||||
load_env_file()
|
||||
|
||||
|
||||
def _env(name: str, default: str = "") -> str:
|
||||
return os.environ.get(name, default).strip()
|
||||
|
||||
|
||||
def _num(name: str, default: float) -> float:
|
||||
raw = _env(name)
|
||||
try:
|
||||
return float(raw) if raw else default
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _int(name: str, default: int) -> int:
|
||||
return int(_num(name, float(default)))
|
||||
|
||||
|
||||
def _flag(name: str, default: bool = False) -> bool:
|
||||
raw = _env(name).lower()
|
||||
if not raw:
|
||||
return default
|
||||
return raw in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _base_url(name: str) -> str:
|
||||
"""Normalised base URL, or `""` when unset/blank. No default host, ever."""
|
||||
return _env(name).rstrip("/")
|
||||
|
||||
|
||||
# A target is `name=url` (comma or whitespace separated). The URL may carry a bearer
|
||||
# token as userinfo (`http://SECRET@host:9120`) so per-host agent tokens need no extra
|
||||
# variable; the token never leaves the backend, it is stripped from every /api payload.
|
||||
_TARGET_SPLIT = re.compile(r"[,\s]+")
|
||||
_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]*$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Target:
|
||||
"""One scrape endpoint. `name` is stable UI text and the history DB's host key."""
|
||||
|
||||
name: str
|
||||
url: str
|
||||
headers: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def display_url(self) -> str:
|
||||
"""URL with any userinfo removed — safe to show in the browser."""
|
||||
return re.sub(r"^(https?://)[^/@]+@", r"\1", self.url)
|
||||
|
||||
|
||||
def parse_targets(raw: str) -> tuple[list[Target], list[str]]:
|
||||
"""Parse `name=url,name=url` into targets, plus a list of problems found.
|
||||
|
||||
Bad entries are reported, not raised: one typo in a list of six hosts must not
|
||||
take the whole dashboard down. The problems surface in /api/config so they are
|
||||
diagnosable without reading container logs.
|
||||
"""
|
||||
targets: list[Target] = []
|
||||
problems: list[str] = []
|
||||
for chunk in _TARGET_SPLIT.split(raw.strip()):
|
||||
if not chunk:
|
||||
continue
|
||||
name, sep, url = chunk.partition("=")
|
||||
name, url = name.strip(), url.strip().rstrip("/")
|
||||
if not sep or not url:
|
||||
problems.append(f"{chunk!r} is not name=url")
|
||||
continue
|
||||
if not _NAME_RE.match(name):
|
||||
problems.append(f"{name!r} is not a valid target name")
|
||||
continue
|
||||
if not url.startswith(("http://", "https://")):
|
||||
problems.append(f"{name}: url must start with http:// or https://")
|
||||
continue
|
||||
headers: dict[str, str] = {}
|
||||
m = re.match(r"^(https?://)([^/@]+)@(.*)$", url)
|
||||
if m:
|
||||
scheme, token, rest = m.groups()
|
||||
url = f"{scheme}{rest}"
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
targets.append(Target(name=name, url=url, headers=headers))
|
||||
return targets, problems
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
# --- server -------------------------------------------------------------------
|
||||
host: str = _env("DASH_HOST", "127.0.0.1")
|
||||
port: int = _int("DASH_PORT", 8090)
|
||||
db_path: str = _env("DASH_DB", str(ROOT / "data" / "dashboards.sqlite3"))
|
||||
read_only: bool = _flag("DASH_READ_ONLY") # kills every mutating endpoint
|
||||
|
||||
# --- system dashboard ---------------------------------------------------------
|
||||
host_targets: tuple[Target, ...] = ()
|
||||
host_target_problems: tuple[str, ...] = ()
|
||||
host_interval: float = _num("HOST_INTERVAL_SECONDS", 10)
|
||||
host_timeout: float = _num("HOST_TIMEOUT_SECONDS", 4)
|
||||
|
||||
# --- llm dashboard ------------------------------------------------------------
|
||||
gufo_url: str = _base_url("GUFO_BASE_URL")
|
||||
gufo_api_key: str = _env("GUFO_API_KEY")
|
||||
vllm_url: str = _base_url("VLLM_BASE_URL")
|
||||
vllm_api_key: str = _env("VLLM_API_KEY")
|
||||
llm_interval: float = _num("LLM_INTERVAL_SECONDS", 2)
|
||||
llm_timeout: float = _num("LLM_TIMEOUT_SECONDS", 3)
|
||||
rate_window: float = _num("RATE_WINDOW_SECONDS", 10)
|
||||
|
||||
# --- ups dashboard ------------------------------------------------------------
|
||||
nut_host: str = _env("NUT_HOST").removeprefix("nut://")
|
||||
nut_port: int = _int("NUT_PORT", 3493)
|
||||
nut_user: str = _env("NUT_USER")
|
||||
nut_password: str = _env("NUT_PASSWORD")
|
||||
nut_interval: float = _num("NUT_INTERVAL_SECONDS", 5)
|
||||
nut_timeout: float = _num("NUT_TIMEOUT_SECONDS", 4)
|
||||
# Instant commands (battery tests, beeper, calibration) are the only write path in
|
||||
# the app. Off unless BOTH the env flag is set and DASH_READ_ONLY is not.
|
||||
nut_allow_actions: bool = _flag("NUT_ALLOW_ACTIONS")
|
||||
|
||||
# --- history ------------------------------------------------------------------
|
||||
raw_keep_hours: float = _num("RETAIN_RAW_HOURS", 48)
|
||||
minute_keep_days: float = _num("RETAIN_MINUTE_DAYS", 30)
|
||||
hour_keep_days: float = _num("RETAIN_HOUR_DAYS", 400)
|
||||
|
||||
# --- browser polling defaults -------------------------------------------------
|
||||
poll_ms_system: int = _int("POLL_MS_SYSTEM", 10_000)
|
||||
poll_ms_llm: int = _int("POLL_MS_LLM", 2_000)
|
||||
poll_ms_ups: int = _int("POLL_MS_UPS", 5_000)
|
||||
|
||||
@property
|
||||
def nut_enabled(self) -> bool:
|
||||
return bool(self.nut_host)
|
||||
|
||||
@property
|
||||
def actions_allowed(self) -> bool:
|
||||
"""Coarse switch. Whether a *specific* command can run also depends on the user
|
||||
having `actions` in upsd.users, which is checked per command (and reported to the
|
||||
UI as a reason) — requiring credentials here would make the label wrong for the
|
||||
simulator and would hide the actual fix from the operator."""
|
||||
return self.nut_allow_actions and not self.read_only
|
||||
|
||||
@property
|
||||
def any_source_enabled(self) -> bool:
|
||||
return bool(self.host_targets) or bool(self.gufo_url or self.vllm_url) or self.nut_enabled
|
||||
|
||||
|
||||
def load() -> Settings:
|
||||
"""Settings from the environment as it stands now.
|
||||
|
||||
Every scalar was captured when this module was imported (see `load_env_file`); only the
|
||||
target list is re-parsed per call, which is what makes a target fixable without a reload
|
||||
in tests and tooling.
|
||||
"""
|
||||
targets, problems = parse_targets(_env("HOST_TARGETS"))
|
||||
return Settings(host_targets=tuple(targets), host_target_problems=tuple(problems))
|
||||
|
||||
|
||||
SETTINGS = load()
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Optional debugpy attach, gated by the DEBUGPY env var.
|
||||
|
||||
Default (unset / DEBUGPY=0): this module does nothing and imports nothing heavy.
|
||||
DEBUGPY=1: listens on DEBUGPY_PORT (default 5678) without blocking startup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def maybe_listen() -> bool:
|
||||
"""Start a debugpy listener when DEBUGPY is truthy. Returns True if it did."""
|
||||
if os.environ.get("DEBUGPY", "").lower() not in ("1", "true", "yes", "on"):
|
||||
return False
|
||||
port = int(os.environ.get("DEBUGPY_PORT", "5678"))
|
||||
try:
|
||||
import debugpy # noqa: PLC0415 - deliberately imported only when requested
|
||||
except ImportError:
|
||||
print(f"[debug] DEBUGPY is set but debugpy is not installed (uv sync --dev); port {port} unused", flush=True)
|
||||
return False
|
||||
debugpy.listen(("0.0.0.0", port))
|
||||
print(f"[debug] debugpy listening on 0.0.0.0:{port}", flush=True)
|
||||
return True
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Polling framework: one asyncio loop per source, independent health for each.
|
||||
|
||||
Rules this enforces, because violating them is what makes dashboards untrustworthy:
|
||||
|
||||
* **Isolation.** A source that throws, hangs, or 500s must never blank, block, or delay
|
||||
another one. Each job has its own task, its own timeout (the collector's problem) and
|
||||
its own health record.
|
||||
* **Last good value wins.** A failed poll keeps the previous payload and marks it stale
|
||||
(`stale 12s` in the UI) instead of showing zeros or `n/a` everywhere.
|
||||
* **Transitions are events, spam is not.** ok→fail and fail→ok write to the event log;
|
||||
500 consecutive failures write one line.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .store import Event, Sample, SampleStore
|
||||
|
||||
logger = logging.getLogger("dashboards.poller")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Result:
|
||||
"""What one successful collection produced."""
|
||||
|
||||
payload: dict[str, Any]
|
||||
samples: list[Sample] = field(default_factory=list)
|
||||
events: list[Event] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceState:
|
||||
key: str # "host:driveripper", "llm:gufo", "nut"
|
||||
dashboard: str # "system" | "llm" | "ups"
|
||||
label: str
|
||||
configured: bool = True
|
||||
ok: bool = False
|
||||
error: str | None = None
|
||||
attempts: int = 0
|
||||
successes: int = 0
|
||||
failures: int = 0
|
||||
consecutive_failures: int = 0
|
||||
last_attempt: float | None = None
|
||||
last_ok: float | None = None
|
||||
latency_ms: float | None = None
|
||||
samples_written: int = 0
|
||||
payload: dict[str, Any] | None = None
|
||||
|
||||
def as_json(self, now: float | None = None) -> dict[str, Any]:
|
||||
"""Health block for the browser. `age_seconds` only counts successes, so a dead
|
||||
source reads as "stale 4m" rather than pretending to be current."""
|
||||
now = time.time() if now is None else now
|
||||
return {
|
||||
"key": self.key,
|
||||
"dashboard": self.dashboard,
|
||||
"label": self.label,
|
||||
"configured": self.configured,
|
||||
"ok": self.ok,
|
||||
"error": self.error,
|
||||
"attempts": self.attempts,
|
||||
"successes": self.successes,
|
||||
"failures": self.failures,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"last_attempt": self.last_attempt,
|
||||
"last_ok": self.last_ok,
|
||||
"age_seconds": (now - self.last_ok) if self.last_ok else None,
|
||||
"latency_ms": self.latency_ms,
|
||||
"samples_written": self.samples_written,
|
||||
}
|
||||
|
||||
|
||||
class Job:
|
||||
"""A named poll loop. `fn` does the work; everything else lives here so every
|
||||
dashboard gets identical health, staleness, and event semantics for free."""
|
||||
|
||||
def __init__(self, key: str, dashboard: str, label: str, interval: float,
|
||||
fn: Callable[[], Awaitable[Result]], *, enabled: bool = True,
|
||||
grace_failures: int = 1) -> None:
|
||||
self.key = key
|
||||
self.dashboard = dashboard
|
||||
self.label = label
|
||||
self.interval = max(0.5, interval)
|
||||
self.fn = fn
|
||||
self.enabled = enabled
|
||||
self.grace_failures = grace_failures
|
||||
self.state = SourceState(key=key, dashboard=dashboard, label=label, configured=enabled)
|
||||
self._down_logged = False
|
||||
self._stop = asyncio.Event()
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._inflight = False
|
||||
|
||||
async def run_once(self, store: SampleStore) -> bool:
|
||||
"""One poll, with the health bookkeeping. Returns True on success."""
|
||||
if self._inflight: # a slow poll must never overlap itself
|
||||
return False
|
||||
self._inflight = True
|
||||
now = time.time()
|
||||
self.state.attempts += 1
|
||||
self.state.last_attempt = now
|
||||
was_ok = self.state.ok
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
result = await self.fn()
|
||||
self.state.latency_ms = round((time.perf_counter() - started) * 1000.0, 1)
|
||||
except Exception as exc: # noqa: BLE001 - a collector must never kill the loop
|
||||
self.state.ok = False
|
||||
self.state.failures += 1
|
||||
self.state.consecutive_failures += 1
|
||||
self.state.error = f"{type(exc).__name__}: {exc}"
|
||||
# One event per outage, not per failed poll. A source that has never answered
|
||||
# gets a warn rather than silence — otherwise a host misconfigured from the first
|
||||
# boot leaves no trace in the log at all.
|
||||
if (self.state.consecutive_failures >= self.grace_failures and not self._down_logged):
|
||||
self._down_logged = True
|
||||
store.log_event(Event(now, self.dashboard, self.label, "unreachable",
|
||||
"error" if self.state.successes else "warn",
|
||||
self.state.error
|
||||
+ ("" if self.state.successes else " (never answered)")))
|
||||
return False
|
||||
finally:
|
||||
self._inflight = False
|
||||
|
||||
self.state.ok = True
|
||||
self.state.successes += 1
|
||||
self.state.payload = result.payload
|
||||
# Recovery is news only if we ever said it was down: on a fresh start every source
|
||||
# would otherwise log "answering again", which reads like an outage happened.
|
||||
if self._down_logged or (not was_ok and self.state.failures):
|
||||
count = self.state.consecutive_failures
|
||||
store.log_event(Event(now, self.dashboard, self.label, "reachable", "info",
|
||||
f"{self.label} is answering again"
|
||||
+ (f" after {count} failure{'s' if count != 1 else ''}"
|
||||
if count else "")))
|
||||
self._down_logged = False
|
||||
self.state.consecutive_failures = 0
|
||||
self.state.last_ok = now
|
||||
self.state.error = None
|
||||
self.state.samples_written += store.add_many(result.samples)
|
||||
for event in result.events:
|
||||
store.log_event(event)
|
||||
return True
|
||||
|
||||
async def _loop(self, store: SampleStore) -> None:
|
||||
while not self._stop.is_set():
|
||||
started = time.perf_counter()
|
||||
await self.run_once(store)
|
||||
remaining = self.interval - (time.perf_counter() - started)
|
||||
try:
|
||||
await asyncio.wait_for(self._stop.wait(), timeout=max(0.2, remaining))
|
||||
except TimeoutError:
|
||||
continue
|
||||
|
||||
def start(self, store: SampleStore) -> None:
|
||||
if self._task is None and self.enabled:
|
||||
self._task = asyncio.create_task(self._loop(store), name=f"poll:{self.key}")
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._stop.set()
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
# the cancellation is the point; the task's own outcome is not our problem here.
|
||||
# CancelledError is a BaseException in 3.8+, so it has to be named explicitly.
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await self._task
|
||||
self._task = None
|
||||
|
||||
|
||||
class Hub:
|
||||
"""Owns the store and every poll loop; the only thing the API layer talks to."""
|
||||
|
||||
def __init__(self, store: SampleStore) -> None:
|
||||
self.store = store
|
||||
self.jobs: dict[str, Job] = {}
|
||||
self.started_at = time.time()
|
||||
self._maintenance: asyncio.Task[None] | None = None
|
||||
|
||||
def add(self, job: Job) -> Job:
|
||||
self.jobs[job.key] = job
|
||||
return job
|
||||
|
||||
def start(self) -> None:
|
||||
for job in self.jobs.values():
|
||||
job.start(self.store)
|
||||
if self._maintenance is None:
|
||||
self._maintenance = asyncio.create_task(self._maintain_loop(), name="maintain")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._maintenance is not None:
|
||||
self._maintenance.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await self._maintenance
|
||||
self._maintenance = None
|
||||
for job in self.jobs.values():
|
||||
await job.stop()
|
||||
|
||||
async def _maintain_loop(self, interval: float = 60.0) -> None:
|
||||
"""Rollup + prune. Runs in a thread: SQLite writes would otherwise stall the loop."""
|
||||
# imported here because only this task needs the thread bridge
|
||||
from anyio.to_thread import run_sync # noqa: PLC0415
|
||||
|
||||
while True:
|
||||
try:
|
||||
await run_sync(self.store.maintain)
|
||||
except Exception:
|
||||
# Logged, never swallowed: a broken rollup means every chart past the raw
|
||||
# window is quietly empty, and "it retried" is not a record of that.
|
||||
logger.exception("store maintenance failed; long-window charts may be stale")
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
def states(self, dashboard: str | None = None) -> list[dict[str, Any]]:
|
||||
now = time.time()
|
||||
return [
|
||||
job.state.as_json(now)
|
||||
for job in self.jobs.values()
|
||||
if dashboard is None or job.dashboard == dashboard
|
||||
]
|
||||
|
||||
def payloads(self, dashboard: str | None = None) -> dict[str, dict[str, Any]]:
|
||||
return {
|
||||
job.key: job.state.payload
|
||||
for job in self.jobs.values()
|
||||
if (dashboard is None or job.dashboard == dashboard) and job.state.payload is not None
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Prometheus text-exposition parser. In-house, zero dependencies.
|
||||
|
||||
Only used for the upstreams we do not control: llama.cpp/gufo (`llamacpp:*`) and vLLM
|
||||
(`vllm:*`). Our own host agent serves JSON, so nothing else here needs this parser and we
|
||||
never pull in `prometheus_client`.
|
||||
|
||||
Ported from llm_monitor, where it survived a year of both engines renaming series under
|
||||
it. Keep the aggregation rules (see `Prom.scalar`): they are what stops a summed ratio
|
||||
from exceeding 1.0 and inventing a percentile.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
_NAME_RE = r"[a-zA-Z_:][a-zA-Z0-9_:]*"
|
||||
# name{label="v",...} value [timestamp]
|
||||
# - the label block is greedy so a '}' inside a quoted value cannot truncate the match;
|
||||
# the anchored value/end-of-line does that instead.
|
||||
# - whitespace before '{' and a trailing '# ...' comment are not part of the official
|
||||
# format, but tolerating them means a quirky exporter shows a number instead of 'n/a'.
|
||||
_LINE_RE = re.compile(
|
||||
rf"^(?P<name>{_NAME_RE})(?:[ \t]*\{{(?P<labels>.*)\}})?[ \t]+"
|
||||
r"(?P<value>[+-]?(\d+\.?\d*(e[+-]?\d+)?|inf|nan))"
|
||||
r"(?:[ \t]+(?P<ts>-?\d+))?[ \t]*(?:#.*)?$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_.:]*)="((?:[^"\\]|\\.)*)"')
|
||||
|
||||
|
||||
def _unquote(value: str) -> str:
|
||||
return re.sub(r"\\(.)", r"\1", value)
|
||||
|
||||
|
||||
class Prom:
|
||||
"""Parsed exposition: name -> [(labels, value), ...]."""
|
||||
|
||||
def __init__(self, text: str) -> None:
|
||||
self.series: dict[str, list[tuple[dict[str, str], float]]] = {}
|
||||
self.parse_errors = 0
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line[0] == "#":
|
||||
continue
|
||||
m = _LINE_RE.match(line)
|
||||
if not m:
|
||||
self.parse_errors += 1
|
||||
continue
|
||||
try:
|
||||
value = float(m.group("value"))
|
||||
except ValueError: # pragma: no cover - regex keeps this unreachable
|
||||
self.parse_errors += 1
|
||||
continue
|
||||
labels = {_unquote(k): _unquote(v) for k, v in _LABEL_RE.findall(m.group("labels") or "")}
|
||||
self.series.setdefault(m.group("name"), []).append((labels, value))
|
||||
|
||||
def resolve(self, names: Sequence[str]) -> str | None:
|
||||
"""First name from `names` that is actually present."""
|
||||
return next((n for n in names if n in self.series), None)
|
||||
|
||||
def resolve_hist(self, names: Sequence[str]) -> str | None:
|
||||
"""Like resolve() for histograms: the bare name never appears, only *_bucket/_sum/_count."""
|
||||
for n in names:
|
||||
if any(f"{n}{suffix}" in self.series for suffix in ("_bucket", "_sum", "_count")):
|
||||
return n
|
||||
return None
|
||||
|
||||
def values(self, names: Sequence[str], label_match: dict[str, str] | None = None) -> list[float]:
|
||||
name = self.resolve(names)
|
||||
if name is None:
|
||||
return []
|
||||
if not label_match:
|
||||
return [v for _labels, v in self.series[name]]
|
||||
return [
|
||||
v for labels, v in self.series[name]
|
||||
if all(labels.get(k) == x for k, x in label_match.items())
|
||||
]
|
||||
|
||||
def scalar(self, names: Sequence[str], agg: str = "sum",
|
||||
label_match: dict[str, str] | None = None) -> float | None:
|
||||
"""Aggregate every matching series, or None when absent.
|
||||
|
||||
agg="sum" counters and per-engine gauges — summing label sets is the fleet total.
|
||||
agg="max" usage ratios — report the most pressured engine, never a sum over 1.0.
|
||||
agg="avg" rate gauges — an average across label sets.
|
||||
"""
|
||||
found = self.values(names, label_match)
|
||||
finite = [v for v in found if math.isfinite(v)]
|
||||
if not finite:
|
||||
return None
|
||||
if agg == "max":
|
||||
return max(finite)
|
||||
if agg == "avg":
|
||||
return sum(finite) / len(finite)
|
||||
return sum(finite)
|
||||
|
||||
def by_label(self, names: Sequence[str], label: str) -> dict[str, float]:
|
||||
name = self.resolve(names)
|
||||
if name is None:
|
||||
return {}
|
||||
out: dict[str, float] = {}
|
||||
for labels, value in self.series[name]:
|
||||
if not math.isfinite(value):
|
||||
continue
|
||||
key = labels.get(label, "")
|
||||
out[key] = out.get(key, 0.0) + value
|
||||
return out
|
||||
|
||||
def histogram(self, names: Sequence[str]) -> dict[str, Any] | None:
|
||||
"""Mean (+ p95 when interpolable) of a `<name>_sum`/`_count`/`_bucket`."""
|
||||
base = self.resolve_hist(names)
|
||||
if base is None:
|
||||
return None
|
||||
total = self.scalar([f"{base}_sum"])
|
||||
count = self.scalar([f"{base}_count"])
|
||||
buckets = self._buckets(base)
|
||||
out: dict[str, Any] = {
|
||||
"count": count,
|
||||
"sum": total,
|
||||
"mean": (total / count) if (total is not None and count) else None,
|
||||
"p95": None,
|
||||
"source": base,
|
||||
}
|
||||
# Interpolate only for a single label set: mixing per-DP-rank histograms into one
|
||||
# cumulative curve invents a bogus p95.
|
||||
if buckets and count and self._bucket_label_groups(base) <= 1:
|
||||
out["p95"] = _quantile(buckets, count, 0.95)
|
||||
return out
|
||||
|
||||
def _bucket_label_groups(self, base: str) -> int:
|
||||
groups = set()
|
||||
for labels, _v in self.series.get(f"{base}_bucket", []):
|
||||
groups.add(tuple(sorted((k, v) for k, v in labels.items() if k != "le")))
|
||||
return len(groups)
|
||||
|
||||
def _buckets(self, base: str) -> list[tuple[float, float]]:
|
||||
out: list[tuple[float, float]] = []
|
||||
for labels, value in self.series.get(f"{base}_bucket", []):
|
||||
le = labels.get("le")
|
||||
if le is None or not math.isfinite(value):
|
||||
continue
|
||||
bound = math.inf if le.lower() in ("+inf", "inf") else float(le)
|
||||
out.append((bound, value))
|
||||
out.sort(key=lambda item: item[0])
|
||||
return out
|
||||
|
||||
|
||||
def _quantile(buckets: list[tuple[float, float]], count: float, q: float) -> float | None:
|
||||
"""Linear interpolation inside the bucket containing the `q` quantile."""
|
||||
target = q * count
|
||||
prev_bound, prev_cum = 0.0, 0.0
|
||||
for bound, cum in buckets:
|
||||
if cum >= target:
|
||||
if math.isinf(bound):
|
||||
return prev_bound if math.isfinite(prev_bound) else None
|
||||
span = cum - prev_cum
|
||||
if span <= 0:
|
||||
return bound
|
||||
return prev_bound + (bound - prev_bound) * ((target - prev_cum) / span)
|
||||
prev_bound, prev_cum = bound, cum
|
||||
return None
|
||||
|
||||
|
||||
def field(value: float | None, source: str | None, derived: bool = False,
|
||||
window: float | None = None) -> dict[str, Any]:
|
||||
"""Uniform metric cell: the UI renders `n/a` whenever value is None.
|
||||
|
||||
`window` is the Δt a rate was measured over, so the UI can say `Δ10.0s` instead of
|
||||
implying a scraped gauge it never got.
|
||||
"""
|
||||
cell: dict[str, Any] = {"value": value, "source": source, "derived": derived}
|
||||
if window is not None:
|
||||
cell["window"] = window
|
||||
return cell
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Δcounter / Δt over a rolling window — the one piece of mutable state in the app.
|
||||
|
||||
Both upstreams export cumulative counters (tokens generated, jiffies spent, bytes read)
|
||||
and the dashboards need per-second rates. A bare 1s delta jitters between 0 and spikes
|
||||
because work arrives in bursts, so the rate is measured over a window (default 10s) and
|
||||
the window that was actually used travels with the value — the UI prints `Δ10.0s` rather
|
||||
than implying a scraped gauge it never got.
|
||||
|
||||
Keyed by the *resolved* upstream series name (e.g. `llamacpp:tokens_predicted_total`, or
|
||||
`driveripper:cpu.total`), so a renamed counter starts a fresh window instead of diffing
|
||||
two different series into a fabricated rate, and two hosts can never read each other's
|
||||
samples.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections import deque
|
||||
|
||||
|
||||
class RateEstimator:
|
||||
def __init__(self, span: float = 10.0, max_samples: int = 512) -> None:
|
||||
self.span = span
|
||||
self.max_samples = max_samples
|
||||
self._samples: dict[str, deque[tuple[float, float]]] = {}
|
||||
|
||||
def rate(self, key: str, value: float | None, now: float | None = None) -> tuple[float | None, float | None]:
|
||||
"""(per-second, window actually used). (None, window|None) when not computable.
|
||||
|
||||
Never returns a negative rate: a counter going backwards means a restart (or a
|
||||
renamed series), so the window is dropped and one scrape answers `n/a`.
|
||||
"""
|
||||
if value is None or not math.isfinite(value):
|
||||
return None, None
|
||||
now = time.time() if now is None else now
|
||||
samples = self._samples.setdefault(key, deque())
|
||||
if samples and value < samples[-1][1]:
|
||||
samples.clear()
|
||||
samples.append((now, value))
|
||||
while len(samples) > self.max_samples:
|
||||
samples.popleft()
|
||||
if len(samples) < 2:
|
||||
return None, None
|
||||
|
||||
inside = [point for point in samples if now - point[0] <= self.span]
|
||||
base = inside[0] if len(inside) > 1 else samples[-2]
|
||||
dt, dv = now - base[0], value - base[1]
|
||||
cutoff = now - 2 * self.span
|
||||
while samples and samples[0][0] < cutoff and len(samples) > 2:
|
||||
samples.popleft()
|
||||
if dt <= 0 or dv < 0:
|
||||
return None, dt if dt > 0 else None
|
||||
return dv / dt, dt
|
||||
|
||||
def forget(self, prefix: str) -> int:
|
||||
"""Drop every window whose key starts with `prefix` (a target that went away)."""
|
||||
keys = [k for k in self._samples if k.startswith(prefix)]
|
||||
for key in keys:
|
||||
del self._samples[key]
|
||||
return len(keys)
|
||||
|
||||
def keys(self) -> list[str]:
|
||||
return list(self._samples)
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Data sources. Each module turns one upstream's raw payload into our cell shape.
|
||||
|
||||
host -> our own stdlib agent (JSON)
|
||||
llm -> gufo / vLLM Prometheus expositions (parsed by our own Prom class)
|
||||
nut -> the upsd wire protocol (reads + instant commands)
|
||||
nut_sim-> a labelled simulator for development, never a substitute for real hardware
|
||||
"""
|
||||
@@ -0,0 +1,416 @@
|
||||
"""Host source: reads `agent/host_agent.py` JSON and turns counters into rates.
|
||||
|
||||
Division of labour, and why:
|
||||
|
||||
* the agent reads /proc and /sys and reports **cumulative counters** (it cannot know how
|
||||
long the interval was, and it must not have to remember who polled it last);
|
||||
* this module owns the **window**: every per-second number is Δcounter/Δt over
|
||||
`RateEstimator`, and carries the window that produced it, so the UI can print `Δ10.0s`.
|
||||
* absent values stay None. A host with no GPU has no gpu series; it does not have a GPU at
|
||||
0% busy (that would say "idle", which is a different claim from "there is nothing here").
|
||||
|
||||
Metric names in the history DB use `@stream` for per-device series:
|
||||
`cpu.usage_pct@core7`, `disk.used_pct@/srv`, `gpu.busy_pct@0`, `net.rx_bytes_per_s@eth0`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from ..poller import Result
|
||||
from ..rates import RateEstimator
|
||||
from ..store import Sample
|
||||
|
||||
# /proc/stat jiffies we treat as "not doing useful work".
|
||||
CPU_IDLE_MODES = ("idle", "iowait")
|
||||
|
||||
|
||||
@dataclass
|
||||
class HostTargetState:
|
||||
"""Per-target memory across polls: only what a *comparison* needs."""
|
||||
|
||||
boot_ts: float | None = None
|
||||
hostname: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class HostCollector:
|
||||
"""Holds the rate windows and the reboot detector for every host."""
|
||||
|
||||
rates: RateEstimator
|
||||
targets: dict[str, HostTargetState] = field(default_factory=dict)
|
||||
|
||||
def target_state(self, name: str) -> HostTargetState:
|
||||
return self.targets.setdefault(name, HostTargetState())
|
||||
|
||||
|
||||
def _rate(collector: HostCollector, host: str, key: str, value: float | None,
|
||||
now: float) -> tuple[float | None, float | None]:
|
||||
return collector.rates.rate(f"{host}:{key}", value, now)
|
||||
|
||||
|
||||
def _ratio_percent(numerator: float | None, denominator: float | None) -> float | None:
|
||||
if numerator is None or denominator is None or denominator <= 0:
|
||||
return None
|
||||
return max(0.0, min(100.0, numerator / denominator * 100.0))
|
||||
|
||||
|
||||
def parse_host_snapshot(name: str, snapshot: dict[str, Any], collector: HostCollector,
|
||||
*, now: float | None = None) -> Result:
|
||||
"""One agent snapshot -> (UI payload, history samples, transition events)."""
|
||||
now = time.time() if now is None else now
|
||||
samples: list[Sample] = []
|
||||
events: list[Any] = []
|
||||
ts = float(snapshot.get("ts") or now)
|
||||
|
||||
info = snapshot.get("host") or {}
|
||||
state = collector.target_state(name)
|
||||
boot_ts = info.get("boot_ts")
|
||||
if boot_ts is not None:
|
||||
if state.boot_ts is not None and abs(boot_ts - state.boot_ts) > 1.0:
|
||||
events.append(_event(now, name, "reboot", "warn",
|
||||
f"{name} rebooted (agent boot time moved from "
|
||||
f"{state.boot_ts:.0f} to {boot_ts:.0f})"))
|
||||
state.boot_ts = boot_ts
|
||||
if info.get("hostname"):
|
||||
state.hostname = str(info["hostname"])
|
||||
|
||||
# ---------------------------------------------------------------- CPU ---- #
|
||||
cpu_raw = snapshot.get("cpu") or {}
|
||||
modes = cpu_raw.get("modes") or {}
|
||||
total_rate, window = _rate(collector, name, "cpu.total", modes.get("total"), now)
|
||||
# Every counter is sampled on every poll, including the first. Gating the idle modes on
|
||||
# `total_rate is not None` (as this used to) meant idle was never seeded on poll 1, so
|
||||
# usage_pct stayed n/a until poll 3 — a dashboard that boots two intervals late, and a
|
||||
# fleet average that counts a live host as missing for its first minute.
|
||||
idle_rate, idle_ok = 0.0, True
|
||||
for mode in CPU_IDLE_MODES:
|
||||
value, _ = _rate(collector, name, f"cpu.{mode}", modes.get(mode), now)
|
||||
if value is None:
|
||||
idle_ok = False
|
||||
else:
|
||||
idle_rate += value
|
||||
busy_rate = max(0.0, total_rate - idle_rate) if (idle_ok and total_rate is not None) else None
|
||||
usage_pct = (busy_rate / total_rate * 100.0) if (busy_rate is not None and total_rate) else None
|
||||
|
||||
mode_pct: dict[str, float | None] = {}
|
||||
for mode in ("user", "system", "iowait", "irq", "softirq", "steal", "nice"):
|
||||
value, _ = _rate(collector, name, f"cpu.{mode}", modes.get(mode), now)
|
||||
mode_pct[mode] = _ratio_percent(value, total_rate) if total_rate else None
|
||||
|
||||
core_usage: list[dict[str, Any]] = []
|
||||
for core in cpu_raw.get("cores") or []:
|
||||
core_modes = core.get("modes") or {}
|
||||
core_total, _ = _rate(collector, name, f"cpu.core{core['core']}.total",
|
||||
core_modes.get("total"), now)
|
||||
core_idle = 0.0
|
||||
core_ok = True
|
||||
for mode in CPU_IDLE_MODES:
|
||||
value, _ = _rate(collector, name, f"cpu.core{core['core']}.{mode}",
|
||||
core_modes.get(mode), now)
|
||||
if value is None:
|
||||
core_ok = False
|
||||
else:
|
||||
core_idle += value
|
||||
core_busy = (core_total - core_idle) if (core_ok and core_total is not None) else None
|
||||
value = _ratio_percent(core_busy, core_total) if core_total else None
|
||||
core_usage.append({"core": core["core"], "usage_pct": value})
|
||||
if value is not None:
|
||||
samples.append(Sample("host", name, f"cpu.usage_pct@core{core['core']}", value, ts))
|
||||
if usage_pct is not None:
|
||||
samples.append(Sample("host", name, "cpu.usage_pct", usage_pct, ts))
|
||||
for mode, value in mode_pct.items():
|
||||
if value is not None:
|
||||
samples.append(Sample("host", name, f"cpu.{mode}_pct", value, ts))
|
||||
|
||||
ctxt_rate, _ = _rate(collector, name, "cpu.ctxt", cpu_raw.get("ctxt_total"), now)
|
||||
forks_rate, _ = _rate(collector, name, "cpu.processes", cpu_raw.get("processes_total"), now)
|
||||
|
||||
# -------------------------------------------------------------- Memory --- #
|
||||
mem = snapshot.get("memory") or {}
|
||||
mem_total = mem.get("total_bytes")
|
||||
mem_used = mem.get("used_bytes")
|
||||
mem_used_pct = _ratio_percent(mem_used, mem_total)
|
||||
swap_total = mem.get("swap_total_bytes") or 0.0
|
||||
swap_used = mem.get("swap_used_bytes") or 0.0
|
||||
swap_used_pct = (swap_used / swap_total * 100.0) if swap_total else None
|
||||
for key in ("total_bytes", "available_bytes", "used_bytes", "cached_bytes", "buffers_bytes",
|
||||
"swap_total_bytes", "swap_used_bytes"):
|
||||
if mem.get(key) is not None:
|
||||
samples.append(Sample("host", name, f"mem.{key}", float(mem[key]), ts))
|
||||
if mem_used_pct is not None:
|
||||
samples.append(Sample("host", name, "mem.used_pct", mem_used_pct, ts))
|
||||
if swap_used_pct is not None:
|
||||
samples.append(Sample("host", name, "mem.swap_used_pct", swap_used_pct, ts))
|
||||
|
||||
# ----------------------------------------------------------- Filesystems - #
|
||||
filesystems = []
|
||||
for fs in snapshot.get("filesystems") or []:
|
||||
used_pct = fs.get("used_pct")
|
||||
filesystems.append(fs)
|
||||
if used_pct is not None:
|
||||
samples.append(Sample("host", name, f"disk.used_pct@{fs['mountpoint']}",
|
||||
float(used_pct), ts))
|
||||
if fs.get("used_bytes") is not None:
|
||||
samples.append(Sample("host", name, f"disk.used_bytes@{fs['mountpoint']}",
|
||||
float(fs["used_bytes"]), ts))
|
||||
scored = [fs for fs in filesystems if fs.get("used_pct") is not None]
|
||||
worst = max(scored, key=lambda fs: fs["used_pct"]) if scored else None
|
||||
|
||||
# -------------------------------------------------------------- Disk IO -- #
|
||||
io_devices = []
|
||||
read_total = write_total = reads_total = writes_total = 0.0
|
||||
any_io = False
|
||||
util_max: float | None = None
|
||||
for dev in (snapshot.get("disk_io") or {}).get("devices") or []:
|
||||
dev_name = dev.get("name") or "?"
|
||||
read_rate, _ = _rate(collector, name, f"disk.{dev_name}.read_bytes",
|
||||
dev.get("read_bytes_total"), now)
|
||||
write_rate, _ = _rate(collector, name, f"disk.{dev_name}.write_bytes",
|
||||
dev.get("write_bytes_total"), now)
|
||||
r_ops, _ = _rate(collector, name, f"disk.{dev_name}.reads", dev.get("reads_total"), now)
|
||||
w_ops, _ = _rate(collector, name, f"disk.{dev_name}.writes", dev.get("writes_total"), now)
|
||||
io_ms_rate, _ = _rate(collector, name, f"disk.{dev_name}.io_time_ms",
|
||||
dev.get("io_time_ms_total"), now)
|
||||
# io_time is milliseconds spent with at least one request outstanding, so
|
||||
# ms-per-second / 10 == percent of the window the device was busy.
|
||||
util = None if io_ms_rate is None else max(0.0, min(100.0, io_ms_rate / 10.0))
|
||||
row = {"name": dev_name, "model": dev.get("model"), "rotational": dev.get("rotational"),
|
||||
"size_bytes": dev.get("size_bytes"),
|
||||
"read_bytes_per_s": read_rate, "write_bytes_per_s": write_rate,
|
||||
"reads_per_s": r_ops, "writes_per_s": w_ops, "util_pct": util,
|
||||
"in_flight": dev.get("in_flight")}
|
||||
io_devices.append(row)
|
||||
if read_rate is not None:
|
||||
read_total += read_rate
|
||||
any_io = True
|
||||
if write_rate is not None:
|
||||
write_total += write_rate
|
||||
any_io = True
|
||||
if r_ops is not None:
|
||||
reads_total += r_ops
|
||||
if w_ops is not None:
|
||||
writes_total += w_ops
|
||||
if util is not None:
|
||||
util_max = util if util_max is None else max(util_max, util)
|
||||
|
||||
# ------------------------------------------------------------- Network --- #
|
||||
net_rows = []
|
||||
rx_total = tx_total = 0.0
|
||||
any_net = False
|
||||
for iface in (snapshot.get("network") or {}).get("interfaces") or []:
|
||||
iname = iface.get("name") or "?"
|
||||
rx, _ = _rate(collector, name, f"net.{iname}.rx", iface.get("rx_bytes_total"), now)
|
||||
tx, _ = _rate(collector, name, f"net.{iname}.tx", iface.get("tx_bytes_total"), now)
|
||||
net_rows.append({"name": iname, "rx_bytes_per_s": rx, "tx_bytes_per_s": tx,
|
||||
"speed_mbps": iface.get("speed_mbps"),
|
||||
"rx_errors_total": iface.get("rx_errors_total"),
|
||||
"tx_errors_total": iface.get("tx_errors_total"),
|
||||
"rx_dropped_total": iface.get("rx_dropped_total"),
|
||||
"tx_dropped_total": iface.get("tx_dropped_total")})
|
||||
if rx is not None:
|
||||
rx_total += rx
|
||||
any_net = True
|
||||
samples.append(Sample("host", name, f"net.rx_bytes_per_s@{iname}", rx, ts))
|
||||
if tx is not None:
|
||||
tx_total += tx
|
||||
samples.append(Sample("host", name, f"net.tx_bytes_per_s@{iname}", tx, ts))
|
||||
|
||||
# ---------------------------------------------------------------- GPU ---- #
|
||||
gpus = []
|
||||
for gpu in snapshot.get("gpus") or []:
|
||||
index = gpu.get("index", len(gpus))
|
||||
stream = f"@{index}"
|
||||
vram_used = gpu.get("vram_used_bytes")
|
||||
vram_total = gpu.get("vram_total_bytes")
|
||||
gtt_used = gpu.get("gtt_used_bytes")
|
||||
gtt_total = gpu.get("gtt_total_bytes")
|
||||
temps = gpu.get("temps") or {}
|
||||
temp_c = max(temps.values()) if temps else None
|
||||
pools_used = None if vram_used is None else vram_used + (gtt_used or 0.0)
|
||||
pools_total = None if vram_total is None else vram_total + (gtt_total or 0.0)
|
||||
row = {
|
||||
"index": index,
|
||||
"card": gpu.get("card"),
|
||||
"model": gpu.get("model") or gpu.get("card") or f"GPU {index}",
|
||||
# the short name is what fits a legend; the lspci string belongs in the tooltip
|
||||
"model_full": gpu.get("model_full") or gpu.get("model"),
|
||||
"vendor": gpu.get("vendor"),
|
||||
"driver": gpu.get("driver"),
|
||||
"pci_address": gpu.get("pci_address"),
|
||||
"busy_pct": gpu.get("busy_pct"),
|
||||
"vram_used_bytes": vram_used,
|
||||
"vram_total_bytes": vram_total,
|
||||
"vram_used_pct": _ratio_percent(vram_used, vram_total),
|
||||
"gtt_used_bytes": gtt_used,
|
||||
"gtt_total_bytes": gtt_total,
|
||||
# an APU's model lives in GTT, so "VRAM 3%" and "memory 78%" can both be true;
|
||||
# this is the honest whole-addressable-pool view, kept beside the two pools
|
||||
"mem_used_bytes": pools_used,
|
||||
"mem_total_bytes": pools_total,
|
||||
"mem_used_pct": _ratio_percent(pools_used, pools_total),
|
||||
"power_watts": gpu.get("power_watts"),
|
||||
"temps": temps,
|
||||
"temp_c": temp_c,
|
||||
"clocks": gpu.get("clocks") or {},
|
||||
}
|
||||
gpus.append(row)
|
||||
for metric, value in (("busy_pct", row["busy_pct"]), ("vram_used_bytes", vram_used),
|
||||
("vram_used_pct", row["vram_used_pct"]),
|
||||
("mem_used_pct", row["mem_used_pct"]),
|
||||
("power_watts", row["power_watts"]), ("temp_c", temp_c)):
|
||||
if value is not None:
|
||||
samples.append(Sample("host", name, f"gpu.{metric}{stream}", float(value), ts))
|
||||
|
||||
# -------------------------------------------------------- Temperatures --- #
|
||||
sensors = snapshot.get("temperatures") or []
|
||||
cpu_temp = snapshot.get("cpu_temperature_celsius")
|
||||
values = [s["celsius"] for s in sensors if s.get("celsius") is not None]
|
||||
max_temp = max(values) if values else None
|
||||
nvme_values = [s["celsius"] for s in sensors
|
||||
if s.get("chip") == "nvme" and s.get("celsius") is not None]
|
||||
nvme_max = max(nvme_values) if nvme_values else None
|
||||
if cpu_temp is not None:
|
||||
samples.append(Sample("host", name, "temp.cpu_c", float(cpu_temp), ts))
|
||||
if max_temp is not None:
|
||||
samples.append(Sample("host", name, "temp.max_c", float(max_temp), ts))
|
||||
if nvme_max is not None:
|
||||
samples.append(Sample("host", name, "temp.nvme_max_c", float(nvme_max), ts))
|
||||
|
||||
# ------------------------------------------------------------ Processes -- #
|
||||
processes = snapshot.get("processes") or {}
|
||||
|
||||
# ------------------------------------------------------------ Payload ---- #
|
||||
for metric, value in (("cpu.load1", cpu_raw.get("load1")),
|
||||
("cpu.load5", cpu_raw.get("load5")),
|
||||
("cpu.load15", cpu_raw.get("load15")),
|
||||
("cpu.freq_mhz", cpu_raw.get("freq_mhz")),
|
||||
("cpu.context_switches_per_s", ctxt_rate),
|
||||
("host.uptime_seconds", info.get("uptime_seconds")),
|
||||
("host.up", 1.0),
|
||||
("disk.read_bytes_per_s", read_total if any_io else None),
|
||||
("disk.write_bytes_per_s", write_total if any_io else None),
|
||||
("disk.reads_per_s", reads_total if any_io else None),
|
||||
("disk.writes_per_s", writes_total if any_io else None),
|
||||
("disk.util_pct", util_max),
|
||||
("net.rx_bytes_per_s", rx_total if any_net else None),
|
||||
("net.tx_bytes_per_s", tx_total if any_net else None)):
|
||||
if value is not None:
|
||||
samples.append(Sample("host", name, metric, float(value), ts))
|
||||
if processes.get("total") is not None:
|
||||
samples.append(Sample("host", name, "host.processes", float(processes["total"]), ts))
|
||||
if worst is not None:
|
||||
samples.append(Sample("host", name, "disk.used_pct", float(worst["used_pct"]), ts))
|
||||
|
||||
payload = {
|
||||
"name": name,
|
||||
"ts": ts,
|
||||
"window_seconds": window,
|
||||
"info": info,
|
||||
"cpu": {
|
||||
"usage_pct": usage_pct,
|
||||
"modes_pct": mode_pct,
|
||||
"load1": cpu_raw.get("load1"),
|
||||
"load5": cpu_raw.get("load5"),
|
||||
"load15": cpu_raw.get("load15"),
|
||||
"cores": len(core_usage) or info.get("cpu_cores_logical"),
|
||||
"core_usage_pct": [c["usage_pct"] for c in core_usage],
|
||||
"freq_mhz": cpu_raw.get("freq_mhz"),
|
||||
"freq_max_mhz": cpu_raw.get("freq_max_mhz"),
|
||||
"context_switches_per_s": ctxt_rate,
|
||||
"forks_per_s": forks_rate,
|
||||
"procs_running": cpu_raw.get("procs_running"),
|
||||
"procs_blocked": cpu_raw.get("procs_blocked"),
|
||||
"temp_c": cpu_temp,
|
||||
},
|
||||
"memory": {
|
||||
"total_bytes": mem_total,
|
||||
"used_bytes": mem_used,
|
||||
"available_bytes": mem.get("available_bytes"),
|
||||
"used_pct": mem_used_pct,
|
||||
"cached_bytes": mem.get("cached_bytes"),
|
||||
"buffers_bytes": mem.get("buffers_bytes"),
|
||||
"swap_total_bytes": swap_total,
|
||||
"swap_used_bytes": swap_used,
|
||||
"swap_used_pct": swap_used_pct,
|
||||
"committed_bytes": mem.get("committed_bytes"),
|
||||
},
|
||||
"disks": {
|
||||
"filesystems": filesystems,
|
||||
"worst_used_pct": worst.get("used_pct") if worst else None,
|
||||
"worst_mount": worst.get("mountpoint") if worst else None,
|
||||
"read_bytes_per_s": read_total if any_io else None,
|
||||
"write_bytes_per_s": write_total if any_io else None,
|
||||
"reads_per_s": reads_total if any_io else None,
|
||||
"writes_per_s": writes_total if any_io else None,
|
||||
"util_pct": util_max,
|
||||
"devices": io_devices,
|
||||
},
|
||||
"network": {
|
||||
"rx_bytes_per_s": rx_total if any_net else None,
|
||||
"tx_bytes_per_s": tx_total if any_net else None,
|
||||
"interfaces": net_rows,
|
||||
},
|
||||
"gpus": gpus,
|
||||
"temperatures": {"cpu_c": cpu_temp, "max_c": max_temp, "nvme_max_c": nvme_max,
|
||||
"sensors": sensors},
|
||||
"processes": {
|
||||
"total": processes.get("total"),
|
||||
"top_cpu": processes.get("top_cpu") or [],
|
||||
"top_rss": processes.get("top_rss") or [],
|
||||
},
|
||||
"agent": {"duration_ms": snapshot.get("duration_ms"), "errors": snapshot.get("errors") or [],
|
||||
"version": snapshot.get("agent_version")},
|
||||
}
|
||||
return Result(payload=payload, samples=samples, events=events)
|
||||
|
||||
|
||||
def fleet_summary(hosts: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Cross-host totals, computed from live payloads only.
|
||||
|
||||
A host whose poll failed is not "0% busy" and is not in the average — it is absent,
|
||||
and `hosts_live` vs `hosts_total` says so. Never silently include a dead host.
|
||||
"""
|
||||
# A host that has never answered is represented by a stub ({"name", "unreachable"}), so
|
||||
# every field here is read with .get() — one unreachable agent must not 500 the fleet card.
|
||||
hosts = [h if isinstance(h, dict) else {} for h in hosts]
|
||||
live = [h for h in hosts if (h.get("cpu") or {}).get("usage_pct") is not None]
|
||||
weighted = [(h["cpu"]["usage_pct"], (h.get("info") or {}).get("cpu_cores_logical") or 1)
|
||||
for h in live]
|
||||
weight_sum = sum(weight for _value, weight in weighted)
|
||||
cpu_avg = (sum(value * weight for value, weight in weighted) / weight_sum) if weight_sum else None
|
||||
mem_total = _sum((h.get("memory") or {}).get("total_bytes") for h in hosts)
|
||||
mem_used = _sum((h.get("memory") or {}).get("used_bytes") for h in hosts)
|
||||
disk_pcts = [p for h in hosts if (p := (h.get("disks") or {}).get("worst_used_pct"))]
|
||||
gpus = [g for h in hosts for g in h.get("gpus") or []]
|
||||
return {
|
||||
"hosts_total": len(hosts),
|
||||
"hosts_live": len(live),
|
||||
"cpu_cores": _sum((h.get("info") or {}).get("cpu_cores_logical") for h in hosts),
|
||||
"cpu_usage_pct_avg": cpu_avg,
|
||||
"memory_total_bytes": mem_total,
|
||||
"memory_used_bytes": mem_used,
|
||||
"memory_used_pct": _ratio_percent(mem_used, mem_total),
|
||||
"disk_worst_used_pct": max(disk_pcts) if disk_pcts else None,
|
||||
"load1_total": _sum((h["cpu"] or {}).get("load1") for h in live),
|
||||
"net_rx_bytes_per_s": _sum((h.get("network") or {}).get("rx_bytes_per_s") for h in hosts),
|
||||
"net_tx_bytes_per_s": _sum((h.get("network") or {}).get("tx_bytes_per_s") for h in hosts),
|
||||
"gpu_count": len(gpus),
|
||||
"gpu_busy_pct_avg": (sum(g["busy_pct"] for g in gpus if g.get("busy_pct") is not None)
|
||||
/ max(1, sum(1 for g in gpus if g.get("busy_pct") is not None))
|
||||
if any(g.get("busy_pct") is not None for g in gpus) else None),
|
||||
"power_estimate_w": None, # no whole-host power source yet; deliberately n/a, not 0
|
||||
}
|
||||
|
||||
|
||||
def _sum(values: Any) -> float | None:
|
||||
numbers = [float(v) for v in values if isinstance(v, int | float)]
|
||||
return sum(numbers) if numbers else None
|
||||
|
||||
|
||||
def _event(ts: float, host: str, kind: str, severity: str, message: str) -> Any:
|
||||
from ..store import Event # noqa: PLC0415 - keeps this module import-light
|
||||
|
||||
return Event(ts, "host", host, kind, severity, message)
|
||||
@@ -0,0 +1,352 @@
|
||||
"""LLM engine source: gufo (llama.cpp-family) and vLLM, read-only.
|
||||
|
||||
Ported from llm_monitor, where these field maps were verified against the deployed
|
||||
servers rather than against documentation — the engines do not export the names people
|
||||
expect. Do not "fix" a name here without scraping the real thing:
|
||||
|
||||
curl -s $GUFO_BASE_URL/metrics | grep -v '^#' | sed -E 's/[{ ].*//' | sort -u
|
||||
curl -s $VLLM_BASE_URL/metrics | grep -v '^#' | sed -E 's/[{ ].*//' | sort -u
|
||||
|
||||
and drop the capture into tests/fixtures/.
|
||||
|
||||
Two rules carried over verbatim, because they are the difference between a truthful and a
|
||||
plausible dashboard:
|
||||
|
||||
1. **Absent = n/a, never 0.** A renamed metric yields `{"value": null}` and the UI shows
|
||||
`n/a` — 0 would claim "no traffic", which is a different statement.
|
||||
2. **Throughput precedence: native aggregate → measured Δcounter/Δt → estimate → n/a.**
|
||||
vLLM's native throughput gauge really is an aggregate, so it wins. gufo's are ONE
|
||||
unlabeled gauge holding the last pass of ONE request, so the summed token counter wins
|
||||
and the gauge survives only as `*_native`. That asymmetry is deliberate; both cases are
|
||||
labelled in the UI with their provenance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from ..poller import Result
|
||||
from ..prom import Prom, field
|
||||
from ..rates import RateEstimator
|
||||
from ..store import Sample
|
||||
|
||||
GUFO_SERIES: dict[str, tuple[tuple[str, ...], str]] = {
|
||||
"requests_processing": (("llamacpp:requests_processing", "llama:requests_processing"), "sum"),
|
||||
"requests_deferred": (("llamacpp:requests_deferred", "llama:requests_deferred"), "sum"),
|
||||
"kv_cache_usage_ratio": (("llamacpp:kv_cache_usage_ratio", "llama:kv_cache_usage_ratio"), "max"),
|
||||
# gufo counts prompt cache hits separately: prompt_tokens_total EXCLUDES cached tokens.
|
||||
"tokens_prompt_total": (("llamacpp:prompt_tokens_total", "llama:prompt_tokens_total",
|
||||
"llamacpp:tokens_prompt_total"), "sum"),
|
||||
"tokens_prompt_cached_total": (("llamacpp:prompt_tokens_cached_total",
|
||||
"llama:prompt_tokens_cached_total"), "sum"),
|
||||
"tokens_generated_total": (("llamacpp:tokens_predicted_total", "llamacpp:tokens_generated_total",
|
||||
"llama:tokens_predicted_total", "llama:tokens_generated_total"), "sum"),
|
||||
# NOT the throughput cell: prefill speed of one request's last pass.
|
||||
"prompt_tokens_per_second_native": (("llamacpp:prompt_tokens_seconds",
|
||||
"llamacpp:prompt_tokens_per_second",
|
||||
"llama:prompt_tokens_seconds"), "avg"),
|
||||
# NOT the throughput cell: one gauge holding one request's last decode pass.
|
||||
"decode_tokens_per_second_native": (("llamacpp:predicted_tokens_seconds",
|
||||
"llamacpp:decode_tokens_per_second",
|
||||
"llama:predicted_tokens_seconds"), "avg"),
|
||||
"n_tokens_max": (("llamacpp:n_tokens_max", "llama:n_tokens_max"), "max"),
|
||||
# GPU context losses (gufo-specific): any non-zero value here is an incident.
|
||||
"device_lost_total": (("gufo_device_lost_total",), "sum"),
|
||||
# speculative decoding (MTP on this fleet): accept rate is the useful number.
|
||||
"spec_drafts_total": (("llamacpp:spec_decode_num_drafts_total",), "sum"),
|
||||
"spec_draft_tokens_total": (("llamacpp:spec_decode_num_draft_tokens_total",), "sum"),
|
||||
"spec_accepted_tokens_total": (("llamacpp:spec_decode_num_accepted_tokens_total",), "sum"),
|
||||
}
|
||||
|
||||
VLLM_SERIES: dict[str, tuple[tuple[str, ...], str]] = {
|
||||
"num_requests_running": (("vllm:num_requests_running",), "sum"),
|
||||
"num_requests_waiting": (("vllm:num_requests_waiting",), "sum"),
|
||||
"gpu_cache_usage": (("vllm:kv_cache_usage_perc", "vllm:gpu_cache_usage"), "max"),
|
||||
"prompt_tokens_total": (("vllm:prompt_tokens_total",), "sum"),
|
||||
"generation_tokens_total": (("vllm:generation_tokens_total",), "sum"),
|
||||
# throughput gauges exist only on some builds; absent -> derived from counters/TPOT
|
||||
"prompt_tokens_per_second": (("vllm:prompt_throughput", "vllm:avg_prompt_throughput_toks_per_s"), "avg"),
|
||||
"decode_tokens_per_second_native": (("vllm:generation_throughput",
|
||||
"vllm:avg_generation_throughput_toks_per_s"), "avg"),
|
||||
"prefix_cache_queries_total": (("vllm:prefix_cache_queries_total",), "sum"),
|
||||
"prefix_cache_hits_total": (("vllm:prefix_cache_hits_total",), "sum"),
|
||||
# preemption = a running request kicked back to the queue: the real KV-pressure signal
|
||||
"num_preemptions_total": (("vllm:num_preemptions_total",), "sum"),
|
||||
"num_requests_swapped": (("vllm:num_requests_swapped",), "sum"),
|
||||
}
|
||||
|
||||
VLLM_HISTOGRAMS: dict[str, tuple[str, ...]] = {
|
||||
"time_to_first_token_seconds": ("vllm:time_to_first_token_seconds",),
|
||||
"time_per_output_token_seconds": ("vllm:time_per_output_token_seconds",
|
||||
"vllm:inter_token_latency_seconds",
|
||||
"vllm:request_time_per_output_token_seconds"),
|
||||
"request_queue_time_seconds": ("vllm:request_queue_time_seconds",),
|
||||
# not shown directly: mean prompt length / mean prefill time = lifetime prefill speed,
|
||||
# which is what keeps prompt tok/s off n/a on a build with no rate gauge
|
||||
"request_prefill_time_seconds": ("vllm:request_prefill_time_seconds",),
|
||||
"request_prompt_tokens": ("vllm:request_prompt_tokens",),
|
||||
"e2e_request_latency_seconds": ("vllm:e2e_request_latency_seconds",),
|
||||
}
|
||||
|
||||
#: history series per engine, in the order the LLM page charts them
|
||||
HISTORY_METRICS: dict[str, tuple[str, ...]] = {
|
||||
"decode_tokens_per_second": ("decode_tokens_per_second",),
|
||||
"prompt_tokens_per_second": ("prompt_tokens_per_second",),
|
||||
"kv_cache_usage": ("kv_cache_usage_ratio", "gpu_cache_usage"),
|
||||
"requests_running": ("requests_processing", "num_requests_running"),
|
||||
"requests_waiting": ("requests_deferred", "num_requests_waiting"),
|
||||
"ttft_seconds": ("time_to_first_token_seconds__mean",),
|
||||
"ttft_p95_seconds": ("time_to_first_token_seconds__p95",),
|
||||
"queue_seconds": ("request_queue_time_seconds__mean",),
|
||||
"prefix_cache_hit_ratio": ("prefix_cache_hit_ratio", "prompt_cache_hit_ratio"),
|
||||
}
|
||||
|
||||
|
||||
def _ratio(numerator: dict[str, Any], denominator: dict[str, Any], source: str) -> dict[str, Any]:
|
||||
num, den = numerator["value"], denominator["value"]
|
||||
if num is None or den is None or den == 0:
|
||||
return field(None, None)
|
||||
return field(num / den, source, derived=True)
|
||||
|
||||
|
||||
def _histogram_mean(out: dict[str, Any], key: str) -> float | None:
|
||||
cell = out.get(key)
|
||||
mean = cell.get("mean") if isinstance(cell, dict) else None
|
||||
return mean if isinstance(mean, float) and math.isfinite(mean) else None
|
||||
|
||||
|
||||
def parse_gufo_metrics(text: str, rates: RateEstimator | None = None,
|
||||
now: float | None = None) -> dict[str, Any]:
|
||||
"""`rates` enables the Δ-based aggregate rates; omitted, the parse stays pure."""
|
||||
parsed = Prom(text)
|
||||
out: dict[str, Any] = {}
|
||||
for key, (names, agg) in GUFO_SERIES.items():
|
||||
out[key] = field(parsed.scalar(names, agg), parsed.resolve(names))
|
||||
|
||||
def measured(total_key: str) -> dict[str, Any] | None:
|
||||
"""Δcounter/Δt for one cumulative counter, keyed by the RESOLVED series name so an
|
||||
alias rename starts a fresh window instead of diffing two different series."""
|
||||
if rates is None:
|
||||
return None
|
||||
source = out[total_key]["source"]
|
||||
if source is None:
|
||||
return None
|
||||
rate, window = rates.rate(source, out[total_key]["value"], now)
|
||||
if rate is None:
|
||||
return None
|
||||
return field(rate, f"computed:Δ{source}/Δt", derived=True, window=window)
|
||||
|
||||
# The counters add every slot; gufo's single gauge cannot. Measured therefore wins for
|
||||
# both speeds here (the inverse of vLLM, where the native gauge is the aggregate) — and
|
||||
# when no window exists yet the cell stays n/a instead of borrowing one request's speed.
|
||||
for ui_key, total_key in (
|
||||
("decode_tokens_per_second", "tokens_generated_total"),
|
||||
("prompt_tokens_per_second", "tokens_prompt_total"),
|
||||
):
|
||||
rate = measured(total_key)
|
||||
out[ui_key] = rate if rate is not None else field(None, None)
|
||||
|
||||
prompt = out["tokens_prompt_total"]["value"]
|
||||
cached = out["tokens_prompt_cached_total"]["value"] or 0.0
|
||||
generated = out["tokens_generated_total"]["value"]
|
||||
parts = [v for v in (prompt, generated) if v is not None]
|
||||
out["tokens_total"] = (field(sum(parts) + cached, "computed:prompt+cached+generated")
|
||||
if parts else field(None, None))
|
||||
out["prompt_cache_hit_ratio"] = (
|
||||
field(cached / (prompt + cached) if (prompt + cached) > 0 else None,
|
||||
"computed:cached/(prompt+cached)", derived=True)
|
||||
if prompt is not None else field(None, None))
|
||||
out["spec_accept_rate"] = _ratio(out["spec_accepted_tokens_total"],
|
||||
out["spec_draft_tokens_total"],
|
||||
"computed:accepted/draft_tokens")
|
||||
out["spec_drafts_per_request"] = _ratio(out["spec_drafts_total"],
|
||||
out["spec_draft_tokens_total"],
|
||||
"computed:drafts/draft_tokens")
|
||||
return out
|
||||
|
||||
|
||||
def parse_vllm_metrics(text: str, rates: RateEstimator | None = None,
|
||||
now: float | None = None) -> dict[str, Any]:
|
||||
parsed = Prom(text)
|
||||
out: dict[str, Any] = {}
|
||||
for key, (names, agg) in VLLM_SERIES.items():
|
||||
out[key] = field(parsed.scalar(names, agg), parsed.resolve(names))
|
||||
|
||||
reason_names = ("vllm:num_requests_waiting_by_reason",
|
||||
"vllm:num_requests_waiting_by_reason_total")
|
||||
reasons = parsed.by_label(reason_names, "reason")
|
||||
out["waiting_by_reason"] = {"value": reasons, "source": parsed.resolve(reason_names),
|
||||
"derived": False}
|
||||
out["waiting_capacity"] = field(reasons.get("capacity"), parsed.resolve(reason_names))
|
||||
|
||||
for ui_key, names in VLLM_HISTOGRAMS.items():
|
||||
out[ui_key] = parsed.histogram(names)
|
||||
out[f"{ui_key}__mean"] = field(out[ui_key]["mean"] if out[ui_key] else None,
|
||||
out[ui_key]["source"] if out[ui_key] else None)
|
||||
out[f"{ui_key}__p95"] = field(out[ui_key]["p95"] if out[ui_key] else None,
|
||||
out[ui_key]["source"] if out[ui_key] else None)
|
||||
|
||||
tpot = out["time_per_output_token_seconds"]["mean"] if out["time_per_output_token_seconds"] else None
|
||||
per_seq = (1.0 / tpot) if tpot else None
|
||||
out["per_request_decode_tokens_per_second"] = field(per_seq, "computed:1/tpot", derived=True)
|
||||
|
||||
def measured(key: str, total_key: str) -> dict[str, Any] | None:
|
||||
if rates is None:
|
||||
return None
|
||||
rate, window = rates.rate(key, out[total_key]["value"], now)
|
||||
if rate is None:
|
||||
return None
|
||||
return field(rate, f"computed:Δ{VLLM_SERIES[total_key][0][0]}/Δt", derived=True, window=window)
|
||||
|
||||
decode = out["decode_tokens_per_second_native"]["value"]
|
||||
running = out["num_requests_running"]["value"]
|
||||
if decode is not None:
|
||||
out["decode_tokens_per_second"] = field(decode,
|
||||
out["decode_tokens_per_second_native"]["source"])
|
||||
else:
|
||||
# measured beats the running/tpot estimate: it counts real output tokens, the
|
||||
# estimate assumes every running request is decoding this instant
|
||||
measured_decode = measured("vllm:generation_tokens_total", "generation_tokens_total")
|
||||
if measured_decode is not None:
|
||||
# `or` here would treat a genuine 0 tok/s as missing and print the lifetime
|
||||
# estimate for an engine that is provably idle
|
||||
out["decode_tokens_per_second"] = measured_decode
|
||||
elif per_seq is not None:
|
||||
out["decode_tokens_per_second"] = field((running or 0) * per_seq,
|
||||
"computed:running/tpot", derived=True)
|
||||
else:
|
||||
out["decode_tokens_per_second"] = field(None, None)
|
||||
|
||||
prefill_avg = None
|
||||
prefill_time = _histogram_mean(out, "request_prefill_time_seconds")
|
||||
prompt_len = _histogram_mean(out, "request_prompt_tokens")
|
||||
if prefill_time and prompt_len:
|
||||
prefill_avg = field(prompt_len / prefill_time,
|
||||
"computed:mean(request_prompt_tokens)/mean(request_prefill_time)",
|
||||
derived=True)
|
||||
if out["prompt_tokens_per_second"]["value"] is None:
|
||||
measured_prefill = measured("vllm:prompt_tokens_total", "prompt_tokens_total")
|
||||
if measured_prefill is not None:
|
||||
out["prompt_tokens_per_second"] = measured_prefill
|
||||
elif prefill_avg is not None:
|
||||
out["prompt_tokens_per_second"] = prefill_avg
|
||||
else:
|
||||
out["prompt_tokens_per_second"] = field(None, None)
|
||||
# a lifetime average is a different claim from current throughput — labelled apart
|
||||
out["prefill_tokens_per_second_lifetime"] = prefill_avg or field(None, None)
|
||||
|
||||
parts = [v for v in (out["prompt_tokens_total"]["value"],
|
||||
out["generation_tokens_total"]["value"]) if v is not None]
|
||||
out["tokens_total"] = field(sum(parts), "computed:prompt+generated") if parts else field(None, None)
|
||||
out["prefix_cache_hit_ratio"] = _ratio(out["prefix_cache_hits_total"],
|
||||
out["prefix_cache_queries_total"],
|
||||
"computed:hits/queries")
|
||||
return out
|
||||
|
||||
|
||||
def engine_result(engine: str, base_url: str, text: str, rates: RateEstimator,
|
||||
*, slots: list[dict[str, Any]] | None = None, now: float | None = None) -> Result:
|
||||
"""Parse one engine's /metrics into a poll Result (payload + history samples)."""
|
||||
now = time.time() if now is None else now
|
||||
if engine == "gufo":
|
||||
metrics = parse_gufo_metrics(text, rates, now)
|
||||
active = metrics["requests_processing"]["value"]
|
||||
else:
|
||||
metrics = parse_vllm_metrics(text, rates, now)
|
||||
active = metrics["num_requests_running"]["value"]
|
||||
|
||||
parsed = Prom(text)
|
||||
sessions = _sessions(engine, slots, active, metrics)
|
||||
payload = {
|
||||
"engine": engine,
|
||||
"upstream": base_url,
|
||||
"scraped_at": now,
|
||||
"metrics": metrics,
|
||||
"sessions": sessions,
|
||||
"series_count": len(parsed.series),
|
||||
"unparsed": parsed.parse_errors,
|
||||
}
|
||||
samples: list[Sample] = []
|
||||
for metric, candidates in HISTORY_METRICS.items():
|
||||
value = next((metrics[key]["value"] for key in candidates
|
||||
if key in metrics and metrics[key]["value"] is not None), None)
|
||||
if value is not None:
|
||||
samples.append(Sample("llm", engine, f"llm.{metric}", float(value), now))
|
||||
return Result(payload=payload, samples=samples)
|
||||
|
||||
|
||||
def _sessions(engine: str, slots: list[dict[str, Any]] | None, active: float | None,
|
||||
metrics: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Active sessions, however this engine expresses them.
|
||||
|
||||
gufo exposes /slots (one row per parallel slot, with per-slot tokens and speed), so it
|
||||
gets a real list. vLLM publishes no per-session endpoint in its /metrics, so it gets
|
||||
counts and says the list is not available rather than inventing rows.
|
||||
"""
|
||||
out: dict[str, Any] = {"active": active, "list": [], "source": None, "detail": None}
|
||||
if engine == "gufo":
|
||||
if slots:
|
||||
out["list"] = slots
|
||||
out["source"] = "gufo /slots"
|
||||
else:
|
||||
out["detail"] = ("no /slots answer — active count still comes from "
|
||||
"llamacpp:requests_processing")
|
||||
else:
|
||||
waiting = metrics.get("num_requests_waiting", {}).get("value")
|
||||
queued = f", {waiting:g} queued" if waiting else ""
|
||||
out["detail"] = (f"vLLM reports counts, not sessions: {active or 0:g} running{queued}. "
|
||||
f"Per-request detail is in the latency histograms.")
|
||||
return out
|
||||
|
||||
|
||||
def fleet_totals(engines: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Cross-engine sums, computed in the API over engines that answered.
|
||||
|
||||
A dead engine keeps its last good values in its own panel but contributes nothing to
|
||||
a total presented as current — and `engines_live` vs `engines_total` makes that visible.
|
||||
"""
|
||||
live = [e for e in engines if e.get("metrics")]
|
||||
|
||||
def sum_metric(key: str) -> float | None:
|
||||
values = [e["metrics"].get(key, {}).get("value") for e in live]
|
||||
numbers = [v for v in values if isinstance(v, int | float)]
|
||||
return sum(numbers) if numbers else None
|
||||
|
||||
def max_metric(key: str) -> float | None:
|
||||
values = [e["metrics"].get(key, {}).get("value") for e in live
|
||||
if isinstance(e["metrics"].get(key, {}).get("value"), int | float)]
|
||||
return max(values) if values else None
|
||||
|
||||
def sum_pairs(keys: Sequence[str]) -> float | None:
|
||||
"""Sum across engines AND across the per-engine key aliases (gufo calls a running
|
||||
request `requests_processing`, vLLM `num_requests_running`)."""
|
||||
values = [sum_metric(key) for key in keys]
|
||||
numbers = [v for v in values if v is not None]
|
||||
return sum(numbers) if numbers else None
|
||||
|
||||
return {
|
||||
"engines_total": len(engines),
|
||||
"engines_live": len(live),
|
||||
"decode_tokens_per_second": sum_metric("decode_tokens_per_second"),
|
||||
"prompt_tokens_per_second": sum_metric("prompt_tokens_per_second"),
|
||||
"requests_active": sum_pairs(("requests_processing", "num_requests_running")),
|
||||
"requests_waiting": sum_pairs(("requests_deferred", "num_requests_waiting")),
|
||||
"tokens_total": sum_metric("tokens_total"),
|
||||
# a real max across both engines' aliases: `a or b` returned gufo's 25% while vLLM
|
||||
# was sitting at 75%, which is the opposite of a pressure signal
|
||||
"kv_cache_usage_max": max(
|
||||
(v for v in (max_metric("kv_cache_usage_ratio"), max_metric("gpu_cache_usage"))
|
||||
if v is not None),
|
||||
default=None,
|
||||
),
|
||||
"seams": ("gufo's prompt counter excludes cache-hit tokens while vLLM's includes "
|
||||
"cached prefix tokens, so the summed prompt rate leans slightly low for "
|
||||
"gufo. Both are counted honestly per engine."),
|
||||
}
|
||||
|
||||
|
||||
def series_names(engine: str) -> Sequence[str]:
|
||||
return tuple(GUFO_SERIES if engine == "gufo" else VLLM_SERIES)
|
||||
@@ -0,0 +1,779 @@
|
||||
"""NUT (Network UPS Tools) client — the upsd wire protocol, asyncio, no dependencies.
|
||||
|
||||
`upsc`/`upspy`/`pynut` are either a subprocess dependency or a stalled third-party lib,
|
||||
and the protocol is a line protocol with ~10 commands. So: talk to upsd on 3493 directly.
|
||||
|
||||
Reading is unauthenticated in stock NUT (`LIST UPS` / `LIST VAR`), so the dashboard works
|
||||
with a read-only upsd. Instant commands (the battery-test buttons) require a user with
|
||||
`actions` in upsd.users — that is the ONLY write path in this whole application, and it is
|
||||
gated three times: `NUT_ALLOW_ACTIONS`, not `DASH_READ_ONLY`, and a confirmation in the UI
|
||||
(destructive ones require typing the UPS name).
|
||||
|
||||
Verified against NUT 2.8.5 (`/usr/bin/upsc` on driveripper). Protocol shape:
|
||||
|
||||
-> LIST UPS
|
||||
<- BEGIN LIST UPS
|
||||
<- UPS "apc" : APC Smart-UPS
|
||||
<- END LIST UPS
|
||||
-> LIST VAR apc
|
||||
<- BEGIN LIST VAR apc
|
||||
<- VAR "ups.status" "OL"
|
||||
<- END LIST VAR apc
|
||||
-> LIST CMD apc ; <- BEGIN LIST CMD apc / CMD test.battery.start.quick / END LIST CMD apc
|
||||
bad -> ERR ACCESS-DENIED (auth or permission problem)
|
||||
ok -> OK ...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_PORT = 3493
|
||||
|
||||
|
||||
class NutError(Exception):
|
||||
"""A NUT protocol error. `code` is the upsd error token (ACCESS-DENIED, ...)."""
|
||||
|
||||
def __init__(self, message: str, *, code: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
#: upsd error tokens -> what the operator should actually do about them.
|
||||
ERROR_HINTS = {
|
||||
"ACCESS-DENIED": "upsd refused: this client is not in an ACL that allows it (upsd.conf `ACL`/"
|
||||
"`LISTEN`, or the user lacks the privilege in upsd.users)",
|
||||
"INVALID-PASSWORD": "wrong NUT_PASSWORD for NUT_USER (upsd.users)",
|
||||
"INVALID-USERNAME": "NUT_USER is not defined in upsd.users",
|
||||
"INVALID-ARGUMENT": "malformed command or unknown UPS",
|
||||
"UNKNOWN-UPSNAME": "that UPS is not exported by upsd (check ups.conf driver instance name)",
|
||||
"UNKNOWN-REQUEST": "this upsd does not implement that command",
|
||||
"CMD-UNSUPP": "the driver does not implement that command",
|
||||
"CMD-NOT-BLOCKING": "not a blocking command",
|
||||
"INVALID-BITMAP": "bad mode bitmap",
|
||||
"DATA-STALE": "the driver stopped updating this UPS (check nut-driver@<ups> on the NUT host)",
|
||||
"DRIVER-NOT-CONNECTED": "no driver is connected to that UPS right now",
|
||||
"PERM-DENIED": "read-only variable, or the user lacks `actions` for that command in upsd.users",
|
||||
"TOO-MANY-CLIENTS": "upsd MAXCONN reached",
|
||||
"ALREADY-SET": "value already set",
|
||||
"FEATURE-NOT-ENABLED": "feature disabled on the driver side",
|
||||
"FEATURE-NOT-SUPPORTED": "driver does not support that feature",
|
||||
"TOO-BUSY": "upsd is busy",
|
||||
"INVALID-VALUE": "value out of range",
|
||||
}
|
||||
|
||||
|
||||
def _unquote(value: str) -> str:
|
||||
if len(value) >= 2 and value[0] == '"' and value[-1] == '"':
|
||||
return value[1:-1].replace('\\"', '"').replace("\\\\", "\\")
|
||||
return value
|
||||
|
||||
|
||||
class NutClient:
|
||||
"""One connection to upsd. Short-lived by design: connect, read, close.
|
||||
|
||||
A persistent connection would be faster but hides restarts of upsd behind a stale
|
||||
socket, and upsd has a per-client limit (MAXCONN) that a long-lived dashboard would
|
||||
sit on.
|
||||
"""
|
||||
|
||||
def __init__(self, host: str, port: int = DEFAULT_PORT, *, timeout: float = 4.0,
|
||||
username: str | None = None, password: str | None = None) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.timeout = timeout
|
||||
self.username = username
|
||||
self.password = password
|
||||
self._reader: asyncio.StreamReader | None = None
|
||||
self._writer = None
|
||||
|
||||
async def __aenter__(self) -> NutClient:
|
||||
await self.connect()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_exc: object) -> None:
|
||||
await self.close()
|
||||
|
||||
async def connect(self) -> None:
|
||||
try:
|
||||
self._reader, self._writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(self.host, self.port), self.timeout
|
||||
)
|
||||
except (TimeoutError, OSError) as exc:
|
||||
raise NutError(
|
||||
f"cannot reach upsd at {self.host}:{self.port} ({type(exc).__name__}: {exc})",
|
||||
code="UNREACHABLE",
|
||||
) from exc
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._writer is None:
|
||||
return
|
||||
with contextlib.suppress(Exception): # logout is best effort; upsd may already be gone
|
||||
if self.username and self.password:
|
||||
await self._send("LOGOUT")
|
||||
await self._read_line(expect_ok=False)
|
||||
self._writer.close()
|
||||
with contextlib.suppress(Exception):
|
||||
await self._writer.wait_closed()
|
||||
self._writer = None
|
||||
self._reader = None
|
||||
|
||||
# -- plumbing ------------------------------------------------------------ #
|
||||
async def _send(self, command: str) -> None:
|
||||
if self._writer is None:
|
||||
raise NutError("not connected", code="NOCONNECTION")
|
||||
self._writer.write((command + "\n").encode())
|
||||
await self._writer.drain()
|
||||
|
||||
async def _read_line(self, *, expect_ok: bool = True) -> str:
|
||||
if self._reader is None:
|
||||
raise NutError("not connected", code="NOCONNECTION")
|
||||
try:
|
||||
raw = await asyncio.wait_for(self._reader.readline(), self.timeout)
|
||||
except TimeoutError as exc:
|
||||
raise NutError(f"upsd at {self.host}:{self.port} did not answer in "
|
||||
f"{self.timeout:g}s", code="TIMEOUT") from exc
|
||||
except (ConnectionResetError, BrokenPipeError, OSError) as exc:
|
||||
# upsd dropping the socket (restart, MAXCONN, firewall) is a read failure with a
|
||||
# name, not a traceback out of the poller
|
||||
raise NutError(f"upsd at {self.host}:{self.port} dropped the connection "
|
||||
f"({type(exc).__name__})", code="CLOSED") from exc
|
||||
if not raw:
|
||||
raise NutError("upsd closed the connection", code="CLOSED")
|
||||
line = raw.decode(errors="replace").rstrip("\r\n")
|
||||
if line.startswith("ERR"):
|
||||
code = line.split(None, 2)[1] if len(line.split()) > 1 else "UNKNOWN"
|
||||
raise NutError(f"upsd: {line} — {ERROR_HINTS.get(code, 'see upsd(8)')}", code=code)
|
||||
if expect_ok and not line.startswith("OK"):
|
||||
raise NutError(f"unexpected upsd reply: {line!r}", code="UNEXPECTED")
|
||||
return line
|
||||
|
||||
async def _list(self, command: str, kind: str) -> list[list[str]]:
|
||||
"""Run a LIST command and collect the lines between BEGIN/END."""
|
||||
begin = f"BEGIN LIST {kind}"
|
||||
end = f"END LIST {kind}"
|
||||
await self._send(command)
|
||||
# LIST replies begin with `BEGIN LIST <kind>`, not `OK`, so this path must not
|
||||
# demand an OK line — upsd would otherwise be "broken" on every read.
|
||||
first = await self._read_line(expect_ok=False)
|
||||
if not first.startswith(begin):
|
||||
raise NutError(f"expected {begin!r}, got {first!r}", code="UNEXPECTED")
|
||||
rows: list[list[str]] = []
|
||||
while True:
|
||||
line = await self._read_line(expect_ok=False)
|
||||
if line.startswith(end):
|
||||
return rows
|
||||
rows.append(_split_fields(line))
|
||||
if len(rows) > 4096: # a wedged upsd must not grow the heap
|
||||
raise NutError(f"{kind} list too long", code="UNEXPECTED")
|
||||
|
||||
async def login(self) -> bool:
|
||||
"""Authenticate for actions. Returns False when no credentials are configured."""
|
||||
if not self.username or not self.password:
|
||||
return False
|
||||
await self._send(f"USER {self.username}")
|
||||
await self._read_line()
|
||||
await self._send(f"PASS {self.password}")
|
||||
await self._read_line()
|
||||
return True
|
||||
|
||||
# -- reads ---------------------------------------------------------------- #
|
||||
async def list_ups(self) -> list[dict[str, str]]:
|
||||
rows = await self._list("LIST UPS", "UPS")
|
||||
out: list[dict[str, str]] = []
|
||||
for row in rows:
|
||||
# UPS <name> ["description"]
|
||||
if len(row) >= 2 and row[0] == "UPS":
|
||||
name = row[1]
|
||||
desc = _unquote(" ".join(row[2:])) if len(row) > 2 else ""
|
||||
out.append({"name": name, "description": desc})
|
||||
return out
|
||||
|
||||
async def list_vars(self, ups: str) -> dict[str, str]:
|
||||
"""`VAR <ups> <name> <value>` — the UPS name is field 1, the variable is field 2.
|
||||
|
||||
Reading field 1 as the name returns the UPS name over and over and every lookup
|
||||
misses; real upsd was the only way to catch that, so the fake in the tests speaks
|
||||
the same shape.
|
||||
"""
|
||||
rows = await self._list(f"LIST VAR {_quote(ups)}", "VAR")
|
||||
out: dict[str, str] = {}
|
||||
for row in rows:
|
||||
if len(row) >= 3 and row[0] == "VAR":
|
||||
out[_unquote(row[2])] = _unquote(" ".join(row[3:]))
|
||||
return out
|
||||
|
||||
async def list_cmds(self, ups: str) -> list[str]:
|
||||
"""`CMD <ups> <command>`."""
|
||||
rows = await self._list(f"LIST CMD {_quote(ups)}", "CMD")
|
||||
return [_unquote(row[2]) for row in rows if len(row) >= 3 and row[0] == "CMD"]
|
||||
|
||||
async def list_rw_vars(self, ups: str) -> list[str]:
|
||||
"""`RW <ups> <variable>`."""
|
||||
rows = await self._list(f"LIST RW {_quote(ups)}", "RW")
|
||||
return [_unquote(row[2]) for row in rows if len(row) >= 3 and row[0] == "RW"]
|
||||
|
||||
# -- writes ---------------------------------------------------------------- #
|
||||
async def instcmd(self, ups: str, command: str) -> str:
|
||||
"""Send an instant command. upsd answers `OK Send shorthand command successful`."""
|
||||
if not (self.username and self.password):
|
||||
raise NutError("instant commands need NUT_USER/NUT_PASSWORD with `actions` "
|
||||
"in upsd.users", code="ACCESS-DENIED")
|
||||
await self._send(f"INSTCMD {_quote(ups)} {_quote(command)}")
|
||||
return await self._read_line()
|
||||
|
||||
|
||||
def _quote(value: str) -> str:
|
||||
"""Quote a protocol argument. upsd splits on whitespace and understands C-style quoting."""
|
||||
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return f'"{escaped}"'
|
||||
|
||||
|
||||
def _split_fields(line: str) -> list[str]:
|
||||
"""Split a protocol line into fields, honouring quoted spans (values contain spaces).
|
||||
|
||||
Fields come back unquoted: upsd quotes every variable name and value, and a name that
|
||||
still carries its quote characters (`"battery.charge"`) silently misses every lookup.
|
||||
"""
|
||||
fields: list[str] = []
|
||||
current: list[str] = []
|
||||
in_quotes = False
|
||||
escaped = False
|
||||
for char in line:
|
||||
if escaped:
|
||||
current.append(char)
|
||||
escaped = False
|
||||
elif in_quotes and char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_quotes = not in_quotes
|
||||
current.append(char)
|
||||
elif char == " " and not in_quotes:
|
||||
if current:
|
||||
fields.append("".join(current))
|
||||
current = []
|
||||
else:
|
||||
current.append(char)
|
||||
if current:
|
||||
fields.append("".join(current))
|
||||
return [_unquote(field) for field in fields]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Variable mapping
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Vendor drivers disagree about names (apc vs cyberpower vs usbhid-ups), so every field
|
||||
# is a candidate list and an absent value is None, never 0.
|
||||
VAR_MAP: dict[str, tuple[str, ...]] = {
|
||||
"mfr": ("ups.mfr",),
|
||||
"model": ("ups.model",),
|
||||
"serial": ("ups.serial", "device.serial"),
|
||||
"firmware": ("ups.firmware", "ups.firmware.aux"),
|
||||
"driver": ("driver.name",),
|
||||
"status": ("ups.status",),
|
||||
"charger_status": ("battery.charger.status",), # NUT 2.8 rich status
|
||||
"alarm": ("ups.alarm",),
|
||||
"load_pct": ("ups.load", "ups.load.instead"),
|
||||
"realpower": ("ups.realpower", "ups.power"),
|
||||
"realpower_nominal": ("ups.realpower.nominal", "ups.power.nominal"),
|
||||
"line_voltage": ("input.voltage", "input.voltage.fault"),
|
||||
"line_voltage_nominal": ("input.voltage.nominal",),
|
||||
"input_min": ("input.voltage.minimum", "input.low.line"),
|
||||
"input_max": ("input.voltage.maximum", "input.high.line"),
|
||||
"transfer_low": ("input.transfer.low",),
|
||||
"transfer_high": ("input.transfer.high",),
|
||||
"frequency": ("input.frequency", "input.frequency.nominal"),
|
||||
"output_voltage": ("output.voltage",),
|
||||
"output_current": ("output.current",),
|
||||
"output_frequency": ("output.frequency",),
|
||||
"charge_pct": ("battery.charge", "battery.capacity:typ"),
|
||||
"battery_voltage": ("battery.voltage",),
|
||||
"battery_voltage_nominal": ("battery.voltage.nominal",),
|
||||
"battery_current": ("battery.current",),
|
||||
"battery_temperature": ("battery.temperature",),
|
||||
"battery_type": ("battery.type",),
|
||||
"battery_runtime": ("battery.runtime", "battery.runtime.low"),
|
||||
"battery_runtime_low": ("battery.runtime.low",),
|
||||
"battery_date": ("battery.mfr.date", "battery.date"),
|
||||
"test_result": ("battery.test.result", "ups.test.result"),
|
||||
"test_date": ("battery.test.date", "ups.test.date"),
|
||||
"start_battery": ("ups.start.battery",),
|
||||
"ups_temperature": ("ups.temperature",),
|
||||
"efficiency": ("ups.efficiency",),
|
||||
}
|
||||
|
||||
# Anything else worth showing verbatim, if the driver happens to export it.
|
||||
EXTRA_VAR_PREFIXES = ("battery.test.", "input.transfer.", "output.", "ups.", "battery.")
|
||||
|
||||
STATE_FLAGS = {
|
||||
"OL": ("online", "ok", "On line — utility power is present"),
|
||||
"OB": ("on battery", "warn", "Running on battery"),
|
||||
"OB LB": ("low battery", "fail", "On battery and low — shutdown imminent"),
|
||||
"LB": ("low battery", "fail", "Battery low"),
|
||||
"RB": ("replace battery", "fail", "The UPS says the battery must be replaced"),
|
||||
"CHRG": ("charging", "info", "Battery charging"),
|
||||
"DISCHRG": ("discharging", "warn", "Battery discharging"),
|
||||
"BYPASS": ("bypass", "warn", "Load is on bypass"),
|
||||
"OFF": ("off", "fail", "Output is off"),
|
||||
"NOBAT": ("no battery", "fail", "No battery detected"),
|
||||
"NULL": ("unknown", "warn", "Driver reports no state"),
|
||||
"TEST": ("testing", "info", "Self-test in progress"),
|
||||
"CAL": ("calibrating", "warn", "Battery calibration — runtime estimates are unreliable"),
|
||||
"OVER": ("overloaded", "fail", "Load exceeds the UPS rating"),
|
||||
# FUB is the UPS telling us it decided to power off (scheduled shutdown, or the driver's
|
||||
# own low-battery policy). Missing it reads as "no state" on the one status that matters.
|
||||
"FUB": ("forced shutdown", "fail", "The UPS has decided to shut down (FUB)"),
|
||||
"OTHER": ("unknown", "warn", "Driver reports an unclassifiable state"),
|
||||
"TRIM": ("trimming", "info", "Trimming high input voltage"),
|
||||
"BOOST": ("boosting", "info", "Boosting low input voltage"),
|
||||
}
|
||||
|
||||
CHARGER_STATUS = {
|
||||
"ol": ("online", "ok"),
|
||||
"ol trim": ("trimming", "info"),
|
||||
"ol boost": ("boosting", "info"),
|
||||
"chg": ("charging", "info"),
|
||||
"dchg": ("discharging", "warn"),
|
||||
"full": ("fully charged", "ok"),
|
||||
"pover": ("overloaded", "fail"),
|
||||
"shutd": ("shutting down", "fail"),
|
||||
"ebypass": ("bypass", "warn"),
|
||||
"ehazard": ("hazard", "fail"),
|
||||
"rec": ("in recalibration", "warn"),
|
||||
"test": ("testing", "info"),
|
||||
"other": ("unknown", "warn"),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class UpsReading:
|
||||
"""One UPS, as reported by one poll."""
|
||||
|
||||
name: str
|
||||
description: str = ""
|
||||
vars: dict[str, str] = field(default_factory=dict)
|
||||
commands: list[str] = field(default_factory=list)
|
||||
writable: list[str] = field(default_factory=list)
|
||||
ts: float = field(default_factory=time.time)
|
||||
|
||||
def get(self, key: str) -> str | None:
|
||||
for candidate in VAR_MAP.get(key, (key,)):
|
||||
value = self.vars.get(candidate)
|
||||
if value not in (None, "", "0.000000"):
|
||||
return value
|
||||
return None
|
||||
|
||||
def number(self, key: str) -> float | None:
|
||||
raw = self.get(key)
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return float(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@property
|
||||
def status_flags(self) -> list[str]:
|
||||
return (self.get("status") or "").split()
|
||||
|
||||
def states(self) -> list[dict[str, str]]:
|
||||
"""Human state list. The 2.8 `battery.charger.status` wins over legacy ups.status
|
||||
when both exist because it distinguishes trim/boost/full/recalibration."""
|
||||
out: list[dict[str, str]] = []
|
||||
charger = (self.get("charger_status") or "").strip().lower()
|
||||
if charger and charger in CHARGER_STATUS:
|
||||
label, severity = CHARGER_STATUS[charger]
|
||||
out.append({"id": charger, "label": label, "severity": severity})
|
||||
raw = " ".join(self.status_flags)
|
||||
if raw:
|
||||
if raw in STATE_FLAGS:
|
||||
label, severity, detail = STATE_FLAGS[raw]
|
||||
out.append({"id": raw, "label": label, "severity": severity, "detail": detail})
|
||||
else:
|
||||
for flag in self.status_flags:
|
||||
if flag in STATE_FLAGS:
|
||||
label, severity, detail = STATE_FLAGS[flag]
|
||||
out.append({"id": flag, "label": label, "severity": severity, "detail": detail})
|
||||
return out
|
||||
|
||||
@property
|
||||
def on_battery(self) -> bool:
|
||||
flags = set(self.status_flags)
|
||||
if "OL" in flags:
|
||||
return False
|
||||
if "OB" in flags:
|
||||
return True
|
||||
return (self.get("charger_status") or "").lower() in ("dchg", "rec")
|
||||
|
||||
@property
|
||||
def alarms(self) -> list[str]:
|
||||
raw = self.get("alarm") or ""
|
||||
return [part.strip() for part in raw.split(",") if part.strip()]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Health assessment
|
||||
# --------------------------------------------------------------------------- #
|
||||
SEVERITY_ORDER = {"ok": 0, "info": 0, "unknown": 1, "warn": 2, "fail": 3}
|
||||
|
||||
|
||||
def normalise_celsius(value: float | None) -> float | None:
|
||||
"""Turn a driver temperature into °C.
|
||||
|
||||
NUT drivers disagree: usbhid-ups usually reports °C, while the powerware/mge heritage
|
||||
and some CyberPower firmware report kelvin. 300 "°C" is not a melting battery, it is
|
||||
27 °C, and a health check that fails on it is worse than no check. Above 150 the value
|
||||
can only be kelvin for a UPS battery, so convert; below that, trust the driver.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
return value - 273.15 if value > 150 else value
|
||||
|
||||
|
||||
def _check(name: str, status: str, detail: str, *, metric: str | None = None,
|
||||
value: float | None = None) -> dict[str, Any]:
|
||||
return {"name": name, "status": status, "detail": detail, "metric": metric, "value": value}
|
||||
|
||||
|
||||
def assess(reading: UpsReading, *, now: float | None = None,
|
||||
battery_replacement_years: float = 4.0, min_runtime_seconds: float = 180.0) -> dict[str, Any]:
|
||||
"""A list of explicit checks, not a mystery score.
|
||||
|
||||
Every line says what it looked at and what it saw, because "health: 87" is the kind
|
||||
of number an operator cannot act on. Absent data is `unknown`, never a pass.
|
||||
"""
|
||||
now = time.time() if now is None else now
|
||||
checks: list[dict[str, Any]] = []
|
||||
|
||||
states = reading.states()
|
||||
if not states:
|
||||
checks.append(_check("Power path", "unknown", "ups.status is not exported"))
|
||||
else:
|
||||
worst = min(states, key=lambda s: -SEVERITY_ORDER.get(s["severity"], 0),
|
||||
default=states[0])
|
||||
worst_sev = max((s["severity"] for s in states), key=lambda s: SEVERITY_ORDER.get(s, 0))
|
||||
checks.append(_check("Power path", worst_sev, ", ".join(s["label"] for s in states),
|
||||
value=None))
|
||||
if worst_sev == "fail" and worst is not None:
|
||||
checks[-1]["detail"] = worst.get("detail", worst["label"])
|
||||
|
||||
charge = reading.number("charge_pct")
|
||||
if charge is None:
|
||||
checks.append(_check("Charge", "unknown", "battery.charge is not exported"))
|
||||
elif reading.on_battery:
|
||||
# On battery this number only falls, and `LB` plus the runtime estimate already say
|
||||
# whether that is urgent. Reporting it as an error here would double-alarm.
|
||||
checks.append(_check("Charge", "warn" if charge < 30 else "ok",
|
||||
f"{charge:.0f}% and discharging", value=charge))
|
||||
elif charge >= 99:
|
||||
checks.append(_check("Charge", "ok", f"{charge:.0f}%", value=charge))
|
||||
elif charge >= 90:
|
||||
# Plenty of drivers float at 90-94 % forever (chemistry, temperature, float voltage).
|
||||
# Calling that "degraded" keeps the page permanently amber and teaches people to
|
||||
# ignore the one badge that is supposed to mean something.
|
||||
checks.append(_check("Charge", "ok", f"{charge:.0f}% — floating", value=charge))
|
||||
elif charge >= 50:
|
||||
checks.append(_check("Charge", "warn", f"{charge:.0f}% — not reaching full charge",
|
||||
value=charge))
|
||||
else:
|
||||
checks.append(_check("Charge", "fail", f"{charge:.0f}% — barely charged", value=charge))
|
||||
|
||||
runtime = reading.number("battery_runtime")
|
||||
if runtime is None:
|
||||
checks.append(_check("Runtime estimate", "unknown",
|
||||
"battery.runtime is not exported (the driver cannot estimate it)"))
|
||||
elif runtime >= min_runtime_seconds * 2:
|
||||
checks.append(_check("Runtime estimate", "ok", f"{_duration(runtime)} at the current load",
|
||||
value=runtime))
|
||||
elif runtime >= min_runtime_seconds:
|
||||
checks.append(_check("Runtime estimate", "warn", f"only {_duration(runtime)} left",
|
||||
value=runtime))
|
||||
else:
|
||||
checks.append(_check("Runtime estimate", "fail",
|
||||
f"{_duration(runtime)} — too little to shut down cleanly",
|
||||
value=runtime))
|
||||
|
||||
if "RB" in reading.status_flags:
|
||||
checks.append(_check("Battery replacement", "fail", "The UPS flags RB — replace the battery"))
|
||||
else:
|
||||
age = _battery_age_years(reading, now)
|
||||
if age is None:
|
||||
checks.append(_check("Battery age", "unknown",
|
||||
"no battery.mfr.date / battery.date from this driver"))
|
||||
elif age >= battery_replacement_years:
|
||||
checks.append(_check("Battery age", "warn",
|
||||
f"{age:.1f} years — past the {battery_replacement_years:.0f}-year "
|
||||
f"replacement window", value=age))
|
||||
else:
|
||||
checks.append(_check("Battery age", "ok", f"{age:.1f} years", value=age))
|
||||
|
||||
result = reading.get("test_result")
|
||||
if result is None:
|
||||
checks.append(_check("Last self-test", "unknown",
|
||||
"no self-test result reported — run a battery test"))
|
||||
else:
|
||||
passed = result.strip().lower() in ("ok", "passed", "pass", "completed")
|
||||
failed = result.strip().lower() in ("failed", "fail", "error", "aborted", "test incomplete")
|
||||
checks.append(_check("Last self-test", "ok" if passed else "fail" if failed else "warn",
|
||||
f"{result}" + (f" ({reading.get('test_date')})" if reading.get("test_date") else "")))
|
||||
|
||||
raw_temp = reading.number("battery_temperature")
|
||||
temp = normalise_celsius(raw_temp)
|
||||
converted = temp is not None and raw_temp is not None and raw_temp > 150
|
||||
suffix = " (driver reports kelvin)" if converted else ""
|
||||
if temp is None:
|
||||
checks.append(_check("Battery temperature", "unknown", "not reported by this driver"))
|
||||
elif temp >= 50:
|
||||
checks.append(_check("Battery temperature", "fail", f"{temp:.1f} °C — VRLA damage risk{suffix}",
|
||||
value=temp))
|
||||
elif temp >= 40:
|
||||
checks.append(_check("Battery temperature", "warn",
|
||||
f"{temp:.1f} °C — every +10 °C roughly halves VRLA life{suffix}",
|
||||
value=temp))
|
||||
elif temp < -10:
|
||||
checks.append(_check("Battery temperature", "warn",
|
||||
f"{temp:.1f} °C looks wrong for a battery indoors", value=temp))
|
||||
else:
|
||||
checks.append(_check("Battery temperature", "ok", f"{temp:.1f} °C{suffix}", value=temp))
|
||||
|
||||
load = reading.number("load_pct")
|
||||
if load is not None:
|
||||
status = "ok" if load < 70 else "warn" if load < 90 else "fail"
|
||||
detail = f"{load:.0f}% of rated capacity"
|
||||
power = reading.number("realpower")
|
||||
nominal = reading.number("realpower_nominal")
|
||||
if power is not None and nominal:
|
||||
detail += f" ({power:.0f} W of {nominal:.0f} W)"
|
||||
checks.append(_check("Load", status, detail, value=load))
|
||||
|
||||
alarms = reading.alarms
|
||||
if alarms:
|
||||
blocking = [a for a in alarms if re.search(r"(LB|RB|OVER|NOBAT|TEMP|SC)\b", a)]
|
||||
checks.append(_check("Alarms", "fail" if blocking else "warn", ", ".join(alarms)))
|
||||
|
||||
line = reading.number("line_voltage")
|
||||
nominal = reading.number("line_voltage_nominal")
|
||||
if line is not None and nominal:
|
||||
drift = abs(line - nominal) / nominal * 100.0
|
||||
status = "ok" if drift <= 10 else "warn" if drift <= 15 else "fail"
|
||||
checks.append(_check("Input voltage", status,
|
||||
f"{line:.1f} V against {nominal:.0f} V nominal ({drift:.1f}%)",
|
||||
value=line))
|
||||
low, high = reading.number("transfer_low"), reading.number("transfer_high")
|
||||
if low is not None and high is not None:
|
||||
checks[-1]["detail"] += f"; transfer at {low:.0f}-{high:.0f} V"
|
||||
elif line is not None:
|
||||
checks.append(_check("Input voltage", "ok", f"{line:.1f} V", value=line))
|
||||
|
||||
worst = max((SEVERITY_ORDER.get(c["status"], 0) for c in checks), default=0)
|
||||
overall = {0: "healthy", 1: "unknown", 2: "degraded", 3: "at risk"}[worst]
|
||||
return {
|
||||
"overall": overall,
|
||||
"severity": {0: "ok", 1: "unknown", 2: "warn", 3: "fail"}[worst],
|
||||
"checks": checks,
|
||||
"counts": {
|
||||
name: sum(1 for c in checks if c["status"] == name)
|
||||
for name in ("ok", "warn", "fail", "unknown")
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _battery_age_years(reading: UpsReading, now: float) -> float | None:
|
||||
raw = reading.get("battery_date")
|
||||
if not raw:
|
||||
return None
|
||||
match = re.match(r"(\d{4})[-/](\d{1,2})(?:[-/](\d{1,2}))?", raw.strip())
|
||||
if not match:
|
||||
return None
|
||||
year, month, day = match.groups()
|
||||
try:
|
||||
created = int(year) * 365.25 + (int(month or 1) - 1) * 30.44 + (int(day or 1) - 1)
|
||||
except ValueError:
|
||||
return None
|
||||
now_years = time.gmtime(now).tm_year * 365.25 + (time.gmtime(now).tm_yday - 1)
|
||||
age = (now_years - created) / 365.25
|
||||
# A clock skew (battery.date defaults to 1970 or the future) is not a 56-year-old battery.
|
||||
if age < 0 or age > 30:
|
||||
return None
|
||||
return age
|
||||
|
||||
|
||||
def _duration(seconds: float) -> str:
|
||||
seconds = max(0.0, seconds)
|
||||
hours, remainder = divmod(int(seconds), 3600)
|
||||
minutes, secs = divmod(remainder, 60)
|
||||
if hours:
|
||||
return f"{hours}h {minutes:02d}m"
|
||||
if minutes:
|
||||
return f"{minutes}m {secs:02d}s"
|
||||
return f"{secs}s"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Instant commands (the app's only write path)
|
||||
# --------------------------------------------------------------------------- #
|
||||
RISK_SAFE = "safe"
|
||||
RISK_CAUTION = "caution"
|
||||
RISK_DANGEROUS = "dangerous"
|
||||
|
||||
#: command -> label, risk, what it really does to the hardware behind the screen.
|
||||
#: Anything not listed here is refused even if the driver advertises it.
|
||||
COMMANDS: dict[str, dict[str, str]] = {
|
||||
"test.battery.start.quick": {
|
||||
"label": "Quick battery test",
|
||||
"risk": RISK_CAUTION,
|
||||
"help": "Transfers the load to battery for a few seconds. Safe on a healthy UPS, "
|
||||
"but connected equipment is on battery for the duration.",
|
||||
},
|
||||
"test.battery.start.deep": {
|
||||
"label": "Deep battery test",
|
||||
"risk": RISK_DANGEROUS,
|
||||
"help": "Drains the battery until it is low or a runtime limit is hit. Everything on "
|
||||
"this UPS runs on battery for minutes, and a worn battery may fail outright. "
|
||||
"Run it deliberately, with an eye on the runtime number.",
|
||||
},
|
||||
"test.battery.stop": {"label": "Stop battery test", "risk": RISK_SAFE,
|
||||
"help": "Aborts a running battery test and returns to line power."},
|
||||
"test.panel.start": {"label": "Panel test", "risk": RISK_SAFE,
|
||||
"help": "Lights and beeps the front panel for a few seconds."},
|
||||
"test.system.start": {"label": "Full system self-test", "risk": RISK_CAUTION,
|
||||
"help": "Vendor self-test; behaviour varies by model and may move "
|
||||
"the load to battery."},
|
||||
"beeper.enable": {"label": "Beeper on", "risk": RISK_SAFE, "help": "Re-enable audible alarms."},
|
||||
"beeper.disable": {"label": "Beeper off", "risk": RISK_SAFE,
|
||||
"help": "Silences audible alarms. Faults are still reported in this dashboard."},
|
||||
"beeper.mute": {"label": "Beeper mute", "risk": RISK_SAFE, "help": "Silence the current alarm only."},
|
||||
"calibrate.start": {
|
||||
"label": "Start calibration",
|
||||
"risk": RISK_DANGEROUS,
|
||||
"help": "Discharges the battery to recalibrate the charge gauge. Long and disruptive; "
|
||||
"only do this after a full charge and when the load can tolerate it.",
|
||||
},
|
||||
"calibrate.stop": {"label": "Stop calibration", "risk": RISK_SAFE, "help": "Ends calibration."},
|
||||
"reset.input.minmax": {"label": "Reset input min/max", "risk": RISK_SAFE,
|
||||
"help": "Clears the recorded input voltage extremes."},
|
||||
"load.on": {"label": "Energise output", "risk": RISK_CAUTION,
|
||||
"help": "Turns the outlets on."},
|
||||
"load.off": {"label": "Cut output power", "risk": RISK_DANGEROUS,
|
||||
"help": "Kills power to everything plugged in, immediately, on battery or not."},
|
||||
"load.reboot": {"label": "Power-cycle outlets", "risk": RISK_DANGEROUS,
|
||||
"help": "Cuts and restores output power. Everything attached reboots."},
|
||||
"shutdown.return": {"label": "Graceful shutdown, then restore", "risk": RISK_DANGEROUS,
|
||||
"help": "Asks attached systems to shut down and powers back on when "
|
||||
"utility returns."},
|
||||
"shutdown.standoff": {"label": "Graceful shutdown, stay off", "risk": RISK_DANGEROUS,
|
||||
"help": "Shuts attached systems down and stays off."},
|
||||
"shutdown.stop": {"label": "Cancel shutdown", "risk": RISK_SAFE,
|
||||
"help": "Aborts a pending shutdown sequence."},
|
||||
}
|
||||
|
||||
|
||||
def command_advice(command: str) -> dict[str, str] | None:
|
||||
return COMMANDS.get(command)
|
||||
|
||||
|
||||
def available_commands(advertised: list[str], *, allow_actions: bool,
|
||||
read_only: bool, has_credentials: bool,
|
||||
on_battery: bool = False) -> list[dict[str, Any]]:
|
||||
"""What the UI may offer, with the reason when it may not.
|
||||
|
||||
Filtering server-side (not just hiding buttons) is the point: the response says
|
||||
`enabled: false` plus `reason`, so the page never offers a button that 403s.
|
||||
"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for name in advertised:
|
||||
advice = COMMANDS.get(name)
|
||||
if advice is None:
|
||||
continue
|
||||
reason = None
|
||||
if read_only:
|
||||
reason = "the server is in DASH_READ_ONLY mode"
|
||||
elif not allow_actions:
|
||||
reason = "set NUT_ALLOW_ACTIONS=1 to enable UPS commands"
|
||||
elif not has_credentials:
|
||||
reason = "NUT_USER / NUT_PASSWORD must name a user with `actions` in upsd.users"
|
||||
elif on_battery and advice["risk"] == RISK_DANGEROUS:
|
||||
reason = "refusing a destructive command while the UPS is on battery"
|
||||
out.append({
|
||||
"command": name,
|
||||
"label": advice["label"],
|
||||
"risk": advice["risk"],
|
||||
"help": advice["help"],
|
||||
"enabled": reason is None,
|
||||
"reason": reason,
|
||||
"confirm_text": None if advice["risk"] != RISK_DANGEROUS else "DANGER",
|
||||
})
|
||||
order = {RISK_SAFE: 0, RISK_CAUTION: 1, RISK_DANGEROUS: 2}
|
||||
out.sort(key=lambda item: (order[item["risk"]], item["label"]))
|
||||
return out
|
||||
|
||||
|
||||
def payload(reading: UpsReading, *, now: float | None = None, **assess_kwargs: Any) -> dict[str, Any]:
|
||||
"""The JSON one UPS card renders from."""
|
||||
now = time.time() if now is None else now
|
||||
health = assess(reading, now=now, **assess_kwargs)
|
||||
numbers = {key: reading.number(key) for key in VAR_MAP}
|
||||
# charts must not plot 300 "°C"; the raw string stays visible in the variable dump
|
||||
for key in ("battery_temperature", "ups_temperature"):
|
||||
if numbers.get(key) is not None:
|
||||
numbers[key] = normalise_celsius(numbers[key])
|
||||
return {
|
||||
"name": reading.name,
|
||||
"description": reading.description,
|
||||
"identity": {
|
||||
"mfr": reading.get("mfr"),
|
||||
"model": reading.get("model"),
|
||||
"serial": reading.get("serial"),
|
||||
"firmware": reading.get("firmware"),
|
||||
"driver": reading.get("driver"),
|
||||
"battery_type": reading.get("battery_type"),
|
||||
},
|
||||
"ts": reading.ts,
|
||||
"status": reading.get("status"),
|
||||
"charger_status": reading.get("charger_status"),
|
||||
"states": reading.states(),
|
||||
"alarms": reading.alarms,
|
||||
"on_battery": reading.on_battery,
|
||||
"numbers": numbers,
|
||||
"health": health,
|
||||
"advertised_commands": reading.commands,
|
||||
"writable": reading.writable,
|
||||
"extra": {
|
||||
key: value for key, value in sorted(reading.vars.items())
|
||||
if key.startswith(EXTRA_VAR_PREFIXES) and key not in {v for vs in VAR_MAP.values() for v in vs}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def history_samples(reading: UpsReading) -> dict[str, float]:
|
||||
"""The numeric series worth keeping a history of."""
|
||||
raw = {
|
||||
"ups.charge_pct": "charge_pct",
|
||||
"ups.battery_voltage": "battery_voltage",
|
||||
"ups.battery_current": "battery_current",
|
||||
"ups.battery_runtime_seconds": "battery_runtime",
|
||||
"ups.battery_temperature_c": "battery_temperature",
|
||||
"ups.line_voltage": "line_voltage",
|
||||
"ups.output_voltage": "output_voltage",
|
||||
"ups.output_current": "output_current",
|
||||
"ups.output_frequency": "output_frequency",
|
||||
"ups.frequency": "frequency",
|
||||
"ups.load_pct": "load_pct",
|
||||
"ups.realpower_w": "realpower",
|
||||
"ups.temperature_c": "ups_temperature",
|
||||
}
|
||||
out: dict[str, float] = {}
|
||||
for metric, key in raw.items():
|
||||
value = reading.number(key)
|
||||
if value is None:
|
||||
continue
|
||||
if metric.endswith("_temperature_c") or metric == "ups.temperature_c":
|
||||
value = normalise_celsius(value)
|
||||
if value is not None:
|
||||
out[metric] = value
|
||||
return out
|
||||
@@ -0,0 +1,214 @@
|
||||
"""A *labelled* UPS simulator, for development when there is no upsd to talk to.
|
||||
|
||||
Enable with `NUT_SIMULATE=1`. It exists so the UPS page can be built and screenshotted
|
||||
before the hardware is cabled — and so the battery-test buttons can be exercised without
|
||||
putting a real load on battery. It is never silent about what it is: the payload carries
|
||||
`simulated: true` and the page shows a watermark on every simulated number.
|
||||
|
||||
The model is deliberately small and physical rather than random:
|
||||
* charge sits at float (100%) while on line, drains at a Peukert-ish rate when on battery
|
||||
* runtime estimates scale with charge and fall off steeply with load (as they do for real
|
||||
VRLA packs, which is why the number jumps around while the load moves)
|
||||
* a quick battery test moves to OB for ~12s and then reports PASSED, so the whole
|
||||
command → state change → event-log path is exercised
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .nut import UpsReading
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimUps:
|
||||
"""One fake UPS: nameplate values plus the state a driver would report."""
|
||||
|
||||
name: str
|
||||
model: str
|
||||
mfr: str
|
||||
serial: str
|
||||
nominal_va: float = 1500.0
|
||||
nominal_w: float = 900.0
|
||||
battery_cells: int = 2 # 2 x 12V blocks
|
||||
battery_nominal: float = 24.0
|
||||
capacity_ah: float = 17.0
|
||||
load_w: float = 210.0
|
||||
charge: float = 100.0
|
||||
on_battery: bool = False
|
||||
temperature_c: float = 27.0
|
||||
mfr_date: str = "2022-04-15"
|
||||
test_result: str = "PASSED"
|
||||
test_date: str = "2026-09-01"
|
||||
firmware: str = "UPS 08.3 / IGR 1.00"
|
||||
driver: str = "usbhid-ups"
|
||||
_test_until: float = 0.0
|
||||
_deep_until: float = 0.0
|
||||
_beeper: bool = True
|
||||
_alarms: list[str] = field(default_factory=list)
|
||||
|
||||
def tick(self, now: float) -> None:
|
||||
"""Advance the model by wall-clock time since the last tick."""
|
||||
# a running self-test pulls the load from the wall, whichever test it is
|
||||
self.on_battery = now < self._test_until or now < self._deep_until
|
||||
|
||||
def step(self, dt: float) -> None:
|
||||
dt = max(0.0, min(dt, 60.0))
|
||||
if self.on_battery:
|
||||
# Ah drawn / capacity, with a mild Peukert exponent so a heavy load costs more
|
||||
hours = dt / 3600.0
|
||||
load_fraction = max(0.05, self.load_w / self.nominal_w)
|
||||
drawn = (self.load_w / self.battery_nominal) * hours * (load_fraction ** 0.15)
|
||||
self.charge = max(0.0, self.charge - (drawn / self.capacity_ah) * 100.0)
|
||||
elif self.charge < 100.0:
|
||||
self.charge = min(100.0, self.charge + dt * (100.0 / 14400.0)) # ~4h recharge
|
||||
self.temperature_c = 27.0 + 0.6 * math.sin(time.time() / 300.0) + self.charge / 400.0
|
||||
|
||||
def runtime_seconds(self) -> float:
|
||||
"""Battery runtime at the present load — the number the UPS itself estimates."""
|
||||
if not self.on_battery and self.charge >= 99.5:
|
||||
# Full and on line: what a *fresh* pack would give at this load, which is what
|
||||
# a UPS reports when it is idle on mains.
|
||||
usable_ah = self.capacity_ah * 0.9
|
||||
else:
|
||||
usable_ah = self.capacity_ah * (self.charge / 100.0)
|
||||
amps = max(0.5, self.load_w / (self.battery_nominal * 0.9))
|
||||
load_penalty = max(0.35, 1.0 - (self.load_w / self.nominal_w) ** 2 * 0.45)
|
||||
return (usable_ah / amps) * 3600.0 * load_penalty
|
||||
|
||||
def vars(self, now: float) -> dict[str, str]:
|
||||
load_pct = self.load_w / self.nominal_w * 100.0
|
||||
flags = ["OL"] if not self.on_battery else (["OB"] + (["LB"] if self.charge < 20 else []))
|
||||
if self.charge >= 99.5 and not self.on_battery:
|
||||
flags.append("CHRG" if self.charge < 99.9 else "")
|
||||
status = " ".join(part for part in flags if part)
|
||||
charger = "ol" if not self.on_battery else ("dchg" if self.charge < 99.5 else "ol")
|
||||
if self.charge >= 99.9 and not self.on_battery:
|
||||
charger = "full"
|
||||
volts = (self.battery_nominal * (0.86 + 0.14 * self.charge / 100.0)
|
||||
if self.on_battery else self.battery_nominal * 1.045)
|
||||
line = 121.4 + 1.1 * math.sin(now / 90.0)
|
||||
out = {
|
||||
"ups.mfr": self.mfr,
|
||||
"ups.model": self.model,
|
||||
"ups.serial": self.serial,
|
||||
"ups.firmware": self.firmware,
|
||||
"ups.status": status,
|
||||
"battery.charger.status": charger,
|
||||
"ups.load": f"{load_pct:.1f}",
|
||||
"ups.realpower": f"{self.load_w:.1f}",
|
||||
"ups.realpower.nominal": f"{self.nominal_w:.0f}",
|
||||
"input.voltage": f"{line:.1f}",
|
||||
"input.voltage.nominal": "120",
|
||||
"input.frequency": f"{60.0 + 0.02 * math.sin(now / 40.0):.2f}",
|
||||
"input.transfer.low": "92",
|
||||
"input.transfer.high": "139",
|
||||
"output.voltage": f"{line if not self.on_battery else 119.8:.1f}",
|
||||
"output.current": f"{self.load_w / 120.0:.2f}",
|
||||
"output.frequency": f"{60.0 + 0.02 * math.sin(now / 40.0):.2f}",
|
||||
"battery.charge": f"{self.charge:.1f}",
|
||||
"battery.voltage": f"{volts:.2f}",
|
||||
"battery.voltage.nominal": f"{self.battery_nominal:.0f}",
|
||||
"battery.current": f"{self.load_w / volts:.2f}",
|
||||
"battery.temperature": f"{self.temperature_c + 273.15:.1f}", # drivers report K
|
||||
"battery.type": "PbAcid: 12V / 9Ah / 17Ah sealed",
|
||||
"battery.runtime": f"{self.runtime_seconds():.0f}",
|
||||
"battery.runtime.low": "120",
|
||||
"battery.mfr.date": self.mfr_date,
|
||||
"battery.capacity": f"{self.capacity_ah:.0f}",
|
||||
"battery.test.result": self.test_result,
|
||||
"battery.test.date": self.test_date,
|
||||
"driver.name": self.driver,
|
||||
"driver.parameter.port": "auto",
|
||||
"ups.id": self.name,
|
||||
}
|
||||
if self._alarms:
|
||||
out["ups.alarm"] = ",".join(self._alarms)
|
||||
return out
|
||||
|
||||
def commands(self) -> list[str]:
|
||||
return ["test.battery.start.quick", "test.battery.start.deep", "test.battery.stop",
|
||||
"test.panel.start", "beeper.enable", "beeper.disable", "beeper.mute",
|
||||
"calibrate.start", "calibrate.stop", "reset.input.minmax",
|
||||
"load.on", "load.off", "shutdown.stop"]
|
||||
|
||||
def run_command(self, command: str, now: float) -> str:
|
||||
"""Apply a command the way a driver would, so the UI's whole path is real."""
|
||||
if command == "test.battery.start.quick":
|
||||
self._test_until = now + 12.0
|
||||
self.on_battery = True
|
||||
self.test_date = time.strftime("%Y-%m-%d", time.gmtime(now))
|
||||
return "OK Start Quick Battery Test"
|
||||
if command == "test.battery.start.deep":
|
||||
self._deep_until = now + 90.0
|
||||
self.on_battery = True
|
||||
return "OK Start Deep Battery Test"
|
||||
if command in ("test.battery.stop", "calibrate.stop"):
|
||||
self._test_until = self._deep_until = 0.0
|
||||
self.on_battery = False
|
||||
return "OK Stop Test"
|
||||
if command == "test.panel.start":
|
||||
return "OK Start Panel Test"
|
||||
if command == "beeper.enable":
|
||||
self._beeper = True
|
||||
return "OK Beeper enabled"
|
||||
if command in ("beeper.disable", "beeper.mute"):
|
||||
self._beeper = False
|
||||
return "OK Beeper muted"
|
||||
if command == "calibrate.start":
|
||||
self._deep_until = now + 180.0
|
||||
self.on_battery = True
|
||||
return "OK Start Calibration"
|
||||
if command == "reset.input.minmax":
|
||||
return "OK Input min/max reset"
|
||||
if command == "load.on":
|
||||
return "OK Load energised"
|
||||
if command == "load.off":
|
||||
return "OK Load switched off (simulated)"
|
||||
if command == "shutdown.stop":
|
||||
return "OK Shutdown cancelled"
|
||||
raise ValueError(f"simulated UPS does not implement {command}")
|
||||
|
||||
|
||||
class NutSimulator:
|
||||
"""Stands in for upsd: same shapes as NutClient reads, no network."""
|
||||
|
||||
def __init__(self, spec: str | None = None) -> None:
|
||||
self.ups = [
|
||||
SimUps("cyberpower", "PR1500LCDRT2U", "Cyber Power Systems", "CPS-04213",
|
||||
nominal_va=1500.0, nominal_w=900.0, load_w=210.0, capacity_ah=17.0,
|
||||
mfr_date="2022-04-15"),
|
||||
SimUps("apc", "Smart-UPS 1500", "APC", "AS-11902",
|
||||
nominal_va=1500.0, nominal_w=865.0, load_w=140.0, capacity_ah=8.0,
|
||||
mfr_date="2019-11-02", driver="usbhid-ups",
|
||||
test_result="PASSED", test_date="2026-06-14"),
|
||||
]
|
||||
self._last = time.time()
|
||||
|
||||
async def read(self) -> list[UpsReading]:
|
||||
now = time.time()
|
||||
dt = now - self._last
|
||||
self._last = now
|
||||
for ups in self.ups:
|
||||
ups.tick(now)
|
||||
ups.step(dt)
|
||||
return [
|
||||
UpsReading(name=ups.name, description=f"{ups.mfr} {ups.model} (simulated)",
|
||||
vars=ups.vars(now), commands=ups.commands(),
|
||||
writable=["beeper.enable"], ts=now)
|
||||
for ups in self.ups
|
||||
]
|
||||
|
||||
def command(self, ups_name: str, command: str) -> str:
|
||||
for ups in self.ups:
|
||||
if ups.name == ups_name:
|
||||
return ups.run_command(command, time.time())
|
||||
raise KeyError(f"unknown simulated UPS {ups_name!r}")
|
||||
|
||||
|
||||
def from_env(value: str | None) -> Any:
|
||||
"""`NUT_SIMULATE=1` -> a simulator; anything else -> None (talk to real upsd)."""
|
||||
return NutSimulator() if (value or "").strip().lower() in ("1", "true", "yes", "on") else None
|
||||
@@ -0,0 +1,348 @@
|
||||
"""SQLite history: one sample table with three resolutions, plus an event log.
|
||||
|
||||
Why SQLite and not Prometheus: the fleet is a handful of boxes, the poll resolution we
|
||||
need is 1-10s, and a single file the container owns beats running a TSDB. Series are
|
||||
addressed by `(source, host, metric, res, ts)`:
|
||||
|
||||
source the dashboard family — "host", "llm", "ups"
|
||||
host stable target key ("driveripper", "gufo", "apc-ups1")
|
||||
metric dotted name, optionally `@stream` for per-device series
|
||||
(cpu.usage_pct, cpu.usage_pct@core7, disk.used_pct@/srv, gpu.busy_pct@0)
|
||||
res "raw" | "1m" | "1h"
|
||||
|
||||
`maintain()` rolls raw up into 1m, 1m into 1h, and prunes past the retention window, so
|
||||
the file stays bounded (a 3-host / 3-engine / 2-UPS fleet is a few hundred MB a year,
|
||||
not a disk emergency).
|
||||
|
||||
The connection is guarded by a lock and every method is synchronous: async callers wrap
|
||||
them in `anyio.to_thread.run_sync`, sync FastAPI handlers run in the threadpool already.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
RES_RAW, RES_MIN, RES_HOUR = "raw", "1m", "1h"
|
||||
MINUTE = 60.0
|
||||
HOUR = 3600.0
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS sample (
|
||||
source TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
metric TEXT NOT NULL,
|
||||
res TEXT NOT NULL,
|
||||
ts REAL NOT NULL,
|
||||
value REAL NOT NULL,
|
||||
PRIMARY KEY (source, host, metric, res, ts) ON CONFLICT REPLACE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS event (
|
||||
id INTEGER PRIMARY KEY,
|
||||
ts REAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
host TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
severity TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
data TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS event_ts ON event (ts DESC);
|
||||
CREATE INDEX IF NOT EXISTS event_source_ts ON event (source, ts DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Sample:
|
||||
source: str
|
||||
host: str
|
||||
metric: str
|
||||
value: float
|
||||
ts: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Event:
|
||||
ts: float
|
||||
source: str
|
||||
host: str
|
||||
kind: str
|
||||
severity: str # info | warn | error
|
||||
message: str
|
||||
data: dict | None = None
|
||||
|
||||
def as_json(self) -> dict:
|
||||
return {
|
||||
"ts": self.ts,
|
||||
"source": self.source,
|
||||
"host": self.host,
|
||||
"kind": self.kind,
|
||||
"severity": self.severity,
|
||||
"message": self.message,
|
||||
"data": self.data,
|
||||
}
|
||||
|
||||
|
||||
class SampleStore:
|
||||
def __init__(self, path: str | Path, *, raw_keep_hours: float = 48,
|
||||
minute_keep_days: float = 30, hour_keep_days: float = 400) -> None:
|
||||
self.path = Path(path)
|
||||
self.raw_keep_seconds = raw_keep_hours * HOUR
|
||||
self.minute_keep_seconds = minute_keep_days * 24 * HOUR
|
||||
self.hour_keep_seconds = hour_keep_days * 24 * HOUR
|
||||
self._lock = threading.Lock()
|
||||
if str(self.path) != ":memory:":
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._db = sqlite3.connect(str(self.path), check_same_thread=False)
|
||||
self._db.execute("PRAGMA journal_mode=WAL")
|
||||
self._db.execute("PRAGMA synchronous=NORMAL") # a lost sample is fine, corruption is not
|
||||
self._db.execute("PRAGMA busy_timeout=5000")
|
||||
self._db.executescript(_SCHEMA)
|
||||
self._db.commit()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
self._db.close()
|
||||
|
||||
# -- writing --------------------------------------------------------------- #
|
||||
def add(self, source: str, host: str, metric: str, value: float, ts: float | None = None) -> None:
|
||||
self.add_many([Sample(source, host, metric, value, time.time() if ts is None else ts)])
|
||||
|
||||
def add_many(self, samples: Iterable[Sample]) -> int:
|
||||
"""Insert samples; NaN/inf are dropped (SQLite would store them and charts choke).
|
||||
|
||||
Returns the number actually written so the caller can report it, never to fail
|
||||
the poll over one bad value.
|
||||
"""
|
||||
rows = [
|
||||
(s.source, s.host, s.metric, RES_RAW, s.ts, s.value)
|
||||
for s in samples
|
||||
if isinstance(s.value, int | float) and math.isfinite(s.value)
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
with self._lock, self._db:
|
||||
self._db.executemany("INSERT OR REPLACE INTO sample VALUES (?,?,?,?,?,?)", rows)
|
||||
return len(rows)
|
||||
|
||||
def log_event(self, event: Event) -> None:
|
||||
with self._lock, self._db:
|
||||
self._db.execute(
|
||||
"INSERT INTO event (ts, source, host, kind, severity, message, data) VALUES (?,?,?,?,?,?,?)",
|
||||
(event.ts, event.source, event.host, event.kind, event.severity, event.message,
|
||||
json.dumps(event.data) if event.data else None),
|
||||
)
|
||||
|
||||
# -- reading --------------------------------------------------------------- #
|
||||
def query(self, source: str, host: str, metric: str, *, res: str = RES_RAW,
|
||||
start: float = 0.0, end: float | None = None, limit: int = 2000,
|
||||
tail: bool = False) -> list[tuple[float, float]]:
|
||||
"""(ts, value) ascending by time within [start, end].
|
||||
|
||||
`tail=True` keeps the *newest* `limit` rows instead of the oldest. A window query
|
||||
that truncates at the head draws the left edge of the chart and silently drops
|
||||
"now", which is the one point a dashboard has to have.
|
||||
"""
|
||||
end = time.time() if end is None else end
|
||||
where = ("source=? AND host=? AND metric=? AND res=? AND ts>=? AND ts<=?")
|
||||
args: tuple[object, ...] = (source, host, metric, res, start, end)
|
||||
with self._lock:
|
||||
if tail:
|
||||
cur = self._db.execute(
|
||||
f"SELECT ts, value FROM (SELECT ts, value FROM sample WHERE {where}"
|
||||
" ORDER BY ts DESC LIMIT ?) ORDER BY ts ASC",
|
||||
(*args, limit),
|
||||
)
|
||||
else:
|
||||
cur = self._db.execute(
|
||||
f"SELECT ts, value FROM sample WHERE {where} ORDER BY ts ASC LIMIT ?",
|
||||
(*args, limit),
|
||||
)
|
||||
return cur.fetchall() # type: ignore[no-any-return]
|
||||
|
||||
def pick_res(self, source: str, host: str, metric: str, span: float, max_points: int) -> str:
|
||||
"""Finest resolution that still fits in `max_points` for this window."""
|
||||
now = time.time()
|
||||
start = now - span
|
||||
finest_with_data: str | None = None
|
||||
for res in (RES_RAW, RES_MIN, RES_HOUR):
|
||||
with self._lock:
|
||||
(count,) = self._db.execute(
|
||||
"SELECT COUNT(*) FROM sample WHERE source=? AND host=? AND metric=? AND res=? AND ts>=?",
|
||||
(source, host, metric, res, start),
|
||||
).fetchone()
|
||||
if not count:
|
||||
continue
|
||||
if finest_with_data is None:
|
||||
# too many points for the budget is fine — history() thins. Returning a
|
||||
# resolution with *no* rows in the window reads as a dead sensor.
|
||||
finest_with_data = res
|
||||
if count <= max_points:
|
||||
return res
|
||||
return finest_with_data or RES_HOUR
|
||||
|
||||
def history(self, source: str, host: str, metric: str, *, span: float = 3600.0,
|
||||
max_points: int = 600) -> list[tuple[float, float]]:
|
||||
"""Windowed series, thinned to `max_points` so the browser never gets 100k rows."""
|
||||
res = self.pick_res(source, host, metric, span, max_points)
|
||||
window_start = time.time() - span
|
||||
points = self.query(source, host, metric, res=res, start=window_start,
|
||||
limit=max_points * 4, tail=True)
|
||||
if not points and res != RES_RAW:
|
||||
# Rollups lag behind (they only cover buckets that are fully in the past, and a
|
||||
# dashboard that has been up for ten minutes has none at all). Falling back to
|
||||
# raw turns "no samples yet" into a real chart instead of a lie by omission.
|
||||
points = self.query(source, host, metric, res=RES_RAW, start=window_start,
|
||||
limit=max_points * 4, tail=True)
|
||||
if len(points) <= max_points:
|
||||
return points
|
||||
stride = math.ceil(len(points) / max_points)
|
||||
thinned = points[::stride]
|
||||
if thinned[-1] != points[-1]:
|
||||
thinned.append(points[-1]) # never lose "now" to the thinning
|
||||
return thinned
|
||||
|
||||
def series_names(self, source: str | None = None, host: str | None = None) -> list[str]:
|
||||
sql = "SELECT DISTINCT metric FROM sample"
|
||||
args: list[object] = []
|
||||
clauses = []
|
||||
if source:
|
||||
clauses.append("source=?")
|
||||
args.append(source)
|
||||
if host:
|
||||
clauses.append("host=?")
|
||||
args.append(host)
|
||||
if clauses:
|
||||
sql += " WHERE " + " AND ".join(clauses)
|
||||
with self._lock:
|
||||
return [row[0] for row in self._db.execute(sql + " ORDER BY metric", args)]
|
||||
|
||||
def hosts(self, source: str) -> list[str]:
|
||||
with self._lock:
|
||||
return [row[0] for row in self._db.execute(
|
||||
"SELECT DISTINCT host FROM sample WHERE source=? ORDER BY host", (source,))]
|
||||
|
||||
def recent_events(self, limit: int = 50, *, source: str | None = None,
|
||||
since: float | None = None) -> list[Event]:
|
||||
sql = "SELECT ts, source, host, kind, severity, message, data FROM event"
|
||||
args: list[object] = []
|
||||
clauses: list[str] = []
|
||||
if source:
|
||||
clauses.append("source=?")
|
||||
args.append(source)
|
||||
if since is not None:
|
||||
clauses.append("ts>=?")
|
||||
args.append(since)
|
||||
if clauses:
|
||||
sql += " WHERE " + " AND ".join(clauses)
|
||||
sql += " ORDER BY ts DESC LIMIT ?"
|
||||
args.append(limit)
|
||||
out: list[Event] = []
|
||||
with self._lock:
|
||||
for ts, src, host, kind, sev, msg, data in self._db.execute(sql, args):
|
||||
out.append(Event(ts, src, host, kind, sev, msg, json.loads(data) if data else None))
|
||||
return out
|
||||
|
||||
def count(self, *, res: str | None = None) -> int:
|
||||
sql = "SELECT COUNT(*) FROM sample"
|
||||
args: tuple[object, ...] = ()
|
||||
if res:
|
||||
sql, args = "SELECT COUNT(*) FROM sample WHERE res=?", (res,)
|
||||
with self._lock:
|
||||
return int(self._db.execute(sql, args).fetchone()[0])
|
||||
|
||||
# -- maintenance ----------------------------------------------------------- #
|
||||
def maintain(self, now: float | None = None) -> dict[str, int]:
|
||||
"""Roll up and prune. Safe to call every minute; each step is idempotent.
|
||||
|
||||
A bucket is only rolled once its whole window is in the past (+60s slack), so a
|
||||
rollup never averages a partial minute and then gets rewritten by a later one.
|
||||
"""
|
||||
now = time.time() if now is None else now
|
||||
counts = {"to_minute": 0, "to_hour": 0, "pruned": 0}
|
||||
with self._lock, self._db:
|
||||
counts["to_minute"] = self._db.execute(
|
||||
"INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)"
|
||||
" SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)"
|
||||
" FROM sample WHERE res=? AND ts < ? AND ts >= ?"
|
||||
" GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)",
|
||||
(RES_MIN, MINUTE, MINUTE, RES_RAW, now - MINUTE - 60,
|
||||
now - self.raw_keep_seconds, MINUTE),
|
||||
).rowcount
|
||||
counts["to_hour"] = self._db.execute(
|
||||
"INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)"
|
||||
" SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)"
|
||||
" FROM sample WHERE res=? AND ts < ? AND ts >= ?"
|
||||
" GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)",
|
||||
(RES_HOUR, HOUR, HOUR, RES_MIN, now - HOUR - 60,
|
||||
now - self.minute_keep_seconds, HOUR),
|
||||
).rowcount
|
||||
counts["pruned"] = self._db.execute(
|
||||
"DELETE FROM sample WHERE (res=? AND ts < ?) OR (res=? AND ts < ?) OR (res=? AND ts < ?)",
|
||||
(RES_RAW, now - self.raw_keep_seconds,
|
||||
RES_MIN, now - self.minute_keep_seconds,
|
||||
RES_HOUR, now - self.hour_keep_seconds),
|
||||
).rowcount
|
||||
self._db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_maintain', ?)", (str(now),))
|
||||
return counts
|
||||
|
||||
def stats(self) -> dict[str, object]:
|
||||
with self._lock:
|
||||
by_res = {row[0]: row[1] for row in self._db.execute(
|
||||
"SELECT res, COUNT(*) FROM sample GROUP BY res")}
|
||||
(events,) = self._db.execute("SELECT COUNT(*) FROM event").fetchone()
|
||||
last = self._db.execute("SELECT value FROM meta WHERE key='last_maintain'").fetchone()
|
||||
size = 0
|
||||
if str(self.path) != ":memory:" and self.path.exists():
|
||||
size = self.path.stat().st_size
|
||||
return {
|
||||
"path": str(self.path),
|
||||
"size_bytes": size,
|
||||
"samples_by_res": by_res,
|
||||
"samples_total": sum(by_res.values()),
|
||||
"events": events,
|
||||
"last_maintain": float(last[0]) if last else None,
|
||||
}
|
||||
|
||||
def delete_series(self, source: str, host: str, metric: str | None = None) -> int:
|
||||
"""Drop history for a series (or a whole host). Used by tests and by the
|
||||
'forget this host' escape hatch when a target is renamed for real."""
|
||||
sql = "DELETE FROM sample WHERE source=? AND host=?"
|
||||
args: list[object] = [source, host]
|
||||
if metric:
|
||||
sql += " AND metric=?"
|
||||
args.append(metric)
|
||||
with self._lock, self._db:
|
||||
return self._db.execute(sql, args).rowcount
|
||||
|
||||
|
||||
def samples_from_flat(source: str, host: str, values: dict[str, float | None],
|
||||
ts: float, *, prefix: str = "") -> list[Sample]:
|
||||
"""Flatten a parse result into samples, skipping absent values.
|
||||
|
||||
`n/a` in the UI means "this exporter does not report it" — it must not become a
|
||||
stored 0, which would look identical on a 30-day chart and lie about history.
|
||||
"""
|
||||
out: list[Sample] = []
|
||||
for key, value in values.items():
|
||||
if value is None or isinstance(value, dict | list | str | bool):
|
||||
continue
|
||||
out.append(Sample(source, host, f"{prefix}{key}", float(value), ts))
|
||||
return out
|
||||
|
||||
|
||||
def select_window(span: float, allowed: Sequence[float]) -> float:
|
||||
"""Closest allowed window >= span (used to map ?range=1h onto the presets)."""
|
||||
for candidate in sorted(allowed):
|
||||
if span <= candidate:
|
||||
return candidate
|
||||
return max(allowed)
|
||||
@@ -0,0 +1,128 @@
|
||||
<!doctype html>
|
||||
<html lang="en" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Homelab · overview</title>
|
||||
<link rel="stylesheet" href="/static/dash.css">
|
||||
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' rx='22' fill='%23ff8a3d'/></svg>">
|
||||
</head>
|
||||
<body>
|
||||
<header class="masthead">
|
||||
<a class="brand" href="/"><span class="brand-mark">◆</span> Homelab</a>
|
||||
<nav class="tabs" aria-label="Dashboards">
|
||||
<a href="/" aria-current="page">Overview</a>
|
||||
<a href="/system">System</a>
|
||||
<a href="/llm">LLM</a>
|
||||
<a href="/ups">UPS</a>
|
||||
</nav>
|
||||
<div class="masthead-right">
|
||||
<div id="nav-status" class="gap" aria-live="polite"></div>
|
||||
<span class="clock" id="clock">--:--:--</span>
|
||||
<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">☾ Dark</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="page-head">
|
||||
<div>
|
||||
<h1>Overview</h1>
|
||||
<div class="sub" id="summary-line">reading fleet…</div>
|
||||
</div>
|
||||
<div class="spacer"></div>
|
||||
<span class="chip" id="config-chip" hidden>loading config</span>
|
||||
</div>
|
||||
|
||||
<div id="banners" class="stack" style="margin-bottom:.85rem"></div>
|
||||
|
||||
<section class="grid auto" aria-label="Dashboards">
|
||||
<article class="card">
|
||||
<header><h2>System resources</h2><span class="spacer"></span><span class="dot idle" id="dot-system"></span></header>
|
||||
<div class="body">
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="label">Hosts answering</div>
|
||||
<div class="value"><span id="sys-hosts-live">n/a</span><span class="unit" id="sys-hosts-total"></span></div></div>
|
||||
<div class="metric"><div class="label">CPU average</div>
|
||||
<div class="value"><span id="sys-cpu">n/a</span><span class="unit">%</span></div>
|
||||
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
|
||||
<div class="metric"><div class="label">Memory</div>
|
||||
<div class="value"><span id="sys-mem">n/a</span><span class="unit" id="sys-mem-unit"></span></div>
|
||||
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
|
||||
<div class="metric"><div class="label">Worst disk</div>
|
||||
<div class="value"><span id="sys-disk">n/a</span><span class="unit">%</span></div>
|
||||
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
|
||||
<div class="metric"><div class="label">GPUs</div>
|
||||
<div class="value sm" id="sys-gpus">n/a</div></div>
|
||||
</div>
|
||||
<div class="gap" style="margin-top:.7rem"><a class="action" href="/system" style="text-decoration:none">Open system dashboard →</a></div>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<article class="card">
|
||||
<header><h2>LLM engines</h2><span class="spacer"></span><span class="dot idle" id="dot-llm"></span></header>
|
||||
<div class="body">
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="label">Engines answering</div>
|
||||
<div class="value"><span id="llm-live">n/a</span><span class="unit" id="llm-total"></span></div></div>
|
||||
<div class="metric"><div class="label">Decode</div>
|
||||
<div class="value"><span id="llm-decode">n/a</span><span class="unit">tok/s</span></div></div>
|
||||
<div class="metric"><div class="label">Prompt</div>
|
||||
<div class="value"><span id="llm-prompt">n/a</span><span class="unit">tok/s</span></div></div>
|
||||
<div class="metric"><div class="label">Active sessions</div>
|
||||
<div class="value" id="llm-active">n/a</div></div>
|
||||
<div class="metric"><div class="label">Queued</div>
|
||||
<div class="value" id="llm-waiting">n/a</div></div>
|
||||
<div class="metric"><div class="label" title="highest KV-cache use across engines right now">KV cache now</div>
|
||||
<div class="value"><span id="llm-kv">n/a</span><span class="unit">%</span></div>
|
||||
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
|
||||
</div>
|
||||
<div class="gap" style="margin-top:.7rem"><a class="action" href="/llm" style="text-decoration:none">Open LLM dashboard →</a></div>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<article class="card">
|
||||
<header><h2>UPS / battery</h2><span class="spacer"></span><span class="dot idle" id="dot-ups"></span></header>
|
||||
<div class="body">
|
||||
<div id="ups-list" class="stack">
|
||||
<div class="muted">loading…</div>
|
||||
</div>
|
||||
<div class="gap" style="margin-top:.7rem"><a class="action" href="/ups" style="text-decoration:none">Open UPS dashboard →</a></div>
|
||||
</div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="grid wide" style="margin-top:.85rem">
|
||||
<article class="card">
|
||||
<header><h2>Fleet</h2><span class="spacer"></span><span class="faint mono" id="fleet-note"></span></header>
|
||||
<div class="body tight">
|
||||
<table class="data" id="fleet-table">
|
||||
<thead><tr><th>Host</th><th>CPU</th><th>Load 1</th><th>Mem used</th><th>Worst disk</th><th>GPU busy</th><th>Temp</th><th>Up</th></tr></thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
<div class="empty-state" id="fleet-empty" hidden>
|
||||
<h3>No hosts configured</h3>
|
||||
<p>Run <code>agent/host_agent.py</code> on each machine and list it in
|
||||
<code>HOST_TARGETS</code>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<article class="card">
|
||||
<header><h2>Recent events</h2><span class="spacer"></span>
|
||||
<span class="faint">transitions only</span></header>
|
||||
<div class="body"><ul class="events" id="events"><li class="muted">loading…</li></ul></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<footer class="pagefoot">
|
||||
<span id="foot-versions"></span>
|
||||
<span class="spacer" style="margin-inline-start:auto"></span>
|
||||
<span>every number is scraped from the box it belongs to · <code>null</code> renders as <code>n/a</code></span>
|
||||
</footer>
|
||||
</main>
|
||||
|
||||
<script src="/static/charts.js"></script>
|
||||
<script src="/static/app.js"></script>
|
||||
<script src="/static/page-index.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,121 @@
|
||||
<!doctype html>
|
||||
<html lang="en" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Homelab · LLM</title>
|
||||
<link rel="stylesheet" href="/static/dash.css">
|
||||
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' rx='22' fill='%23ff8a3d'/></svg>">
|
||||
</head>
|
||||
<body>
|
||||
<header class="masthead">
|
||||
<a class="brand" href="/"><span class="brand-mark">◆</span> Homelab</a>
|
||||
<nav class="tabs" aria-label="Dashboards">
|
||||
<a href="/">Overview</a>
|
||||
<a href="/system">System</a>
|
||||
<a href="/llm" aria-current="page">LLM</a>
|
||||
<a href="/ups">UPS</a>
|
||||
</nav>
|
||||
<div class="masthead-right">
|
||||
<div id="nav-status" class="gap" aria-live="polite"></div>
|
||||
<span class="clock" id="clock">--:--:--</span>
|
||||
<button class="ghost" id="pause" type="button" aria-pressed="false">⏸ Pause</button>
|
||||
<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">☾ Dark</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="page-head">
|
||||
<div>
|
||||
<h1>LLM engines</h1>
|
||||
<div class="sub" id="sub">reading engines…</div>
|
||||
</div>
|
||||
<div class="spacer"></div>
|
||||
<div class="gap"><span class="faint">window</span><div id="range"></div></div>
|
||||
</div>
|
||||
|
||||
<div id="banners" class="stack" style="margin-bottom:.85rem"></div>
|
||||
|
||||
<section class="card flat" aria-label="Engine totals">
|
||||
<div class="body">
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="label">Engines answering</div>
|
||||
<div class="value"><span id="f-live">n/a</span><span class="unit" id="f-total"></span></div></div>
|
||||
<div class="metric"><div class="label">Decode (sum)</div>
|
||||
<div class="value"><span id="f-decode">n/a</span><span class="unit">tok/s</span></div></div>
|
||||
<div class="metric"><div class="label">Prompt (sum)</div>
|
||||
<div class="value"><span id="f-prompt">n/a</span><span class="unit">tok/s</span></div></div>
|
||||
<div class="metric"><div class="label">Active sessions</div>
|
||||
<div class="value" id="f-active">n/a</div></div>
|
||||
<div class="metric"><div class="label">Queued</div>
|
||||
<div class="value" id="f-waiting">n/a</div></div>
|
||||
<div class="metric"><div class="label">Tokens lifetime</div>
|
||||
<div class="value"><span id="f-tokens">n/a</span><span class="unit"></span></div></div>
|
||||
<div class="metric"><div class="label" title="highest KV-cache use across engines right now, not a window peak">KV cache now</div>
|
||||
<div class="value"><span id="f-kv">n/a</span><span class="unit">%</span></div>
|
||||
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
|
||||
</div>
|
||||
<div class="faint" id="seam-note" style="margin-top:.5rem;font-size:11.5px"></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="grid cols-2" style="margin-top:.85rem" aria-label="Throughput charts">
|
||||
<article class="card">
|
||||
<header><h3>Decode throughput</h3><span class="spacer"></span>
|
||||
<span class="faint">tok/s · every engine, summed slots</span></header>
|
||||
<div class="body"><div class="chart" id="chart-decode"></div><div class="legend" id="legend-decode"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Prompt / prefill throughput</h3><span class="spacer"></span>
|
||||
<span class="faint">tok/s · newly computed tokens</span></header>
|
||||
<div class="body"><div class="chart" id="chart-prompt"></div><div class="legend" id="legend-prompt"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>KV / prefix cache pressure</h3><span class="spacer"></span><span class="faint">% of cache in use</span></header>
|
||||
<div class="body"><div class="chart" id="chart-kv"></div><div class="legend" id="legend-kv"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Sessions & queue</h3><span class="spacer"></span><span class="faint">running vs waiting</span></header>
|
||||
<div class="body"><div class="chart" id="chart-queue"></div><div class="legend" id="legend-queue"></div></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section style="margin-top:.85rem">
|
||||
<div class="grid cols-2" id="engines"></div>
|
||||
<div class="empty-state" id="engines-empty" hidden>
|
||||
<h3>No engines configured</h3>
|
||||
<p>Set <code>GUFO_BASE_URL</code> (llama.cpp / gufo) and/or <code>VLLM_BASE_URL</code>.
|
||||
Either half appears only when its variable is set — a server you do not run
|
||||
should not have a red panel on your dashboard.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="grid cols-2" style="margin-top:.85rem">
|
||||
<article class="card" id="slots-card">
|
||||
<header><h3>gufo slots</h3><span class="spacer"></span>
|
||||
<span class="faint">one row per parallel slot</span></header>
|
||||
<div class="body tight"><table class="data" id="slots-table">
|
||||
<thead><tr><th>Slot</th><th>State</th><th>Tokens</th><th>N prompt</th><th>N gen</th>
|
||||
<th>Cache</th><th>Speed</th></tr></thead><tbody></tbody></table>
|
||||
<div class="empty-state" id="slots-empty" hidden><p>No <code>/slots</code> rows.</p></div>
|
||||
</div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Events</h3><span class="spacer"></span><span class="faint">engine up / down</span></header>
|
||||
<div class="body"><ul class="events" id="events"><li class="muted">loading…</li></ul></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<footer class="pagefoot">
|
||||
<span>each cell carries its provenance: scraped gauge, <span class="mono">Δ10s</span> measured
|
||||
counter, or lifetime estimate</span>
|
||||
<span class="spacer" style="margin-inline-start:auto"></span>
|
||||
<span class="mono" id="foot-note"></span>
|
||||
</footer>
|
||||
</main>
|
||||
|
||||
<script src="/static/charts.js"></script>
|
||||
<script src="/static/app.js"></script>
|
||||
<script src="/static/page-llm.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,136 @@
|
||||
<!doctype html>
|
||||
<html lang="en" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Homelab · system</title>
|
||||
<link rel="stylesheet" href="/static/dash.css">
|
||||
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' rx='22' fill='%23ff8a3d'/></svg>">
|
||||
</head>
|
||||
<body>
|
||||
<header class="masthead">
|
||||
<a class="brand" href="/"><span class="brand-mark">◆</span> Homelab</a>
|
||||
<nav class="tabs" aria-label="Dashboards">
|
||||
<a href="/">Overview</a>
|
||||
<a href="/system" aria-current="page">System</a>
|
||||
<a href="/llm">LLM</a>
|
||||
<a href="/ups">UPS</a>
|
||||
</nav>
|
||||
<div class="masthead-right">
|
||||
<div id="nav-status" class="gap" aria-live="polite"></div>
|
||||
<span class="clock" id="clock">--:--:--</span>
|
||||
<button class="ghost" id="pause" type="button" aria-pressed="false">⏸ Pause</button>
|
||||
<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">☾ Dark</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="page-head">
|
||||
<div>
|
||||
<h1>System resources</h1>
|
||||
<div class="sub" id="sub">reading hosts…</div>
|
||||
</div>
|
||||
<div class="spacer"></div>
|
||||
<div class="gap">
|
||||
<span class="faint">window</span>
|
||||
<div id="range"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section class="card flat" aria-label="Fleet totals">
|
||||
<div class="body">
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="label">Hosts answering</div>
|
||||
<div class="value"><span id="f-hosts">n/a</span><span class="unit" id="f-hosts-of"></span></div></div>
|
||||
<div class="metric"><div class="label">CPU average</div>
|
||||
<div class="value"><span id="f-cpu">n/a</span><span class="unit">%</span></div></div>
|
||||
<div class="metric"><div class="label">Load 1 (sum)</div>
|
||||
<div class="value" id="f-load">n/a</div></div>
|
||||
<div class="metric"><div class="label">Memory in use</div>
|
||||
<div class="value"><span id="f-mem">n/a</span><span class="unit" id="f-mem-of"></span></div></div>
|
||||
<div class="metric"><div class="label">Worst filesystem</div>
|
||||
<div class="value"><span id="f-disk">n/a</span><span class="unit">%</span></div></div>
|
||||
<div class="metric"><div class="label">Network in</div>
|
||||
<div class="value"><span id="f-rx">n/a</span><span class="unit" id="f-rx-unit"></span></div></div>
|
||||
<div class="metric"><div class="label">Network out</div>
|
||||
<div class="value"><span id="f-tx">n/a</span><span class="unit" id="f-tx-unit"></span></div></div>
|
||||
<div class="metric"><div class="label">GPU busy (avg)</div>
|
||||
<div class="value"><span id="f-gpu">n/a</span><span class="unit">%</span></div></div>
|
||||
<div class="metric"><div class="label">Power</div>
|
||||
<div class="value sm" id="f-power" title="No whole-host power source is wired up yet; this stays n/a rather than guessing.">n/a</div></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="grid cols-2" style="margin-top:.85rem" aria-label="Fleet charts">
|
||||
<article class="card">
|
||||
<header><h3>CPU utilisation</h3><span class="spacer"></span><span class="faint">% · per host</span></header>
|
||||
<div class="body"><div class="chart" id="chart-cpu"></div><div class="legend" id="legend-cpu"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Memory used</h3><span class="spacer"></span><span class="faint">% of MemAvailable basis</span></header>
|
||||
<div class="body"><div class="chart" id="chart-mem"></div><div class="legend" id="legend-mem"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Disk I/O throughput</h3><span class="spacer"></span>
|
||||
<span class="faint">selected host · all physical devices summed</span></header>
|
||||
<div class="body"><div class="chart" id="chart-disk"></div><div class="legend" id="legend-disk"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Network throughput</h3><span class="spacer"></span><span class="faint">selected host · physical interfaces</span></header>
|
||||
<div class="body"><div class="chart" id="chart-net"></div><div class="legend" id="legend-net"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>GPU utilisation</h3><span class="spacer"></span><span class="faint">one series per host GPU</span></header>
|
||||
<div class="body"><div class="chart" id="chart-gpu"></div><div class="legend" id="legend-gpu"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Temperature</h3><span class="spacer"></span><span class="faint">CPU package · °C</span></header>
|
||||
<div class="body"><div class="chart" id="chart-temp"></div><div class="legend" id="legend-temp"></div></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="card" style="margin-top:.85rem" aria-label="Series detail">
|
||||
<header>
|
||||
<h3>Detail</h3>
|
||||
<span class="spacer"></span>
|
||||
<label class="field" for="detail-host">host</label>
|
||||
<select id="detail-host"></select>
|
||||
<label class="field" for="detail-metric">metric</label>
|
||||
<select id="detail-metric"></select>
|
||||
</header>
|
||||
<div class="body">
|
||||
<div class="chart" id="chart-detail" style="height:200px"></div>
|
||||
<div class="legend" id="legend-detail"></div>
|
||||
<div class="faint" id="detail-note" style="margin-top:.4rem;font-size:11.5px"></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section style="margin-top:.85rem">
|
||||
<div class="page-head"><h2 style="font-size:.95rem;margin:0">Per host</h2>
|
||||
<span class="sub">agent-reported values; rates measured over the window in each header</span></div>
|
||||
<div class="grid auto" id="hosts"></div>
|
||||
<div class="empty-state" id="hosts-empty" hidden>
|
||||
<h3>No hosts reporting</h3>
|
||||
<p>Start the agent on a machine and add it to <code>HOST_TARGETS</code>:</p>
|
||||
<p class="mono">python3 agent/host_agent.py --host 0.0.0.0 --port 9120 --token …</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="card" style="margin-top:.85rem">
|
||||
<header><h3>Events</h3><span class="spacer"></span><span class="faint">unreachable / reachable / reboots</span></header>
|
||||
<div class="body"><ul class="events" id="events"><li class="muted">loading…</li></ul></div>
|
||||
</section>
|
||||
|
||||
<footer class="pagefoot">
|
||||
<span>numbers are measured, not inferred: rates are Δcounter/Δt over the poll window</span>
|
||||
<span class="spacer" style="margin-inline-start:auto"></span>
|
||||
<span class="mono" id="foot-note"></span>
|
||||
</footer>
|
||||
</main>
|
||||
|
||||
<script src="/static/charts.js"></script>
|
||||
<script src="/static/app.js"></script>
|
||||
<script src="/static/page-system.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,101 @@
|
||||
<!doctype html>
|
||||
<html lang="en" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Homelab · UPS</title>
|
||||
<link rel="stylesheet" href="/static/dash.css">
|
||||
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' rx='22' fill='%23ff8a3d'/></svg>">
|
||||
</head>
|
||||
<body>
|
||||
<header class="masthead">
|
||||
<a class="brand" href="/"><span class="brand-mark">◆</span> Homelab</a>
|
||||
<nav class="tabs" aria-label="Dashboards">
|
||||
<a href="/">Overview</a>
|
||||
<a href="/system">System</a>
|
||||
<a href="/llm">LLM</a>
|
||||
<a href="/ups" aria-current="page">UPS</a>
|
||||
</nav>
|
||||
<div class="masthead-right">
|
||||
<div id="nav-status" class="gap" aria-live="polite"></div>
|
||||
<span class="clock" id="clock">--:--:--</span>
|
||||
<button class="ghost" id="pause" type="button" aria-pressed="false">⏸ Pause</button>
|
||||
<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">☾ Dark</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="page-head">
|
||||
<div>
|
||||
<h1>UPS & battery health</h1>
|
||||
<div class="sub" id="sub">reading upsd…</div>
|
||||
</div>
|
||||
<div class="spacer"></div>
|
||||
<div class="gap"><span class="faint">window</span><div id="range"></div></div>
|
||||
</div>
|
||||
|
||||
<div id="banners" class="stack" style="margin-bottom:.85rem"></div>
|
||||
|
||||
<section class="grid auto" id="ups-cards"></section>
|
||||
<div class="empty-state" id="ups-empty" hidden>
|
||||
<h3>No UPS reporting</h3>
|
||||
<p>The dashboard speaks the NUT protocol directly to <code>upsd</code>. On the NUT host:</p>
|
||||
<p class="mono">/etc/ups/upsd.conf → <code>LISTEN 0.0.0.0</code> · <code>PORT 3493</code></p>
|
||||
<p class="mono">/etc/ups/upsd.users → <code>[dashboards]</code> with
|
||||
<code>password = …</code> and <code>actions = -</code></p>
|
||||
<p>then <code>systemctl restart nut-server</code> and open the port to this server.</p>
|
||||
</div>
|
||||
|
||||
<section class="grid cols-2" style="margin-top:.85rem" aria-label="Battery charts">
|
||||
<article class="card">
|
||||
<header><h3>Battery charge</h3><span class="spacer"></span><span class="faint">%</span></header>
|
||||
<div class="body"><div class="chart" id="chart-charge"></div><div class="legend" id="legend-charge"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Runtime estimate</h3><span class="spacer"></span>
|
||||
<span class="faint">seconds of backup at the current load</span></header>
|
||||
<div class="body"><div class="chart" id="chart-runtime"></div><div class="legend" id="legend-runtime"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Voltages</h3><span class="spacer"></span><span class="faint">battery and incoming line</span></header>
|
||||
<div class="body"><div class="chart" id="chart-voltage"></div><div class="legend" id="legend-voltage"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Load on the UPS</h3><span class="spacer"></span><span class="faint">% of rated capacity</span></header>
|
||||
<div class="body"><div class="chart" id="chart-load"></div><div class="legend" id="legend-load"></div></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="card" style="margin-top:.85rem">
|
||||
<header><h3>Power events</h3><span class="spacer"></span>
|
||||
<span class="faint">state changes, self-test results, commands sent</span></header>
|
||||
<div class="body"><ul class="events" id="events"><li class="muted">loading…</li></ul></div>
|
||||
</section>
|
||||
|
||||
<footer class="pagefoot">
|
||||
<span>commands are the only write path in this app, and every one of them is confirmed here first</span>
|
||||
<span class="spacer" style="margin-inline-start:auto"></span>
|
||||
<span class="mono" id="foot-note"></span>
|
||||
</footer>
|
||||
</main>
|
||||
|
||||
<dialog id="confirm-dialog">
|
||||
<header><span id="dialog-risk" class="chip">confirm</span><h2 id="dialog-title">Run this command?</h2></header>
|
||||
<div class="body">
|
||||
<p id="dialog-help"></p>
|
||||
<p id="dialog-target" class="mono" style="font-size:12px"></p>
|
||||
<p id="dialog-confirm-wrap" hidden>Type <strong>DANGER</strong> to enable the button:</p>
|
||||
<p hidden id="dialog-input-wrap"><input class="confirm" id="dialog-input" type="text"
|
||||
autocomplete="off" spellcheck="false" placeholder="DANGER"></p>
|
||||
</div>
|
||||
<footer>
|
||||
<button class="ghost" id="dialog-cancel" type="button">Cancel</button>
|
||||
<button class="action" id="dialog-run" type="button">Run command</button>
|
||||
</footer>
|
||||
</dialog>
|
||||
|
||||
<script src="/static/charts.js"></script>
|
||||
<script src="/static/app.js"></script>
|
||||
<script src="/static/page-ups.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,341 @@
|
||||
/* app.js — shared behaviour for every page: formatting, polling, theme, status.
|
||||
*
|
||||
* Two rules the whole UI follows:
|
||||
* - `null`/`undefined` renders as `n/a`, never as `0`, `""`, or a leftover number.
|
||||
* A dash that changes to another dash silently is how dashboards lie.
|
||||
* - a stale value is shown *and labelled*: the dot goes amber/red, `stale 12s` appears,
|
||||
* and the card dims. Hiding the failure is worse than the failure.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const Dash = (() => {
|
||||
const $ = (sel, root) => (root || document).querySelector(sel);
|
||||
const $$ = (sel, root) => Array.from((root || document).querySelectorAll(sel));
|
||||
|
||||
// ---- formatting ---------------------------------------------------------
|
||||
const BYTE_UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
|
||||
|
||||
function fmtBytes(value, digits) {
|
||||
if (value == null || !isFinite(value)) return null;
|
||||
let v = Math.abs(value);
|
||||
let i = 0;
|
||||
while (v >= 1024 && i < BYTE_UNITS.length - 1) { v /= 1024; i += 1; }
|
||||
const d = digits != null ? digits : (v >= 100 || i === 0 ? 0 : v >= 10 ? 1 : 2);
|
||||
return { text: v.toFixed(d), unit: BYTE_UNITS[i] };
|
||||
}
|
||||
|
||||
function bytes(value, digits) {
|
||||
const parts = fmtBytes(value, digits);
|
||||
return parts ? `${parts.text} ${parts.unit}` : naText;
|
||||
}
|
||||
|
||||
function rate(value, digits) {
|
||||
const parts = fmtBytes(value, digits);
|
||||
return parts ? `${parts.text} ${parts.unit}/s` : naText;
|
||||
}
|
||||
|
||||
function pct(value, digits) {
|
||||
if (value == null || !isFinite(value)) return naText;
|
||||
return `${value.toFixed(digits == null ? 1 : digits)}%`;
|
||||
}
|
||||
|
||||
function num(value, digits) {
|
||||
if (value == null || !isFinite(value)) return naText;
|
||||
const d = digits == null ? (Math.abs(value) >= 100 ? 0 : Math.abs(value) >= 10 ? 1 : 2) : digits;
|
||||
return value.toFixed(d);
|
||||
}
|
||||
|
||||
function duration(seconds) {
|
||||
if (seconds == null || !isFinite(seconds)) return naText;
|
||||
const s = Math.max(0, seconds);
|
||||
if (s < 90) return `${Math.round(s)}s`;
|
||||
const m = s / 60;
|
||||
if (m < 90) return `${m.toFixed(m < 10 ? 1 : 0)}m`;
|
||||
const h = m / 60;
|
||||
if (h < 48) return `${h.toFixed(h < 10 ? 1 : 0)}h`;
|
||||
const d = h / 24;
|
||||
if (d < 60) return `${d.toFixed(d < 10 ? 1 : 0)}d`;
|
||||
return `${(d / 365.25).toFixed(1)}y`;
|
||||
}
|
||||
|
||||
function age(ts) {
|
||||
if (ts == null) return 'never';
|
||||
const seconds = Math.max(0, Date.now() / 1000 - ts);
|
||||
if (seconds < 2) return 'now';
|
||||
return duration(seconds).replace(/(h|m|s|d|y)$/, '$') + ' ago';
|
||||
}
|
||||
|
||||
const naText = 'n/a';
|
||||
|
||||
// ---- DOM helpers --------------------------------------------------------
|
||||
/** Write text only when it changed: avoids killing transitions and focus. */
|
||||
function setTxt(el, value) {
|
||||
if (!el) return;
|
||||
const text = value == null ? naText : String(value);
|
||||
if (el.textContent !== text) el.textContent = text;
|
||||
el.classList.toggle('is-null', value == null);
|
||||
el.classList.toggle('na', value == null);
|
||||
}
|
||||
|
||||
function setValue(el, value, digits, unit) {
|
||||
if (!el) return;
|
||||
const parent = el.parentElement;
|
||||
if (value == null || !isFinite(value)) {
|
||||
setTxt(el, naText);
|
||||
if (parent) parent.classList.add('is-null');
|
||||
return;
|
||||
}
|
||||
if (parent) parent.classList.remove('is-null');
|
||||
const text = typeof value === 'string' ? value : num(value, digits);
|
||||
setTxt(el, text);
|
||||
const unitEl = el.nextElementSibling;
|
||||
if (unit && unitEl && unitEl.classList.contains('unit')) setTxt(unitEl, unit);
|
||||
}
|
||||
|
||||
function setBar(el, value, kind) {
|
||||
if (!el) return;
|
||||
const fill = el.tagName === 'I' ? el : $('i', el);
|
||||
const pctValue = value == null ? 0 : Math.max(0, Math.min(100, value));
|
||||
if (fill) fill.style.width = `${pctValue}%`;
|
||||
const threshold = kind || severityFor(pctValue);
|
||||
el.classList.remove('ok', 'warn', 'fail');
|
||||
el.classList.add(threshold);
|
||||
}
|
||||
|
||||
function severityFor(value, warnAt, failAt) {
|
||||
const warn = warnAt == null ? 75 : warnAt;
|
||||
const fail = failAt == null ? 90 : failAt;
|
||||
if (value == null) return '';
|
||||
if (value >= fail) return 'fail';
|
||||
if (value >= warn) return 'warn';
|
||||
return 'ok';
|
||||
}
|
||||
|
||||
function setDot(el, status) {
|
||||
if (!el) return;
|
||||
el.classList.remove('ok', 'warn', 'fail', 'idle');
|
||||
el.classList.add(status || 'idle');
|
||||
}
|
||||
|
||||
function chip(text, kind, title) {
|
||||
const span = document.createElement('span');
|
||||
span.className = 'chip' + (kind ? ' ' + kind : '');
|
||||
span.textContent = text;
|
||||
if (title) span.title = title;
|
||||
return span;
|
||||
}
|
||||
|
||||
// ---- networking ---------------------------------------------------------
|
||||
async function fetchJSON(url, options) {
|
||||
const response = await fetch(url, Object.assign({ headers: { accept: 'application/json' } },
|
||||
options || {}));
|
||||
let body = null;
|
||||
try { body = await response.json(); } catch (err) { body = null; }
|
||||
if (!response.ok) {
|
||||
const detail = body && body.detail
|
||||
? (typeof body.detail === 'string' ? body.detail : body.detail.detail || JSON.stringify(body.detail))
|
||||
: `${response.status} ${response.statusText}`;
|
||||
const error = new Error(detail);
|
||||
error.status = response.status;
|
||||
error.body = body;
|
||||
throw error;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll `fn` every `ms`, skipping overlapping runs and pausing when the tab is hidden
|
||||
* or the user hit pause. Errors are reported, never swallowed silently.
|
||||
*/
|
||||
function poll(fn, ms) {
|
||||
let stopped = false;
|
||||
let timer = null;
|
||||
let inFlight = false;
|
||||
const state = {
|
||||
get paused() { return state._paused; },
|
||||
_paused: false,
|
||||
interval: ms,
|
||||
setInterval(next) { state.interval = next; if (timer) { clearTimeout(timer); tick(); } },
|
||||
setPaused(next) {
|
||||
state._paused = next;
|
||||
if (next) { if (timer) clearTimeout(timer); timer = null; }
|
||||
else if (!timer) tick();
|
||||
},
|
||||
stop() { stopped = true; if (timer) clearTimeout(timer); },
|
||||
runNow() { tick(); },
|
||||
};
|
||||
async function tick() {
|
||||
if (stopped || inFlight || state._paused || document.hidden) { schedule(); return; }
|
||||
inFlight = true;
|
||||
try { await fn(); } finally { inFlight = false; schedule(); }
|
||||
}
|
||||
function schedule() {
|
||||
if (stopped || state._paused) return;
|
||||
timer = setTimeout(tick, state.interval);
|
||||
}
|
||||
document.addEventListener('visibilitychange', () => { if (!document.hidden && !state._paused) tick(); });
|
||||
tick();
|
||||
return state;
|
||||
}
|
||||
|
||||
// ---- chrome -------------------------------------------------------------
|
||||
function initChrome() {
|
||||
const root = document.documentElement;
|
||||
const stored = localStorage.getItem('dash-theme');
|
||||
const prefersLight = window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches;
|
||||
root.dataset.theme = stored || (prefersLight ? 'light' : 'dark');
|
||||
|
||||
const themeButton = $('#theme-toggle');
|
||||
if (themeButton) {
|
||||
const paint = () => {
|
||||
themeButton.textContent = root.dataset.theme === 'light' ? '☀ Light' : '☾ Dark';
|
||||
themeButton.setAttribute('aria-pressed', root.dataset.theme === 'light' ? 'true' : 'false');
|
||||
};
|
||||
themeButton.addEventListener('click', () => {
|
||||
root.dataset.theme = root.dataset.theme === 'light' ? 'dark' : 'light';
|
||||
localStorage.setItem('dash-theme', root.dataset.theme);
|
||||
paint();
|
||||
document.dispatchEvent(new CustomEvent('dash:theme'));
|
||||
});
|
||||
paint();
|
||||
}
|
||||
|
||||
const clock = $('#clock');
|
||||
if (clock) {
|
||||
const tickClock = () => {
|
||||
const now = new Date();
|
||||
const target = document.getElementById('sim-flag');
|
||||
clock.textContent = now.toLocaleTimeString(undefined, { hour12: false });
|
||||
void target;
|
||||
};
|
||||
tickClock();
|
||||
setInterval(tickClock, 1000);
|
||||
}
|
||||
|
||||
const nav = $('#nav-status');
|
||||
if (nav) {
|
||||
const jobs = [
|
||||
{ key: 'system', label: 'hosts', href: '/system' },
|
||||
{ key: 'llm', label: 'engines', href: '/llm' },
|
||||
{ key: 'ups', label: 'UPS', href: '/ups' },
|
||||
];
|
||||
nav.innerHTML = jobs.map((job) => `<span class="chip" title="${job.label}">
|
||||
<span class="dot idle" data-dot="${job.key}"></span>${job.label}</span>`).join('');
|
||||
const refreshStatus = async () => {
|
||||
try {
|
||||
const health = await fetchJSON('/api/health');
|
||||
for (const job of jobs) {
|
||||
const dot = nav.querySelector(`[data-dot="${job.key}"]`);
|
||||
const states = (health.sources || []).filter((s) => s.dashboard === job.key && s.configured);
|
||||
if (!states.length) { setDot(dot, 'idle'); continue; }
|
||||
const ok = states.filter((s) => s.ok).length;
|
||||
setDot(dot, ok === states.length ? 'ok' : ok ? 'warn' : 'fail');
|
||||
dot.title = `${ok}/${states.length} ${job.label} answering`;
|
||||
}
|
||||
} catch (err) {
|
||||
for (const job of jobs) setDot(nav.querySelector(`[data-dot="${job.key}"]`), 'fail');
|
||||
}
|
||||
};
|
||||
refreshStatus();
|
||||
setInterval(refreshStatus, 15000);
|
||||
}
|
||||
}
|
||||
|
||||
/** Range buttons (5m … 30d). Calls onChange(seconds, key). */
|
||||
function rangeControl(container, current, onChange, choices) {
|
||||
const options = choices || [['5m', 300], ['15m', 900], ['1h', 3600], ['6h', 21600],
|
||||
['24h', 86400], ['7d', 604800], ['30d', 2592000]];
|
||||
let value = current || 3600;
|
||||
container.className = 'gap';
|
||||
container.innerHTML = '';
|
||||
const buttons = options.map(([key, seconds]) => {
|
||||
const button = document.createElement('button');
|
||||
button.className = 'ghost';
|
||||
button.type = 'button';
|
||||
button.textContent = key;
|
||||
button.dataset.seconds = String(seconds);
|
||||
button.setAttribute('aria-pressed', seconds === value ? 'true' : 'false');
|
||||
button.addEventListener('click', () => {
|
||||
value = seconds;
|
||||
buttons.forEach((b) => b.setAttribute('aria-pressed',
|
||||
Number(b.dataset.seconds) === value ? 'true' : 'false'));
|
||||
onChange(seconds, key);
|
||||
});
|
||||
container.appendChild(button);
|
||||
return button;
|
||||
});
|
||||
return { get value() { return value; } };
|
||||
}
|
||||
|
||||
let toastTimer = null;
|
||||
function toast(message, kind) {
|
||||
let host = $('#toasts');
|
||||
if (!host) {
|
||||
host = document.createElement('div');
|
||||
host.id = 'toasts';
|
||||
host.style.cssText = 'position:fixed;right:1rem;bottom:1rem;z-index:60;display:grid;gap:.4rem';
|
||||
document.body.appendChild(host);
|
||||
}
|
||||
const div = document.createElement('div');
|
||||
div.className = 'banner ' + (kind || 'info');
|
||||
div.textContent = message;
|
||||
host.appendChild(div);
|
||||
setTimeout(() => div.remove(), kind === 'fail' ? 9000 : 4500);
|
||||
if (toastTimer) clearTimeout(toastTimer);
|
||||
toastTimer = setTimeout(() => { host.remove(); host = null; }, 10000);
|
||||
}
|
||||
|
||||
/** Pause button + Space, wired to a poll handle. */
|
||||
function bindPause(poller) {
|
||||
const button = $('#pause');
|
||||
if (!button || !poller) return;
|
||||
const paint = () => {
|
||||
button.setAttribute('aria-pressed', poller.paused ? 'true' : 'false');
|
||||
button.textContent = poller.paused ? '▶ Paused' : '⏸ Pause';
|
||||
};
|
||||
button.addEventListener('click', () => { poller.setPaused(!poller.paused); paint(); });
|
||||
document.addEventListener('keydown', (event) => {
|
||||
const tag = (event.target && event.target.tagName || '').toLowerCase();
|
||||
if (tag === 'input' || tag === 'textarea' || event.metaKey || event.ctrlKey) return;
|
||||
if (event.code === 'Space') { event.preventDefault(); poller.setPaused(!poller.paused); paint(); }
|
||||
if (event.key === 't' || event.key === 'T') {
|
||||
const theme = $('#theme-toggle');
|
||||
if (theme) theme.click();
|
||||
}
|
||||
});
|
||||
paint();
|
||||
}
|
||||
|
||||
/** The "last poll" line every card header shows. */
|
||||
function staleness(el, state) {
|
||||
if (!el) return;
|
||||
if (!state) { setTxt(el, '—'); return; }
|
||||
if (!state.configured) { setTxt(el, 'not configured'); return; }
|
||||
if (!state.ok) {
|
||||
setTxt(el, state.last_ok ? `stale ${duration(ageSeconds(state.last_ok))} · ${state.error || 'down'}`
|
||||
: (state.error || 'never reached'));
|
||||
setDot(el.parentElement && el.parentElement.querySelector('.dot'), 'fail');
|
||||
return;
|
||||
}
|
||||
const seconds = ageSeconds(state.last_ok);
|
||||
setTxt(el, seconds > 30 ? `stale ${duration(seconds)}` : `updated ${duration(seconds)} ago`);
|
||||
setDot(el.parentElement && el.parentElement.querySelector('.dot'),
|
||||
seconds > 30 ? 'warn' : 'ok');
|
||||
}
|
||||
|
||||
function ageSeconds(ts) {
|
||||
return ts == null ? Infinity : Math.max(0, Date.now() / 1000 - ts);
|
||||
}
|
||||
|
||||
/** Cards whose data is older than the poll interval get dimmed + labelled. */
|
||||
function markStale(card, stale) {
|
||||
if (!card) return;
|
||||
card.classList.toggle('stale', !!stale);
|
||||
}
|
||||
|
||||
return { $, $$, fetchJSON, poll, fmtBytes, bytes, rate, pct, num, duration, age, setTxt,
|
||||
setValue, setBar, setDot, chip, initChrome, rangeControl, toast, bindPause,
|
||||
staleness, markStale, severityFor, ageSeconds, naText };
|
||||
})();
|
||||
|
||||
document.addEventListener('DOMContentLoaded', () => Dash.initChrome());
|
||||
@@ -0,0 +1,378 @@
|
||||
/* charts.js — hand-rolled SVG time-series charts. No CDN, no chart library.
|
||||
*
|
||||
* Coordinate space is real pixels (viewBox = the element's measured size, no
|
||||
* preserveAspectRatio="none"), which is what keeps axis text crisp: a stretched viewBox
|
||||
* smears every glyph drawn inside it. Redraw happens on ResizeObserver, not on a timer.
|
||||
*
|
||||
* Deliberate behaviours:
|
||||
* - A series with no points draws nothing and the chart says why. It never draws 0,
|
||||
* because "no data" and "zero" are different claims.
|
||||
* - The y-range comes from the data actually plotted, and the labels are generated from
|
||||
* the same numbers the path used, so a label can never drift from the curve.
|
||||
* - Gaps are gaps: a sample missing in the middle breaks the line rather than drawing a
|
||||
* straight lie across the hole.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const CHART_COLORS = ['--series-1', '--series-2', '--series-3', '--series-4',
|
||||
'--series-5', '--series-6', '--series-7', '--series-8'];
|
||||
|
||||
function cssVar(name) {
|
||||
return getComputedStyle(document.documentElement).getPropertyValue(name).trim();
|
||||
}
|
||||
|
||||
/** Round a range to a "nice" tick step: 1, 2, 2.5, 5, 10 x 10^n. */
|
||||
function niceTicks(min, max, count) {
|
||||
if (!isFinite(min) || !isFinite(max) || min === max) {
|
||||
const base = isFinite(min) ? min : 0;
|
||||
return { lo: base - 1, hi: base + 1, step: 1, values: [base - 1, base, base + 1] };
|
||||
}
|
||||
const raw = (max - min) / Math.max(1, count);
|
||||
const mag = Math.pow(10, Math.floor(Math.log10(Math.abs(raw) || 1)));
|
||||
const norm = raw / mag;
|
||||
const step = (norm <= 1 ? 1 : norm <= 2 ? 2 : norm <= 2.5 ? 2.5 : norm <= 5 ? 5 : 10) * mag;
|
||||
const lo = Math.floor(min / step) * step;
|
||||
const hi = Math.ceil(max / step) * step;
|
||||
const values = [];
|
||||
for (let v = lo; v <= hi + step * 0.5; v += step) values.push(Number(v.toFixed(10)));
|
||||
return { lo, hi, step, values };
|
||||
}
|
||||
|
||||
const DEFAULT_FORMAT = (v) => (Math.abs(v) >= 100 ? v.toFixed(0) : v.toFixed(1));
|
||||
|
||||
class LineChart {
|
||||
/**
|
||||
* container: element to fill (its width is used; height comes from opts.height)
|
||||
* opts: {height, yFormat, xSpan, yMin, yMax, area, legend, showAxis, min, unit}
|
||||
*/
|
||||
constructor(container, opts) {
|
||||
this.el = typeof container === 'string' ? document.querySelector(container) : container;
|
||||
this.opts = Object.assign({
|
||||
height: 150, area: true, legend: true, showAxis: true,
|
||||
yFormat: DEFAULT_FORMAT, xSpan: 3600, yMin: null, yMax: null, unit: '',
|
||||
}, opts || {});
|
||||
this.series = [];
|
||||
this.hoverTs = null;
|
||||
this._build();
|
||||
if (typeof ResizeObserver !== 'undefined') {
|
||||
this._ro = new ResizeObserver(() => this.draw());
|
||||
this._ro.observe(this.el);
|
||||
} else {
|
||||
window.addEventListener('resize', () => this.draw());
|
||||
}
|
||||
}
|
||||
|
||||
_build() {
|
||||
this.el.classList.add('chart');
|
||||
this.el.style.height = this.opts.height + 'px';
|
||||
this.el.innerHTML = '';
|
||||
this.svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
|
||||
this.svg.setAttribute('width', '100%');
|
||||
this.svg.setAttribute('height', this.opts.height);
|
||||
this.el.appendChild(this.svg);
|
||||
this.empty = document.createElement('div');
|
||||
this.empty.className = 'empty';
|
||||
this.el.appendChild(this.empty);
|
||||
this.tooltip = document.createElement('div');
|
||||
this.tooltip.className = 'tooltip';
|
||||
this.el.appendChild(this.tooltip);
|
||||
this.svg.addEventListener('mousemove', (e) => this._onMove(e));
|
||||
this.svg.addEventListener('mouseleave', () => { this.hoverTs = null; this._hideTip(); this._drawCursor(); });
|
||||
}
|
||||
|
||||
/** series: [{name, points: [{ts, value}], color?, yMax?}] — replaces previous data. */
|
||||
setData(series) {
|
||||
this.series = (series || []).map((s, i) => ({
|
||||
name: s.name,
|
||||
color: s.color || cssVar(CHART_COLORS[i % CHART_COLORS.length]),
|
||||
points: (s.points || []).filter((p) => p && Number.isFinite(p.value) && Number.isFinite(p.ts))
|
||||
.sort((a, b) => a.ts - b.ts),
|
||||
}));
|
||||
this.draw();
|
||||
return this;
|
||||
}
|
||||
|
||||
_layout() {
|
||||
const w = Math.max(80, this.el.clientWidth);
|
||||
const h = this.opts.height;
|
||||
const pad = this.opts.showAxis ? { l: 44, r: 8, t: 8, b: 16 } : { l: 2, r: 2, t: 3, b: 3 };
|
||||
return { w, h, pad, plotW: Math.max(10, w - pad.l - pad.r), plotH: Math.max(10, h - pad.t - pad.b) };
|
||||
}
|
||||
|
||||
_xExtent() {
|
||||
const span = this.opts.xSpan;
|
||||
const end = Date.now() / 1000;
|
||||
return [end - span, end];
|
||||
}
|
||||
|
||||
_yExtent() {
|
||||
if (this.opts.yMin !== null && this.opts.yMax !== null) {
|
||||
return [this.opts.yMin, this.opts.yMax];
|
||||
}
|
||||
let lo = Infinity, hi = -Infinity;
|
||||
for (const s of this.series) {
|
||||
for (const p of s.points) {
|
||||
if (p.value < lo) lo = p.value;
|
||||
if (p.value > hi) hi = p.value;
|
||||
}
|
||||
}
|
||||
if (!isFinite(lo)) return [0, 1];
|
||||
if (lo === hi) { const pad = Math.abs(lo) * 0.1 || 1; return [lo - pad, hi + pad]; }
|
||||
const margin = (hi - lo) * 0.12;
|
||||
lo = this.opts.yMin !== null ? this.opts.yMin : lo - margin;
|
||||
hi = this.opts.yMax !== null ? this.opts.yMax : hi + margin;
|
||||
return [lo, hi];
|
||||
}
|
||||
|
||||
draw() {
|
||||
if (!this.el.clientWidth) return;
|
||||
const plotted = this.series.filter((s) => s.points.length);
|
||||
const { w, h, pad, plotW, plotH } = this._layout();
|
||||
this.svg.setAttribute('viewBox', `0 0 ${w} ${h}`);
|
||||
this.svg.innerHTML = '';
|
||||
if (!plotted.length) {
|
||||
this.empty.textContent = 'no samples yet';
|
||||
this.empty.style.display = 'grid';
|
||||
return;
|
||||
}
|
||||
this.empty.style.display = 'none';
|
||||
const [x0, x1] = this._xExtent();
|
||||
const [ylo, yhi] = this._yExtent();
|
||||
const ticks = niceTicks(ylo, yhi, 4);
|
||||
this.scales = {
|
||||
x: (ts) => pad.l + ((ts - x0) / (x1 - x0)) * plotW,
|
||||
y: (v) => pad.t + plotH - ((Math.min(Math.max(v, ticks.lo), ticks.hi) - ticks.lo) / (ticks.hi - ticks.lo)) * plotH,
|
||||
ticks, x0, x1, ylo: ticks.lo, yhi: ticks.hi,
|
||||
};
|
||||
const ns = 'http://www.w3.org/2000/svg';
|
||||
const g = (name) => {
|
||||
const grp = document.createElementNS(ns, 'g');
|
||||
grp.setAttribute('class', name);
|
||||
this.svg.appendChild(grp);
|
||||
return grp;
|
||||
};
|
||||
if (this.opts.showAxis) {
|
||||
const grid = g('grid');
|
||||
for (const t of ticks.values) {
|
||||
const y = this.scales.y(t);
|
||||
if (y < pad.t - 1 || y > pad.t + plotH + 1) continue;
|
||||
const line = document.createElementNS(ns, 'line');
|
||||
line.setAttribute('x1', pad.l); line.setAttribute('x2', pad.l + plotW);
|
||||
line.setAttribute('y1', y); line.setAttribute('y2', y);
|
||||
grid.appendChild(line);
|
||||
const text = document.createElementNS(ns, 'text');
|
||||
text.setAttribute('x', pad.l - 6); text.setAttribute('y', y + 3.5);
|
||||
text.setAttribute('text-anchor', 'end');
|
||||
text.textContent = this.opts.yFormat(t);
|
||||
grid.appendChild(text);
|
||||
}
|
||||
const axis = g('axis');
|
||||
for (let i = 0; i <= 4; i++) {
|
||||
const ts = x0 + ((x1 - x0) * i) / 4;
|
||||
const x = this.scales.x(ts);
|
||||
const text = document.createElementNS(ns, 'text');
|
||||
text.setAttribute('x', x); text.setAttribute('y', h - 3);
|
||||
text.setAttribute('text-anchor', i === 0 ? 'start' : i === 4 ? 'end' : 'middle');
|
||||
text.textContent = shortTime(ts, x1 - x0);
|
||||
axis.appendChild(text);
|
||||
}
|
||||
}
|
||||
for (const s of plotted) {
|
||||
const paths = buildPath(s.points, this.scales, plotH + pad.t);
|
||||
if (this.opts.area && paths.length) {
|
||||
for (const seg of paths) {
|
||||
const area = document.createElementNS(ns, 'path');
|
||||
area.setAttribute('class', 'area');
|
||||
area.setAttribute('d', seg.area);
|
||||
area.setAttribute('fill', s.color);
|
||||
this.svg.appendChild(area);
|
||||
}
|
||||
}
|
||||
for (const seg of paths) {
|
||||
const line = document.createElementNS(ns, 'path');
|
||||
line.setAttribute('class', 'line');
|
||||
line.setAttribute('d', seg.line);
|
||||
line.setAttribute('stroke', s.color);
|
||||
this.svg.appendChild(line);
|
||||
}
|
||||
}
|
||||
const cursor = document.createElementNS(ns, 'g');
|
||||
cursor.setAttribute('class', 'cursor');
|
||||
this.svg.appendChild(cursor);
|
||||
this.cursorEl = cursor;
|
||||
this._drawCursor();
|
||||
}
|
||||
|
||||
_drawCursor() {
|
||||
if (!this.cursorEl || !this.scales || this.hoverTs == null) {
|
||||
if (this.cursorEl) this.cursorEl.innerHTML = '';
|
||||
return;
|
||||
}
|
||||
const { pad } = this._layout();
|
||||
const x = this.scales.x(this.hoverTs);
|
||||
this.cursorEl.innerHTML = '';
|
||||
const line = document.createElementNS('http://www.w3.org/2000/svg', 'line');
|
||||
line.setAttribute('x1', x); line.setAttribute('x2', x);
|
||||
line.setAttribute('y1', pad.t); line.setAttribute('y2', this.opts.height - pad.b);
|
||||
this.cursorEl.appendChild(line);
|
||||
}
|
||||
|
||||
_onMove(event) {
|
||||
if (!this.scales) return;
|
||||
const rect = this.svg.getBoundingClientRect();
|
||||
const box = (this.svg.getAttribute('viewBox') || '0 0 1 1').split(' ');
|
||||
const px = ((event.clientX - rect.left) / rect.width) * Number(box[2]);
|
||||
const { pad, plotW } = this._layout();
|
||||
const { x0, x1 } = this.scales;
|
||||
const frac = Math.min(1, Math.max(0, (px - pad.l) / plotW));
|
||||
const at = x0 + frac * (x1 - x0);
|
||||
this.hoverTs = at;
|
||||
this._drawCursor();
|
||||
this._showTip(at, event);
|
||||
}
|
||||
|
||||
_showTip(at, event) {
|
||||
const rows = this.series.map((s) => {
|
||||
const near = nearest(s.points, at);
|
||||
if (!near) return null;
|
||||
return `<div class="row"><span class="swatch" style="background:${s.color}"></span>` +
|
||||
`<span>${escapeHtml(s.name)}</span>` +
|
||||
`<span class="val">${this.opts.yFormat(near.value)}${this.opts.unit ? ' ' + this.opts.unit : ''}</span></div>`;
|
||||
}).filter(Boolean);
|
||||
if (!rows.length) { this._hideTip(); return; }
|
||||
this.tooltip.innerHTML = `<div class="ts">${fullTime(at)}</div>${rows.join('')}`;
|
||||
this.tooltip.style.opacity = '1';
|
||||
const rect = this.el.getBoundingClientRect();
|
||||
const left = Math.min(rect.width - 150, Math.max(4, event.clientX - rect.left + 12));
|
||||
this.tooltip.style.left = left + 'px';
|
||||
this.tooltip.style.top = '8px';
|
||||
}
|
||||
|
||||
_hideTip() { this.tooltip.style.opacity = '0'; }
|
||||
}
|
||||
|
||||
/** Split points into contiguous runs; a gap > 3x the median step breaks the line. */
|
||||
function buildPath(points, scales, baseline) {
|
||||
if (!points.length) return [];
|
||||
const steps = [];
|
||||
for (let i = 1; i < points.length; i++) steps.push(points[i].ts - points[i - 1].ts);
|
||||
steps.sort((a, b) => a - b);
|
||||
const median = steps.length ? steps[Math.floor(steps.length / 2)] : 0;
|
||||
const maxGap = median > 0 ? median * 3 : Infinity;
|
||||
const runs = [];
|
||||
let current = [points[0]];
|
||||
for (let i = 1; i < points.length; i++) {
|
||||
if (points[i].ts - points[i - 1].ts > maxGap) { runs.push(current); current = [points[i]]; }
|
||||
else current.push(points[i]);
|
||||
}
|
||||
runs.push(current);
|
||||
return runs.map((run) => {
|
||||
let line = '';
|
||||
run.forEach((p, i) => {
|
||||
const x = scales.x(p.ts).toFixed(1);
|
||||
const y = scales.y(p.value).toFixed(1);
|
||||
line += (i === 0 ? `M${x} ${y}` : ` L${x} ${y}`);
|
||||
});
|
||||
const first = scales.x(run[0].ts).toFixed(1);
|
||||
const last = scales.x(run[run.length - 1].ts).toFixed(1);
|
||||
const area = `${line} L${last} ${baseline.toFixed(1)} L${first} ${baseline.toFixed(1)} Z`;
|
||||
return { line, area };
|
||||
});
|
||||
}
|
||||
|
||||
function nearest(points, ts) {
|
||||
if (!points.length) return null;
|
||||
let lo = 0, hi = points.length - 1;
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
if (points[mid].ts < ts) lo = mid + 1; else hi = mid;
|
||||
}
|
||||
const a = points[Math.max(0, lo - 1)], b = points[lo];
|
||||
return Math.abs(a.ts - ts) < Math.abs(b.ts - ts) ? a : b;
|
||||
}
|
||||
|
||||
function shortTime(ts, span) {
|
||||
const d = new Date(ts * 1000);
|
||||
if (span > 86400 * 3) return d.toLocaleDateString(undefined, { month: 'short', day: 'numeric' });
|
||||
if (span > 86400) return d.toLocaleDateString(undefined, { weekday: 'short' }) + ' ' +
|
||||
d.toLocaleTimeString(undefined, { hour: '2-digit' });
|
||||
return d.toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' });
|
||||
}
|
||||
|
||||
function fullTime(ts) {
|
||||
return new Date(ts * 1000).toLocaleString(undefined, {
|
||||
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit',
|
||||
});
|
||||
}
|
||||
|
||||
function escapeHtml(value) {
|
||||
return String(value).replace(/[&<>"']/g, (c) => ({
|
||||
'&': '&', '<': '<', '>': '>', '"': '"', "'": ''',
|
||||
}[c]));
|
||||
}
|
||||
|
||||
/** Legend with the live last value per series, under the chart. */
|
||||
/**
|
||||
* Legend with the live last value per series, under the chart.
|
||||
*
|
||||
* `showPeak` adds the window maximum too. Throughput is bursty: a legend that reads
|
||||
* `0.0 tok/s` next to a chart that clearly went to 60 twice is read as a broken dashboard,
|
||||
* even though "last poll was idle" is the truth. Saying both keeps it honest and legible.
|
||||
*/
|
||||
function renderLegend(container, series, format, showPeak) {
|
||||
container.innerHTML = series.map((s, i) => {
|
||||
const color = s.color || cssVar(CHART_COLORS[i % CHART_COLORS.length]);
|
||||
const points = (s.points || []).filter((p) => Number.isFinite(p.value));
|
||||
const last = points[points.length - 1];
|
||||
let peakHtml = '';
|
||||
if (showPeak && points.length) {
|
||||
const peak = points.reduce((best, p) => (p.value > best.value ? p : best), points[0]);
|
||||
if (peak.value !== last.value && peak.value > 0) {
|
||||
peakHtml = `<span class="lv faint" title="highest value in the window">peak ${format(peak.value)}</span>`;
|
||||
}
|
||||
}
|
||||
return `<span class="item"><span class="swatch" style="background:${color}"></span>` +
|
||||
`<span>${escapeHtml(s.name)}</span>` +
|
||||
`<span class="lv">${last ? format(last.value) : 'n/a'}</span>${peakHtml}</span>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
/** Column heat bars for per-core usage: 48 cores read as a pattern, not a table. */
|
||||
function renderCores(container, values, format) {
|
||||
const f = format || ((v) => (v == null ? 'n/a' : v.toFixed(0) + '%'));
|
||||
container.innerHTML = (values || []).map((v, i) => {
|
||||
const pct = v == null ? 0 : Math.max(0, Math.min(100, v));
|
||||
const hue = pct > 85 ? 'var(--fail)' : pct > 60 ? 'var(--warn)' : 'var(--accent)';
|
||||
return `<span class="core" title="core ${i}: ${escapeHtml(f(v))}">` +
|
||||
`<i style="height:${pct.toFixed(1)}%;background:${hue}"></i></span>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
/** Small non-interactive sparkline for cards where a full chart would shout. */
|
||||
function sparkline(container, points, opts) {
|
||||
const o = Object.assign({ height: 34, color: cssVar('--series-1'), min: null, max: null }, opts || {});
|
||||
const pts = (points || []).filter((p) => p && Number.isFinite(p.value));
|
||||
const w = Math.max(60, container.clientWidth || 160);
|
||||
const h = o.height;
|
||||
container.innerHTML = '';
|
||||
if (pts.length < 2) {
|
||||
container.innerHTML = '<div class="faint mono" style="font-size:11px">no history</div>';
|
||||
return;
|
||||
}
|
||||
const xs = pts.map((p) => p.ts), ys = pts.map((p) => p.value);
|
||||
const x0 = Math.min(...xs), x1 = Math.max(...xs);
|
||||
let lo = o.min !== null ? o.min : Math.min(...ys);
|
||||
let hi = o.max !== null ? o.max : Math.max(...ys);
|
||||
if (lo === hi) { const pad = Math.abs(lo) * 0.1 || 1; lo -= pad; hi += pad; }
|
||||
const sx = (ts) => ((ts - x0) / (x1 - x0 || 1)) * (w - 2) + 1;
|
||||
const sy = (v) => h - 2 - ((v - lo) / (hi - lo)) * (h - 4);
|
||||
const d = pts.map((p, i) => `${i ? 'L' : 'M'}${sx(p.ts).toFixed(1)} ${sy(p.value).toFixed(1)}`).join(' ');
|
||||
const svg = `<svg viewBox="0 0 ${w} ${h}" width="100%" height="${h}" preserveAspectRatio="none"
|
||||
role="img" aria-label="trend from ${lo.toFixed(2)} to ${hi.toFixed(2)}">
|
||||
<path d="${d} L${sx(x1).toFixed(1)} ${h} L${sx(x0).toFixed(1)} ${h} Z" fill="${o.color}" opacity=".14"/>
|
||||
<path d="${d}" fill="none" stroke="${o.color}" stroke-width="1.4"/></svg>`;
|
||||
container.innerHTML = svg;
|
||||
}
|
||||
|
||||
window.DashCharts = { LineChart, niceTicks, renderLegend, renderCores, sparkline, cssVar,
|
||||
CHART_COLORS, DEFAULT_FORMAT, escapeHtml };
|
||||
@@ -0,0 +1,442 @@
|
||||
/* Homelab dashboards — design system.
|
||||
*
|
||||
* No CDN, no webfonts, no framework: this runs on a homelab network that may have no
|
||||
* route to the internet, and a font that fails to load is a page that reflows in front
|
||||
* of you. Everything here is system fonts and hand-written SVG.
|
||||
*
|
||||
* Two themes via [data-theme] on <html> (persisted in localStorage), defaulting to the
|
||||
* OS preference. All colours are tokens; nothing in a component hardcodes a colour.
|
||||
*/
|
||||
|
||||
:root {
|
||||
--bg: #0d0f13;
|
||||
--bg-raised: #14171e;
|
||||
--bg-sunken: #0a0c0f;
|
||||
--border: #232836;
|
||||
--border-strong: #333b4d;
|
||||
--text: #e7eaf0;
|
||||
--text-muted: #8b94a7;
|
||||
--text-faint: #5c6478;
|
||||
--accent: #ff8a3d;
|
||||
--accent-soft: rgba(255, 138, 61, 0.14);
|
||||
--ok: #3fb950;
|
||||
--ok-soft: rgba(63, 185, 80, 0.15);
|
||||
--warn: #d29922;
|
||||
--warn-soft: rgba(210, 153, 34, 0.16);
|
||||
--fail: #f85149;
|
||||
--fail-soft: rgba(248, 81, 73, 0.16);
|
||||
--info: #58a6ff;
|
||||
--info-soft: rgba(88, 166, 255, 0.15);
|
||||
--series-1: #ff8a3d;
|
||||
--series-2: #58a6ff;
|
||||
--series-3: #3fb950;
|
||||
--series-4: #bc8cff;
|
||||
--series-5: #f778ba;
|
||||
--series-6: #d29922;
|
||||
--series-7: #39c5cf;
|
||||
--series-8: #8b94a7;
|
||||
--radius: 10px;
|
||||
--radius-sm: 6px;
|
||||
--shadow: 0 1px 2px rgba(0, 0, 0, 0.4), 0 8px 24px rgba(0, 0, 0, 0.25);
|
||||
--mono: ui-monospace, "SF Mono", "JetBrains Mono", "Cascadia Mono", Menlo, Consolas, monospace;
|
||||
--sans: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
|
||||
--header-h: 58px;
|
||||
}
|
||||
|
||||
[data-theme="light"] {
|
||||
--bg: #f4f5f8;
|
||||
--bg-raised: #ffffff;
|
||||
--bg-sunken: #e9ebf0;
|
||||
--border: #dfe3ea;
|
||||
--border-strong: #c3cad6;
|
||||
--text: #16191f;
|
||||
--text-muted: #5b6473;
|
||||
--text-faint: #8a93a3;
|
||||
--accent: #d2691e;
|
||||
--accent-soft: rgba(210, 105, 30, 0.12);
|
||||
--ok: #1a7f37;
|
||||
--ok-soft: rgba(26, 127, 55, 0.12);
|
||||
--warn: #9a6700;
|
||||
--warn-soft: rgba(154, 103, 0, 0.12);
|
||||
--fail: #cf222e;
|
||||
--fail-soft: rgba(207, 34, 46, 0.1);
|
||||
--info: #0969da;
|
||||
--info-soft: rgba(9, 105, 218, 0.1);
|
||||
--series-1: #d2691e;
|
||||
--series-2: #0969da;
|
||||
--series-3: #1a7f37;
|
||||
--series-4: #8250df;
|
||||
--series-5: #bf3989;
|
||||
--series-6: #9a6700;
|
||||
--series-7: #1b7c84;
|
||||
--series-8: #5b6473;
|
||||
--shadow: 0 1px 2px rgba(16, 24, 40, 0.06), 0 6px 20px rgba(16, 24, 40, 0.06);
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
html, body { height: 100%; }
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font: 14px/1.5 var(--sans);
|
||||
-webkit-font-smoothing: antialiased;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
|
||||
:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 4px; }
|
||||
|
||||
/* ---- header ------------------------------------------------------------- */
|
||||
.masthead {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 40;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1.1rem;
|
||||
height: var(--header-h);
|
||||
padding: 0 1.25rem;
|
||||
background: color-mix(in srgb, var(--bg-raised) 92%, transparent);
|
||||
backdrop-filter: blur(8px);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.brand {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: .55rem;
|
||||
font-weight: 650;
|
||||
letter-spacing: .01em;
|
||||
color: var(--text);
|
||||
white-space: nowrap;
|
||||
}
|
||||
.brand:hover { text-decoration: none; }
|
||||
.brand-mark {
|
||||
width: 22px; height: 22px;
|
||||
border-radius: 6px;
|
||||
background: linear-gradient(140deg, var(--accent), color-mix(in srgb, var(--accent) 45%, var(--info)));
|
||||
box-shadow: inset 0 0 0 1px rgba(255, 255, 255, .12);
|
||||
display: grid;
|
||||
place-items: center;
|
||||
font-size: 12px;
|
||||
color: #14171e;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
nav.tabs { display: flex; gap: .15rem; margin-inline-start: .4rem; }
|
||||
nav.tabs a {
|
||||
padding: .38rem .7rem;
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--text-muted);
|
||||
font-weight: 550;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: .4rem;
|
||||
}
|
||||
nav.tabs a:hover { background: var(--bg-sunken); color: var(--text); text-decoration: none; }
|
||||
nav.tabs a[aria-current="page"] { background: var(--accent-soft); color: var(--accent); }
|
||||
|
||||
.masthead-right { margin-inline-start: auto; display: flex; align-items: center; gap: .55rem; }
|
||||
|
||||
.clock { font-family: var(--mono); font-size: 12.5px; color: var(--text-muted); }
|
||||
@media (max-width: 900px) { .clock { display: none; } }
|
||||
|
||||
button.ghost {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text-muted);
|
||||
border-radius: var(--radius-sm);
|
||||
padding: .32rem .6rem;
|
||||
font: 550 12.5px var(--sans);
|
||||
cursor: pointer;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: .35rem;
|
||||
}
|
||||
button.ghost:hover { border-color: var(--border-strong); color: var(--text); background: var(--bg-sunken); }
|
||||
button.ghost[aria-pressed="true"] { border-color: var(--accent); color: var(--accent); background: var(--accent-soft); }
|
||||
|
||||
/* ---- layout ------------------------------------------------------------- */
|
||||
main { padding: 1.15rem 1.25rem 3rem; max-width: 1680px; margin: 0 auto; }
|
||||
|
||||
.page-head {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
gap: 1rem;
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
.page-head h1 { font-size: 1.15rem; margin: 0; font-weight: 650; letter-spacing: .01em; }
|
||||
.page-head .sub { color: var(--text-muted); font-size: 12.5px; }
|
||||
.page-head .spacer { margin-inline-start: auto; }
|
||||
|
||||
.grid { display: grid; gap: .85rem; }
|
||||
.grid.auto { grid-template-columns: repeat(auto-fit, minmax(min(360px, 100%), 1fr)); }
|
||||
.grid.cols-2 { grid-template-columns: repeat(auto-fit, minmax(min(480px, 100%), 1fr)); }
|
||||
.grid.cols-3 { grid-template-columns: repeat(auto-fit, minmax(min(300px, 100%), 1fr)); }
|
||||
.grid.wide { grid-template-columns: minmax(0, 2fr) minmax(0, 1fr); }
|
||||
@media (max-width: 1150px) { .grid.wide { grid-template-columns: 1fr; } }
|
||||
.stack { display: grid; gap: .85rem; align-content: start; }
|
||||
|
||||
/* ---- cards -------------------------------------------------------------- */
|
||||
.card {
|
||||
background: var(--bg-raised);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
box-shadow: var(--shadow);
|
||||
overflow: hidden;
|
||||
}
|
||||
.card > header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: .55rem;
|
||||
padding: .6rem .8rem;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: color-mix(in srgb, var(--bg-raised) 70%, var(--bg-sunken));
|
||||
}
|
||||
.card > header h2, .card > header h3 {
|
||||
margin: 0; font-size: 12.5px; font-weight: 650; letter-spacing: .04em; text-transform: uppercase;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
.card > header .spacer { margin-inline-start: auto; }
|
||||
.card > .body { padding: .8rem; }
|
||||
.card > .body.tight { padding: .45rem .8rem .6rem; }
|
||||
.card.flat { box-shadow: none; }
|
||||
|
||||
.cardset { display: grid; gap: .85rem; }
|
||||
|
||||
/* ---- metrics ------------------------------------------------------------ */
|
||||
.metrics { display: grid; gap: .7rem .9rem; grid-template-columns: repeat(auto-fit, minmax(104px, 1fr)); }
|
||||
.metric { min-width: 0; }
|
||||
.metric .label {
|
||||
font-size: 10.5px; text-transform: uppercase; letter-spacing: .06em; color: var(--text-faint);
|
||||
white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||||
}
|
||||
.metric .value {
|
||||
font-family: var(--mono); font-size: 1.3rem; font-weight: 600; line-height: 1.25;
|
||||
display: flex; align-items: baseline; gap: .22rem; row-gap: .1rem;
|
||||
/* the provenance chip has to be able to drop under the number: with nowrap it painted
|
||||
over the neighbouring column, which reads as a wrong value on the next metric */
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.metric .value > span, .metric .value > .unit { white-space: nowrap; }
|
||||
.metric .value .unit { font-size: .62em; color: var(--text-muted); font-weight: 500; }
|
||||
.metric .value.sm { font-size: 1.02rem; }
|
||||
.metric .note { font-size: 11px; color: var(--text-faint); }
|
||||
.metric .value.is-null { color: var(--text-faint); font-size: 1rem; }
|
||||
|
||||
.prov {
|
||||
font-family: var(--mono); font-size: 9.5px; letter-spacing: .02em;
|
||||
color: var(--text-faint); border: 1px solid var(--border); border-radius: 999px;
|
||||
padding: 0 .32rem; margin-inline-start: .28rem; vertical-align: 2px; white-space: nowrap;
|
||||
}
|
||||
|
||||
.na { color: var(--text-faint); }
|
||||
|
||||
/* ---- bars, dots, chips -------------------------------------------------- */
|
||||
.bar {
|
||||
position: relative; height: 7px; border-radius: 999px; background: var(--bg-sunken);
|
||||
overflow: hidden; border: 1px solid var(--border);
|
||||
}
|
||||
.bar > i {
|
||||
position: absolute; inset: 0 auto 0 0; width: 0; border-radius: 999px;
|
||||
background: var(--accent); transition: width .35s ease;
|
||||
}
|
||||
.bar.ok > i { background: var(--ok); }
|
||||
.bar.warn > i { background: var(--warn); }
|
||||
.bar.fail > i { background: var(--fail); }
|
||||
.bar.thin { height: 5px; }
|
||||
.bar.tall { height: 10px; }
|
||||
|
||||
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--text-faint); flex: none; }
|
||||
.dot.ok { background: var(--ok); box-shadow: 0 0 0 3px var(--ok-soft); }
|
||||
.dot.warn { background: var(--warn); box-shadow: 0 0 0 3px var(--warn-soft); }
|
||||
.dot.fail { background: var(--fail); box-shadow: 0 0 0 3px var(--fail-soft); }
|
||||
.dot.idle { background: var(--text-faint); }
|
||||
|
||||
.chip {
|
||||
display: inline-flex; align-items: center; gap: .3rem;
|
||||
font-size: 11px; font-weight: 600; letter-spacing: .02em;
|
||||
padding: .16rem .45rem; border-radius: 999px;
|
||||
background: var(--bg-sunken); border: 1px solid var(--border); color: var(--text-muted);
|
||||
white-space: nowrap;
|
||||
}
|
||||
.chip.ok { color: var(--ok); background: var(--ok-soft); border-color: color-mix(in srgb, var(--ok) 35%, transparent); }
|
||||
.chip.warn { color: var(--warn); background: var(--warn-soft); border-color: color-mix(in srgb, var(--warn) 35%, transparent); }
|
||||
.chip.fail { color: var(--fail); background: var(--fail-soft); border-color: color-mix(in srgb, var(--fail) 35%, transparent); }
|
||||
.chip.info { color: var(--info); background: var(--info-soft); border-color: color-mix(in srgb, var(--info) 35%, transparent); }
|
||||
.chip.accent { color: var(--accent); background: var(--accent-soft); border-color: color-mix(in srgb, var(--accent) 35%, transparent); }
|
||||
|
||||
.banner {
|
||||
display: flex; gap: .55rem; align-items: center;
|
||||
padding: .5rem .7rem; border-radius: var(--radius-sm);
|
||||
background: var(--bg-sunken); border: 1px solid var(--border);
|
||||
color: var(--text-muted); font-size: 12.5px;
|
||||
}
|
||||
.banner.warn { background: var(--warn-soft); border-color: color-mix(in srgb, var(--warn) 35%, transparent); color: var(--warn); }
|
||||
.banner.fail { background: var(--fail-soft); border-color: color-mix(in srgb, var(--fail) 35%, transparent); color: var(--fail); }
|
||||
.banner.ok { background: var(--ok-soft); border-color: color-mix(in srgb, var(--ok) 35%, transparent); color: var(--ok); }
|
||||
.banner.info { background: var(--info-soft); border-color: color-mix(in srgb, var(--info) 30%, transparent); color: var(--info); }
|
||||
.banner code { font-family: var(--mono); font-size: 11.5px; background: var(--bg-sunken); padding: 0 .25rem; border-radius: 3px; }
|
||||
|
||||
/* simulated data is watermarked everywhere it appears */
|
||||
.sim-badge {
|
||||
font: 700 10px/1 var(--mono); letter-spacing: .1em; text-transform: uppercase;
|
||||
color: var(--warn); background: var(--warn-soft);
|
||||
border: 1px dashed color-mix(in srgb, var(--warn) 55%, transparent);
|
||||
padding: .2rem .4rem; border-radius: 4px;
|
||||
}
|
||||
|
||||
/* ---- tables ------------------------------------------------------------- */
|
||||
table.data { width: 100%; border-collapse: collapse; font-size: 12.5px; }
|
||||
table.data th, table.data td { padding: .34rem .5rem; text-align: right; white-space: nowrap; }
|
||||
table.data th:first-child, table.data td:first-child { text-align: left; }
|
||||
table.data thead th {
|
||||
font-size: 10.5px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint);
|
||||
border-bottom: 1px solid var(--border); position: sticky; top: 0; background: var(--bg-raised);
|
||||
}
|
||||
table.data tbody tr + tr td { border-top: 1px solid color-mix(in srgb, var(--border) 55%, transparent); }
|
||||
table.data tbody tr:hover td { background: var(--bg-sunken); }
|
||||
table.data td.name { font-weight: 600; }
|
||||
table.data .num { font-family: var(--mono); }
|
||||
table.data .mute { color: var(--text-muted); }
|
||||
table.data.scroll { display: block; overflow: auto; max-height: 320px; }
|
||||
|
||||
/* ---- charts ------------------------------------------------------------- */
|
||||
.chart { position: relative; width: 100%; }
|
||||
.chart svg { display: block; width: 100%; height: 100%; overflow: visible; }
|
||||
.chart .axis text { fill: var(--text-faint); font-family: var(--mono); font-size: 10px; }
|
||||
.chart .axis line, .chart .grid line { stroke: var(--border); stroke-width: 1; }
|
||||
.chart .grid line { stroke-dasharray: 2 4; opacity: .75; }
|
||||
.chart .zero line { stroke: var(--border-strong); }
|
||||
.chart .line { fill: none; stroke-width: 1.75; stroke-linejoin: round; stroke-linecap: round; }
|
||||
.chart .area { opacity: .16; }
|
||||
.chart .cursor line { stroke: var(--text-faint); stroke-width: 1; stroke-dasharray: 3 3; }
|
||||
.chart .empty {
|
||||
position: absolute; inset: 0; display: grid; place-items: center;
|
||||
color: var(--text-faint); font-size: 12px; pointer-events: none;
|
||||
}
|
||||
.legend { display: flex; flex-wrap: wrap; gap: .35rem .8rem; font-size: 11.5px; color: var(--text-muted); }
|
||||
.legend .item { display: inline-flex; align-items: center; gap: .35rem; }
|
||||
.legend .swatch { width: 9px; height: 9px; border-radius: 2px; }
|
||||
.legend .lv { font-family: var(--mono); color: var(--text); }
|
||||
|
||||
.tooltip {
|
||||
position: absolute; z-index: 30; pointer-events: none; opacity: 0;
|
||||
background: color-mix(in srgb, var(--bg-raised) 95%, transparent);
|
||||
border: 1px solid var(--border-strong); border-radius: var(--radius-sm);
|
||||
box-shadow: var(--shadow); padding: .35rem .5rem; font-size: 11.5px; min-width: 118px;
|
||||
transition: opacity .12s ease;
|
||||
}
|
||||
.tooltip .ts { color: var(--text-faint); font-family: var(--mono); font-size: 10.5px; margin-bottom: .18rem; }
|
||||
.tooltip .row { display: flex; align-items: center; gap: .35rem; }
|
||||
.tooltip .row .val { margin-inline-start: auto; font-family: var(--mono); }
|
||||
|
||||
/* per-core usage heat bars */
|
||||
.cores { display: grid; grid-template-columns: repeat(auto-fill, minmax(9px, 1fr)); gap: 2px; }
|
||||
.cores .core { height: 22px; border-radius: 2px; background: var(--bg-sunken); position: relative; }
|
||||
.cores .core > i { position: absolute; bottom: 0; left: 0; right: 0; background: var(--accent); border-radius: 2px; }
|
||||
|
||||
/* ---- lists -------------------------------------------------------------- */
|
||||
ul.events { list-style: none; margin: 0; padding: 0; display: grid; gap: .1rem; }
|
||||
ul.events li {
|
||||
display: grid; grid-template-columns: 62px 1fr; gap: .55rem; align-items: baseline;
|
||||
padding: .3rem .1rem; border-bottom: 1px solid color-mix(in srgb, var(--border) 50%, transparent);
|
||||
font-size: 12.5px;
|
||||
}
|
||||
ul.events li:last-child { border-bottom: 0; }
|
||||
ul.events time { font-family: var(--mono); font-size: 11px; color: var(--text-faint); }
|
||||
ul.events .sev { font-weight: 600; }
|
||||
ul.events .sev.info { color: var(--text-muted); }
|
||||
ul.events .sev.warn { color: var(--warn); }
|
||||
ul.events .sev.error { color: var(--fail); }
|
||||
|
||||
ul.checks { list-style: none; margin: 0; padding: 0; display: grid; gap: .4rem; }
|
||||
ul.checks li { display: grid; grid-template-columns: 18px 1fr; gap: .5rem; align-items: start; }
|
||||
ul.checks .what { font-weight: 600; font-size: 12.5px; }
|
||||
ul.checks .detail { color: var(--text-muted); font-size: 12px; }
|
||||
ul.checks .mark { font-family: var(--mono); font-size: 12px; text-align: center; }
|
||||
ul.checks li.ok .mark { color: var(--ok); }
|
||||
ul.checks li.warn .mark { color: var(--warn); }
|
||||
ul.checks li.fail .mark { color: var(--fail); }
|
||||
ul.checks li.unknown .mark { color: var(--text-faint); }
|
||||
|
||||
.kv { display: grid; grid-template-columns: max-content 1fr; gap: .18rem .8rem; font-size: 12.5px; }
|
||||
.kv dt { color: var(--text-faint); }
|
||||
.kv dd { margin: 0; font-family: var(--mono); text-align: right; overflow-wrap: anywhere; }
|
||||
|
||||
/* ---- buttons ------------------------------------------------------------ */
|
||||
button.action {
|
||||
font: 600 12.5px var(--sans);
|
||||
padding: .42rem .7rem; border-radius: var(--radius-sm);
|
||||
border: 1px solid var(--border-strong); background: var(--bg-sunken); color: var(--text);
|
||||
cursor: pointer;
|
||||
}
|
||||
button.action:hover:not(:disabled) { border-color: var(--accent); color: var(--accent); }
|
||||
button.action:disabled { opacity: .5; cursor: not-allowed; }
|
||||
button.action.caution { border-color: color-mix(in srgb, var(--warn) 45%, var(--border-strong)); }
|
||||
button.action.dangerous { border-color: color-mix(in srgb, var(--fail) 45%, var(--border-strong)); color: var(--fail); }
|
||||
button.action.dangerous:hover:not(:disabled) { background: var(--fail-soft); border-color: var(--fail); color: var(--fail); }
|
||||
|
||||
fieldset.commands { border: 1px solid var(--border); border-radius: var(--radius-sm); padding: .6rem .7rem; margin: 0; }
|
||||
fieldset.commands legend { font-size: 11px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint); padding: 0 .3rem; }
|
||||
fieldset.commands .row { display: flex; flex-wrap: wrap; gap: .4rem; }
|
||||
|
||||
input[type="text"].confirm {
|
||||
font-family: var(--mono); background: var(--bg-sunken); color: var(--text);
|
||||
border: 1px solid var(--fail); border-radius: var(--radius-sm); padding: .35rem .5rem; width: 11ch;
|
||||
}
|
||||
|
||||
/* ---- dialog ------------------------------------------------------------- */
|
||||
dialog {
|
||||
border: 1px solid var(--border-strong); border-radius: var(--radius); background: var(--bg-raised);
|
||||
color: var(--text); padding: 0; max-width: 34rem; width: calc(100% - 2rem); box-shadow: var(--shadow);
|
||||
}
|
||||
dialog::backdrop { background: rgba(0, 0, 0, .55); }
|
||||
dialog h2 { margin: 0; font-size: 1rem; }
|
||||
dialog > header { padding: .8rem .9rem; border-bottom: 1px solid var(--border); display: flex; gap: .5rem; align-items: center; }
|
||||
dialog .body { padding: .9rem; font-size: 13px; color: var(--text-muted); }
|
||||
dialog footer { padding: .7rem .9rem; border-top: 1px solid var(--border); display: flex; gap: .5rem; justify-content: flex-end; }
|
||||
|
||||
/* ---- misc --------------------------------------------------------------- */
|
||||
.muted { color: var(--text-muted); }
|
||||
.faint { color: var(--text-faint); }
|
||||
.mono { font-family: var(--mono); }
|
||||
.nowrap { white-space: nowrap; }
|
||||
.right { text-align: right; }
|
||||
.center { text-align: center; }
|
||||
.gap { display: flex; gap: .5rem; align-items: center; flex-wrap: wrap; }
|
||||
/* `[hidden]` must beat any display set on the same element — .banner is display:flex, so a
|
||||
* hidden banner used to occupy a grey strip above the KPIs on every card. */
|
||||
[hidden] { display: none !important; }
|
||||
|
||||
.vh { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); }
|
||||
.wrap { overflow-wrap: anywhere; }
|
||||
|
||||
.empty-state { padding: 2rem 1rem; text-align: center; color: var(--text-muted); }
|
||||
.empty-state h3 { margin: 0 0 .4rem; font-size: 1rem; color: var(--text); }
|
||||
.empty-state code { font-family: var(--mono); font-size: 12px; background: var(--bg-sunken); padding: .1rem .3rem; border-radius: 4px; }
|
||||
.empty-state ul { text-align: left; display: inline-block; margin: .6rem 0 0; }
|
||||
|
||||
.stale { opacity: .62; }
|
||||
.stale-flag { color: var(--warn); font-size: 11px; }
|
||||
|
||||
select, input[type="text"] {
|
||||
background: var(--bg-sunken); color: var(--text); border: 1px solid var(--border);
|
||||
border-radius: var(--radius-sm); padding: .3rem .45rem; font: 500 12.5px var(--sans);
|
||||
}
|
||||
label.field { font-size: 11px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint); }
|
||||
|
||||
footer.pagefoot {
|
||||
margin-top: 1.6rem; padding-top: .8rem; border-top: 1px solid var(--border);
|
||||
color: var(--text-faint); font-size: 11.5px; display: flex; gap: 1rem; flex-wrap: wrap;
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
* { transition: none !important; animation: none !important; }
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
/* Overview page: one /api/summary call, plus the config-driven setup hints. */
|
||||
'use strict';
|
||||
|
||||
const { $, fetchJSON, setTxt, setValue, setBar, setDot, bytes, pct, num, duration, poll } = Dash;
|
||||
|
||||
async function loadConfig() {
|
||||
try {
|
||||
const config = await fetchJSON('/api/config');
|
||||
const banners = [];
|
||||
if (!config.system.enabled) {
|
||||
banners.push(['warn', 'System dashboard has no targets — set <code>HOST_TARGETS</code> ' +
|
||||
'(run <code>agent/host_agent.py</code> on each host first).']);
|
||||
}
|
||||
if (config.system.target_problems && config.system.target_problems.length) {
|
||||
banners.push(['fail', 'Bad HOST_TARGETS entries: ' +
|
||||
config.system.target_problems.map((p) => `<code>${DashCharts.escapeHtml(p)}</code>`).join(' ')]);
|
||||
}
|
||||
if (!config.llm.gufo.enabled && !config.llm.vllm.enabled) {
|
||||
banners.push(['warn', 'No LLM engines configured — set <code>GUFO_BASE_URL</code> and/or ' +
|
||||
'<code>VLLM_BASE_URL</code>.']);
|
||||
}
|
||||
if (!config.ups.enabled) {
|
||||
banners.push(['warn', 'No NUT server configured — set <code>NUT_HOST</code> (upsd must ' +
|
||||
'have a <code>LISTEN</code> line).']);
|
||||
} else if (!config.ups.actions_enabled) {
|
||||
banners.push(['info', 'UPS commands are off — set <code>NUT_ALLOW_ACTIONS=1</code> to enable them.']);
|
||||
} else if (!config.ups.auth && !config.ups.simulated) {
|
||||
// the simulator needs no credentials: asking for them there sends the operator fixing
|
||||
// something that is not broken
|
||||
banners.push(['info', 'UPS commands need credentials: set <code>NUT_USER</code>/' +
|
||||
'<code>NUT_PASSWORD</code> with <code>actions</code> in <code>upsd.users</code>.']);
|
||||
}
|
||||
if (config.ups.simulated) {
|
||||
banners.push(['warn', '<strong>SIMULATED UPS DATA.</strong> <code>NUT_SIMULATE=1</code> is set: ' +
|
||||
'the UPS numbers are a development model, not your batteries.']);
|
||||
}
|
||||
const host = $('#banners');
|
||||
host.innerHTML = banners.map(([kind, html]) =>
|
||||
`<div class="banner ${kind}">${html}</div>`).join('');
|
||||
$('#foot-versions').textContent =
|
||||
`dashboards ${config.app.version} · poll system ${config.system.interval_seconds}s · ` +
|
||||
`llm ${config.llm.interval_seconds}s · ups ${config.ups.interval_seconds}s · ` +
|
||||
`uptime ${duration(config.app.started_at ? Date.now() / 1000 - config.app.started_at : null)}` +
|
||||
(config.app.read_only ? ' · read-only' : '');
|
||||
return config;
|
||||
} catch (err) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function renderEvents(events) {
|
||||
const list = $('#events');
|
||||
if (!events || !events.length) {
|
||||
list.innerHTML = '<li class="muted" style="grid-template-columns:1fr">Nothing has changed ' +
|
||||
'since this server started — the log only records transitions.</li>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = events.map((event) => {
|
||||
const when = new Date(event.ts * 1000).toLocaleTimeString(undefined,
|
||||
{ hour: '2-digit', minute: '2-digit', hour12: false });
|
||||
return `<li><time>${when}</time><span><span class="sev ${event.severity}">${event.host}</span> ` +
|
||||
`${DashCharts.escapeHtml(event.message.replace(event.host + ':', '').trim() || event.kind)}</span></li>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function renderSummary(summary) {
|
||||
const system = summary.system || {};
|
||||
const fleet = system.fleet || {};
|
||||
setDot($('#dot-system'), system.enabled ? (fleet.hosts_live === fleet.hosts_total ? 'ok'
|
||||
: fleet.hosts_live ? 'warn' : 'fail') : 'idle');
|
||||
setTxt($('#sys-hosts-live'), fleet.hosts_live == null ? 'n/a' : String(fleet.hosts_live));
|
||||
setTxt($('#sys-hosts-total'), fleet.hosts_total == null ? '' : `of ${fleet.hosts_total}`);
|
||||
setValue($('#sys-cpu'), fleet.cpu_usage_pct_avg, 0);
|
||||
setBar($('#sys-cpu').closest('.metric').querySelector('.bar'), fleet.cpu_usage_pct_avg);
|
||||
if (fleet.memory_used_bytes != null) {
|
||||
const used = Dash.fmtBytes(fleet.memory_used_bytes, 0);
|
||||
const total = Dash.fmtBytes(fleet.memory_total_bytes, 0);
|
||||
// "200 / 253 GiB", not "200 GiB of 253 GiB": the card column is narrow and the long form
|
||||
// ran into the worst-disk number next to it
|
||||
setTxt($('#sys-mem'), total ? `${used.text}/${total.text}` : used.text);
|
||||
setTxt($('#sys-mem-unit'), used.unit);
|
||||
} else {
|
||||
setTxt($('#sys-mem'), 'n/a');
|
||||
setTxt($('#sys-mem-unit'), '');
|
||||
}
|
||||
setBar($('#sys-mem').closest('.metric').querySelector('.bar'), fleet.memory_used_pct);
|
||||
setValue($('#sys-disk'), fleet.disk_worst_used_pct, 0);
|
||||
setBar($('#sys-disk').closest('.metric').querySelector('.bar'), fleet.disk_worst_used_pct,
|
||||
Dash.severityFor(fleet.disk_worst_used_pct, 80, 92));
|
||||
setTxt($('#sys-gpus'), fleet.gpu_count == null ? 'n/a'
|
||||
: `${fleet.gpu_count}${fleet.gpu_busy_pct_avg != null ? ' · ' + num(fleet.gpu_busy_pct_avg, 0) + '%' : ''}`);
|
||||
|
||||
const llm = summary.llm || {};
|
||||
const engines = llm.fleet || {};
|
||||
setDot($('#dot-llm'), !llm.enabled ? 'idle'
|
||||
: (engines.engines_live === engines.engines_total ? 'ok' : engines.engines_live ? 'warn' : 'fail'));
|
||||
setTxt($('#llm-live'), engines.engines_live == null ? 'n/a' : String(engines.engines_live));
|
||||
setTxt($('#llm-total'), engines.engines_total == null ? '' : `of ${engines.engines_total}`);
|
||||
setValue($('#llm-decode'), engines.decode_tokens_per_second, 1);
|
||||
setValue($('#llm-prompt'), engines.prompt_tokens_per_second, 0);
|
||||
setTxt($('#llm-active'), engines.requests_active == null ? 'n/a' : num(engines.requests_active, 0));
|
||||
setTxt($('#llm-waiting'), engines.requests_waiting == null ? 'n/a' : num(engines.requests_waiting, 0));
|
||||
setValue($('#llm-kv'), engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100, 0);
|
||||
setBar($('#llm-kv').closest('.metric').querySelector('.bar'),
|
||||
engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100,
|
||||
Dash.severityFor(engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100, 75, 90));
|
||||
|
||||
const ups = summary.ups || {};
|
||||
setDot($('#dot-ups'), !ups.enabled ? 'idle'
|
||||
: (ups.ups || []).some((u) => u.overall === 'at risk') ? 'fail'
|
||||
: (ups.ups || []).some((u) => u.overall !== 'healthy') ? 'warn' : 'ok');
|
||||
const list = $('#ups-list');
|
||||
if (!ups.enabled) {
|
||||
list.innerHTML = '<div class="muted">No NUT server configured.</div>';
|
||||
} else if (!(ups.ups || []).length) {
|
||||
list.innerHTML = '<div class="muted">No UPS answering from this upsd yet.</div>';
|
||||
} else {
|
||||
list.innerHTML = ups.ups.map((item) => {
|
||||
const kind = item.overall === 'healthy' ? 'ok' : item.overall === 'at risk' ? 'fail'
|
||||
: item.overall === 'degraded' ? 'warn' : '';
|
||||
return `<div class="gap" style="justify-content:space-between">
|
||||
<span class="gap"><span class="chip ${kind}">${DashCharts.escapeHtml(item.name)}</span>
|
||||
<span class="mono faint">${DashCharts.escapeHtml(item.status || 'no state')}</span>
|
||||
${ups.simulated ? '<span class="sim-badge">sim</span>' : ''}</span>
|
||||
<span class="mono">${item.charge_pct == null ? 'n/a' : num(item.charge_pct, 0) + '%'} ·
|
||||
${item.load_pct == null ? 'n/a' : num(item.load_pct, 0) + '% load'} ·
|
||||
${item.runtime_seconds == null ? 'n/a' : duration(item.runtime_seconds) + ' runtime'}</span>
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
const rows = system.hosts || [];
|
||||
const tbody = $('#fleet-table tbody');
|
||||
$('#fleet-empty').hidden = rows.length > 0;
|
||||
if (!rows.length) { tbody.innerHTML = ''; return; }
|
||||
tbody.innerHTML = rows.map((host) => {
|
||||
const gpus = (host.gpu_busy_pct || []).filter((v) => v != null);
|
||||
return `<tr>
|
||||
<td class="name">${DashCharts.escapeHtml(host.name)}</td>
|
||||
<td class="num">${host.cpu_usage_pct == null ? 'n/a' : num(host.cpu_usage_pct, 0) + '%'}</td>
|
||||
<td class="num mute">${host.load1 == null ? 'n/a' : num(host.load1, 2)}</td>
|
||||
<td class="num">${host.mem_used_pct == null ? 'n/a' : num(host.mem_used_pct, 0) + '%'}</td>
|
||||
<td class="num">${host.disk_worst_used_pct == null ? 'n/a' : num(host.disk_worst_used_pct, 0) + '%'}</td>
|
||||
<td class="num">${gpus.length ? num(gpus[0], 0) + '%' : '<span class="mute">—</span>'}</td>
|
||||
<td class="num">${host.temp_c == null ? '<span class="mute">n/a</span>' : num(host.temp_c, 0) + '°C'}</td>
|
||||
<td class="num mute">${host.uptime_seconds == null ? 'n/a' : duration(host.uptime_seconds)}</td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
try {
|
||||
const summary = await fetchJSON('/api/summary');
|
||||
const total = (summary.system.hosts || []).length;
|
||||
const live = summary.system.fleet ? summary.system.fleet.hosts_live : 0;
|
||||
const engines = summary.llm.fleet ? `${summary.llm.fleet.engines_live}/${summary.llm.fleet.engines_total} engines` : 'no engines';
|
||||
const ups = summary.ups.ups ? `${summary.ups.ups.length} UPS` : 'no UPS';
|
||||
$('#summary-line').textContent = `${live}/${total} hosts · ${engines} · ${ups} · ` +
|
||||
`${(summary.recent_events || []).length} recent events`;
|
||||
renderSummary(summary);
|
||||
renderEvents(summary.recent_events);
|
||||
} catch (err) {
|
||||
$('#summary-line').textContent = `summary failed: ${err.message}`;
|
||||
}
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', async () => {
|
||||
await loadConfig();
|
||||
await refresh();
|
||||
poll(refresh, 10000);
|
||||
});
|
||||
@@ -0,0 +1,382 @@
|
||||
/* LLM dashboard.
|
||||
*
|
||||
* Every throughput cell shows where its number came from, because "78 tok/s" means
|
||||
* different things depending on whether it was scraped from an aggregate gauge, measured as
|
||||
* Δtokens/Δt across all slots, or estimated from TPOT over the process lifetime. The tag is
|
||||
* generated from the cell's `source`, never hardcoded in markup, so a parser change cannot
|
||||
* leave a stale label behind.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, pct, duration, poll,
|
||||
severityFor } = Dash;
|
||||
|
||||
const state = { range: 3600, panels: new Map(), charts: {}, poller: null, pollMs: 2000 };
|
||||
|
||||
const CELLS = [
|
||||
{ key: 'decode_tokens_per_second', label: 'Decode', unit: 'tok/s', digits: 1,
|
||||
tip: 'Aggregate generation speed across every busy slot.' },
|
||||
{ key: 'prompt_tokens_per_second', label: 'Prompt', unit: 'tok/s', digits: 1,
|
||||
tip: 'Prefill speed for newly computed tokens.' },
|
||||
{ key: 'sessions', label: 'Sessions', unit: 'active', digits: 0 },
|
||||
{ key: 'queued', label: 'Queued', unit: 'waiting', digits: 0 },
|
||||
{ key: 'cache', label: 'KV cache', unit: '%', digits: 0, bar: true },
|
||||
{ key: 'ttft_mean', label: 'TTFT mean', unit: 'ms', digits: 0 },
|
||||
{ key: 'ttft_p95', label: 'TTFT p95', unit: 'ms', digits: 0 },
|
||||
{ key: 'tpot_mean', label: 'TPOT mean', unit: 'ms', digits: 1 },
|
||||
{ key: 'queue_mean', label: 'Queue mean', unit: 'ms', digits: 0 },
|
||||
{ key: 'cache_hit', label: 'Prefix cache hit', unit: '%', digits: 0 },
|
||||
{ key: 'spec_accept', label: 'Spec accept', unit: '%', digits: 0 },
|
||||
{ key: 'tokens_total', label: 'Tokens lifetime', unit: '', digits: 0, human: true },
|
||||
{ key: 'preemptions', label: 'Preemptions', unit: '', digits: 0 },
|
||||
{ key: 'device_lost', label: 'GPU context lost', unit: '', digits: 0 },
|
||||
];
|
||||
|
||||
function cellValue(engine, metrics, sessions, key) {
|
||||
const get = (k) => (metrics[k] && metrics[k].value != null ? metrics[k] : null);
|
||||
switch (key) {
|
||||
case 'sessions': {
|
||||
const value = sessions && sessions.active != null ? sessions.active : null;
|
||||
return { value, source: 'session count', derived: false };
|
||||
}
|
||||
case 'queued': {
|
||||
const cell = get('requests_deferred') || get('num_requests_waiting');
|
||||
return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'cache': {
|
||||
const cell = get('kv_cache_usage_ratio') || get('gpu_cache_usage');
|
||||
return cell ? { value: cell.value * 100, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'ttft_mean': {
|
||||
const cell = get('time_to_first_token_seconds__mean');
|
||||
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'ttft_p95': {
|
||||
const cell = get('time_to_first_token_seconds__p95');
|
||||
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'tpot_mean': {
|
||||
const cell = get('time_per_output_token_seconds__mean');
|
||||
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'queue_mean': {
|
||||
const cell = get('request_queue_time_seconds__mean');
|
||||
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'cache_hit': {
|
||||
const cell = get('prefix_cache_hit_ratio') || get('prompt_cache_hit_ratio');
|
||||
return cell ? { value: cell.value * 100, source: cell.source, derived: true } : { value: null };
|
||||
}
|
||||
case 'spec_accept': {
|
||||
const cell = get('spec_accept_rate');
|
||||
return cell ? { value: cell.value * 100, source: cell.source, derived: true } : { value: null };
|
||||
}
|
||||
case 'tokens_total': {
|
||||
const cell = get('tokens_total');
|
||||
return cell ? { value: cell.value, source: cell.source, derived: true } : { value: null };
|
||||
}
|
||||
case 'preemptions': {
|
||||
const cell = get('num_preemptions_total');
|
||||
return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'device_lost': {
|
||||
const cell = get('device_lost_total');
|
||||
return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
default: {
|
||||
const cell = get(key);
|
||||
return cell ? { value: cell.value, source: cell.source, derived: cell.derived,
|
||||
window: cell.window } : { value: null };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The provenance tag: Δ10.0s for measured rates, the formula for estimates, nothing for a
|
||||
* plain scraped gauge (which would only add noise). */
|
||||
function provenance(cell) {
|
||||
if (!cell || !cell.source) return '';
|
||||
if (!String(cell.source).startsWith('computed:')) return '';
|
||||
if (cell.window) return `Δ${cell.window.toFixed(1)}s`;
|
||||
const text = String(cell.source).replace('computed:', '');
|
||||
return text.length > 16 ? text.slice(0, 15) + '…' : text;
|
||||
}
|
||||
|
||||
function humanNumber(value) {
|
||||
if (value == null || !isFinite(value)) return null;
|
||||
if (value >= 1e9) return { text: (value / 1e9).toFixed(2), unit: 'B tok' };
|
||||
if (value >= 1e6) return { text: (value / 1e6).toFixed(2), unit: 'M tok' };
|
||||
if (value >= 1e3) return { text: (value / 1e3).toFixed(1), unit: 'k tok' };
|
||||
return { text: value.toFixed(0), unit: 'tok' };
|
||||
}
|
||||
|
||||
function enginePanel(engine) {
|
||||
const card = document.createElement('article');
|
||||
card.className = 'card';
|
||||
card.dataset.engine = engine;
|
||||
card.innerHTML = `
|
||||
<header>
|
||||
<span class="dot idle"></span>
|
||||
<h3>${DashCharts.escapeHtml(engine)}</h3>
|
||||
<span class="chip upstream mono" hidden></span>
|
||||
<span class="spacer"></span>
|
||||
<span class="faint mono updated"></span>
|
||||
</header>
|
||||
<div class="body stack">
|
||||
<div class="banner err" hidden></div>
|
||||
<div class="metrics">${CELLS.map((cell) => `
|
||||
<div class="metric" data-cell="${cell.key}" ${cell.tip ? `title="${cell.tip}"` : ''}>
|
||||
<div class="label">${cell.label}</div>
|
||||
<div class="value${cell.digits === 0 ? '' : ''}"><span data-v>n/a</span><span class="unit">${cell.unit}</span></div>
|
||||
${cell.bar ? '<div class="bar" style="margin-top:.25rem"><i></i></div>' : ''}
|
||||
</div>`).join('')}</div>
|
||||
<div class="faint mono sessions-note" style="font-size:11.5px"></div>
|
||||
<div class="faint mono debug" style="font-size:11px"></div>
|
||||
</div>`;
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderPanel(card, enginePayload, stateRow) {
|
||||
const metrics = enginePayload.metrics || {};
|
||||
const sessions = enginePayload.sessions || {};
|
||||
const q = (sel) => card.querySelector(sel);
|
||||
|
||||
const dot = q('.dot');
|
||||
setDot(dot, stateRow && stateRow.ok ? (Dash.ageSeconds(stateRow.last_ok) > 15 ? 'warn' : 'ok') : 'fail');
|
||||
setTxt(q('.updated'), stateRow ? (stateRow.last_ok == null
|
||||
? (stateRow.error || 'never reached')
|
||||
: (Dash.ageSeconds(stateRow.last_ok) > 15 ? `stale ${duration(Dash.ageSeconds(stateRow.last_ok))}`
|
||||
: `updated ${duration(Dash.ageSeconds(stateRow.last_ok))}`)) : '—');
|
||||
Dash.markStale(card, stateRow && (!stateRow.ok || Dash.ageSeconds(stateRow.last_ok) > 15));
|
||||
|
||||
const banner = q('.banner.err');
|
||||
if (stateRow && !stateRow.ok) {
|
||||
banner.hidden = false;
|
||||
banner.className = 'banner fail err';
|
||||
banner.textContent = stateRow.error || 'engine is not answering';
|
||||
} else {
|
||||
banner.hidden = true;
|
||||
}
|
||||
|
||||
for (const cell of CELLS) {
|
||||
const wrapper = q(`[data-cell="${cell.key}"]`);
|
||||
if (!wrapper) continue;
|
||||
const value = cellValue(enginePayload.engine, metrics, sessions, cell.key);
|
||||
const valueEl = wrapper.querySelector('[data-v]');
|
||||
const unitEl = wrapper.querySelector('.unit');
|
||||
if (value.value == null) {
|
||||
setTxt(valueEl, 'n/a');
|
||||
setTxt(unitEl, ''); // "n/a %" reads like a measurement of nothing
|
||||
wrapper.classList.add('is-null');
|
||||
} else {
|
||||
wrapper.classList.remove('is-null');
|
||||
if (cell.human) {
|
||||
const parts = humanNumber(value.value);
|
||||
setTxt(valueEl, parts ? parts.text : 'n/a');
|
||||
setTxt(unitEl, parts ? parts.unit : '');
|
||||
} else {
|
||||
setTxt(valueEl, num(value.value, cell.digits));
|
||||
setTxt(unitEl, cell.unit);
|
||||
}
|
||||
}
|
||||
let tag = wrapper.querySelector('.prov');
|
||||
const label = provenance(value);
|
||||
if (label) {
|
||||
if (!tag) {
|
||||
tag = document.createElement('span');
|
||||
tag.className = 'prov';
|
||||
wrapper.querySelector('.value').appendChild(tag);
|
||||
}
|
||||
setTxt(tag, label);
|
||||
tag.title = `${value.source || ''}${value.window ? ` · measured over ${value.window.toFixed(1)}s` : ''}`;
|
||||
} else if (tag) {
|
||||
tag.remove();
|
||||
}
|
||||
if (cell.bar) setBar(wrapper.querySelector('.bar'), value.value,
|
||||
severityFor(value.value, 75, 90));
|
||||
if (cell.key === 'device_lost' && value.value != null && value.value > 0) {
|
||||
wrapper.querySelector('.value').classList.add('is-null');
|
||||
wrapper.classList.add('fail');
|
||||
wrapper.title = `GPU context has been lost ${value.value} times — an incident, not a stat`;
|
||||
}
|
||||
}
|
||||
|
||||
const note = q('.sessions-note');
|
||||
setTxt(note, sessions.detail || `${sessions.active == null ? 'n/a' : sessions.active} active sessions`);
|
||||
const upstream = q('.chip.upstream');
|
||||
if (enginePayload.upstream) {
|
||||
upstream.hidden = false;
|
||||
upstream.textContent = enginePayload.upstream.replace(/^https?:\/\//, '');
|
||||
upstream.title = enginePayload.upstream;
|
||||
}
|
||||
setTxt(q('.debug'), enginePayload.unparsed != null
|
||||
? `${enginePayload.series_count} series · ${enginePayload.unparsed} unparsed`
|
||||
: '');
|
||||
}
|
||||
|
||||
function renderSlots(payloads) {
|
||||
const gufo = payloads.find((p) => p.engine === 'gufo');
|
||||
const rows = gufo && gufo.sessions ? gufo.sessions.list || [] : [];
|
||||
const tbody = $('#slots-table tbody');
|
||||
$('#slots-empty').hidden = rows.length > 0;
|
||||
$('#slots-card').hidden = !gufo;
|
||||
tbody.innerHTML = rows.map((slot) => {
|
||||
const id = slot.id != null ? slot.id : slot.slot != null ? slot.slot : '—';
|
||||
const stateText = slot.state || (slot.idle === true ? 'idle' : slot.idle === false ? 'busy' : '—');
|
||||
const withText = (slot.with_messages && slot.with_messages.length) || slot.with_message;
|
||||
const nPrompt = slot.n_prompt != null ? slot.n_prompt : (slot.prompt_tokens != null ? slot.prompt_tokens : null);
|
||||
const nPredict = slot.n_predict != null ? slot.n_predict : (slot.generated_tokens != null ? slot.generated_tokens : null);
|
||||
const cache = slot.cache != null ? slot.cache : slot.cache_usable;
|
||||
const speed = slot.decode_per_second || slot.tokens_per_second || null;
|
||||
const cell = (value, digits) => `<td class="num">${value == null ? '<span class="mute">—</span>' : num(value, digits == null ? 0 : digits)}</td>`;
|
||||
return `<tr>
|
||||
<td class="name mono" style="font-size:11.5px">${DashCharts.escapeHtml(id)}</td>
|
||||
<td>${stateText === 'busy' || stateText === 'processing' ? '<span class="chip ok">busy</span>'
|
||||
: stateText === 'idle' ? '<span class="chip">idle</span>'
|
||||
: `<span class="chip warn">${DashCharts.escapeHtml(stateText)}</span>`}</td>
|
||||
<td>${withText ? '<span class="chip info">has msg</span>' : '<span class="mute">—</span>'}</td>
|
||||
${cell(nPrompt)}${cell(nPredict)}${cell(cache)}${cell(speed, 1)}</tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function makeCharts() {
|
||||
const common = { height: 165, xSpan: state.range };
|
||||
state.charts.decode = new DashCharts.LineChart('#chart-decode', Object.assign({}, common, { yMin: 0 }));
|
||||
state.charts.prompt = new DashCharts.LineChart('#chart-prompt', Object.assign({}, common, { yMin: 0 }));
|
||||
state.charts.kv = new DashCharts.LineChart('#chart-kv', Object.assign({}, common,
|
||||
{ yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
|
||||
state.charts.queue = new DashCharts.LineChart('#chart-queue', Object.assign({}, common,
|
||||
{ yMin: 0, yFormat: (v) => num(v, 0) }));
|
||||
}
|
||||
|
||||
function pointsFor(response, engine, metric) {
|
||||
const found = (response.series || []).find((s) => s.host === engine && s.metric === `llm.${metric}`);
|
||||
return found ? found.points.map((p) => ({ ts: p.ts, value: p.value })) : [];
|
||||
}
|
||||
|
||||
async function refreshCharts(engineNames) {
|
||||
if (!engineNames.length) return;
|
||||
const list = engineNames.join(',');
|
||||
const [decode, prompt, queue] = await Promise.all([
|
||||
fetchJSON(`/api/llm/history?engines=${list}&metrics=decode_tokens_per_second&range=${state.range}`)
|
||||
.catch(() => null),
|
||||
fetchJSON(`/api/llm/history?engines=${list}&metrics=prompt_tokens_per_second&range=${state.range}`)
|
||||
.catch(() => null),
|
||||
fetchJSON(`/api/llm/history?engines=${list}&metrics=requests_running,requests_waiting,` +
|
||||
`kv_cache_usage&range=${state.range}`).catch(() => null),
|
||||
]);
|
||||
if (decode) {
|
||||
const series = engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(decode, engine, 'decode_tokens_per_second') }));
|
||||
state.charts.decode.setData(series);
|
||||
DashCharts.renderLegend($('#legend-decode'), series, (v) => num(v, 1) + ' tok/s', true);
|
||||
}
|
||||
if (prompt) {
|
||||
const series = engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(prompt, engine, 'prompt_tokens_per_second') }));
|
||||
state.charts.prompt.setData(series);
|
||||
DashCharts.renderLegend($('#legend-prompt'), series, (v) => num(v, 0) + ' tok/s', true);
|
||||
}
|
||||
if (queue) {
|
||||
const series = [];
|
||||
for (const engine of engineNames) {
|
||||
series.push({ name: `${engine} running`, points: pointsFor(queue, engine, 'requests_running') });
|
||||
series.push({ name: `${engine} waiting`, points: pointsFor(queue, engine, 'requests_waiting') });
|
||||
}
|
||||
state.charts.queue.setData(series);
|
||||
DashCharts.renderLegend($('#legend-queue'), series, (v) => num(v, 0));
|
||||
const kv = engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(queue, engine, 'kv_cache_usage').map((p) => ({ ts: p.ts, value: p.value * 100 })) }));
|
||||
state.charts.kv.setData(kv);
|
||||
DashCharts.renderLegend($('#legend-kv'), kv, (v) => num(v, 1) + '%', true);
|
||||
}
|
||||
}
|
||||
|
||||
function renderFleet(fleet) {
|
||||
if (!fleet) return;
|
||||
setTxt($('#f-live'), fleet.engines_live == null ? 'n/a' : String(fleet.engines_live));
|
||||
setTxt($('#f-total'), fleet.engines_total == null ? '' : `of ${fleet.engines_total}`);
|
||||
setValue($('#f-decode'), fleet.decode_tokens_per_second, 1);
|
||||
setValue($('#f-prompt'), fleet.prompt_tokens_per_second, 0);
|
||||
setTxt($('#f-active'), fleet.requests_active == null ? 'n/a' : num(fleet.requests_active, 0));
|
||||
setTxt($('#f-waiting'), fleet.requests_waiting == null ? 'n/a' : num(fleet.requests_waiting, 0));
|
||||
const tokens = humanNumber(fleet.tokens_total);
|
||||
setTxt($('#f-tokens'), tokens ? tokens.text : 'n/a');
|
||||
$('#f-tokens').nextElementSibling && setTxt($('#f-tokens').nextElementSibling, tokens ? tokens.unit : '');
|
||||
const kv = fleet.kv_cache_usage_max == null ? null : fleet.kv_cache_usage_max * 100;
|
||||
setValue($('#f-kv'), kv, 0);
|
||||
setBar($('#f-kv').closest('.metric').querySelector('.bar'), kv, severityFor(kv, 75, 90));
|
||||
$('#seam-note').textContent = fleet.seams || '';
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
let snapshot;
|
||||
try {
|
||||
snapshot = await fetchJSON('/api/llm/snapshot');
|
||||
} catch (err) {
|
||||
$('#sub').textContent = err.message;
|
||||
return;
|
||||
}
|
||||
const rows = {};
|
||||
(snapshot.states || []).forEach((row) => { rows[row.label.toLowerCase()] = row; });
|
||||
const payloads = snapshot.engines || [];
|
||||
const names = payloads.map((p) => p.engine);
|
||||
payloads.forEach((payload) => {
|
||||
let card = state.panels.get(payload.engine);
|
||||
if (!card) {
|
||||
card = enginePanel(payload.engine);
|
||||
state.panels.set(payload.engine, card);
|
||||
$('#engines').appendChild(card);
|
||||
}
|
||||
renderPanel(card, payload, rows[payload.engine] ||
|
||||
(snapshot.states || []).find((row) => row.key.endsWith(payload.engine)));
|
||||
});
|
||||
$('#engines-empty').hidden = payloads.length > 0;
|
||||
renderFleet(snapshot.fleet);
|
||||
renderSlots(payloads);
|
||||
const fleet = snapshot.fleet || {};
|
||||
$('#sub').textContent = `${fleet.engines_live || 0}/${fleet.engines_total || 0} engines · ` +
|
||||
`${fleet.requests_active == null ? 'n/a' : num(fleet.requests_active, 0)} active · ` +
|
||||
`window ${duration(state.range)}`;
|
||||
$('#foot-note').textContent = `poll ${Math.round(state.pollMs / 1000)}s · rates over the ` +
|
||||
`Δ window in each tag`;
|
||||
await refreshCharts(names);
|
||||
}
|
||||
|
||||
async function refreshEvents() {
|
||||
try {
|
||||
const data = await fetchJSON('/api/events?limit=20&source=llm');
|
||||
const list = $('#events');
|
||||
if (!(data.events || []).length) {
|
||||
list.innerHTML = '<li class="muted" style="grid-template-columns:1fr">No engine transitions yet.</li>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = data.events.map((event) => `<li><time>${
|
||||
new Date(event.ts * 1000).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit', hour12: false })
|
||||
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
|
||||
`${DashCharts.escapeHtml(event.message)}</span></li>`).join('');
|
||||
} catch (err) { /* next poll */ }
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', async () => {
|
||||
makeCharts();
|
||||
try {
|
||||
const config = await fetchJSON('/api/config');
|
||||
state.pollMs = config.llm.poll_ms || 2000;
|
||||
const banners = [];
|
||||
if (!config.llm.gufo.enabled && !config.llm.vllm.enabled) {
|
||||
banners.push(['warn', 'Neither engine is configured.']);
|
||||
}
|
||||
if (config.ups.simulated) banners.push(['warn', 'UPS numbers on other pages are simulated.']);
|
||||
$('#banners').innerHTML = banners.map(([kind, html]) => `<div class="banner ${kind}">${html}</div>`)
|
||||
.join('');
|
||||
} catch (err) { /* not fatal */ }
|
||||
|
||||
Dash.rangeControl($('#range'), state.range, (seconds) => {
|
||||
state.range = seconds;
|
||||
Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = seconds; });
|
||||
refresh();
|
||||
});
|
||||
|
||||
state.poller = poll(async () => { await refresh(); await refreshEvents(); }, state.pollMs);
|
||||
Dash.bindPause(state.poller);
|
||||
});
|
||||
@@ -0,0 +1,503 @@
|
||||
/* System dashboard page.
|
||||
*
|
||||
* Host cards are created once and patched in place (keyed by host name) — rebuilding the
|
||||
* DOM every 10s kills hover, transitions, and any scroll position inside the tables.
|
||||
* Charts get their history from /api/system/history: one request per chart, N hosts each.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const { $, $$, fetchJSON, setTxt, setValue, setBar, setDot, bytes, pct, num, duration, poll,
|
||||
staleness, markStale, severityFor } = Dash;
|
||||
|
||||
const state = {
|
||||
range: 3600,
|
||||
detailHost: null,
|
||||
detailMetric: 'cpu.usage_pct',
|
||||
hosts: new Map(), // name -> card element
|
||||
charts: {},
|
||||
poller: null,
|
||||
pollMs: 10000,
|
||||
metricList: [],
|
||||
};
|
||||
|
||||
const DETAIL_METRICS = [
|
||||
['cpu.usage_pct', 'CPU utilisation', 'pct'],
|
||||
['cpu.user_pct', 'CPU user', 'pct'],
|
||||
['cpu.system_pct', 'CPU system', 'pct'],
|
||||
['cpu.iowait_pct', 'CPU iowait', 'pct'],
|
||||
['cpu.steal_pct', 'CPU steal', 'pct'],
|
||||
['cpu.load1', 'Load average (1m)', 'num'],
|
||||
['cpu.load15', 'Load average (15m)', 'num'],
|
||||
['cpu.freq_mhz', 'CPU frequency', 'mhz'],
|
||||
['cpu.context_switches_per_s', 'Context switches', 'num'],
|
||||
['mem.used_pct', 'Memory used', 'pct'],
|
||||
['mem.used_bytes', 'Memory used', 'bytes'],
|
||||
['mem.available_bytes', 'Memory available', 'bytes'],
|
||||
['mem.swap_used_pct', 'Swap used', 'pct'],
|
||||
['disk.used_pct', 'Filesystem used (worst)', 'pct'],
|
||||
['disk.read_bytes_per_s', 'Disk read', 'bytes'],
|
||||
['disk.write_bytes_per_s', 'Disk write', 'bytes'],
|
||||
['disk.util_pct', 'Disk busy', 'pct'],
|
||||
['net.rx_bytes_per_s', 'Network in', 'bytes'],
|
||||
['net.tx_bytes_per_s', 'Network out', 'bytes'],
|
||||
['temp.cpu_c', 'CPU temperature', 'temp'],
|
||||
['temp.nvme_max_c', 'NVMe temperature', 'temp'],
|
||||
['host.uptime_seconds', 'Uptime', 'dur'],
|
||||
['host.processes', 'Process count', 'num'],
|
||||
];
|
||||
|
||||
const FORMATS = {
|
||||
pct: { fn: (v) => num(v, 1), unit: '%', axis: (v) => num(v, 0) },
|
||||
bytes: { fn: (v) => bytes(v, 1), unit: 'B/s', axis: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text +
|
||||
((Dash.fmtBytes(v, 0) || { unit: '' }).unit) },
|
||||
mhz: { fn: (v) => num(v, 0) + ' MHz', unit: 'MHz', axis: (v) => num(v, 0) },
|
||||
temp: { fn: (v) => num(v, 1) + ' °C', unit: '°C', axis: (v) => num(v, 0) },
|
||||
dur: { fn: (v) => duration(v), unit: '', axis: (v) => duration(v) },
|
||||
num: { fn: (v) => num(v, 1), unit: '', axis: (v) => num(v, 0) },
|
||||
};
|
||||
|
||||
function formatFor(kind) { return FORMATS[kind] || FORMATS.num; }
|
||||
|
||||
function metricKind(name) {
|
||||
if (name.endsWith('_pct')) return 'pct';
|
||||
if (name.endsWith('_bytes') || name.includes('bytes_per_s')) return 'bytes';
|
||||
if (name.includes('_c') && name.startsWith('temp.')) return 'temp';
|
||||
if (name === 'host.uptime_seconds') return 'dur';
|
||||
return 'num';
|
||||
}
|
||||
|
||||
// ---- host cards -----------------------------------------------------------
|
||||
function hostCard(name) {
|
||||
const card = document.createElement('article');
|
||||
card.className = 'card';
|
||||
card.dataset.host = name;
|
||||
card.innerHTML = `
|
||||
<header>
|
||||
<span class="dot idle"></span>
|
||||
<h3></h3>
|
||||
<span class="chip os" hidden></span>
|
||||
<span class="spacer"></span>
|
||||
<span class="faint mono updated"></span>
|
||||
</header>
|
||||
<div class="body stack">
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="label">CPU</div><div class="value"><span data-k="cpu">n/a</span><span class="unit">%</span></div>
|
||||
<div class="bar" data-bar="cpu"><i></i></div></div>
|
||||
<div class="metric"><div class="label">Load 1 / 15</div><div class="value sm" data-k="load">n/a</div></div>
|
||||
<div class="metric"><div class="label">CPU temp</div><div class="value sm"><span data-k="temp">n/a</span><span class="unit">°C</span></div></div>
|
||||
<div class="metric"><div class="label">Clock</div><div class="value sm"><span data-k="freq">n/a</span><span class="unit">GHz</span></div></div>
|
||||
<div class="metric"><div class="label">Memory</div><div class="value sm"><span data-k="mem">n/a</span><span class="unit"></span></div>
|
||||
<div class="bar" data-bar="mem"><i></i></div></div>
|
||||
<div class="metric"><div class="label">Worst fs</div><div class="value sm"><span data-k="disk">n/a</span><span class="unit"></span></div>
|
||||
<div class="bar" data-bar="disk"><i></i></div></div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="label faint" style="font-size:10.5px;letter-spacing:.06em;text-transform:uppercase">Per-core</div>
|
||||
<div class="cores" data-k="cores" style="margin-top:.25rem"></div>
|
||||
</div>
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="label">Disk read</div><div class="value sm"><span data-k="dr">n/a</span><span class="unit"></span></div></div>
|
||||
<div class="metric"><div class="label">Disk write</div><div class="value sm"><span data-k="dw">n/a</span><span class="unit"></span></div></div>
|
||||
<div class="metric"><div class="label">Disk busy</div><div class="value sm"><span data-k="dutil">n/a</span><span class="unit">%</span></div></div>
|
||||
<div class="metric"><div class="label">Net in</div><div class="value sm"><span data-k="rx">n/a</span><span class="unit"></span></div></div>
|
||||
<div class="metric"><div class="label">Net out</div><div class="value sm"><span data-k="tx">n/a</span><span class="unit"></span></div></div>
|
||||
<div class="metric"><div class="label">Up</div><div class="value sm" data-k="uptime">n/a</div></div>
|
||||
</div>
|
||||
<div data-k="gpus"></div>
|
||||
<details>
|
||||
<summary class="faint" style="cursor:pointer;font-size:12px">Filesystems & processes</summary>
|
||||
<div style="margin-top:.6rem" data-k="details"></div>
|
||||
</details>
|
||||
</div>`;
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderHostCard(card, host) {
|
||||
const q = (key) => card.querySelector(`[data-k="${key}"]`);
|
||||
const stateRow = host.state || {};
|
||||
card.querySelector('h3').textContent = host.name;
|
||||
const osChip = card.querySelector('.os');
|
||||
const osName = (host.info || {}).os_pretty || (host.info || {}).os_id;
|
||||
if (osName) { osChip.hidden = false; osChip.textContent = osName; }
|
||||
setDot(card.querySelector('.dot'), stateRow.ok === false ? 'fail'
|
||||
: (Dash.ageSeconds(stateRow.last_ok) > 3 * (host.window || 10) ? 'warn' : 'ok'));
|
||||
setTxt(card.querySelector('.updated'), stalenessText(stateRow));
|
||||
markStale(card, stateRow.ok === false || Dash.ageSeconds(stateRow.last_ok) > 5 * (host.window || 10));
|
||||
|
||||
const cpu = host.cpu || {};
|
||||
setValue(q('cpu'), cpu.usage_pct, 0);
|
||||
setBar(card.querySelector('[data-bar="cpu"]'), cpu.usage_pct, severityFor(cpu.usage_pct, 75, 90));
|
||||
setTxt(q('load'), cpu.load1 == null ? 'n/a' : `${num(cpu.load1, 2)} / ${num(cpu.load15, 2)}`);
|
||||
setValue(q('temp'), cpu.temp_c, 1);
|
||||
setValue(q('freq'), cpu.freq_mhz == null ? null : cpu.freq_mhz / 1000, 2);
|
||||
|
||||
const mem = host.memory || {};
|
||||
const memText = Dash.fmtBytes(mem.used_bytes, 1);
|
||||
const memTotal = Dash.fmtBytes(mem.total_bytes, 0);
|
||||
setTxt(q('mem'), memText ? memText.text : 'n/a');
|
||||
setTxt(q('mem').nextElementSibling, memText ? `${memText.unit}/${memTotal ? memTotal.text + memTotal.unit : ''}` : '');
|
||||
setBar(card.querySelector('[data-bar="mem"]'), mem.used_pct, severityFor(mem.used_pct, 80, 92));
|
||||
|
||||
const disks = host.disks || {};
|
||||
setTxt(q('disk'), disks.worst_used_pct == null ? 'n/a' : num(disks.worst_used_pct, 0) + '%');
|
||||
setTxt(q('disk').nextElementSibling, disks.worst_mount ? ' ' + disks.worst_mount : '');
|
||||
setBar(card.querySelector('[data-bar="disk"]'), disks.worst_used_pct, severityFor(disks.worst_used_pct, 80, 92));
|
||||
|
||||
DashCharts.renderCores(q('cores'), cpu.core_usage_pct || []);
|
||||
|
||||
setValue(q('dr'), disks.read_bytes_per_s == null ? null : disks.read_bytes_per_s, null);
|
||||
if (disks.read_bytes_per_s != null) {
|
||||
const parts = Dash.fmtBytes(disks.read_bytes_per_s, 1);
|
||||
setTxt(q('dr'), parts.text); setTxt(q('dr').nextElementSibling, parts.unit + '/s');
|
||||
const w = Dash.fmtBytes(disks.write_bytes_per_s, 1);
|
||||
setTxt(q('dw'), w ? w.text : 'n/a'); setTxt(q('dw').nextElementSibling, w ? w.unit + '/s' : '');
|
||||
} else { setTxt(q('dr'), 'n/a'); setTxt(q('dw'), 'n/a'); }
|
||||
setValue(q('dutil'), disks.util_pct, 0);
|
||||
const rx = Dash.fmtBytes((host.network || {}).rx_bytes_per_s, 1);
|
||||
const tx = Dash.fmtBytes((host.network || {}).tx_bytes_per_s, 1);
|
||||
setTxt(q('rx'), rx ? rx.text : 'n/a'); setTxt(q('rx').nextElementSibling, rx ? rx.unit + '/s' : '');
|
||||
setTxt(q('tx'), tx ? tx.text : 'n/a'); setTxt(q('tx').nextElementSibling, tx ? tx.unit + '/s' : '');
|
||||
setTxt(q('uptime'), duration((host.info || {}).uptime_seconds));
|
||||
|
||||
// GPUs: one row each, with both memory pools named (an APU's model lives in GTT)
|
||||
const gpus = host.gpus || [];
|
||||
const gpuHost = q('gpus');
|
||||
if (!gpus.length) {
|
||||
gpuHost.innerHTML = '<div class="faint" style="font-size:12px">No GPU reported by this agent</div>';
|
||||
} else {
|
||||
gpuHost.innerHTML = gpus.map((gpu) => `
|
||||
<div class="stack" style="gap:.2rem;margin-top:.35rem">
|
||||
<div class="gap" style="justify-content:space-between">
|
||||
<span class="mono" style="font-size:12px">${DashCharts.escapeHtml(gpu.model || 'GPU')}</span>
|
||||
<span class="mono" style="font-size:12px">
|
||||
${gpu.busy_pct == null ? 'n/a' : num(gpu.busy_pct, 0) + '% busy'} ·
|
||||
${gpu.power_watts == null ? 'power n/a' : num(gpu.power_watts, 0) + ' W'} ·
|
||||
${gpu.temp_c == null ? 'temp n/a' : num(gpu.temp_c, 0) + '°C'}</span>
|
||||
</div>
|
||||
<div class="bar thin"><i style="width:${(gpu.busy_pct || 0).toFixed(1)}%"></i></div>
|
||||
<div class="gap" style="justify-content:space-between;font-size:11.5px" class="faint">
|
||||
<span class="faint">VRAM ${gpu.vram_used_bytes == null ? 'n/a' : bytes(gpu.vram_used_bytes, 0)}
|
||||
${gpu.vram_total_bytes ? 'of ' + bytes(gpu.vram_total_bytes, 0) : ''}</span>
|
||||
<span class="faint">shared (GTT) ${gpu.gtt_used_bytes == null ? 'n/a' : bytes(gpu.gtt_used_bytes, 0)}
|
||||
${gpu.mem_used_pct == null ? '' : ' · pool ' + num(gpu.mem_used_pct, 0) + '%'}</span>
|
||||
</div>
|
||||
<div class="bar thin"><i style="width:${(gpu.mem_used_pct || 0).toFixed(1)}%;background:var(--series-2)"></i></div>
|
||||
</div>`).join('');
|
||||
}
|
||||
|
||||
const filesystems = (disks.filesystems || []).filter((fs) => fs.used_pct != null);
|
||||
const procs = host.processes || {};
|
||||
const details = q('details');
|
||||
details.innerHTML = `
|
||||
<table class="data">
|
||||
<thead><tr><th>Mount</th><th>Type</th><th>Used</th><th>Free</th><th>Total</th></tr></thead>
|
||||
<tbody>${filesystems.map((fs) => `<tr>
|
||||
<td class="name mono" style="font-size:11.5px">${DashCharts.escapeHtml(fs.mountpoint)}</td>
|
||||
<td class="mute">${DashCharts.escapeHtml(fs.fstype)}</td>
|
||||
<td class="num">${num(fs.used_pct, 1)}%</td>
|
||||
<td class="num mute">${bytes(fs.free_bytes, 0)}</td>
|
||||
<td class="num mute">${bytes(fs.total_bytes, 0)}</td></tr>`).join('') ||
|
||||
'<tr><td colspan="5" class="mute">no filesystems reported</td></tr>'}</tbody>
|
||||
</table>
|
||||
<div class="grid cols-2" style="margin-top:.6rem">
|
||||
<div><div class="faint" style="font-size:11px;margin-bottom:.2rem">Top by CPU</div>
|
||||
<table class="data"><tbody>${(procs.top_cpu || []).slice(0, 5).map((p) => `<tr>
|
||||
<td class="mono" style="font-size:11.5px">${DashCharts.escapeHtml(p.name)}</td>
|
||||
<td class="num">${num(p.cpu_pct, 0)}%</td><td class="num mute">pid ${p.pid}</td></tr>`).join('')
|
||||
|| '<tr><td class="mute">needs two samples</td></tr>'}</tbody></table></div>
|
||||
<div><div class="faint" style="font-size:11px;margin-bottom:.2rem">Top by RSS</div>
|
||||
<table class="data"><tbody>${(procs.top_rss || []).slice(0, 5).map((p) => `<tr>
|
||||
<td class="mono" style="font-size:11.5px">${DashCharts.escapeHtml(p.name)}</td>
|
||||
<td class="num">${bytes(p.rss_bytes, 0)}</td><td class="num mute">pid ${p.pid}</td></tr>`).join('')
|
||||
|| '<tr><td class="mute">—</td></tr>'}</tbody></table></div>
|
||||
</div>
|
||||
${(host.agent && host.agent.errors && host.agent.errors.length)
|
||||
? `<div class="banner warn" style="margin-top:.6rem">agent degraded: ${DashCharts.escapeHtml(host.agent.errors.join('; '))}</div>`
|
||||
: ''}
|
||||
${(host.degraded && host.degraded.length)
|
||||
? `<div class="banner warn" style="margin-top:.4rem">collector degraded: ${DashCharts.escapeHtml(host.degraded.join('; '))}</div>`
|
||||
: ''}`;
|
||||
}
|
||||
|
||||
function stalenessText(row) {
|
||||
if (!row || !row.configured) return 'not configured';
|
||||
if (row.last_ok == null) return row.error || 'never reached';
|
||||
const seconds = Dash.ageSeconds(row.last_ok);
|
||||
return seconds > 30 ? `stale ${duration(seconds)}` : `updated ${seconds < 2 ? 'now' : duration(seconds)}`;
|
||||
}
|
||||
|
||||
// ---- charts ---------------------------------------------------------------
|
||||
function makeCharts() {
|
||||
const common = { height: 165, xSpan: state.range };
|
||||
state.charts.cpu = new DashCharts.LineChart('#chart-cpu', Object.assign({}, common, {
|
||||
yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
|
||||
state.charts.mem = new DashCharts.LineChart('#chart-mem', Object.assign({}, common, {
|
||||
yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
|
||||
state.charts.disk = new DashCharts.LineChart('#chart-disk', Object.assign({}, common, {
|
||||
yMin: 0, yFormat: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text }));
|
||||
state.charts.net = new DashCharts.LineChart('#chart-net', Object.assign({}, common, {
|
||||
yMin: 0, yFormat: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text }));
|
||||
state.charts.gpu = new DashCharts.LineChart('#chart-gpu', Object.assign({}, common, {
|
||||
yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
|
||||
state.charts.temp = new DashCharts.LineChart('#chart-temp', Object.assign({}, common, {
|
||||
yFormat: (v) => num(v, 0) }));
|
||||
state.charts.detail = new DashCharts.LineChart('#chart-detail', Object.assign({}, common, {
|
||||
height: 200, yFormat: (v) => num(v, 1) }));
|
||||
}
|
||||
|
||||
function setChartSpan(span) {
|
||||
Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = span; });
|
||||
}
|
||||
|
||||
function seriesFor(response, metric) {
|
||||
return (response.series || []).filter((s) => s.metric === metric).map((s) => ({
|
||||
name: s.host, points: s.points.map((p) => ({ ts: p.ts, value: p.value })),
|
||||
}));
|
||||
}
|
||||
|
||||
async function refreshCharts(snapshot) {
|
||||
const hostNames = (snapshot.hosts || []).map((h) => h.name);
|
||||
const detailHost = state.detailHost || hostNames[0] || null;
|
||||
const span = state.range;
|
||||
const requests = {
|
||||
cpu: fetchJSON(`/api/system/history?metrics=cpu.usage_pct&range=${span}`).catch(() => null),
|
||||
mem: fetchJSON(`/api/system/history?metrics=mem.used_pct&range=${span}`).catch(() => null),
|
||||
temp: fetchJSON(`/api/system/history?metrics=temp.cpu_c&range=${span}`).catch(() => null),
|
||||
disk: detailHost ? fetchJSON(`/api/system/history?hosts=${encodeURIComponent(detailHost)}&` +
|
||||
`metrics=disk.read_bytes_per_s,disk.write_bytes_per_s&range=${span}`).catch(() => null) : null,
|
||||
net: detailHost ? fetchJSON(`/api/system/history?hosts=${encodeURIComponent(detailHost)}&` +
|
||||
`metrics=net.rx_bytes_per_s,net.tx_bytes_per_s&range=${span}`).catch(() => null) : null,
|
||||
};
|
||||
const [cpu, mem, temp, disk, net] = await Promise.all([requests.cpu, requests.mem,
|
||||
requests.temp, requests.disk, requests.net]);
|
||||
|
||||
if (cpu) {
|
||||
const series = seriesFor(cpu, 'cpu.usage_pct');
|
||||
state.charts.cpu.setData(series);
|
||||
DashCharts.renderLegend($('#legend-cpu'), series, (v) => num(v, 1) + '%', true);
|
||||
}
|
||||
if (mem) {
|
||||
const series = seriesFor(mem, 'mem.used_pct');
|
||||
state.charts.mem.setData(series);
|
||||
DashCharts.renderLegend($('#legend-mem'), series, (v) => num(v, 1) + '%');
|
||||
}
|
||||
if (temp) {
|
||||
const series = seriesFor(temp, 'temp.cpu_c');
|
||||
state.charts.temp.setData(series);
|
||||
DashCharts.renderLegend($('#legend-temp'), series, (v) => num(v, 1) + '°C', true);
|
||||
}
|
||||
if (disk) {
|
||||
const read = (disk.series.find((s) => s.metric === 'disk.read_bytes_per_s') || { points: [] })
|
||||
.points.map((p) => ({ ts: p.ts, value: p.value }));
|
||||
const write = (disk.series.find((s) => s.metric === 'disk.write_bytes_per_s') || { points: [] })
|
||||
.points.map((p) => ({ ts: p.ts, value: p.value }));
|
||||
const series = [{ name: 'read', points: read }, { name: 'write', points: write }];
|
||||
state.charts.disk.setData(series);
|
||||
DashCharts.renderLegend($('#legend-disk'), series, (v) => bytes(v, 1) + '/s', true);
|
||||
}
|
||||
if (net) {
|
||||
const rx = (net.series.find((s) => s.metric === 'net.rx_bytes_per_s') || { points: [] })
|
||||
.points.map((p) => ({ ts: p.ts, value: p.value }));
|
||||
const tx = (net.series.find((s) => s.metric === 'net.tx_bytes_per_s') || { points: [] })
|
||||
.points.map((p) => ({ ts: p.ts, value: p.value }));
|
||||
const series = [{ name: 'in', points: rx }, { name: 'out', points: tx }];
|
||||
state.charts.net.setData(series);
|
||||
DashCharts.renderLegend($('#legend-net'), series, (v) => bytes(v, 1) + '/s', true);
|
||||
}
|
||||
await refreshGpuChart(hostNames);
|
||||
await refreshDetailChart();
|
||||
}
|
||||
|
||||
async function refreshGpuChart(hostNames) {
|
||||
if (!hostNames.length) {
|
||||
state.charts.gpu.setData([]);
|
||||
$('#legend-gpu').innerHTML = '<span class="faint">no hosts</span>';
|
||||
return;
|
||||
}
|
||||
const series = [];
|
||||
for (const host of hostNames) {
|
||||
// per-stream names are discovered, not guessed: gpu.busy_pct@0, @1, ...
|
||||
const found = await fetchJSON(`/api/metrics?source=host&host=${encodeURIComponent(host)}`)
|
||||
.catch(() => ({ metrics: [] }));
|
||||
const names = (found.metrics || []).filter((m) => m.startsWith('gpu.busy_pct@'));
|
||||
for (const name of names) {
|
||||
const data = await fetchJSON(`/api/history?source=host&host=${encodeURIComponent(host)}` +
|
||||
`&metric=${encodeURIComponent(name)}&range=${state.range}`).catch(() => null);
|
||||
if (data) series.push({ name: `${host} ${name.split('@')[1]}`,
|
||||
points: data.points.map((p) => ({ ts: p.ts, value: p.value })) });
|
||||
}
|
||||
}
|
||||
state.charts.gpu.setData(series);
|
||||
DashCharts.renderLegend($('#legend-gpu'), series, (v) => num(v, 0) + '%', true);
|
||||
}
|
||||
|
||||
async function refreshDetailChart() {
|
||||
const host = state.detailHost;
|
||||
if (!host) { state.charts.detail.setData([]); return; }
|
||||
const kind = metricKind(state.detailMetric);
|
||||
const format = formatFor(kind);
|
||||
const data = await fetchJSON(`/api/history?source=host&host=${encodeURIComponent(host)}` +
|
||||
`&metric=${encodeURIComponent(state.detailMetric)}&range=${state.range}`).catch(() => null);
|
||||
if (!data) return;
|
||||
const series = [{ name: state.detailMetric,
|
||||
points: data.points.map((p) => ({ ts: p.ts, value: p.value })) }];
|
||||
state.charts.detail.opts.yFormat = format.axis;
|
||||
state.charts.detail.opts.yMin = state.detailMetric.endsWith('_pct') ? 0 : null;
|
||||
state.charts.detail.opts.yMax = state.detailMetric.endsWith('_pct') ? 100 : null;
|
||||
state.charts.detail.setData(series);
|
||||
DashCharts.renderLegend($('#legend-detail'), series, format.fn);
|
||||
const last = series[0].points.slice(-1)[0];
|
||||
$('#detail-note').textContent = `${series[0].points.length} samples over ${duration(state.range)} · ` +
|
||||
`last ${last ? format.fn(last.value) : 'n/a'} · ${state.range <= 86400 ? 'raw resolution' : 'rolled up'}`;
|
||||
}
|
||||
|
||||
async function populateDetailMetrics(host) {
|
||||
const select = $('#detail-metric');
|
||||
const found = await fetchJSON(`/api/metrics?source=host&host=${encodeURIComponent(host)}`)
|
||||
.catch(() => ({ metrics: [] }));
|
||||
const curated = new Set(DETAIL_METRICS.map(([name]) => name));
|
||||
const extras = (found.metrics || []).filter((m) => !curated.has(m)).sort();
|
||||
select.innerHTML = '';
|
||||
const groupMain = document.createElement('optgroup');
|
||||
groupMain.label = 'Common';
|
||||
DETAIL_METRICS.forEach(([name, label]) => {
|
||||
const option = document.createElement('option');
|
||||
option.value = name; option.textContent = label;
|
||||
groupMain.appendChild(option);
|
||||
});
|
||||
select.appendChild(groupMain);
|
||||
if (extras.length) {
|
||||
const groupExtra = document.createElement('optgroup');
|
||||
groupExtra.label = `Per-device (${extras.length})`;
|
||||
extras.forEach((name) => {
|
||||
const option = document.createElement('option');
|
||||
option.value = name; option.textContent = name;
|
||||
groupExtra.appendChild(option);
|
||||
});
|
||||
select.appendChild(groupExtra);
|
||||
}
|
||||
select.value = DETAIL_METRICS.some(([name]) => name === state.detailMetric)
|
||||
? state.detailMetric : (extras[0] || DETAIL_METRICS[0][0]);
|
||||
state.detailMetric = select.value || DETAIL_METRICS[0][0];
|
||||
}
|
||||
|
||||
// ---- snapshot -------------------------------------------------------------
|
||||
function renderFleet(fleet) {
|
||||
if (!fleet) return;
|
||||
setTxt($('#f-hosts'), fleet.hosts_live == null ? 'n/a' : String(fleet.hosts_live));
|
||||
setTxt($('#f-hosts-of'), fleet.hosts_total == null ? '' : `of ${fleet.hosts_total}`);
|
||||
setValue($('#f-cpu'), fleet.cpu_usage_pct_avg, 0);
|
||||
setValue($('#f-load'), fleet.load1_total, 2);
|
||||
const used = Dash.fmtBytes(fleet.memory_used_bytes, 0);
|
||||
const total = Dash.fmtBytes(fleet.memory_total_bytes, 0);
|
||||
setTxt($('#f-mem'), used ? used.text : 'n/a');
|
||||
setTxt($('#f-mem-of'), used && total ? `${used.unit} of ${total.text} ${total.unit}` : '');
|
||||
setValue($('#f-disk'), fleet.disk_worst_used_pct, 0);
|
||||
const rx = Dash.fmtBytes(fleet.net_rx_bytes_per_s, 1);
|
||||
const tx = Dash.fmtBytes(fleet.net_tx_bytes_per_s, 1);
|
||||
setTxt($('#f-rx'), rx ? rx.text : 'n/a'); setTxt($('#f-rx-unit'), rx ? rx.unit + '/s' : '');
|
||||
setTxt($('#f-tx'), tx ? tx.text : 'n/a'); setTxt($('#f-tx-unit'), tx ? tx.unit + '/s' : '');
|
||||
setValue($('#f-gpu'), fleet.gpu_busy_pct_avg, 0);
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
let snapshot;
|
||||
try {
|
||||
snapshot = await fetchJSON('/api/system/snapshot');
|
||||
} catch (err) {
|
||||
$('#sub').textContent = err.message;
|
||||
return;
|
||||
}
|
||||
const states = {};
|
||||
(snapshot.states || []).forEach((row) => { states[row.key.split(':')[1]] = row; });
|
||||
const hostNames = [];
|
||||
(snapshot.hosts || []).forEach((host) => {
|
||||
hostNames.push(host.name);
|
||||
host.state = states[host.name];
|
||||
host.window = snapshot.window_seconds;
|
||||
let card = state.hosts.get(host.name);
|
||||
if (!card) {
|
||||
card = hostCard(host.name);
|
||||
state.hosts.set(host.name, card);
|
||||
$('#hosts').appendChild(card);
|
||||
card.addEventListener('click', () => {
|
||||
if (state.detailHost !== host.name) {
|
||||
state.detailHost = host.name;
|
||||
$('#detail-host').value = host.name;
|
||||
populateDetailMetrics(host.name).then(refreshDetailChart);
|
||||
}
|
||||
});
|
||||
}
|
||||
renderHostCard(card, host);
|
||||
});
|
||||
$('#hosts-empty').hidden = hostNames.length > 0;
|
||||
|
||||
const hostSelect = $('#detail-host');
|
||||
if (hostSelect.dataset.names !== hostNames.join(',')) {
|
||||
hostSelect.dataset.names = hostNames.join(',');
|
||||
hostSelect.innerHTML = hostNames.map((name) =>
|
||||
`<option value="${DashCharts.escapeHtml(name)}">${DashCharts.escapeHtml(name)}</option>`).join('');
|
||||
if (!state.detailHost || !hostNames.includes(state.detailHost)) {
|
||||
state.detailHost = hostNames[0] || null;
|
||||
}
|
||||
if (state.detailHost) hostSelect.value = state.detailHost;
|
||||
if (state.detailHost) await populateDetailMetrics(state.detailHost);
|
||||
}
|
||||
|
||||
renderFleet(snapshot.fleet);
|
||||
const fleet = snapshot.fleet || {};
|
||||
$('#sub').textContent = `${fleet.hosts_live || 0}/${fleet.hosts_total || 0} hosts answering · ` +
|
||||
`${fleet.cpu_cores || 0} cores · ${fleet.gpu_count || 0} GPUs · ` +
|
||||
`window ${duration(state.range)}`;
|
||||
$('#foot-note').textContent = `rates over Δ${snapshot.window_seconds}s · ` +
|
||||
`poll ${Math.round(state.pollMs / 1000)}s`;
|
||||
await refreshCharts(snapshot);
|
||||
}
|
||||
|
||||
async function refreshEvents() {
|
||||
try {
|
||||
const data = await fetchJSON('/api/events?limit=25&source=host');
|
||||
const list = $('#events');
|
||||
if (!(data.events || []).length) {
|
||||
list.innerHTML = '<li class="muted" style="grid-template-columns:1fr">No host transitions ' +
|
||||
'recorded yet — this log fills when a host goes away, comes back, or reboots.</li>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = data.events.map((event) => `<li><time>${
|
||||
new Date(event.ts * 1000).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit', hour12: false })
|
||||
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
|
||||
`${DashCharts.escapeHtml(event.message)}</span></li>`).join('');
|
||||
} catch (err) { /* the next poll will try again */ }
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', async () => {
|
||||
makeCharts();
|
||||
let pollMs = 10000;
|
||||
try {
|
||||
const config = await fetchJSON('/api/config');
|
||||
pollMs = config.system.poll_ms || 10000;
|
||||
if (!config.system.enabled) $('#sub').textContent = config.system.target_problems.length
|
||||
? `bad HOST_TARGETS: ${config.system.target_problems.join('; ')}`
|
||||
: 'no hosts configured — set HOST_TARGETS';
|
||||
} catch (err) { /* config is nice-to-have */ }
|
||||
state.pollMs = pollMs;
|
||||
|
||||
Dash.rangeControl($('#range'), state.range, (seconds) => {
|
||||
state.range = seconds;
|
||||
setChartSpan(seconds);
|
||||
refresh();
|
||||
});
|
||||
|
||||
$('#detail-host').addEventListener('change', async (event) => {
|
||||
state.detailHost = event.target.value;
|
||||
await populateDetailMetrics(state.detailHost);
|
||||
await refreshDetailChart();
|
||||
});
|
||||
$('#detail-metric').addEventListener('change', async (event) => {
|
||||
state.detailMetric = event.target.value;
|
||||
await refreshDetailChart();
|
||||
});
|
||||
|
||||
state.poller = poll(async () => { await refresh(); await refreshEvents(); }, state.pollMs);
|
||||
Dash.bindPause(state.poller);
|
||||
});
|
||||
@@ -0,0 +1,358 @@
|
||||
/* UPS dashboard: health assessment, NUT instant commands, history.
|
||||
*
|
||||
* The health list is deliberately explicit checks rather than a score: every row names the
|
||||
* variable it read and what it saw, because "health: 87" is not actionable. The backend
|
||||
* owns that logic (sources/nut.py::assess) so the same rules apply in tests and API clients.
|
||||
*
|
||||
* Commands are the only write path in the product. They are offered only when the backend
|
||||
* says `enabled`, and anything that can cut power requires typing DANGER here *and* the
|
||||
* server re-checks the same gate — a hidden button is not a safety control.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, duration, poll, severityFor } = Dash;
|
||||
|
||||
const state = { range: 86400, cards: new Map(), charts: {}, poller: null, pollMs: 5000,
|
||||
simulated: false, pending: null };
|
||||
|
||||
const MARKS = { ok: '✓', warn: '!', fail: '✕', unknown: '?' };
|
||||
|
||||
/**
|
||||
* Runtime left as "1h 32m", not "1.5h". The health check next to it spells the same number
|
||||
* out in h+m; two formats for one value invites a misread while the UPS is discharging.
|
||||
*/
|
||||
function runtimeText(seconds) {
|
||||
if (seconds == null || !isFinite(seconds)) return 'n/a';
|
||||
if (seconds < 0) return 'on battery';
|
||||
const total = Math.round(seconds);
|
||||
if (total < 120) return `${total}s`;
|
||||
const minutes = Math.round(total / 60);
|
||||
if (minutes < 60) return `${minutes}m`;
|
||||
return `${Math.floor(minutes / 60)}h ${String(minutes % 60).padStart(2, '0')}m`;
|
||||
}
|
||||
|
||||
function upsCard(name) {
|
||||
const card = document.createElement('article');
|
||||
card.className = 'card';
|
||||
card.dataset.ups = name;
|
||||
card.innerHTML = `
|
||||
<header>
|
||||
<span class="dot idle"></span>
|
||||
<h3></h3>
|
||||
<span class="chip model" hidden></span>
|
||||
<span class="sim-badge" hidden>simulated</span>
|
||||
<span class="spacer"></span>
|
||||
<span class="chip health">unknown</span>
|
||||
<span class="faint mono updated"></span>
|
||||
</header>
|
||||
<div class="body stack">
|
||||
<div class="gap states"></div>
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="label">Charge</div>
|
||||
<div class="value"><span data-k="charge">n/a</span><span class="unit">%</span></div>
|
||||
<div class="bar" data-bar="charge"><i></i></div></div>
|
||||
<div class="metric"><div class="label">Runtime left</div>
|
||||
<div class="value sm" data-k="runtime">n/a</div></div>
|
||||
<div class="metric"><div class="label">Load</div>
|
||||
<div class="value sm"><span data-k="load">n/a</span><span class="unit">%</span></div>
|
||||
<div class="bar" data-bar="load"><i></i></div></div>
|
||||
<div class="metric"><div class="label">Line in</div>
|
||||
<div class="value sm"><span data-k="line">n/a</span><span class="unit">V</span></div></div>
|
||||
<div class="metric"><div class="label">Out</div>
|
||||
<div class="value sm"><span data-k="out">n/a</span><span class="unit">V</span></div></div>
|
||||
<div class="metric"><div class="label">Battery</div>
|
||||
<div class="value sm"><span data-k="batt">n/a</span><span class="unit">V</span></div></div>
|
||||
<div class="metric"><div class="label">Battery A</div>
|
||||
<div class="value sm"><span data-k="amps">n/a</span><span class="unit">A</span></div></div>
|
||||
<div class="metric"><div class="label">Battery °C</div>
|
||||
<div class="value sm"><span data-k="temp">n/a</span><span class="unit">°C</span></div></div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="faint" style="font-size:10.5px;letter-spacing:.06em;text-transform:uppercase;margin-bottom:.3rem">
|
||||
Health checks</div>
|
||||
<ul class="checks" data-k="checks"></ul>
|
||||
</div>
|
||||
<div data-k="commands"></div>
|
||||
<details>
|
||||
<summary class="faint" style="cursor:pointer;font-size:12px">NUT variables</summary>
|
||||
<div class="body tight" data-k="raw" style="padding:0;margin-top:.5rem"></div>
|
||||
</details>
|
||||
</div>`;
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderCard(card, item, stateRow) {
|
||||
const q = (key) => card.querySelector(`[data-k="${key}"]`);
|
||||
card.querySelector('h3').textContent = item.name;
|
||||
const model = [item.identity && item.identity.mfr, item.identity && item.identity.model]
|
||||
.filter(Boolean).join(' ');
|
||||
const modelChip = card.querySelector('.model');
|
||||
if (model) { modelChip.hidden = false; modelChip.textContent = model; }
|
||||
card.querySelector('.sim-badge').hidden = !state.simulated;
|
||||
|
||||
const overall = (item.health && item.health.overall) || 'unknown';
|
||||
const healthChip = card.querySelector('.health');
|
||||
healthChip.textContent = overall;
|
||||
healthChip.className = 'chip health ' + ({ healthy: 'ok', degraded: 'warn', 'at risk': 'fail',
|
||||
unknown: '' }[overall] || '');
|
||||
const severity = stateRow && !stateRow.ok ? 'fail'
|
||||
: (item.health && item.health.severity) || 'idle';
|
||||
setDot(card.querySelector('.dot'), severity === 'ok' ? 'ok' : severity === 'warn' ? 'warn'
|
||||
: severity === 'fail' ? 'fail' : 'idle');
|
||||
setTxt(card.querySelector('.updated'), stateRow && stateRow.last_ok != null
|
||||
? (Dash.ageSeconds(stateRow.last_ok) > 30 ? `stale ${duration(Dash.ageSeconds(stateRow.last_ok))}`
|
||||
: `updated ${duration(Dash.ageSeconds(stateRow.last_ok))}`)
|
||||
: (stateRow && stateRow.error) || 'never reached');
|
||||
|
||||
const states = card.querySelector('.states');
|
||||
states.innerHTML = (item.states || []).map((s) => {
|
||||
const kind = s.severity === 'ok' ? 'ok' : s.severity === 'warn' ? 'warn'
|
||||
: s.severity === 'fail' ? 'fail' : 'info';
|
||||
return `<span class="chip ${kind}" title="${DashCharts.escapeHtml(s.detail || s.label)}">${
|
||||
DashCharts.escapeHtml(s.label)}</span>`;
|
||||
}).join('') + (item.status ? `<span class="chip mono" title="raw ups.status">${
|
||||
DashCharts.escapeHtml(item.status)}</span>` : '') +
|
||||
(item.alarms || []).map((alarm) => `<span class="chip warn">alarm: ${
|
||||
DashCharts.escapeHtml(alarm)}</span>`).join('');
|
||||
|
||||
const n = item.numbers || {};
|
||||
const cell = (key) => q(key);
|
||||
setValue(cell('charge'), n.charge_pct, 0);
|
||||
setBar(card.querySelector('[data-bar="charge"]'), n.charge_pct,
|
||||
n.charge_pct == null ? '' : n.charge_pct >= 95 ? 'ok' : n.charge_pct >= 60 ? 'warn' : 'fail');
|
||||
setTxt(cell('runtime'), runtimeText(n.battery_runtime));
|
||||
setValue(cell('load'), n.load_pct, 1);
|
||||
setBar(card.querySelector('[data-bar="load"]'), n.load_pct, severityFor(n.load_pct, 70, 90));
|
||||
setValue(cell('line'), n.line_voltage, 1);
|
||||
setValue(cell('out'), n.output_voltage, 1);
|
||||
setValue(cell('batt'), n.battery_voltage, 2);
|
||||
setValue(cell('amps'), n.battery_current, 2);
|
||||
setValue(cell('temp'), n.battery_temperature, 1);
|
||||
|
||||
const health = item.health || { checks: [] };
|
||||
q('checks').innerHTML = (health.checks || []).map((check) => `
|
||||
<li class="${check.status}">
|
||||
<span class="mark" title="${check.status}">${MARKS[check.status] || '?'}</span>
|
||||
<span><span class="what">${DashCharts.escapeHtml(check.name)}</span><br>
|
||||
<span class="detail">${DashCharts.escapeHtml(check.detail)}</span></span>
|
||||
</li>`).join('') || '<li class="unknown"><span class="mark">?</span><span>No checks available</span></li>';
|
||||
|
||||
const commands = item.commands || [];
|
||||
const groups = [['safe', 'Diagnostics'], ['caution', 'Affects the load briefly'],
|
||||
['dangerous', 'Can cut power']];
|
||||
const host = q('commands');
|
||||
host.innerHTML = groups.map(([risk, legend]) => {
|
||||
const inGroup = commands.filter((command) => command.risk === risk);
|
||||
if (!inGroup.length) return '';
|
||||
return `<fieldset class="commands" style="margin-top:.5rem"><legend>${legend}</legend>
|
||||
<div class="row">${inGroup.map((command) => `
|
||||
<button class="action ${risk}" type="button" data-command="${command.command}"
|
||||
${command.enabled ? '' : 'disabled'}
|
||||
title="${DashCharts.escapeHtml(command.enabled ? command.help : command.reason)}">${
|
||||
DashCharts.escapeHtml(command.label)}</button>`).join('')}</div></fieldset>`;
|
||||
}).join('') || '<div class="faint" style="font-size:12px;margin-top:.4rem">' +
|
||||
'The driver advertises no commands we know how to send.</div>';
|
||||
host.querySelectorAll('button[data-command]').forEach((button) => {
|
||||
button.addEventListener('click', () => openCommandDialog(item.name, button.dataset.command,
|
||||
commands));
|
||||
});
|
||||
|
||||
const extra = item.extra || {};
|
||||
const entries = Object.entries(extra);
|
||||
q('raw').innerHTML = `<table class="data"><tbody>${entries.map(([key, value]) =>
|
||||
`<tr><td class="mono" style="font-size:11px">${DashCharts.escapeHtml(key)}</td>
|
||||
<td class="mono" style="font-size:11px">${DashCharts.escapeHtml(value)}</td></tr>`).join('')
|
||||
|| '<tr><td class="mute">no extra variables</td></tr>'}</tbody></table>` +
|
||||
`<div class="faint mono" style="font-size:11px;margin-top:.4rem">driver ${
|
||||
DashCharts.escapeHtml((item.identity || {}).driver || '?')} · firmware ${
|
||||
DashCharts.escapeHtml((item.identity || {}).firmware || '?')} · serial ${
|
||||
DashCharts.escapeHtml((item.identity || {}).serial || '?')} · ${
|
||||
(item.advertised_commands || []).length} commands advertised</div>`;
|
||||
}
|
||||
|
||||
// ---- command dialog -------------------------------------------------------
|
||||
function openCommandDialog(upsName, commandName, commands) {
|
||||
const command = commands.find((c) => c.command === commandName);
|
||||
if (!command) return;
|
||||
const dialog = $('#confirm-dialog');
|
||||
const dangerous = command.risk === 'dangerous';
|
||||
$('#dialog-title').textContent = command.label;
|
||||
$('#dialog-help').textContent = command.help;
|
||||
$('#dialog-target').textContent = `UPS ${upsName} · ${command.command} · risk: ${command.risk}` +
|
||||
(state.simulated ? ' · this UPS is simulated' : '');
|
||||
const riskChip = $('#dialog-risk');
|
||||
riskChip.textContent = command.risk;
|
||||
riskChip.className = 'chip ' + (command.risk === 'dangerous' ? 'fail'
|
||||
: command.risk === 'caution' ? 'warn' : 'ok');
|
||||
$('#dialog-confirm-wrap').hidden = !dangerous;
|
||||
$('#dialog-input-wrap').hidden = !dangerous;
|
||||
$('#dialog-input').value = '';
|
||||
const run = $('#dialog-run');
|
||||
run.classList.toggle('dangerous', dangerous);
|
||||
run.disabled = dangerous;
|
||||
state.pending = { ups: upsName, command: commandName, dangerous };
|
||||
dialog.showModal();
|
||||
if (dangerous) $('#dialog-input').focus();
|
||||
}
|
||||
|
||||
async function runPending() {
|
||||
const pending = state.pending;
|
||||
if (!pending) return;
|
||||
const body = { ups: pending.ups, command: pending.command };
|
||||
if (pending.dangerous) body.confirm = $('#dialog-input').value.trim();
|
||||
$('#confirm-dialog').close();
|
||||
try {
|
||||
const result = await fetchJSON('/api/ups/command', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
Dash.toast(`sent ${result.command} to ${result.ups}: ${result.detail}`, 'ok');
|
||||
} catch (err) {
|
||||
Dash.toast(`refused: ${err.message}`, 'fail');
|
||||
}
|
||||
state.pending = null;
|
||||
refresh();
|
||||
}
|
||||
|
||||
// ---- charts ---------------------------------------------------------------
|
||||
function makeCharts() {
|
||||
const common = { height: 160, xSpan: state.range };
|
||||
state.charts.charge = new DashCharts.LineChart('#chart-charge', Object.assign({}, common,
|
||||
{ yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
|
||||
state.charts.runtime = new DashCharts.LineChart('#chart-runtime', Object.assign({}, common,
|
||||
{ yMin: 0, yFormat: (v) => duration(v) }));
|
||||
state.charts.voltage = new DashCharts.LineChart('#chart-voltage', Object.assign({}, common,
|
||||
{ yFormat: (v) => num(v, 1) }));
|
||||
state.charts.load = new DashCharts.LineChart('#chart-load', Object.assign({}, common,
|
||||
{ yMin: 0, yFormat: (v) => num(v, 0) }));
|
||||
}
|
||||
|
||||
function points(response, upsName, metric) {
|
||||
const found = (response.series || []).find((s) => s.host === upsName && s.metric === metric);
|
||||
return found ? found.points.map((p) => ({ ts: p.ts, value: p.value })) : [];
|
||||
}
|
||||
|
||||
async function refreshCharts(names) {
|
||||
if (!names.length) return;
|
||||
const list = names.join(',');
|
||||
const metrics = 'ups.charge_pct,ups.battery_runtime_seconds,ups.battery_voltage,' +
|
||||
'ups.line_voltage,ups.load_pct';
|
||||
const data = await fetchJSON(`/api/ups/history?ups=${encodeURIComponent(list)}&metrics=${metrics}` +
|
||||
`&range=${state.range}`).catch(() => null);
|
||||
if (!data) return;
|
||||
const build = (metric, scale) => names.map((name) => ({
|
||||
name, points: points(data, name, metric).map((p) => (scale ? { ts: p.ts, value: scale(p.value) } : p)),
|
||||
}));
|
||||
const charge = build('ups.charge_pct');
|
||||
state.charts.charge.setData(charge);
|
||||
DashCharts.renderLegend($('#legend-charge'), charge, (v) => num(v, 0) + '%');
|
||||
const runtime = build('ups.battery_runtime_seconds');
|
||||
state.charts.runtime.setData(runtime);
|
||||
DashCharts.renderLegend($('#legend-runtime'), runtime, (v) => duration(v), true);
|
||||
const battery = build('ups.battery_voltage');
|
||||
const line = build('ups.line_voltage');
|
||||
const voltage = battery.map((series, index) => ({ name: `${series.name} battery`,
|
||||
points: series.points })).concat(line.map((series, index) => ({ name: `${series.name} line`,
|
||||
points: series.points, color: undefined })));
|
||||
state.charts.voltage.setData(voltage);
|
||||
DashCharts.renderLegend($('#legend-voltage'), voltage, (v) => num(v, 1) + ' V');
|
||||
const load = build('ups.load_pct');
|
||||
state.charts.load.setData(load);
|
||||
DashCharts.renderLegend($('#legend-load'), load, (v) => num(v, 0) + '%', true);
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
let snapshot;
|
||||
try {
|
||||
snapshot = await fetchJSON('/api/ups/snapshot');
|
||||
} catch (err) {
|
||||
$('#sub').textContent = err.message;
|
||||
return;
|
||||
}
|
||||
state.simulated = !!snapshot.simulated;
|
||||
const stateRow = (snapshot.states || [])[0];
|
||||
const items = snapshot.ups || [];
|
||||
items.forEach((item) => {
|
||||
let card = state.cards.get(item.name);
|
||||
if (!card) {
|
||||
card = upsCard(item.name);
|
||||
state.cards.set(item.name, card);
|
||||
$('#ups-cards').appendChild(card);
|
||||
}
|
||||
renderCard(card, item, stateRow);
|
||||
});
|
||||
$('#ups-empty').hidden = items.length > 0;
|
||||
|
||||
const banners = [];
|
||||
if (state.simulated) {
|
||||
banners.push(['warn', '<strong>SIMULATED DATA.</strong> <code>NUT_SIMULATE=1</code> — these are ' +
|
||||
'modelled UPSes for development, not your batteries. Unset it to read real upsd.']);
|
||||
}
|
||||
if (!snapshot.actions_enabled) {
|
||||
banners.push(['info', 'Command buttons are disabled. Set <code>NUT_ALLOW_ACTIONS=1</code> plus ' +
|
||||
'<code>NUT_USER</code>/<code>NUT_PASSWORD</code> for a user with <code>actions</code> in ' +
|
||||
'<code>upsd.users</code>.']);
|
||||
}
|
||||
if (stateRow && !stateRow.ok && stateRow.error) {
|
||||
banners.push(['fail', `upsd: ${DashCharts.escapeHtml(stateRow.error)}`]);
|
||||
}
|
||||
$('#banners').innerHTML = banners.map(([kind, html]) => `<div class="banner ${kind}">${html}</div>`)
|
||||
.join('');
|
||||
|
||||
// "0 healthy" reads like an outage when every UPS is merely past its replacement window;
|
||||
// name the states that are actually present instead of counting the absence of one.
|
||||
const counts = { healthy: 0, degraded: 0, 'at risk': 0, unknown: 0 };
|
||||
items.forEach((item) => { counts[(item.health && item.health.overall) || 'unknown'] += 1; });
|
||||
const breakdown = Object.entries(counts).filter(([, n]) => n > 0)
|
||||
.map(([name, n]) => `${n} ${name}`).join(', ') || 'none reporting';
|
||||
$('#sub').textContent = `${items.length} UPS${items.length === 1 ? '' : 'es'} · ${breakdown} · ` +
|
||||
(snapshot.server ? `upsd ${snapshot.server} · ` : '') + `window ${duration(state.range)}`;
|
||||
$('#foot-note').textContent = `poll ${Math.round(state.pollMs / 1000)}s · ` +
|
||||
`${items.reduce((total, item) => total + (item.advertised_commands || []).length, 0)} commands advertised`;
|
||||
await refreshCharts(items.map((item) => item.name));
|
||||
await refreshEvents();
|
||||
}
|
||||
|
||||
async function refreshEvents() {
|
||||
try {
|
||||
const data = await fetchJSON('/api/events?limit=40&source=ups');
|
||||
const list = $('#events');
|
||||
if (!(data.events || []).length) {
|
||||
list.innerHTML = '<li class="muted" style="grid-template-columns:1fr">No power events yet — ' +
|
||||
'state changes, self-test results, and commands you send all land here.</li>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = data.events.map((event) => `<li><time>${
|
||||
new Date(event.ts * 1000).toLocaleString(undefined,
|
||||
{ month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', hour12: false })
|
||||
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
|
||||
`${DashCharts.escapeHtml(event.message.replace(event.host + ':', '').trim())}` +
|
||||
`${event.data && event.data.runtime_seconds ? ` <span class="faint mono">(${
|
||||
duration(event.data.runtime_seconds)} runtime)</span>` : ''}</span></li>`).join('');
|
||||
} catch (err) { /* next poll */ }
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', async () => {
|
||||
makeCharts();
|
||||
try {
|
||||
const config = await fetchJSON('/api/config');
|
||||
state.pollMs = config.ups.poll_ms || 5000;
|
||||
} catch (err) { /* default poll rate */ }
|
||||
|
||||
Dash.rangeControl($('#range'), state.range, (seconds) => {
|
||||
state.range = seconds;
|
||||
Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = seconds; });
|
||||
refresh();
|
||||
}, [['1h', 3600], ['6h', 21600], ['24h', 86400], ['7d', 604800], ['30d', 2592000],
|
||||
['90d', 7776000]]);
|
||||
|
||||
$('#dialog-cancel').addEventListener('click', () => { $('#confirm-dialog').close(); state.pending = null; });
|
||||
$('#dialog-run').addEventListener('click', runPending);
|
||||
$('#dialog-input').addEventListener('input', (event) => {
|
||||
$('#dialog-run').disabled = event.target.value.trim() !== 'DANGER';
|
||||
});
|
||||
|
||||
state.poller = poll(refresh, state.pollMs);
|
||||
Dash.bindPause(state.poller);
|
||||
});
|
||||
Reference in New Issue
Block a user