Add homelab dashboards: system, LLM, and UPS on one domain

Three dashboards served by one FastAPI app with a SQLite history store
(raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana,
no CDN, no build step. Nothing is fetched from the internet at runtime.

* system: a stdlib-only agent (agent/host_agent.py) reports /proc and
  /sys counters; the dashboard turns them into rates over a labelled
  window, so every number says how it was measured.
* llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house
  exposition parser. Measured delta/elapsed throughput beats a native
  per-request gauge, and each value carries its provenance.
* ups: a NUT wire-protocol client plus health checks that name what
  they looked at. Commands are gated by NUT_ALLOW_ACTIONS,
  DASH_READ_ONLY and a DANGER confirmation, re-checked server-side.

Missing values render as n/a rather than 0, and an unconfigured source
answers 404 {"configured": false} without dialling anything, so "not
deployed here" never looks like an outage.

253 tests at 94% coverage against captured real fixtures; ruff and
pyright clean.

LLM-Generated: true
This commit is contained in:
2026-10-16 17:20:45 -04:00
commit c4bb374ccc
56 changed files with 13018 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
"""Homelab dashboards: system resources, LLM engines, and UPS battery health.
One FastAPI app, one domain, one SQLite file. See AGENTS.md for the rules and
.agent(s)/PLAN.md for the locked decisions.
"""
from .config import SETTINGS
__version__ = "0.1.0"
__all__ = ["SETTINGS", "__version__"]
+28
View File
@@ -0,0 +1,28 @@
"""`python -m dashboards` — the same server as `uv run uvicorn dashboards.app:app`.
DASH_PORT=8090 DEBUGPY=1 python -m dashboards
"""
from __future__ import annotations
import os
from .app import create_app
from .config import SETTINGS
from .debug import maybe_listen
def main() -> None:
import uvicorn # noqa: PLC0415 - only the entry point needs uvicorn
maybe_listen()
reload = os.environ.get("DASH_RELOAD", "").lower() in ("1", "true", "yes", "on")
if reload:
uvicorn.run("dashboards.app:app", host=SETTINGS.host, port=SETTINGS.port, reload=True)
else:
# pass the object: create_app() is cheap and avoids a second import of config
uvicorn.run(create_app(), host=SETTINGS.host, port=SETTINGS.port)
if __name__ == "__main__":
main()
+399
View File
@@ -0,0 +1,399 @@
"""HTTP API. GET everywhere except one endpoint: `POST /api/ups/command`.
Everything a browser can see is derived from the in-memory payloads (fast, no DB on the
hot path) plus SQLite for anything older than a few minutes. Two rules from llm_monitor that
are worth restating here because they are easy to break by accident:
* `/api/config` never leaks a token or password — target URLs are passed through
`Target.display_url`, which strips userinfo.
* an unconfigured source answers 404 `{"configured": false}` without dialling anything, so
the page can tell "not deployed here" apart from "deployed and broken".
"""
from __future__ import annotations
import time
from dataclasses import dataclass
from typing import Any
from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request
from .collectors import NutMonitor
from .config import Settings
from .poller import Hub
from .sources import nut
from .sources.host import fleet_summary
from .sources.llm import fleet_totals
from .store import Event, SampleStore
NO_STORE = {"cache-control": "no-store, max-age=0"}
RANGE_SECONDS = {"5m": 300, "15m": 900, "30m": 1800, "1h": 3600, "3h": 10_800,
"6h": 21_600, "12h": 43_200, "24h": 86_400, "7d": 604_800,
"30d": 2_592_000, "90d": 7_776_000}
@dataclass
class AppContext:
"""Everything the routes need, assembled once at startup and passed around."""
settings: Settings
store: SampleStore
hub: Hub
nut_monitor: NutMonitor | None = None
started_at: float = 0.0
def get_context(request: Request) -> AppContext:
"""The context built during lifespan. FastAPI hands it to every route."""
return request.app.state.ctx # type: ignore[no-any-return]
def parse_range(value: str) -> float:
"""`1h` / `7d` / `45m` / raw seconds. Unknown values are a 400, not a silent default:
a chart that quietly shows the wrong window is worse than an error."""
raw = (value or "1h").strip().lower()
if raw in RANGE_SECONDS:
return float(RANGE_SECONDS[raw])
if raw[-1] in ("s", "m", "h", "d") and raw[:-1].isdigit():
multiplier = {"s": 1, "m": 60, "h": 3600, "d": 86_400}[raw[-1]]
return float(int(raw[:-1]) * multiplier)
if raw.isdigit():
return float(raw)
raise HTTPException(400, f"unknown range {value!r}; try 5m 1h 24h 7d 30d")
def _history(store: SampleStore, source: str, host: str, metric: str, span: float,
points: int) -> dict[str, Any]:
series = store.history(source, host, metric, span=span, max_points=points)
return {
"source": source,
"host": host,
"metric": metric,
"range_seconds": span,
"points": [{"ts": ts, "value": value} for ts, value in series],
}
# --------------------------------------------------------------------------- #
# Common routes
# --------------------------------------------------------------------------- #
def common_router() -> APIRouter:
router = APIRouter(prefix="/api", tags=["meta"])
@router.get("/config")
async def config(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
settings = ctx.settings
return {
"app": {"name": "dashboards", "version": "0.1.0",
"started_at": ctx.started_at, "read_only": settings.read_only},
"system": {
"enabled": bool(settings.host_targets),
"interval_seconds": settings.host_interval,
"poll_ms": settings.poll_ms_system,
"targets": [{"name": t.name, "url": t.display_url} for t in settings.host_targets],
"target_problems": list(settings.host_target_problems),
},
"llm": {
"gufo": {"enabled": bool(settings.gufo_url), "base_url": settings.gufo_url,
"auth": bool(settings.gufo_api_key)},
"vllm": {"enabled": bool(settings.vllm_url), "base_url": settings.vllm_url,
"auth": bool(settings.vllm_api_key)},
"interval_seconds": settings.llm_interval,
"poll_ms": settings.poll_ms_llm,
"rate_window_seconds": settings.rate_window,
},
"ups": {
"enabled": settings.nut_enabled,
"server": f"{settings.nut_host}:{settings.nut_port}" if settings.nut_enabled else None,
"auth": bool(settings.nut_user),
"interval_seconds": settings.nut_interval,
"poll_ms": settings.poll_ms_ups,
"actions_enabled": settings.actions_allowed,
"simulated": bool(ctx.nut_monitor and ctx.nut_monitor.simulated),
},
"ranges": sorted(RANGE_SECONDS, key=lambda key: RANGE_SECONDS[key]),
}
@router.get("/health")
async def health(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
states = ctx.hub.states()
enabled = [s for s in states if s["configured"]]
healthy = [s for s in enabled if s["ok"]]
status = "ok" if enabled and len(healthy) == len(enabled) else (
"degraded" if healthy else ("empty" if not enabled else "down"))
# a fresh install with nothing configured is empty, not broken — `ok` answers "is the
# app working", the banner on the page answers "is there anything to look at"
return {"ok": status in ("ok", "empty"), "status": status, "sources": states,
"db": ctx.store.stats(), "uptime_seconds": time.time() - ctx.started_at}
@router.get("/events")
def events(limit: int = Query(60, ge=1, le=500), source: str | None = None,
since: float | None = None,
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
"""Event log (transitions, UPS state changes, actions taken).
Served by a sync handler on purpose: SQLite reads belong in Starlette's
threadpool, not on the event loop the collectors share.
"""
rows = ctx.store.recent_events(limit, source=source, since=since)
return {"events": [row.as_json() for row in rows]}
@router.get("/history")
def history(source: str = Query(...), host: str = Query(...), metric: str = Query(...),
range: str = Query("1h"), points: int = Query(600, ge=10, le=4000),
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
return _history(ctx.store, source, host, metric, parse_range(range), points)
@router.get("/metrics")
def metrics(source: str | None = None, host: str | None = None,
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
"""Which series actually have history — the honest answer to "why is this chart empty"."""
return {"metrics": ctx.store.series_names(source, host)}
@router.get("/summary")
async def summary(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
"""One call for the landing page: is each dashboard alive, and what stands out."""
hosts = [job.state.payload for job in ctx.hub.jobs.values()
if job.dashboard == "system" and job.state.payload]
engines = [job.state.payload for job in ctx.hub.jobs.values()
if job.dashboard == "llm" and job.state.payload]
ups_payload = None
nut_job = ctx.hub.jobs.get("nut")
if nut_job and nut_job.state.payload:
ups_payload = nut_job.state.payload
return {
"ts": time.time(),
"system": {"enabled": bool(ctx.settings.host_targets),
"fleet": fleet_summary(hosts) if hosts else None,
"hosts": [{"name": h["name"], "os": (h.get("info") or {}).get("os_pretty"),
"cpu_usage_pct": h["cpu"]["usage_pct"],
"load1": h["cpu"].get("load1"),
"cores": (h.get("info") or {}).get("cpu_cores_logical"),
"mem_used_pct": h["memory"]["used_pct"],
"disk_worst_used_pct": h["disks"]["worst_used_pct"],
"gpu_busy_pct": [g.get("busy_pct") for g in h.get("gpus") or []],
"temp_c": h["cpu"].get("temp_c"),
"uptime_seconds": (h.get("info") or {}).get("uptime_seconds")}
for h in hosts],
"states": ctx.hub.states("system")},
"llm": {"enabled": bool(ctx.settings.gufo_url or ctx.settings.vllm_url),
"fleet": fleet_totals(engines) if engines else None,
"engines": [{"engine": e["engine"],
"decode_tokens_per_second": e["metrics"].get(
"decode_tokens_per_second", {}).get("value"),
"active": e["sessions"].get("active")}
for e in engines],
"states": ctx.hub.states("llm")},
"ups": {"enabled": ctx.settings.nut_enabled,
"simulated": bool(ups_payload and ups_payload.get("simulated")),
"ups": [{"name": card["name"], "overall": card["health"]["overall"],
"status": card["status"],
"charge_pct": card["numbers"].get("charge_pct"),
"load_pct": card["numbers"].get("load_pct"),
"runtime_seconds": card["numbers"].get("battery_runtime")}
for card in (ups_payload or {}).get("ups", [])],
"states": ctx.hub.states("ups")},
"recent_events": [event.as_json() for event in ctx.store.recent_events(12)],
}
return router
# --------------------------------------------------------------------------- #
# System dashboard
# --------------------------------------------------------------------------- #
def system_router() -> APIRouter:
router = APIRouter(prefix="/api/system", tags=["system"])
@router.get("/snapshot")
async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
if not ctx.settings.host_targets:
raise HTTPException(404, detail={"configured": False,
"detail": "no hosts configured — set HOST_TARGETS"})
hosts = []
for job in ctx.hub.jobs.values():
if job.dashboard != "system":
continue
payload = job.state.payload
if payload is None:
# keep an unreachable host in the list: absent from the list and present-but-
# red are different statements, and only one of them is a config problem
payload = {"name": job.label, "ts": job.state.last_attempt, "unreachable": True}
hosts.append(payload)
hosts.sort(key=lambda item: item.get("name") or "")
return {"ts": time.time(), "hosts": hosts, "fleet": fleet_summary(hosts),
"states": ctx.hub.states("system"),
"window_seconds": ctx.settings.host_interval}
@router.get("/history")
def history(metrics: str = Query("cpu.usage_pct"), hosts: str = Query(""),
range: str = Query("1h"), points: int = Query(600, ge=10, le=4000),
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
"""One request for a whole chart: N hosts x M metrics.
`hosts` is comma-separated, or empty for every configured target. Charting three
hosts should cost one round trip, not three, because the browser polls this.
"""
span = parse_range(range)
wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()]
wanted_hosts = [item.strip() for item in hosts.split(",") if item.strip()] or [
target.name for target in ctx.settings.host_targets]
series = [_history(ctx.store, "host", host, metric, span, points)
for host in wanted_hosts for metric in wanted_metrics]
return {"hosts": wanted_hosts, "range_seconds": span, "series": series}
return router
# --------------------------------------------------------------------------- #
# LLM dashboard
# --------------------------------------------------------------------------- #
def llm_router() -> APIRouter:
router = APIRouter(prefix="/api/llm", tags=["llm"])
@router.get("/snapshot")
async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
if not (ctx.settings.gufo_url or ctx.settings.vllm_url):
raise HTTPException(404, detail={"configured": False,
"detail": "no engines configured — set "
"GUFO_BASE_URL / VLLM_BASE_URL"})
engines = []
for job in ctx.hub.jobs.values():
if job.dashboard != "llm":
continue
payload = job.state.payload or {
"engine": job.label, "metrics": {}, "sessions": {"active": None, "list": []},
"unreachable": True}
engines.append(payload)
return {"ts": time.time(), "engines": engines, "fleet": fleet_totals(engines),
"states": ctx.hub.states("llm")}
@router.get("/history")
def history(metrics: str = Query("decode_tokens_per_second"), engines: str = Query(""),
range: str = Query("1h"), points: int = Query(600, ge=10, le=4000),
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
span = parse_range(range)
wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()]
wanted_engines = [item.strip() for item in engines.split(",") if item.strip()] or [
name for name, url in (("gufo", ctx.settings.gufo_url), ("vllm", ctx.settings.vllm_url)) if url]
return {"engines": wanted_engines, "range_seconds": span,
"series": [_history(ctx.store, "llm", engine, f"llm.{metric}", span, points)
for engine in wanted_engines for metric in wanted_metrics]}
return router
# --------------------------------------------------------------------------- #
# UPS dashboard
# --------------------------------------------------------------------------- #
def ups_router() -> APIRouter:
router = APIRouter(prefix="/api/ups", tags=["ups"])
def require_monitor(ctx: AppContext) -> NutMonitor:
if ctx.nut_monitor is None or not ctx.settings.nut_enabled:
raise HTTPException(404, detail={"configured": False,
"detail": "no NUT server configured — set NUT_HOST"})
return ctx.nut_monitor
@router.get("/snapshot")
async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
monitor = require_monitor(ctx)
job = ctx.hub.jobs.get("nut")
payload = (job.state.payload if job else None) or {
"ts": time.time(), "ups": [], "simulated": monitor.simulated,
"actions_enabled": ctx.settings.actions_allowed}
events = ctx.store.recent_events(40, source="ups")
return {**payload, "states": ctx.hub.states("ups"),
"events": [event.as_json() for event in events]}
@router.get("/history")
def history(metrics: str = Query("ups.charge_pct"), ups: str = Query(""),
range: str = Query("24h"), points: int = Query(600, ge=10, le=4000),
ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
span = parse_range(range)
wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()]
job = ctx.hub.jobs.get("nut")
wanted_ups = [item.strip() for item in ups.split(",") if item.strip()] or [
card["name"] for card in ((job.state.payload if job else None) or {}).get("ups", [])]
return {"ups": wanted_ups, "range_seconds": span,
"series": [_history(ctx.store, "ups", name, metric, span, points)
for name in wanted_ups for metric in wanted_metrics]}
@router.get("/commands")
async def commands(ctx: AppContext = Depends(get_context)) -> dict[str, Any]:
"""What may be sent right now, and why not when it may not."""
require_monitor(ctx)
job = ctx.hub.jobs.get("nut")
payload = (job.state.payload if job else None) or {"ups": []}
return {"actions_enabled": ctx.settings.actions_allowed,
"read_only": ctx.settings.read_only,
"risk_levels": [nut.RISK_SAFE, nut.RISK_CAUTION, nut.RISK_DANGEROUS],
"catalog": [{"command": key, **value} for key, value in nut.COMMANDS.items()],
"ups": [{"name": card["name"], "commands": card["commands"]}
for card in payload.get("ups", [])]}
@router.post("/command")
async def send_command(ctx: AppContext = Depends(get_context),
payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
"""The one write endpoint in the application. Deliberately loud about it.
Gated on: NUT_ALLOW_ACTIONS, not DASH_READ_ONLY, credentials that carry `actions`
in upsd.users, the command being advertised by the driver for that UPS, and — for
anything that can cut power — the browser echoing the confirmation word back.
"""
monitor = require_monitor(ctx)
ups_name = str(payload.get("ups") or "").strip()
command = str(payload.get("command") or "").strip()
confirm = str(payload.get("confirm") or "").strip()
if not ups_name or not command:
raise HTTPException(400, "both `ups` and `command` are required")
if not ctx.settings.actions_allowed:
raise HTTPException(403, "UPS actions are disabled — set NUT_ALLOW_ACTIONS=1 "
"(and don't run with DASH_READ_ONLY=1)")
job = ctx.hub.jobs.get("nut")
card = next((item for item in ((job.state.payload or {}).get("ups", []) if job else [])
if item["name"] == ups_name), None)
if card is None:
raise HTTPException(404, f"unknown UPS {ups_name!r} — not in the last poll")
advice = nut.command_advice(command)
if advice is None:
raise HTTPException(400, f"{command} is not a command this dashboard will send")
if command not in card.get("advertised_commands", []):
raise HTTPException(400, f"the driver for {ups_name} does not advertise {command}")
if advice["risk"] == nut.RISK_DANGEROUS and confirm != "DANGER":
raise HTTPException(422, f"{advice['label']} can cut power: send "
f"confirm=\"DANGER\" to prove you read the warning")
if any(item["command"] == command and not item["enabled"] for item in card["commands"]):
blocked = next(item for item in card["commands"] if item["command"] == command)
raise HTTPException(409, blocked["reason"] or f"{command} is not available")
try:
if monitor.simulated:
detail = monitor.simulator.command(ups_name, command) # type: ignore[union-attr]
else:
async with nut.NutClient(
ctx.settings.nut_host, ctx.settings.nut_port,
timeout=ctx.settings.nut_timeout,
username=ctx.settings.nut_user, password=ctx.settings.nut_password,
) as client:
await client.login()
detail = await client.instcmd(ups_name, command)
except nut.NutError as exc:
ctx.store.log_event(Event(time.time(), "ups", ups_name, "action", "error",
f"{advice['label']} on {ups_name} failed: {exc}",
{"command": command, "code": exc.code}))
raise HTTPException(502, f"upsd refused {command}: {exc}") from exc
ctx.store.log_event(_action_event(ups_name, command, advice, detail))
return {"ok": True, "ups": ups_name, "command": command, "detail": detail,
"risk": advice["risk"], "simulated": monitor.simulated,
"hint": "watch the state card — the driver answers over the next few polls"}
return router
def _action_event(ups_name: str, command: str, advice: dict[str, str], detail: str) -> Event:
severity = "warn" if advice["risk"] != nut.RISK_SAFE else "info"
return Event(time.time(), "ups", ups_name, "action", severity,
f"{advice['label']} sent to {ups_name}: {detail}", {"command": command})
+178
View File
@@ -0,0 +1,178 @@
"""Application assembly: one FastAPI app, four pages, one SQLite file, N poll loops.
`create_app()` takes injectable settings and store so tests can build an app with an
in-memory DB and no targets — the same code path production uses, without the network.
"""
from __future__ import annotations
import os
import time
from collections.abc import AsyncIterator, Awaitable, Callable
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
import httpx
from fastapi import FastAPI, Request
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from starlette.exceptions import HTTPException as StarletteHTTPException
from .api import NO_STORE, AppContext, common_router, llm_router, system_router, ups_router
from .collectors import NutMonitor, UpstreamError, host_jobs, llm_jobs, nut_job
from .config import SETTINGS as DEFAULT_SETTINGS
from .config import Settings
from .poller import Hub
from .rates import RateEstimator
from .sources.host import HostCollector
from .sources.nut_sim import NutSimulator
from .store import SampleStore
WEB = Path(__file__).resolve().parent / "web"
PAGES = {"/": "index.html", "/system": "system.html", "/llm": "llm.html", "/ups": "ups.html"}
VERSION = "0.1.0"
def build_context(settings: Settings, client: httpx.AsyncClient) -> tuple[
AppContext, SampleStore, Callable[[], None]]:
"""Create the store, the collectors, and the hub's jobs. Returns (ctx, store, start)."""
store = SampleStore(
settings.db_path,
raw_keep_hours=settings.raw_keep_hours,
minute_keep_days=settings.minute_keep_days,
hour_keep_days=settings.hour_keep_days,
)
rates = RateEstimator(span=settings.rate_window)
host_collector = HostCollector(rates)
nut_monitor: NutMonitor | None = None
if settings.nut_enabled:
simulator = NutSimulator() if _truthy(os.environ.get("NUT_SIMULATE")) else None
nut_monitor = NutMonitor(settings, simulator=simulator)
hub = Hub(store)
for job in host_jobs(list(settings.host_targets), client, host_collector,
settings.host_interval):
hub.add(job)
for job in llm_jobs(settings, client, rates):
hub.add(job)
if nut_monitor is not None:
hub.add(nut_job(settings, nut_monitor, client))
ctx = AppContext(settings=settings, store=store, hub=hub, nut_monitor=nut_monitor,
started_at=time.time())
return ctx, store, hub.start
def _truthy(value: str | None) -> bool:
return (value or "").strip().lower() in ("1", "true", "yes", "on")
def create_app(settings: Settings | None = None, store: SampleStore | None = None) -> FastAPI:
"""`store` is for tests: when supplied it is not created and not closed here."""
settings = settings or DEFAULT_SETTINGS
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
owned_store: SampleStore | None = store
ctx: AppContext | None = None
client = httpx.AsyncClient(
timeout=httpx.Timeout(max(settings.host_timeout, settings.llm_timeout)),
follow_redirects=False,
limits=httpx.Limits(max_connections=32, max_keepalive_connections=16),
)
try:
if store is not None:
# injected: rebuild everything except the DB
rates = RateEstimator(span=settings.rate_window)
hub = Hub(store)
nut_monitor = None
if settings.nut_enabled:
simulator = NutSimulator() if _truthy(os.environ.get("NUT_SIMULATE")) else None
nut_monitor = NutMonitor(settings, simulator=simulator)
for job in host_jobs(list(settings.host_targets), client,
HostCollector(rates), settings.host_interval):
hub.add(job)
for job in llm_jobs(settings, client, rates):
hub.add(job)
if nut_monitor is not None:
hub.add(nut_job(settings, nut_monitor, client))
ctx = AppContext(settings=settings, store=store, hub=hub,
nut_monitor=nut_monitor, started_at=time.time())
start = hub.start
else:
ctx, owned_store, start = build_context(settings, client)
app.state.ctx = ctx
app.state.hub = ctx.hub
start()
yield
finally:
# `ctx` is None when startup itself failed; the client and DB still need closing
if ctx is not None:
await ctx.hub.stop()
await client.aclose()
if owned_store is not None and store is None:
owned_store.close()
app = FastAPI(title="Homelab Dashboards", version=VERSION, lifespan=lifespan,
docs_url=None, redoc_url=None, openapi_url=None)
app.mount("/static", StaticFiles(directory=str(WEB / "static")), name="static")
for path, filename in PAGES.items():
app.add_api_route(path, _page_handler(filename), methods=["GET"], name=f"page:{path}")
app.include_router(common_router())
app.include_router(system_router())
app.include_router(llm_router())
app.include_router(ups_router())
@app.exception_handler(UpstreamError)
async def upstream_error_handler(request: Request, exc: UpstreamError) -> JSONResponse:
return JSONResponse({"ok": False, "detail": exc.message}, status_code=exc.status,
headers=NO_STORE)
@app.middleware("http")
async def no_store_for_api(request: Request, call_next):
"""Every /api response is `no-store`, success or failure.
A dashboard is a polling app: a cached snapshot in a proxy or the browser makes a
dead host look alive for as long as the TTL. Static assets keep their caching.
"""
response = await call_next(request)
if request.url.path.startswith("/api"):
response.headers["cache-control"] = "no-store, max-age=0"
return response
@app.exception_handler(StarletteHTTPException)
async def http_error_handler(request: Request,
exc: StarletteHTTPException) -> JSONResponse:
"""One JSON error shape for every failure, deliberate or not.
A dict detail is a structured answer (`{"configured": false, ...}`) and is spread
into the body so clients can branch on it. A status-specific 404 handler must NOT be
registered: Starlette resolves handlers by status code first, so it would also swallow
the deliberate `404 {"configured": false}` from an unconfigured source — the difference
between "not deployed here" and "deployed and broken" on the page. Unmatched routes
arrive here as Starlette's plain "Not Found" and get relabelled.
"""
body: dict[str, Any] = {"ok": False}
detail: Any = exc.detail # Starlette types this as str | dict | list | None
if isinstance(detail, dict):
body.update(detail)
elif exc.status_code == 404 and str(detail) in ("Not Found", "Not Found."):
body["detail"] = f"no such route: {request.url.path}"
else:
body["detail"] = detail
return JSONResponse(body, status_code=exc.status_code, headers=NO_STORE)
return app
def _page_handler(filename: str) -> Callable[..., Awaitable[FileResponse]]:
async def handler() -> FileResponse:
return FileResponse(WEB / "pages" / filename, media_type="text/html", headers=NO_STORE)
return handler
app = create_app()
+244
View File
@@ -0,0 +1,244 @@
"""Collectors: the fetch half of each poll. Parse + store live in the Job wrapper.
Every collector returns a `Result`; every failure is an exception the Job turns into a
red dot plus an event. Nothing here writes to any upstream — with one deliberate exception,
`run_ups_command`, which is the app's only mutating path and is gated in three places.
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field
from typing import Any
import httpx
from .config import Target
from .poller import Job, Result
from .rates import RateEstimator
from .sources import llm as llm_source
from .sources import nut
from .sources.host import HostCollector, parse_host_snapshot
from .store import Event, Sample
class UpstreamError(Exception):
"""An upstream answered badly or not at all; 502/504 in the API, red dot in the UI."""
def __init__(self, message: str, *, status: int = 502) -> None:
super().__init__(message)
self.message = message
self.status = status
# --------------------------------------------------------------------------- #
# System hosts
# --------------------------------------------------------------------------- #
async def fetch_host_snapshot(client: httpx.AsyncClient, target: Target,
host_collector: HostCollector) -> Result:
url = f"{target.url}/snapshot"
try:
response = await client.get(url, headers=target.headers)
except httpx.TimeoutException as exc:
raise UpstreamError(f"{target.name}: {url} timed out", status=504) from exc
except httpx.HTTPError as exc:
raise UpstreamError(f"{target.name}: {url} unreachable ({type(exc).__name__})") from exc
if response.status_code == 401:
raise UpstreamError(f"{target.name}: agent rejected the token (401) — check the "
f"userinfo token in HOST_TARGETS")
if response.status_code >= 400:
raise UpstreamError(f"{target.name}: {url} -> HTTP {response.status_code}")
try:
body = response.json()
except ValueError as exc:
raise UpstreamError(f"{target.name}: agent returned non-JSON "
f"({response.text[:80]!r})") from exc
if not body.get("ok", False):
raise UpstreamError(f"{target.name}: agent said not ok: {body.get('detail')}")
result = parse_host_snapshot(target.name, body, host_collector)
if body.get("errors"):
# The agent degraded a section (a /proc read failed) but still answered. Say so
# in the payload rather than presenting a partial snapshot as a complete one.
result.payload["degraded"] = body["errors"]
return result
# --------------------------------------------------------------------------- #
# LLM engines
# --------------------------------------------------------------------------- #
async def fetch_engine(client: httpx.AsyncClient, engine: str, base_url: str,
api_key: str, rates: RateEstimator, *,
with_slots: bool = False) -> Result:
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
url = f"{base_url}/metrics"
try:
response = await client.get(url, headers=headers)
except httpx.TimeoutException as exc:
raise UpstreamError(f"{engine}: {url} timed out", status=504) from exc
except httpx.HTTPError as exc:
raise UpstreamError(f"{engine}: {url} unreachable ({type(exc).__name__})") from exc
if response.status_code >= 400:
raise UpstreamError(f"{engine}: {url} -> HTTP {response.status_code} "
f"{response.reason_phrase}")
text = response.text
slots: list[dict[str, Any]] | None = None
if with_slots:
# /slots is gufo's; /v1/slots is the documented alias, so a server that only
# exposes the alias still gives us the per-session list.
for path in ("/slots", "/v1/slots"):
try:
slots_response = await client.get(f"{base_url}{path}", headers=headers)
except httpx.HTTPError:
continue
if slots_response.status_code == 200:
try:
data = slots_response.json()
except ValueError:
continue
slots = data if isinstance(data, list) else data.get("slots", [])
break
return llm_source.engine_result(engine, base_url, text, rates, slots=slots)
# --------------------------------------------------------------------------- #
# UPS / NUT
# --------------------------------------------------------------------------- #
@dataclass
class NutState:
"""Previous poll's per-UPS status, so transitions are events and not spam."""
status: dict[str, str] = field(default_factory=dict)
test_results: dict[str, str] = field(default_factory=dict)
async def read(self, client: httpx.AsyncClient | None, settings: Any) -> Result:
raise NotImplementedError
class NutMonitor:
"""Polls upsd once and returns every UPS. Also owns the transition bookkeeping."""
def __init__(self, settings: Any, simulator: Any | None = None) -> None:
self.settings = settings
self.simulator = simulator
self._status: dict[str, str] = {}
self._test_results: dict[str, str] = {}
self.simulated = simulator is not None
async def collect(self) -> Result:
now = time.time()
readings: list[nut.UpsReading] = []
if self.simulator is not None:
readings = await self.simulator.read()
else:
async with nut.NutClient(
self.settings.nut_host, self.settings.nut_port,
timeout=self.settings.nut_timeout,
username=self.settings.nut_user or None,
password=self.settings.nut_password or None,
) as client:
for entry in await client.list_ups():
ups = entry["name"]
reading = nut.UpsReading(name=ups, description=entry.get("description", ""),
ts=now)
reading.vars = await client.list_vars(ups)
try:
reading.commands = await client.list_cmds(ups)
except nut.NutError:
reading.commands = []
try:
reading.writable = await client.list_rw_vars(ups)
except nut.NutError:
reading.writable = []
readings.append(reading)
samples = []
events = []
payloads = []
for reading in readings:
card = nut.payload(reading, now=now)
card["commands"] = nut.available_commands(
reading.commands,
allow_actions=self.settings.actions_allowed,
read_only=self.settings.read_only,
# the simulator needs no credentials; refusing its commands would make the
# dev buttons dead while claiming a permissions problem that does not exist
has_credentials=bool(self.settings.nut_user and self.settings.nut_password) or self.simulated,
on_battery=reading.on_battery,
)
payloads.append(card)
samples.extend(
Sample("ups", reading.name, metric, value, reading.ts)
for metric, value in nut.history_samples(reading).items()
)
events.extend(self._transitions(reading, now))
payload = {
"ts": now,
"server": None if self.simulated else f"{self.settings.nut_host}:{self.settings.nut_port}",
"simulated": self.simulated,
"ups": payloads,
"actions_enabled": self.settings.actions_allowed,
}
return Result(payload=payload, samples=samples, events=events)
def _transitions(self, reading: nut.UpsReading, now: float) -> list[Event]:
"""Log a state change once, at the severity of the state we landed in."""
out: list[Event] = []
status = reading.get("status") or ""
previous = self._status.get(reading.name)
if previous is None:
self._status[reading.name] = status
return out
if status != previous:
severities = {s["severity"] for s in reading.states()}
severity = ("error" if "fail" in severities else "warn" if "warn" in severities
else "info")
out.append(Event(now, "ups", reading.name, "status", severity,
f"{reading.name}: {previous or '(no state)'} → {status or '(no state)'}",
{"from": previous, "to": status,
"states": reading.states(),
"charge_pct": reading.number("charge_pct"),
"runtime_seconds": reading.number("battery_runtime")}))
self._status[reading.name] = status
result = reading.get("test_result")
if result and self._test_results.get(reading.name) != result:
first_seen = reading.name not in self._test_results
self._test_results[reading.name] = result
if not first_seen: # the first sighting is not news, a change is
passed = result.strip().lower() in ("ok", "passed", "pass", "completed")
out.append(Event(now, "ups", reading.name, "test",
"info" if passed else "warn",
f"{reading.name}: self-test reported {result}"))
return out
# --------------------------------------------------------------------------- #
# Job factories (what app.py wires into the Hub)
# --------------------------------------------------------------------------- #
def host_jobs(targets: list[Target], client: httpx.AsyncClient, host_collector: HostCollector,
interval: float) -> list[Job]:
return [
Job(f"host:{target.name}", "system", target.name, interval,
lambda t=target: fetch_host_snapshot(client, t, host_collector))
for target in targets
]
def llm_jobs(settings: Any, client: httpx.AsyncClient, rates: RateEstimator) -> list[Job]:
jobs: list[Job] = []
if settings.gufo_url:
jobs.append(Job("llm:gufo", "llm", "gufo", settings.llm_interval,
lambda: fetch_engine(client, "gufo", settings.gufo_url,
settings.gufo_api_key, rates, with_slots=True)))
if settings.vllm_url:
jobs.append(Job("llm:vllm", "llm", "vLLM", settings.llm_interval,
lambda: fetch_engine(client, "vllm", settings.vllm_url,
settings.vllm_api_key, rates)))
return jobs
def nut_job(settings: Any, monitor: NutMonitor, client: httpx.AsyncClient) -> Job:
return Job("nut", "ups", "NUT server", settings.nut_interval, monitor.collect,
enabled=settings.nut_enabled)
+199
View File
@@ -0,0 +1,199 @@
"""Environment-driven configuration. Read once at import; restart to pick up changes.
There is no config file format and no CLI flags on purpose: deployment is a container with
env vars (see Containerfile + the deployment repo's quadlet env template). A `.env` file is
only a convenience for the dev loop — it seeds the environment and never overrides a real
variable. Every value has a safe default EXCEPT the sources, which default to *unset* — an
unset source means "this deployment does not run it", which must render as "not configured",
never as a red panel for a server the operator never installed.
"""
from __future__ import annotations
import os
import re
from dataclasses import dataclass, field
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def load_env_file() -> Path | None:
"""Seed the environment from `DASH_ENV_FILE` (default: `.env` at the repo root).
`override=False`, so a variable the container, systemd unit, or shell already set always
beats the file. `.env.example` is documentation and is never loaded, and a missing file is
the normal case in production — not a warning.
"""
raw = os.environ.get("DASH_ENV_FILE", "").strip()
path = Path(raw).expanduser() if raw else ROOT / ".env"
if not path.is_file() or path.name == ".env.example":
return None
from dotenv import load_dotenv # noqa: PLC0415 - only needed when a file exists
load_dotenv(path, override=False)
return path
# This runs here, at the top of the module, because `Settings` below reads the environment
# in its field defaults — which are evaluated once, when the class body executes. Loading the
# file from `load()` instead would seed the environment after those defaults had already been
# taken, and the file would silently do nothing.
load_env_file()
def _env(name: str, default: str = "") -> str:
return os.environ.get(name, default).strip()
def _num(name: str, default: float) -> float:
raw = _env(name)
try:
return float(raw) if raw else default
except ValueError:
return default
def _int(name: str, default: int) -> int:
return int(_num(name, float(default)))
def _flag(name: str, default: bool = False) -> bool:
raw = _env(name).lower()
if not raw:
return default
return raw in ("1", "true", "yes", "on")
def _base_url(name: str) -> str:
"""Normalised base URL, or `""` when unset/blank. No default host, ever."""
return _env(name).rstrip("/")
# A target is `name=url` (comma or whitespace separated). The URL may carry a bearer
# token as userinfo (`http://SECRET@host:9120`) so per-host agent tokens need no extra
# variable; the token never leaves the backend, it is stripped from every /api payload.
_TARGET_SPLIT = re.compile(r"[,\s]+")
_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]*$")
@dataclass(frozen=True)
class Target:
"""One scrape endpoint. `name` is stable UI text and the history DB's host key."""
name: str
url: str
headers: dict[str, str] = field(default_factory=dict)
@property
def display_url(self) -> str:
"""URL with any userinfo removed — safe to show in the browser."""
return re.sub(r"^(https?://)[^/@]+@", r"\1", self.url)
def parse_targets(raw: str) -> tuple[list[Target], list[str]]:
"""Parse `name=url,name=url` into targets, plus a list of problems found.
Bad entries are reported, not raised: one typo in a list of six hosts must not
take the whole dashboard down. The problems surface in /api/config so they are
diagnosable without reading container logs.
"""
targets: list[Target] = []
problems: list[str] = []
for chunk in _TARGET_SPLIT.split(raw.strip()):
if not chunk:
continue
name, sep, url = chunk.partition("=")
name, url = name.strip(), url.strip().rstrip("/")
if not sep or not url:
problems.append(f"{chunk!r} is not name=url")
continue
if not _NAME_RE.match(name):
problems.append(f"{name!r} is not a valid target name")
continue
if not url.startswith(("http://", "https://")):
problems.append(f"{name}: url must start with http:// or https://")
continue
headers: dict[str, str] = {}
m = re.match(r"^(https?://)([^/@]+)@(.*)$", url)
if m:
scheme, token, rest = m.groups()
url = f"{scheme}{rest}"
headers["Authorization"] = f"Bearer {token}"
targets.append(Target(name=name, url=url, headers=headers))
return targets, problems
@dataclass(frozen=True)
class Settings:
# --- server -------------------------------------------------------------------
host: str = _env("DASH_HOST", "127.0.0.1")
port: int = _int("DASH_PORT", 8090)
db_path: str = _env("DASH_DB", str(ROOT / "data" / "dashboards.sqlite3"))
read_only: bool = _flag("DASH_READ_ONLY") # kills every mutating endpoint
# --- system dashboard ---------------------------------------------------------
host_targets: tuple[Target, ...] = ()
host_target_problems: tuple[str, ...] = ()
host_interval: float = _num("HOST_INTERVAL_SECONDS", 10)
host_timeout: float = _num("HOST_TIMEOUT_SECONDS", 4)
# --- llm dashboard ------------------------------------------------------------
gufo_url: str = _base_url("GUFO_BASE_URL")
gufo_api_key: str = _env("GUFO_API_KEY")
vllm_url: str = _base_url("VLLM_BASE_URL")
vllm_api_key: str = _env("VLLM_API_KEY")
llm_interval: float = _num("LLM_INTERVAL_SECONDS", 2)
llm_timeout: float = _num("LLM_TIMEOUT_SECONDS", 3)
rate_window: float = _num("RATE_WINDOW_SECONDS", 10)
# --- ups dashboard ------------------------------------------------------------
nut_host: str = _env("NUT_HOST").removeprefix("nut://")
nut_port: int = _int("NUT_PORT", 3493)
nut_user: str = _env("NUT_USER")
nut_password: str = _env("NUT_PASSWORD")
nut_interval: float = _num("NUT_INTERVAL_SECONDS", 5)
nut_timeout: float = _num("NUT_TIMEOUT_SECONDS", 4)
# Instant commands (battery tests, beeper, calibration) are the only write path in
# the app. Off unless BOTH the env flag is set and DASH_READ_ONLY is not.
nut_allow_actions: bool = _flag("NUT_ALLOW_ACTIONS")
# --- history ------------------------------------------------------------------
raw_keep_hours: float = _num("RETAIN_RAW_HOURS", 48)
minute_keep_days: float = _num("RETAIN_MINUTE_DAYS", 30)
hour_keep_days: float = _num("RETAIN_HOUR_DAYS", 400)
# --- browser polling defaults -------------------------------------------------
poll_ms_system: int = _int("POLL_MS_SYSTEM", 10_000)
poll_ms_llm: int = _int("POLL_MS_LLM", 2_000)
poll_ms_ups: int = _int("POLL_MS_UPS", 5_000)
@property
def nut_enabled(self) -> bool:
return bool(self.nut_host)
@property
def actions_allowed(self) -> bool:
"""Coarse switch. Whether a *specific* command can run also depends on the user
having `actions` in upsd.users, which is checked per command (and reported to the
UI as a reason) — requiring credentials here would make the label wrong for the
simulator and would hide the actual fix from the operator."""
return self.nut_allow_actions and not self.read_only
@property
def any_source_enabled(self) -> bool:
return bool(self.host_targets) or bool(self.gufo_url or self.vllm_url) or self.nut_enabled
def load() -> Settings:
"""Settings from the environment as it stands now.
Every scalar was captured when this module was imported (see `load_env_file`); only the
target list is re-parsed per call, which is what makes a target fixable without a reload
in tests and tooling.
"""
targets, problems = parse_targets(_env("HOST_TARGETS"))
return Settings(host_targets=tuple(targets), host_target_problems=tuple(problems))
SETTINGS = load()
+24
View File
@@ -0,0 +1,24 @@
"""Optional debugpy attach, gated by the DEBUGPY env var.
Default (unset / DEBUGPY=0): this module does nothing and imports nothing heavy.
DEBUGPY=1: listens on DEBUGPY_PORT (default 5678) without blocking startup.
"""
from __future__ import annotations
import os
def maybe_listen() -> bool:
"""Start a debugpy listener when DEBUGPY is truthy. Returns True if it did."""
if os.environ.get("DEBUGPY", "").lower() not in ("1", "true", "yes", "on"):
return False
port = int(os.environ.get("DEBUGPY_PORT", "5678"))
try:
import debugpy # noqa: PLC0415 - deliberately imported only when requested
except ImportError:
print(f"[debug] DEBUGPY is set but debugpy is not installed (uv sync --dev); port {port} unused", flush=True)
return False
debugpy.listen(("0.0.0.0", port))
print(f"[debug] debugpy listening on 0.0.0.0:{port}", flush=True)
return True
+230
View File
@@ -0,0 +1,230 @@
"""Polling framework: one asyncio loop per source, independent health for each.
Rules this enforces, because violating them is what makes dashboards untrustworthy:
* **Isolation.** A source that throws, hangs, or 500s must never blank, block, or delay
another one. Each job has its own task, its own timeout (the collector's problem) and
its own health record.
* **Last good value wins.** A failed poll keeps the previous payload and marks it stale
(`stale 12s` in the UI) instead of showing zeros or `n/a` everywhere.
* **Transitions are events, spam is not.** ok→fail and fail→ok write to the event log;
500 consecutive failures write one line.
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any
from .store import Event, Sample, SampleStore
logger = logging.getLogger("dashboards.poller")
@dataclass
class Result:
"""What one successful collection produced."""
payload: dict[str, Any]
samples: list[Sample] = field(default_factory=list)
events: list[Event] = field(default_factory=list)
@dataclass
class SourceState:
key: str # "host:driveripper", "llm:gufo", "nut"
dashboard: str # "system" | "llm" | "ups"
label: str
configured: bool = True
ok: bool = False
error: str | None = None
attempts: int = 0
successes: int = 0
failures: int = 0
consecutive_failures: int = 0
last_attempt: float | None = None
last_ok: float | None = None
latency_ms: float | None = None
samples_written: int = 0
payload: dict[str, Any] | None = None
def as_json(self, now: float | None = None) -> dict[str, Any]:
"""Health block for the browser. `age_seconds` only counts successes, so a dead
source reads as "stale 4m" rather than pretending to be current."""
now = time.time() if now is None else now
return {
"key": self.key,
"dashboard": self.dashboard,
"label": self.label,
"configured": self.configured,
"ok": self.ok,
"error": self.error,
"attempts": self.attempts,
"successes": self.successes,
"failures": self.failures,
"consecutive_failures": self.consecutive_failures,
"last_attempt": self.last_attempt,
"last_ok": self.last_ok,
"age_seconds": (now - self.last_ok) if self.last_ok else None,
"latency_ms": self.latency_ms,
"samples_written": self.samples_written,
}
class Job:
"""A named poll loop. `fn` does the work; everything else lives here so every
dashboard gets identical health, staleness, and event semantics for free."""
def __init__(self, key: str, dashboard: str, label: str, interval: float,
fn: Callable[[], Awaitable[Result]], *, enabled: bool = True,
grace_failures: int = 1) -> None:
self.key = key
self.dashboard = dashboard
self.label = label
self.interval = max(0.5, interval)
self.fn = fn
self.enabled = enabled
self.grace_failures = grace_failures
self.state = SourceState(key=key, dashboard=dashboard, label=label, configured=enabled)
self._down_logged = False
self._stop = asyncio.Event()
self._task: asyncio.Task[None] | None = None
self._inflight = False
async def run_once(self, store: SampleStore) -> bool:
"""One poll, with the health bookkeeping. Returns True on success."""
if self._inflight: # a slow poll must never overlap itself
return False
self._inflight = True
now = time.time()
self.state.attempts += 1
self.state.last_attempt = now
was_ok = self.state.ok
try:
started = time.perf_counter()
result = await self.fn()
self.state.latency_ms = round((time.perf_counter() - started) * 1000.0, 1)
except Exception as exc: # noqa: BLE001 - a collector must never kill the loop
self.state.ok = False
self.state.failures += 1
self.state.consecutive_failures += 1
self.state.error = f"{type(exc).__name__}: {exc}"
# One event per outage, not per failed poll. A source that has never answered
# gets a warn rather than silence — otherwise a host misconfigured from the first
# boot leaves no trace in the log at all.
if (self.state.consecutive_failures >= self.grace_failures and not self._down_logged):
self._down_logged = True
store.log_event(Event(now, self.dashboard, self.label, "unreachable",
"error" if self.state.successes else "warn",
self.state.error
+ ("" if self.state.successes else " (never answered)")))
return False
finally:
self._inflight = False
self.state.ok = True
self.state.successes += 1
self.state.payload = result.payload
# Recovery is news only if we ever said it was down: on a fresh start every source
# would otherwise log "answering again", which reads like an outage happened.
if self._down_logged or (not was_ok and self.state.failures):
count = self.state.consecutive_failures
store.log_event(Event(now, self.dashboard, self.label, "reachable", "info",
f"{self.label} is answering again"
+ (f" after {count} failure{'s' if count != 1 else ''}"
if count else "")))
self._down_logged = False
self.state.consecutive_failures = 0
self.state.last_ok = now
self.state.error = None
self.state.samples_written += store.add_many(result.samples)
for event in result.events:
store.log_event(event)
return True
async def _loop(self, store: SampleStore) -> None:
while not self._stop.is_set():
started = time.perf_counter()
await self.run_once(store)
remaining = self.interval - (time.perf_counter() - started)
try:
await asyncio.wait_for(self._stop.wait(), timeout=max(0.2, remaining))
except TimeoutError:
continue
def start(self, store: SampleStore) -> None:
if self._task is None and self.enabled:
self._task = asyncio.create_task(self._loop(store), name=f"poll:{self.key}")
async def stop(self) -> None:
self._stop.set()
if self._task is not None:
self._task.cancel()
# the cancellation is the point; the task's own outcome is not our problem here.
# CancelledError is a BaseException in 3.8+, so it has to be named explicitly.
with contextlib.suppress(asyncio.CancelledError, Exception):
await self._task
self._task = None
class Hub:
"""Owns the store and every poll loop; the only thing the API layer talks to."""
def __init__(self, store: SampleStore) -> None:
self.store = store
self.jobs: dict[str, Job] = {}
self.started_at = time.time()
self._maintenance: asyncio.Task[None] | None = None
def add(self, job: Job) -> Job:
self.jobs[job.key] = job
return job
def start(self) -> None:
for job in self.jobs.values():
job.start(self.store)
if self._maintenance is None:
self._maintenance = asyncio.create_task(self._maintain_loop(), name="maintain")
async def stop(self) -> None:
if self._maintenance is not None:
self._maintenance.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await self._maintenance
self._maintenance = None
for job in self.jobs.values():
await job.stop()
async def _maintain_loop(self, interval: float = 60.0) -> None:
"""Rollup + prune. Runs in a thread: SQLite writes would otherwise stall the loop."""
# imported here because only this task needs the thread bridge
from anyio.to_thread import run_sync # noqa: PLC0415
while True:
try:
await run_sync(self.store.maintain)
except Exception:
# Logged, never swallowed: a broken rollup means every chart past the raw
# window is quietly empty, and "it retried" is not a record of that.
logger.exception("store maintenance failed; long-window charts may be stale")
await asyncio.sleep(interval)
def states(self, dashboard: str | None = None) -> list[dict[str, Any]]:
now = time.time()
return [
job.state.as_json(now)
for job in self.jobs.values()
if dashboard is None or job.dashboard == dashboard
]
def payloads(self, dashboard: str | None = None) -> dict[str, dict[str, Any]]:
return {
job.key: job.state.payload
for job in self.jobs.values()
if (dashboard is None or job.dashboard == dashboard) and job.state.payload is not None
}
+177
View File
@@ -0,0 +1,177 @@
"""Prometheus text-exposition parser. In-house, zero dependencies.
Only used for the upstreams we do not control: llama.cpp/gufo (`llamacpp:*`) and vLLM
(`vllm:*`). Our own host agent serves JSON, so nothing else here needs this parser and we
never pull in `prometheus_client`.
Ported from llm_monitor, where it survived a year of both engines renaming series under
it. Keep the aggregation rules (see `Prom.scalar`): they are what stops a summed ratio
from exceeding 1.0 and inventing a percentile.
"""
from __future__ import annotations
import math
import re
from collections.abc import Sequence
from typing import Any
_NAME_RE = r"[a-zA-Z_:][a-zA-Z0-9_:]*"
# name{label="v",...} value [timestamp]
# - the label block is greedy so a '}' inside a quoted value cannot truncate the match;
# the anchored value/end-of-line does that instead.
# - whitespace before '{' and a trailing '# ...' comment are not part of the official
# format, but tolerating them means a quirky exporter shows a number instead of 'n/a'.
_LINE_RE = re.compile(
rf"^(?P<name>{_NAME_RE})(?:[ \t]*\{{(?P<labels>.*)\}})?[ \t]+"
r"(?P<value>[+-]?(\d+\.?\d*(e[+-]?\d+)?|inf|nan))"
r"(?:[ \t]+(?P<ts>-?\d+))?[ \t]*(?:#.*)?$",
re.IGNORECASE,
)
_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_.:]*)="((?:[^"\\]|\\.)*)"')
def _unquote(value: str) -> str:
return re.sub(r"\\(.)", r"\1", value)
class Prom:
"""Parsed exposition: name -> [(labels, value), ...]."""
def __init__(self, text: str) -> None:
self.series: dict[str, list[tuple[dict[str, str], float]]] = {}
self.parse_errors = 0
for raw in text.splitlines():
line = raw.strip()
if not line or line[0] == "#":
continue
m = _LINE_RE.match(line)
if not m:
self.parse_errors += 1
continue
try:
value = float(m.group("value"))
except ValueError: # pragma: no cover - regex keeps this unreachable
self.parse_errors += 1
continue
labels = {_unquote(k): _unquote(v) for k, v in _LABEL_RE.findall(m.group("labels") or "")}
self.series.setdefault(m.group("name"), []).append((labels, value))
def resolve(self, names: Sequence[str]) -> str | None:
"""First name from `names` that is actually present."""
return next((n for n in names if n in self.series), None)
def resolve_hist(self, names: Sequence[str]) -> str | None:
"""Like resolve() for histograms: the bare name never appears, only *_bucket/_sum/_count."""
for n in names:
if any(f"{n}{suffix}" in self.series for suffix in ("_bucket", "_sum", "_count")):
return n
return None
def values(self, names: Sequence[str], label_match: dict[str, str] | None = None) -> list[float]:
name = self.resolve(names)
if name is None:
return []
if not label_match:
return [v for _labels, v in self.series[name]]
return [
v for labels, v in self.series[name]
if all(labels.get(k) == x for k, x in label_match.items())
]
def scalar(self, names: Sequence[str], agg: str = "sum",
label_match: dict[str, str] | None = None) -> float | None:
"""Aggregate every matching series, or None when absent.
agg="sum" counters and per-engine gauges — summing label sets is the fleet total.
agg="max" usage ratios — report the most pressured engine, never a sum over 1.0.
agg="avg" rate gauges — an average across label sets.
"""
found = self.values(names, label_match)
finite = [v for v in found if math.isfinite(v)]
if not finite:
return None
if agg == "max":
return max(finite)
if agg == "avg":
return sum(finite) / len(finite)
return sum(finite)
def by_label(self, names: Sequence[str], label: str) -> dict[str, float]:
name = self.resolve(names)
if name is None:
return {}
out: dict[str, float] = {}
for labels, value in self.series[name]:
if not math.isfinite(value):
continue
key = labels.get(label, "")
out[key] = out.get(key, 0.0) + value
return out
def histogram(self, names: Sequence[str]) -> dict[str, Any] | None:
"""Mean (+ p95 when interpolable) of a `<name>_sum`/`_count`/`_bucket`."""
base = self.resolve_hist(names)
if base is None:
return None
total = self.scalar([f"{base}_sum"])
count = self.scalar([f"{base}_count"])
buckets = self._buckets(base)
out: dict[str, Any] = {
"count": count,
"sum": total,
"mean": (total / count) if (total is not None and count) else None,
"p95": None,
"source": base,
}
# Interpolate only for a single label set: mixing per-DP-rank histograms into one
# cumulative curve invents a bogus p95.
if buckets and count and self._bucket_label_groups(base) <= 1:
out["p95"] = _quantile(buckets, count, 0.95)
return out
def _bucket_label_groups(self, base: str) -> int:
groups = set()
for labels, _v in self.series.get(f"{base}_bucket", []):
groups.add(tuple(sorted((k, v) for k, v in labels.items() if k != "le")))
return len(groups)
def _buckets(self, base: str) -> list[tuple[float, float]]:
out: list[tuple[float, float]] = []
for labels, value in self.series.get(f"{base}_bucket", []):
le = labels.get("le")
if le is None or not math.isfinite(value):
continue
bound = math.inf if le.lower() in ("+inf", "inf") else float(le)
out.append((bound, value))
out.sort(key=lambda item: item[0])
return out
def _quantile(buckets: list[tuple[float, float]], count: float, q: float) -> float | None:
"""Linear interpolation inside the bucket containing the `q` quantile."""
target = q * count
prev_bound, prev_cum = 0.0, 0.0
for bound, cum in buckets:
if cum >= target:
if math.isinf(bound):
return prev_bound if math.isfinite(prev_bound) else None
span = cum - prev_cum
if span <= 0:
return bound
return prev_bound + (bound - prev_bound) * ((target - prev_cum) / span)
prev_bound, prev_cum = bound, cum
return None
def field(value: float | None, source: str | None, derived: bool = False,
window: float | None = None) -> dict[str, Any]:
"""Uniform metric cell: the UI renders `n/a` whenever value is None.
`window` is the Δt a rate was measured over, so the UI can say `Δ10.0s` instead of
implying a scraped gauge it never got.
"""
cell: dict[str, Any] = {"value": value, "source": source, "derived": derived}
if window is not None:
cell["window"] = window
return cell
+64
View File
@@ -0,0 +1,64 @@
"""Δcounter / Δt over a rolling window — the one piece of mutable state in the app.
Both upstreams export cumulative counters (tokens generated, jiffies spent, bytes read)
and the dashboards need per-second rates. A bare 1s delta jitters between 0 and spikes
because work arrives in bursts, so the rate is measured over a window (default 10s) and
the window that was actually used travels with the value — the UI prints `Δ10.0s` rather
than implying a scraped gauge it never got.
Keyed by the *resolved* upstream series name (e.g. `llamacpp:tokens_predicted_total`, or
`driveripper:cpu.total`), so a renamed counter starts a fresh window instead of diffing
two different series into a fabricated rate, and two hosts can never read each other's
samples.
"""
from __future__ import annotations
import math
import time
from collections import deque
class RateEstimator:
def __init__(self, span: float = 10.0, max_samples: int = 512) -> None:
self.span = span
self.max_samples = max_samples
self._samples: dict[str, deque[tuple[float, float]]] = {}
def rate(self, key: str, value: float | None, now: float | None = None) -> tuple[float | None, float | None]:
"""(per-second, window actually used). (None, window|None) when not computable.
Never returns a negative rate: a counter going backwards means a restart (or a
renamed series), so the window is dropped and one scrape answers `n/a`.
"""
if value is None or not math.isfinite(value):
return None, None
now = time.time() if now is None else now
samples = self._samples.setdefault(key, deque())
if samples and value < samples[-1][1]:
samples.clear()
samples.append((now, value))
while len(samples) > self.max_samples:
samples.popleft()
if len(samples) < 2:
return None, None
inside = [point for point in samples if now - point[0] <= self.span]
base = inside[0] if len(inside) > 1 else samples[-2]
dt, dv = now - base[0], value - base[1]
cutoff = now - 2 * self.span
while samples and samples[0][0] < cutoff and len(samples) > 2:
samples.popleft()
if dt <= 0 or dv < 0:
return None, dt if dt > 0 else None
return dv / dt, dt
def forget(self, prefix: str) -> int:
"""Drop every window whose key starts with `prefix` (a target that went away)."""
keys = [k for k in self._samples if k.startswith(prefix)]
for key in keys:
del self._samples[key]
return len(keys)
def keys(self) -> list[str]:
return list(self._samples)
+7
View File
@@ -0,0 +1,7 @@
"""Data sources. Each module turns one upstream's raw payload into our cell shape.
host -> our own stdlib agent (JSON)
llm -> gufo / vLLM Prometheus expositions (parsed by our own Prom class)
nut -> the upsd wire protocol (reads + instant commands)
nut_sim-> a labelled simulator for development, never a substitute for real hardware
"""
+416
View File
@@ -0,0 +1,416 @@
"""Host source: reads `agent/host_agent.py` JSON and turns counters into rates.
Division of labour, and why:
* the agent reads /proc and /sys and reports **cumulative counters** (it cannot know how
long the interval was, and it must not have to remember who polled it last);
* this module owns the **window**: every per-second number is Δcounter/Δt over
`RateEstimator`, and carries the window that produced it, so the UI can print `Δ10.0s`.
* absent values stay None. A host with no GPU has no gpu series; it does not have a GPU at
0% busy (that would say "idle", which is a different claim from "there is nothing here").
Metric names in the history DB use `@stream` for per-device series:
`cpu.usage_pct@core7`, `disk.used_pct@/srv`, `gpu.busy_pct@0`, `net.rx_bytes_per_s@eth0`.
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field
from typing import Any
from ..poller import Result
from ..rates import RateEstimator
from ..store import Sample
# /proc/stat jiffies we treat as "not doing useful work".
CPU_IDLE_MODES = ("idle", "iowait")
@dataclass
class HostTargetState:
"""Per-target memory across polls: only what a *comparison* needs."""
boot_ts: float | None = None
hostname: str | None = None
@dataclass
class HostCollector:
"""Holds the rate windows and the reboot detector for every host."""
rates: RateEstimator
targets: dict[str, HostTargetState] = field(default_factory=dict)
def target_state(self, name: str) -> HostTargetState:
return self.targets.setdefault(name, HostTargetState())
def _rate(collector: HostCollector, host: str, key: str, value: float | None,
now: float) -> tuple[float | None, float | None]:
return collector.rates.rate(f"{host}:{key}", value, now)
def _ratio_percent(numerator: float | None, denominator: float | None) -> float | None:
if numerator is None or denominator is None or denominator <= 0:
return None
return max(0.0, min(100.0, numerator / denominator * 100.0))
def parse_host_snapshot(name: str, snapshot: dict[str, Any], collector: HostCollector,
*, now: float | None = None) -> Result:
"""One agent snapshot -> (UI payload, history samples, transition events)."""
now = time.time() if now is None else now
samples: list[Sample] = []
events: list[Any] = []
ts = float(snapshot.get("ts") or now)
info = snapshot.get("host") or {}
state = collector.target_state(name)
boot_ts = info.get("boot_ts")
if boot_ts is not None:
if state.boot_ts is not None and abs(boot_ts - state.boot_ts) > 1.0:
events.append(_event(now, name, "reboot", "warn",
f"{name} rebooted (agent boot time moved from "
f"{state.boot_ts:.0f} to {boot_ts:.0f})"))
state.boot_ts = boot_ts
if info.get("hostname"):
state.hostname = str(info["hostname"])
# ---------------------------------------------------------------- CPU ---- #
cpu_raw = snapshot.get("cpu") or {}
modes = cpu_raw.get("modes") or {}
total_rate, window = _rate(collector, name, "cpu.total", modes.get("total"), now)
# Every counter is sampled on every poll, including the first. Gating the idle modes on
# `total_rate is not None` (as this used to) meant idle was never seeded on poll 1, so
# usage_pct stayed n/a until poll 3 — a dashboard that boots two intervals late, and a
# fleet average that counts a live host as missing for its first minute.
idle_rate, idle_ok = 0.0, True
for mode in CPU_IDLE_MODES:
value, _ = _rate(collector, name, f"cpu.{mode}", modes.get(mode), now)
if value is None:
idle_ok = False
else:
idle_rate += value
busy_rate = max(0.0, total_rate - idle_rate) if (idle_ok and total_rate is not None) else None
usage_pct = (busy_rate / total_rate * 100.0) if (busy_rate is not None and total_rate) else None
mode_pct: dict[str, float | None] = {}
for mode in ("user", "system", "iowait", "irq", "softirq", "steal", "nice"):
value, _ = _rate(collector, name, f"cpu.{mode}", modes.get(mode), now)
mode_pct[mode] = _ratio_percent(value, total_rate) if total_rate else None
core_usage: list[dict[str, Any]] = []
for core in cpu_raw.get("cores") or []:
core_modes = core.get("modes") or {}
core_total, _ = _rate(collector, name, f"cpu.core{core['core']}.total",
core_modes.get("total"), now)
core_idle = 0.0
core_ok = True
for mode in CPU_IDLE_MODES:
value, _ = _rate(collector, name, f"cpu.core{core['core']}.{mode}",
core_modes.get(mode), now)
if value is None:
core_ok = False
else:
core_idle += value
core_busy = (core_total - core_idle) if (core_ok and core_total is not None) else None
value = _ratio_percent(core_busy, core_total) if core_total else None
core_usage.append({"core": core["core"], "usage_pct": value})
if value is not None:
samples.append(Sample("host", name, f"cpu.usage_pct@core{core['core']}", value, ts))
if usage_pct is not None:
samples.append(Sample("host", name, "cpu.usage_pct", usage_pct, ts))
for mode, value in mode_pct.items():
if value is not None:
samples.append(Sample("host", name, f"cpu.{mode}_pct", value, ts))
ctxt_rate, _ = _rate(collector, name, "cpu.ctxt", cpu_raw.get("ctxt_total"), now)
forks_rate, _ = _rate(collector, name, "cpu.processes", cpu_raw.get("processes_total"), now)
# -------------------------------------------------------------- Memory --- #
mem = snapshot.get("memory") or {}
mem_total = mem.get("total_bytes")
mem_used = mem.get("used_bytes")
mem_used_pct = _ratio_percent(mem_used, mem_total)
swap_total = mem.get("swap_total_bytes") or 0.0
swap_used = mem.get("swap_used_bytes") or 0.0
swap_used_pct = (swap_used / swap_total * 100.0) if swap_total else None
for key in ("total_bytes", "available_bytes", "used_bytes", "cached_bytes", "buffers_bytes",
"swap_total_bytes", "swap_used_bytes"):
if mem.get(key) is not None:
samples.append(Sample("host", name, f"mem.{key}", float(mem[key]), ts))
if mem_used_pct is not None:
samples.append(Sample("host", name, "mem.used_pct", mem_used_pct, ts))
if swap_used_pct is not None:
samples.append(Sample("host", name, "mem.swap_used_pct", swap_used_pct, ts))
# ----------------------------------------------------------- Filesystems - #
filesystems = []
for fs in snapshot.get("filesystems") or []:
used_pct = fs.get("used_pct")
filesystems.append(fs)
if used_pct is not None:
samples.append(Sample("host", name, f"disk.used_pct@{fs['mountpoint']}",
float(used_pct), ts))
if fs.get("used_bytes") is not None:
samples.append(Sample("host", name, f"disk.used_bytes@{fs['mountpoint']}",
float(fs["used_bytes"]), ts))
scored = [fs for fs in filesystems if fs.get("used_pct") is not None]
worst = max(scored, key=lambda fs: fs["used_pct"]) if scored else None
# -------------------------------------------------------------- Disk IO -- #
io_devices = []
read_total = write_total = reads_total = writes_total = 0.0
any_io = False
util_max: float | None = None
for dev in (snapshot.get("disk_io") or {}).get("devices") or []:
dev_name = dev.get("name") or "?"
read_rate, _ = _rate(collector, name, f"disk.{dev_name}.read_bytes",
dev.get("read_bytes_total"), now)
write_rate, _ = _rate(collector, name, f"disk.{dev_name}.write_bytes",
dev.get("write_bytes_total"), now)
r_ops, _ = _rate(collector, name, f"disk.{dev_name}.reads", dev.get("reads_total"), now)
w_ops, _ = _rate(collector, name, f"disk.{dev_name}.writes", dev.get("writes_total"), now)
io_ms_rate, _ = _rate(collector, name, f"disk.{dev_name}.io_time_ms",
dev.get("io_time_ms_total"), now)
# io_time is milliseconds spent with at least one request outstanding, so
# ms-per-second / 10 == percent of the window the device was busy.
util = None if io_ms_rate is None else max(0.0, min(100.0, io_ms_rate / 10.0))
row = {"name": dev_name, "model": dev.get("model"), "rotational": dev.get("rotational"),
"size_bytes": dev.get("size_bytes"),
"read_bytes_per_s": read_rate, "write_bytes_per_s": write_rate,
"reads_per_s": r_ops, "writes_per_s": w_ops, "util_pct": util,
"in_flight": dev.get("in_flight")}
io_devices.append(row)
if read_rate is not None:
read_total += read_rate
any_io = True
if write_rate is not None:
write_total += write_rate
any_io = True
if r_ops is not None:
reads_total += r_ops
if w_ops is not None:
writes_total += w_ops
if util is not None:
util_max = util if util_max is None else max(util_max, util)
# ------------------------------------------------------------- Network --- #
net_rows = []
rx_total = tx_total = 0.0
any_net = False
for iface in (snapshot.get("network") or {}).get("interfaces") or []:
iname = iface.get("name") or "?"
rx, _ = _rate(collector, name, f"net.{iname}.rx", iface.get("rx_bytes_total"), now)
tx, _ = _rate(collector, name, f"net.{iname}.tx", iface.get("tx_bytes_total"), now)
net_rows.append({"name": iname, "rx_bytes_per_s": rx, "tx_bytes_per_s": tx,
"speed_mbps": iface.get("speed_mbps"),
"rx_errors_total": iface.get("rx_errors_total"),
"tx_errors_total": iface.get("tx_errors_total"),
"rx_dropped_total": iface.get("rx_dropped_total"),
"tx_dropped_total": iface.get("tx_dropped_total")})
if rx is not None:
rx_total += rx
any_net = True
samples.append(Sample("host", name, f"net.rx_bytes_per_s@{iname}", rx, ts))
if tx is not None:
tx_total += tx
samples.append(Sample("host", name, f"net.tx_bytes_per_s@{iname}", tx, ts))
# ---------------------------------------------------------------- GPU ---- #
gpus = []
for gpu in snapshot.get("gpus") or []:
index = gpu.get("index", len(gpus))
stream = f"@{index}"
vram_used = gpu.get("vram_used_bytes")
vram_total = gpu.get("vram_total_bytes")
gtt_used = gpu.get("gtt_used_bytes")
gtt_total = gpu.get("gtt_total_bytes")
temps = gpu.get("temps") or {}
temp_c = max(temps.values()) if temps else None
pools_used = None if vram_used is None else vram_used + (gtt_used or 0.0)
pools_total = None if vram_total is None else vram_total + (gtt_total or 0.0)
row = {
"index": index,
"card": gpu.get("card"),
"model": gpu.get("model") or gpu.get("card") or f"GPU {index}",
# the short name is what fits a legend; the lspci string belongs in the tooltip
"model_full": gpu.get("model_full") or gpu.get("model"),
"vendor": gpu.get("vendor"),
"driver": gpu.get("driver"),
"pci_address": gpu.get("pci_address"),
"busy_pct": gpu.get("busy_pct"),
"vram_used_bytes": vram_used,
"vram_total_bytes": vram_total,
"vram_used_pct": _ratio_percent(vram_used, vram_total),
"gtt_used_bytes": gtt_used,
"gtt_total_bytes": gtt_total,
# an APU's model lives in GTT, so "VRAM 3%" and "memory 78%" can both be true;
# this is the honest whole-addressable-pool view, kept beside the two pools
"mem_used_bytes": pools_used,
"mem_total_bytes": pools_total,
"mem_used_pct": _ratio_percent(pools_used, pools_total),
"power_watts": gpu.get("power_watts"),
"temps": temps,
"temp_c": temp_c,
"clocks": gpu.get("clocks") or {},
}
gpus.append(row)
for metric, value in (("busy_pct", row["busy_pct"]), ("vram_used_bytes", vram_used),
("vram_used_pct", row["vram_used_pct"]),
("mem_used_pct", row["mem_used_pct"]),
("power_watts", row["power_watts"]), ("temp_c", temp_c)):
if value is not None:
samples.append(Sample("host", name, f"gpu.{metric}{stream}", float(value), ts))
# -------------------------------------------------------- Temperatures --- #
sensors = snapshot.get("temperatures") or []
cpu_temp = snapshot.get("cpu_temperature_celsius")
values = [s["celsius"] for s in sensors if s.get("celsius") is not None]
max_temp = max(values) if values else None
nvme_values = [s["celsius"] for s in sensors
if s.get("chip") == "nvme" and s.get("celsius") is not None]
nvme_max = max(nvme_values) if nvme_values else None
if cpu_temp is not None:
samples.append(Sample("host", name, "temp.cpu_c", float(cpu_temp), ts))
if max_temp is not None:
samples.append(Sample("host", name, "temp.max_c", float(max_temp), ts))
if nvme_max is not None:
samples.append(Sample("host", name, "temp.nvme_max_c", float(nvme_max), ts))
# ------------------------------------------------------------ Processes -- #
processes = snapshot.get("processes") or {}
# ------------------------------------------------------------ Payload ---- #
for metric, value in (("cpu.load1", cpu_raw.get("load1")),
("cpu.load5", cpu_raw.get("load5")),
("cpu.load15", cpu_raw.get("load15")),
("cpu.freq_mhz", cpu_raw.get("freq_mhz")),
("cpu.context_switches_per_s", ctxt_rate),
("host.uptime_seconds", info.get("uptime_seconds")),
("host.up", 1.0),
("disk.read_bytes_per_s", read_total if any_io else None),
("disk.write_bytes_per_s", write_total if any_io else None),
("disk.reads_per_s", reads_total if any_io else None),
("disk.writes_per_s", writes_total if any_io else None),
("disk.util_pct", util_max),
("net.rx_bytes_per_s", rx_total if any_net else None),
("net.tx_bytes_per_s", tx_total if any_net else None)):
if value is not None:
samples.append(Sample("host", name, metric, float(value), ts))
if processes.get("total") is not None:
samples.append(Sample("host", name, "host.processes", float(processes["total"]), ts))
if worst is not None:
samples.append(Sample("host", name, "disk.used_pct", float(worst["used_pct"]), ts))
payload = {
"name": name,
"ts": ts,
"window_seconds": window,
"info": info,
"cpu": {
"usage_pct": usage_pct,
"modes_pct": mode_pct,
"load1": cpu_raw.get("load1"),
"load5": cpu_raw.get("load5"),
"load15": cpu_raw.get("load15"),
"cores": len(core_usage) or info.get("cpu_cores_logical"),
"core_usage_pct": [c["usage_pct"] for c in core_usage],
"freq_mhz": cpu_raw.get("freq_mhz"),
"freq_max_mhz": cpu_raw.get("freq_max_mhz"),
"context_switches_per_s": ctxt_rate,
"forks_per_s": forks_rate,
"procs_running": cpu_raw.get("procs_running"),
"procs_blocked": cpu_raw.get("procs_blocked"),
"temp_c": cpu_temp,
},
"memory": {
"total_bytes": mem_total,
"used_bytes": mem_used,
"available_bytes": mem.get("available_bytes"),
"used_pct": mem_used_pct,
"cached_bytes": mem.get("cached_bytes"),
"buffers_bytes": mem.get("buffers_bytes"),
"swap_total_bytes": swap_total,
"swap_used_bytes": swap_used,
"swap_used_pct": swap_used_pct,
"committed_bytes": mem.get("committed_bytes"),
},
"disks": {
"filesystems": filesystems,
"worst_used_pct": worst.get("used_pct") if worst else None,
"worst_mount": worst.get("mountpoint") if worst else None,
"read_bytes_per_s": read_total if any_io else None,
"write_bytes_per_s": write_total if any_io else None,
"reads_per_s": reads_total if any_io else None,
"writes_per_s": writes_total if any_io else None,
"util_pct": util_max,
"devices": io_devices,
},
"network": {
"rx_bytes_per_s": rx_total if any_net else None,
"tx_bytes_per_s": tx_total if any_net else None,
"interfaces": net_rows,
},
"gpus": gpus,
"temperatures": {"cpu_c": cpu_temp, "max_c": max_temp, "nvme_max_c": nvme_max,
"sensors": sensors},
"processes": {
"total": processes.get("total"),
"top_cpu": processes.get("top_cpu") or [],
"top_rss": processes.get("top_rss") or [],
},
"agent": {"duration_ms": snapshot.get("duration_ms"), "errors": snapshot.get("errors") or [],
"version": snapshot.get("agent_version")},
}
return Result(payload=payload, samples=samples, events=events)
def fleet_summary(hosts: list[dict[str, Any]]) -> dict[str, Any]:
"""Cross-host totals, computed from live payloads only.
A host whose poll failed is not "0% busy" and is not in the average — it is absent,
and `hosts_live` vs `hosts_total` says so. Never silently include a dead host.
"""
# A host that has never answered is represented by a stub ({"name", "unreachable"}), so
# every field here is read with .get() — one unreachable agent must not 500 the fleet card.
hosts = [h if isinstance(h, dict) else {} for h in hosts]
live = [h for h in hosts if (h.get("cpu") or {}).get("usage_pct") is not None]
weighted = [(h["cpu"]["usage_pct"], (h.get("info") or {}).get("cpu_cores_logical") or 1)
for h in live]
weight_sum = sum(weight for _value, weight in weighted)
cpu_avg = (sum(value * weight for value, weight in weighted) / weight_sum) if weight_sum else None
mem_total = _sum((h.get("memory") or {}).get("total_bytes") for h in hosts)
mem_used = _sum((h.get("memory") or {}).get("used_bytes") for h in hosts)
disk_pcts = [p for h in hosts if (p := (h.get("disks") or {}).get("worst_used_pct"))]
gpus = [g for h in hosts for g in h.get("gpus") or []]
return {
"hosts_total": len(hosts),
"hosts_live": len(live),
"cpu_cores": _sum((h.get("info") or {}).get("cpu_cores_logical") for h in hosts),
"cpu_usage_pct_avg": cpu_avg,
"memory_total_bytes": mem_total,
"memory_used_bytes": mem_used,
"memory_used_pct": _ratio_percent(mem_used, mem_total),
"disk_worst_used_pct": max(disk_pcts) if disk_pcts else None,
"load1_total": _sum((h["cpu"] or {}).get("load1") for h in live),
"net_rx_bytes_per_s": _sum((h.get("network") or {}).get("rx_bytes_per_s") for h in hosts),
"net_tx_bytes_per_s": _sum((h.get("network") or {}).get("tx_bytes_per_s") for h in hosts),
"gpu_count": len(gpus),
"gpu_busy_pct_avg": (sum(g["busy_pct"] for g in gpus if g.get("busy_pct") is not None)
/ max(1, sum(1 for g in gpus if g.get("busy_pct") is not None))
if any(g.get("busy_pct") is not None for g in gpus) else None),
"power_estimate_w": None, # no whole-host power source yet; deliberately n/a, not 0
}
def _sum(values: Any) -> float | None:
numbers = [float(v) for v in values if isinstance(v, int | float)]
return sum(numbers) if numbers else None
def _event(ts: float, host: str, kind: str, severity: str, message: str) -> Any:
from ..store import Event # noqa: PLC0415 - keeps this module import-light
return Event(ts, "host", host, kind, severity, message)
+352
View File
@@ -0,0 +1,352 @@
"""LLM engine source: gufo (llama.cpp-family) and vLLM, read-only.
Ported from llm_monitor, where these field maps were verified against the deployed
servers rather than against documentation — the engines do not export the names people
expect. Do not "fix" a name here without scraping the real thing:
curl -s $GUFO_BASE_URL/metrics | grep -v '^#' | sed -E 's/[{ ].*//' | sort -u
curl -s $VLLM_BASE_URL/metrics | grep -v '^#' | sed -E 's/[{ ].*//' | sort -u
and drop the capture into tests/fixtures/.
Two rules carried over verbatim, because they are the difference between a truthful and a
plausible dashboard:
1. **Absent = n/a, never 0.** A renamed metric yields `{"value": null}` and the UI shows
`n/a` — 0 would claim "no traffic", which is a different statement.
2. **Throughput precedence: native aggregate → measured Δcounter/Δt → estimate → n/a.**
vLLM's native throughput gauge really is an aggregate, so it wins. gufo's are ONE
unlabeled gauge holding the last pass of ONE request, so the summed token counter wins
and the gauge survives only as `*_native`. That asymmetry is deliberate; both cases are
labelled in the UI with their provenance.
"""
from __future__ import annotations
import math
import time
from collections.abc import Sequence
from typing import Any
from ..poller import Result
from ..prom import Prom, field
from ..rates import RateEstimator
from ..store import Sample
GUFO_SERIES: dict[str, tuple[tuple[str, ...], str]] = {
"requests_processing": (("llamacpp:requests_processing", "llama:requests_processing"), "sum"),
"requests_deferred": (("llamacpp:requests_deferred", "llama:requests_deferred"), "sum"),
"kv_cache_usage_ratio": (("llamacpp:kv_cache_usage_ratio", "llama:kv_cache_usage_ratio"), "max"),
# gufo counts prompt cache hits separately: prompt_tokens_total EXCLUDES cached tokens.
"tokens_prompt_total": (("llamacpp:prompt_tokens_total", "llama:prompt_tokens_total",
"llamacpp:tokens_prompt_total"), "sum"),
"tokens_prompt_cached_total": (("llamacpp:prompt_tokens_cached_total",
"llama:prompt_tokens_cached_total"), "sum"),
"tokens_generated_total": (("llamacpp:tokens_predicted_total", "llamacpp:tokens_generated_total",
"llama:tokens_predicted_total", "llama:tokens_generated_total"), "sum"),
# NOT the throughput cell: prefill speed of one request's last pass.
"prompt_tokens_per_second_native": (("llamacpp:prompt_tokens_seconds",
"llamacpp:prompt_tokens_per_second",
"llama:prompt_tokens_seconds"), "avg"),
# NOT the throughput cell: one gauge holding one request's last decode pass.
"decode_tokens_per_second_native": (("llamacpp:predicted_tokens_seconds",
"llamacpp:decode_tokens_per_second",
"llama:predicted_tokens_seconds"), "avg"),
"n_tokens_max": (("llamacpp:n_tokens_max", "llama:n_tokens_max"), "max"),
# GPU context losses (gufo-specific): any non-zero value here is an incident.
"device_lost_total": (("gufo_device_lost_total",), "sum"),
# speculative decoding (MTP on this fleet): accept rate is the useful number.
"spec_drafts_total": (("llamacpp:spec_decode_num_drafts_total",), "sum"),
"spec_draft_tokens_total": (("llamacpp:spec_decode_num_draft_tokens_total",), "sum"),
"spec_accepted_tokens_total": (("llamacpp:spec_decode_num_accepted_tokens_total",), "sum"),
}
VLLM_SERIES: dict[str, tuple[tuple[str, ...], str]] = {
"num_requests_running": (("vllm:num_requests_running",), "sum"),
"num_requests_waiting": (("vllm:num_requests_waiting",), "sum"),
"gpu_cache_usage": (("vllm:kv_cache_usage_perc", "vllm:gpu_cache_usage"), "max"),
"prompt_tokens_total": (("vllm:prompt_tokens_total",), "sum"),
"generation_tokens_total": (("vllm:generation_tokens_total",), "sum"),
# throughput gauges exist only on some builds; absent -> derived from counters/TPOT
"prompt_tokens_per_second": (("vllm:prompt_throughput", "vllm:avg_prompt_throughput_toks_per_s"), "avg"),
"decode_tokens_per_second_native": (("vllm:generation_throughput",
"vllm:avg_generation_throughput_toks_per_s"), "avg"),
"prefix_cache_queries_total": (("vllm:prefix_cache_queries_total",), "sum"),
"prefix_cache_hits_total": (("vllm:prefix_cache_hits_total",), "sum"),
# preemption = a running request kicked back to the queue: the real KV-pressure signal
"num_preemptions_total": (("vllm:num_preemptions_total",), "sum"),
"num_requests_swapped": (("vllm:num_requests_swapped",), "sum"),
}
VLLM_HISTOGRAMS: dict[str, tuple[str, ...]] = {
"time_to_first_token_seconds": ("vllm:time_to_first_token_seconds",),
"time_per_output_token_seconds": ("vllm:time_per_output_token_seconds",
"vllm:inter_token_latency_seconds",
"vllm:request_time_per_output_token_seconds"),
"request_queue_time_seconds": ("vllm:request_queue_time_seconds",),
# not shown directly: mean prompt length / mean prefill time = lifetime prefill speed,
# which is what keeps prompt tok/s off n/a on a build with no rate gauge
"request_prefill_time_seconds": ("vllm:request_prefill_time_seconds",),
"request_prompt_tokens": ("vllm:request_prompt_tokens",),
"e2e_request_latency_seconds": ("vllm:e2e_request_latency_seconds",),
}
#: history series per engine, in the order the LLM page charts them
HISTORY_METRICS: dict[str, tuple[str, ...]] = {
"decode_tokens_per_second": ("decode_tokens_per_second",),
"prompt_tokens_per_second": ("prompt_tokens_per_second",),
"kv_cache_usage": ("kv_cache_usage_ratio", "gpu_cache_usage"),
"requests_running": ("requests_processing", "num_requests_running"),
"requests_waiting": ("requests_deferred", "num_requests_waiting"),
"ttft_seconds": ("time_to_first_token_seconds__mean",),
"ttft_p95_seconds": ("time_to_first_token_seconds__p95",),
"queue_seconds": ("request_queue_time_seconds__mean",),
"prefix_cache_hit_ratio": ("prefix_cache_hit_ratio", "prompt_cache_hit_ratio"),
}
def _ratio(numerator: dict[str, Any], denominator: dict[str, Any], source: str) -> dict[str, Any]:
num, den = numerator["value"], denominator["value"]
if num is None or den is None or den == 0:
return field(None, None)
return field(num / den, source, derived=True)
def _histogram_mean(out: dict[str, Any], key: str) -> float | None:
cell = out.get(key)
mean = cell.get("mean") if isinstance(cell, dict) else None
return mean if isinstance(mean, float) and math.isfinite(mean) else None
def parse_gufo_metrics(text: str, rates: RateEstimator | None = None,
now: float | None = None) -> dict[str, Any]:
"""`rates` enables the Δ-based aggregate rates; omitted, the parse stays pure."""
parsed = Prom(text)
out: dict[str, Any] = {}
for key, (names, agg) in GUFO_SERIES.items():
out[key] = field(parsed.scalar(names, agg), parsed.resolve(names))
def measured(total_key: str) -> dict[str, Any] | None:
"""Δcounter/Δt for one cumulative counter, keyed by the RESOLVED series name so an
alias rename starts a fresh window instead of diffing two different series."""
if rates is None:
return None
source = out[total_key]["source"]
if source is None:
return None
rate, window = rates.rate(source, out[total_key]["value"], now)
if rate is None:
return None
return field(rate, f"computed:Δ{source}/Δt", derived=True, window=window)
# The counters add every slot; gufo's single gauge cannot. Measured therefore wins for
# both speeds here (the inverse of vLLM, where the native gauge is the aggregate) — and
# when no window exists yet the cell stays n/a instead of borrowing one request's speed.
for ui_key, total_key in (
("decode_tokens_per_second", "tokens_generated_total"),
("prompt_tokens_per_second", "tokens_prompt_total"),
):
rate = measured(total_key)
out[ui_key] = rate if rate is not None else field(None, None)
prompt = out["tokens_prompt_total"]["value"]
cached = out["tokens_prompt_cached_total"]["value"] or 0.0
generated = out["tokens_generated_total"]["value"]
parts = [v for v in (prompt, generated) if v is not None]
out["tokens_total"] = (field(sum(parts) + cached, "computed:prompt+cached+generated")
if parts else field(None, None))
out["prompt_cache_hit_ratio"] = (
field(cached / (prompt + cached) if (prompt + cached) > 0 else None,
"computed:cached/(prompt+cached)", derived=True)
if prompt is not None else field(None, None))
out["spec_accept_rate"] = _ratio(out["spec_accepted_tokens_total"],
out["spec_draft_tokens_total"],
"computed:accepted/draft_tokens")
out["spec_drafts_per_request"] = _ratio(out["spec_drafts_total"],
out["spec_draft_tokens_total"],
"computed:drafts/draft_tokens")
return out
def parse_vllm_metrics(text: str, rates: RateEstimator | None = None,
now: float | None = None) -> dict[str, Any]:
parsed = Prom(text)
out: dict[str, Any] = {}
for key, (names, agg) in VLLM_SERIES.items():
out[key] = field(parsed.scalar(names, agg), parsed.resolve(names))
reason_names = ("vllm:num_requests_waiting_by_reason",
"vllm:num_requests_waiting_by_reason_total")
reasons = parsed.by_label(reason_names, "reason")
out["waiting_by_reason"] = {"value": reasons, "source": parsed.resolve(reason_names),
"derived": False}
out["waiting_capacity"] = field(reasons.get("capacity"), parsed.resolve(reason_names))
for ui_key, names in VLLM_HISTOGRAMS.items():
out[ui_key] = parsed.histogram(names)
out[f"{ui_key}__mean"] = field(out[ui_key]["mean"] if out[ui_key] else None,
out[ui_key]["source"] if out[ui_key] else None)
out[f"{ui_key}__p95"] = field(out[ui_key]["p95"] if out[ui_key] else None,
out[ui_key]["source"] if out[ui_key] else None)
tpot = out["time_per_output_token_seconds"]["mean"] if out["time_per_output_token_seconds"] else None
per_seq = (1.0 / tpot) if tpot else None
out["per_request_decode_tokens_per_second"] = field(per_seq, "computed:1/tpot", derived=True)
def measured(key: str, total_key: str) -> dict[str, Any] | None:
if rates is None:
return None
rate, window = rates.rate(key, out[total_key]["value"], now)
if rate is None:
return None
return field(rate, f"computed:Δ{VLLM_SERIES[total_key][0][0]}/Δt", derived=True, window=window)
decode = out["decode_tokens_per_second_native"]["value"]
running = out["num_requests_running"]["value"]
if decode is not None:
out["decode_tokens_per_second"] = field(decode,
out["decode_tokens_per_second_native"]["source"])
else:
# measured beats the running/tpot estimate: it counts real output tokens, the
# estimate assumes every running request is decoding this instant
measured_decode = measured("vllm:generation_tokens_total", "generation_tokens_total")
if measured_decode is not None:
# `or` here would treat a genuine 0 tok/s as missing and print the lifetime
# estimate for an engine that is provably idle
out["decode_tokens_per_second"] = measured_decode
elif per_seq is not None:
out["decode_tokens_per_second"] = field((running or 0) * per_seq,
"computed:running/tpot", derived=True)
else:
out["decode_tokens_per_second"] = field(None, None)
prefill_avg = None
prefill_time = _histogram_mean(out, "request_prefill_time_seconds")
prompt_len = _histogram_mean(out, "request_prompt_tokens")
if prefill_time and prompt_len:
prefill_avg = field(prompt_len / prefill_time,
"computed:mean(request_prompt_tokens)/mean(request_prefill_time)",
derived=True)
if out["prompt_tokens_per_second"]["value"] is None:
measured_prefill = measured("vllm:prompt_tokens_total", "prompt_tokens_total")
if measured_prefill is not None:
out["prompt_tokens_per_second"] = measured_prefill
elif prefill_avg is not None:
out["prompt_tokens_per_second"] = prefill_avg
else:
out["prompt_tokens_per_second"] = field(None, None)
# a lifetime average is a different claim from current throughput — labelled apart
out["prefill_tokens_per_second_lifetime"] = prefill_avg or field(None, None)
parts = [v for v in (out["prompt_tokens_total"]["value"],
out["generation_tokens_total"]["value"]) if v is not None]
out["tokens_total"] = field(sum(parts), "computed:prompt+generated") if parts else field(None, None)
out["prefix_cache_hit_ratio"] = _ratio(out["prefix_cache_hits_total"],
out["prefix_cache_queries_total"],
"computed:hits/queries")
return out
def engine_result(engine: str, base_url: str, text: str, rates: RateEstimator,
*, slots: list[dict[str, Any]] | None = None, now: float | None = None) -> Result:
"""Parse one engine's /metrics into a poll Result (payload + history samples)."""
now = time.time() if now is None else now
if engine == "gufo":
metrics = parse_gufo_metrics(text, rates, now)
active = metrics["requests_processing"]["value"]
else:
metrics = parse_vllm_metrics(text, rates, now)
active = metrics["num_requests_running"]["value"]
parsed = Prom(text)
sessions = _sessions(engine, slots, active, metrics)
payload = {
"engine": engine,
"upstream": base_url,
"scraped_at": now,
"metrics": metrics,
"sessions": sessions,
"series_count": len(parsed.series),
"unparsed": parsed.parse_errors,
}
samples: list[Sample] = []
for metric, candidates in HISTORY_METRICS.items():
value = next((metrics[key]["value"] for key in candidates
if key in metrics and metrics[key]["value"] is not None), None)
if value is not None:
samples.append(Sample("llm", engine, f"llm.{metric}", float(value), now))
return Result(payload=payload, samples=samples)
def _sessions(engine: str, slots: list[dict[str, Any]] | None, active: float | None,
metrics: dict[str, Any]) -> dict[str, Any]:
"""Active sessions, however this engine expresses them.
gufo exposes /slots (one row per parallel slot, with per-slot tokens and speed), so it
gets a real list. vLLM publishes no per-session endpoint in its /metrics, so it gets
counts and says the list is not available rather than inventing rows.
"""
out: dict[str, Any] = {"active": active, "list": [], "source": None, "detail": None}
if engine == "gufo":
if slots:
out["list"] = slots
out["source"] = "gufo /slots"
else:
out["detail"] = ("no /slots answer — active count still comes from "
"llamacpp:requests_processing")
else:
waiting = metrics.get("num_requests_waiting", {}).get("value")
queued = f", {waiting:g} queued" if waiting else ""
out["detail"] = (f"vLLM reports counts, not sessions: {active or 0:g} running{queued}. "
f"Per-request detail is in the latency histograms.")
return out
def fleet_totals(engines: list[dict[str, Any]]) -> dict[str, Any]:
"""Cross-engine sums, computed in the API over engines that answered.
A dead engine keeps its last good values in its own panel but contributes nothing to
a total presented as current — and `engines_live` vs `engines_total` makes that visible.
"""
live = [e for e in engines if e.get("metrics")]
def sum_metric(key: str) -> float | None:
values = [e["metrics"].get(key, {}).get("value") for e in live]
numbers = [v for v in values if isinstance(v, int | float)]
return sum(numbers) if numbers else None
def max_metric(key: str) -> float | None:
values = [e["metrics"].get(key, {}).get("value") for e in live
if isinstance(e["metrics"].get(key, {}).get("value"), int | float)]
return max(values) if values else None
def sum_pairs(keys: Sequence[str]) -> float | None:
"""Sum across engines AND across the per-engine key aliases (gufo calls a running
request `requests_processing`, vLLM `num_requests_running`)."""
values = [sum_metric(key) for key in keys]
numbers = [v for v in values if v is not None]
return sum(numbers) if numbers else None
return {
"engines_total": len(engines),
"engines_live": len(live),
"decode_tokens_per_second": sum_metric("decode_tokens_per_second"),
"prompt_tokens_per_second": sum_metric("prompt_tokens_per_second"),
"requests_active": sum_pairs(("requests_processing", "num_requests_running")),
"requests_waiting": sum_pairs(("requests_deferred", "num_requests_waiting")),
"tokens_total": sum_metric("tokens_total"),
# a real max across both engines' aliases: `a or b` returned gufo's 25% while vLLM
# was sitting at 75%, which is the opposite of a pressure signal
"kv_cache_usage_max": max(
(v for v in (max_metric("kv_cache_usage_ratio"), max_metric("gpu_cache_usage"))
if v is not None),
default=None,
),
"seams": ("gufo's prompt counter excludes cache-hit tokens while vLLM's includes "
"cached prefix tokens, so the summed prompt rate leans slightly low for "
"gufo. Both are counted honestly per engine."),
}
def series_names(engine: str) -> Sequence[str]:
return tuple(GUFO_SERIES if engine == "gufo" else VLLM_SERIES)
+779
View File
@@ -0,0 +1,779 @@
"""NUT (Network UPS Tools) client — the upsd wire protocol, asyncio, no dependencies.
`upsc`/`upspy`/`pynut` are either a subprocess dependency or a stalled third-party lib,
and the protocol is a line protocol with ~10 commands. So: talk to upsd on 3493 directly.
Reading is unauthenticated in stock NUT (`LIST UPS` / `LIST VAR`), so the dashboard works
with a read-only upsd. Instant commands (the battery-test buttons) require a user with
`actions` in upsd.users — that is the ONLY write path in this whole application, and it is
gated three times: `NUT_ALLOW_ACTIONS`, not `DASH_READ_ONLY`, and a confirmation in the UI
(destructive ones require typing the UPS name).
Verified against NUT 2.8.5 (`/usr/bin/upsc` on driveripper). Protocol shape:
-> LIST UPS
<- BEGIN LIST UPS
<- UPS "apc" : APC Smart-UPS
<- END LIST UPS
-> LIST VAR apc
<- BEGIN LIST VAR apc
<- VAR "ups.status" "OL"
<- END LIST VAR apc
-> LIST CMD apc ; <- BEGIN LIST CMD apc / CMD test.battery.start.quick / END LIST CMD apc
bad -> ERR ACCESS-DENIED (auth or permission problem)
ok -> OK ...
"""
from __future__ import annotations
import asyncio
import contextlib
import re
import time
from dataclasses import dataclass, field
from typing import Any
DEFAULT_PORT = 3493
class NutError(Exception):
"""A NUT protocol error. `code` is the upsd error token (ACCESS-DENIED, ...)."""
def __init__(self, message: str, *, code: str | None = None) -> None:
super().__init__(message)
self.code = code
#: upsd error tokens -> what the operator should actually do about them.
ERROR_HINTS = {
"ACCESS-DENIED": "upsd refused: this client is not in an ACL that allows it (upsd.conf `ACL`/"
"`LISTEN`, or the user lacks the privilege in upsd.users)",
"INVALID-PASSWORD": "wrong NUT_PASSWORD for NUT_USER (upsd.users)",
"INVALID-USERNAME": "NUT_USER is not defined in upsd.users",
"INVALID-ARGUMENT": "malformed command or unknown UPS",
"UNKNOWN-UPSNAME": "that UPS is not exported by upsd (check ups.conf driver instance name)",
"UNKNOWN-REQUEST": "this upsd does not implement that command",
"CMD-UNSUPP": "the driver does not implement that command",
"CMD-NOT-BLOCKING": "not a blocking command",
"INVALID-BITMAP": "bad mode bitmap",
"DATA-STALE": "the driver stopped updating this UPS (check nut-driver@<ups> on the NUT host)",
"DRIVER-NOT-CONNECTED": "no driver is connected to that UPS right now",
"PERM-DENIED": "read-only variable, or the user lacks `actions` for that command in upsd.users",
"TOO-MANY-CLIENTS": "upsd MAXCONN reached",
"ALREADY-SET": "value already set",
"FEATURE-NOT-ENABLED": "feature disabled on the driver side",
"FEATURE-NOT-SUPPORTED": "driver does not support that feature",
"TOO-BUSY": "upsd is busy",
"INVALID-VALUE": "value out of range",
}
def _unquote(value: str) -> str:
if len(value) >= 2 and value[0] == '"' and value[-1] == '"':
return value[1:-1].replace('\\"', '"').replace("\\\\", "\\")
return value
class NutClient:
"""One connection to upsd. Short-lived by design: connect, read, close.
A persistent connection would be faster but hides restarts of upsd behind a stale
socket, and upsd has a per-client limit (MAXCONN) that a long-lived dashboard would
sit on.
"""
def __init__(self, host: str, port: int = DEFAULT_PORT, *, timeout: float = 4.0,
username: str | None = None, password: str | None = None) -> None:
self.host = host
self.port = port
self.timeout = timeout
self.username = username
self.password = password
self._reader: asyncio.StreamReader | None = None
self._writer = None
async def __aenter__(self) -> NutClient:
await self.connect()
return self
async def __aexit__(self, *_exc: object) -> None:
await self.close()
async def connect(self) -> None:
try:
self._reader, self._writer = await asyncio.wait_for(
asyncio.open_connection(self.host, self.port), self.timeout
)
except (TimeoutError, OSError) as exc:
raise NutError(
f"cannot reach upsd at {self.host}:{self.port} ({type(exc).__name__}: {exc})",
code="UNREACHABLE",
) from exc
async def close(self) -> None:
if self._writer is None:
return
with contextlib.suppress(Exception): # logout is best effort; upsd may already be gone
if self.username and self.password:
await self._send("LOGOUT")
await self._read_line(expect_ok=False)
self._writer.close()
with contextlib.suppress(Exception):
await self._writer.wait_closed()
self._writer = None
self._reader = None
# -- plumbing ------------------------------------------------------------ #
async def _send(self, command: str) -> None:
if self._writer is None:
raise NutError("not connected", code="NOCONNECTION")
self._writer.write((command + "\n").encode())
await self._writer.drain()
async def _read_line(self, *, expect_ok: bool = True) -> str:
if self._reader is None:
raise NutError("not connected", code="NOCONNECTION")
try:
raw = await asyncio.wait_for(self._reader.readline(), self.timeout)
except TimeoutError as exc:
raise NutError(f"upsd at {self.host}:{self.port} did not answer in "
f"{self.timeout:g}s", code="TIMEOUT") from exc
except (ConnectionResetError, BrokenPipeError, OSError) as exc:
# upsd dropping the socket (restart, MAXCONN, firewall) is a read failure with a
# name, not a traceback out of the poller
raise NutError(f"upsd at {self.host}:{self.port} dropped the connection "
f"({type(exc).__name__})", code="CLOSED") from exc
if not raw:
raise NutError("upsd closed the connection", code="CLOSED")
line = raw.decode(errors="replace").rstrip("\r\n")
if line.startswith("ERR"):
code = line.split(None, 2)[1] if len(line.split()) > 1 else "UNKNOWN"
raise NutError(f"upsd: {line} — {ERROR_HINTS.get(code, 'see upsd(8)')}", code=code)
if expect_ok and not line.startswith("OK"):
raise NutError(f"unexpected upsd reply: {line!r}", code="UNEXPECTED")
return line
async def _list(self, command: str, kind: str) -> list[list[str]]:
"""Run a LIST command and collect the lines between BEGIN/END."""
begin = f"BEGIN LIST {kind}"
end = f"END LIST {kind}"
await self._send(command)
# LIST replies begin with `BEGIN LIST <kind>`, not `OK`, so this path must not
# demand an OK line — upsd would otherwise be "broken" on every read.
first = await self._read_line(expect_ok=False)
if not first.startswith(begin):
raise NutError(f"expected {begin!r}, got {first!r}", code="UNEXPECTED")
rows: list[list[str]] = []
while True:
line = await self._read_line(expect_ok=False)
if line.startswith(end):
return rows
rows.append(_split_fields(line))
if len(rows) > 4096: # a wedged upsd must not grow the heap
raise NutError(f"{kind} list too long", code="UNEXPECTED")
async def login(self) -> bool:
"""Authenticate for actions. Returns False when no credentials are configured."""
if not self.username or not self.password:
return False
await self._send(f"USER {self.username}")
await self._read_line()
await self._send(f"PASS {self.password}")
await self._read_line()
return True
# -- reads ---------------------------------------------------------------- #
async def list_ups(self) -> list[dict[str, str]]:
rows = await self._list("LIST UPS", "UPS")
out: list[dict[str, str]] = []
for row in rows:
# UPS <name> ["description"]
if len(row) >= 2 and row[0] == "UPS":
name = row[1]
desc = _unquote(" ".join(row[2:])) if len(row) > 2 else ""
out.append({"name": name, "description": desc})
return out
async def list_vars(self, ups: str) -> dict[str, str]:
"""`VAR <ups> <name> <value>` — the UPS name is field 1, the variable is field 2.
Reading field 1 as the name returns the UPS name over and over and every lookup
misses; real upsd was the only way to catch that, so the fake in the tests speaks
the same shape.
"""
rows = await self._list(f"LIST VAR {_quote(ups)}", "VAR")
out: dict[str, str] = {}
for row in rows:
if len(row) >= 3 and row[0] == "VAR":
out[_unquote(row[2])] = _unquote(" ".join(row[3:]))
return out
async def list_cmds(self, ups: str) -> list[str]:
"""`CMD <ups> <command>`."""
rows = await self._list(f"LIST CMD {_quote(ups)}", "CMD")
return [_unquote(row[2]) for row in rows if len(row) >= 3 and row[0] == "CMD"]
async def list_rw_vars(self, ups: str) -> list[str]:
"""`RW <ups> <variable>`."""
rows = await self._list(f"LIST RW {_quote(ups)}", "RW")
return [_unquote(row[2]) for row in rows if len(row) >= 3 and row[0] == "RW"]
# -- writes ---------------------------------------------------------------- #
async def instcmd(self, ups: str, command: str) -> str:
"""Send an instant command. upsd answers `OK Send shorthand command successful`."""
if not (self.username and self.password):
raise NutError("instant commands need NUT_USER/NUT_PASSWORD with `actions` "
"in upsd.users", code="ACCESS-DENIED")
await self._send(f"INSTCMD {_quote(ups)} {_quote(command)}")
return await self._read_line()
def _quote(value: str) -> str:
"""Quote a protocol argument. upsd splits on whitespace and understands C-style quoting."""
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
return f'"{escaped}"'
def _split_fields(line: str) -> list[str]:
"""Split a protocol line into fields, honouring quoted spans (values contain spaces).
Fields come back unquoted: upsd quotes every variable name and value, and a name that
still carries its quote characters (`"battery.charge"`) silently misses every lookup.
"""
fields: list[str] = []
current: list[str] = []
in_quotes = False
escaped = False
for char in line:
if escaped:
current.append(char)
escaped = False
elif in_quotes and char == "\\":
escaped = True
elif char == '"':
in_quotes = not in_quotes
current.append(char)
elif char == " " and not in_quotes:
if current:
fields.append("".join(current))
current = []
else:
current.append(char)
if current:
fields.append("".join(current))
return [_unquote(field) for field in fields]
# --------------------------------------------------------------------------- #
# Variable mapping
# --------------------------------------------------------------------------- #
# Vendor drivers disagree about names (apc vs cyberpower vs usbhid-ups), so every field
# is a candidate list and an absent value is None, never 0.
VAR_MAP: dict[str, tuple[str, ...]] = {
"mfr": ("ups.mfr",),
"model": ("ups.model",),
"serial": ("ups.serial", "device.serial"),
"firmware": ("ups.firmware", "ups.firmware.aux"),
"driver": ("driver.name",),
"status": ("ups.status",),
"charger_status": ("battery.charger.status",), # NUT 2.8 rich status
"alarm": ("ups.alarm",),
"load_pct": ("ups.load", "ups.load.instead"),
"realpower": ("ups.realpower", "ups.power"),
"realpower_nominal": ("ups.realpower.nominal", "ups.power.nominal"),
"line_voltage": ("input.voltage", "input.voltage.fault"),
"line_voltage_nominal": ("input.voltage.nominal",),
"input_min": ("input.voltage.minimum", "input.low.line"),
"input_max": ("input.voltage.maximum", "input.high.line"),
"transfer_low": ("input.transfer.low",),
"transfer_high": ("input.transfer.high",),
"frequency": ("input.frequency", "input.frequency.nominal"),
"output_voltage": ("output.voltage",),
"output_current": ("output.current",),
"output_frequency": ("output.frequency",),
"charge_pct": ("battery.charge", "battery.capacity:typ"),
"battery_voltage": ("battery.voltage",),
"battery_voltage_nominal": ("battery.voltage.nominal",),
"battery_current": ("battery.current",),
"battery_temperature": ("battery.temperature",),
"battery_type": ("battery.type",),
"battery_runtime": ("battery.runtime", "battery.runtime.low"),
"battery_runtime_low": ("battery.runtime.low",),
"battery_date": ("battery.mfr.date", "battery.date"),
"test_result": ("battery.test.result", "ups.test.result"),
"test_date": ("battery.test.date", "ups.test.date"),
"start_battery": ("ups.start.battery",),
"ups_temperature": ("ups.temperature",),
"efficiency": ("ups.efficiency",),
}
# Anything else worth showing verbatim, if the driver happens to export it.
EXTRA_VAR_PREFIXES = ("battery.test.", "input.transfer.", "output.", "ups.", "battery.")
STATE_FLAGS = {
"OL": ("online", "ok", "On line — utility power is present"),
"OB": ("on battery", "warn", "Running on battery"),
"OB LB": ("low battery", "fail", "On battery and low — shutdown imminent"),
"LB": ("low battery", "fail", "Battery low"),
"RB": ("replace battery", "fail", "The UPS says the battery must be replaced"),
"CHRG": ("charging", "info", "Battery charging"),
"DISCHRG": ("discharging", "warn", "Battery discharging"),
"BYPASS": ("bypass", "warn", "Load is on bypass"),
"OFF": ("off", "fail", "Output is off"),
"NOBAT": ("no battery", "fail", "No battery detected"),
"NULL": ("unknown", "warn", "Driver reports no state"),
"TEST": ("testing", "info", "Self-test in progress"),
"CAL": ("calibrating", "warn", "Battery calibration — runtime estimates are unreliable"),
"OVER": ("overloaded", "fail", "Load exceeds the UPS rating"),
# FUB is the UPS telling us it decided to power off (scheduled shutdown, or the driver's
# own low-battery policy). Missing it reads as "no state" on the one status that matters.
"FUB": ("forced shutdown", "fail", "The UPS has decided to shut down (FUB)"),
"OTHER": ("unknown", "warn", "Driver reports an unclassifiable state"),
"TRIM": ("trimming", "info", "Trimming high input voltage"),
"BOOST": ("boosting", "info", "Boosting low input voltage"),
}
CHARGER_STATUS = {
"ol": ("online", "ok"),
"ol trim": ("trimming", "info"),
"ol boost": ("boosting", "info"),
"chg": ("charging", "info"),
"dchg": ("discharging", "warn"),
"full": ("fully charged", "ok"),
"pover": ("overloaded", "fail"),
"shutd": ("shutting down", "fail"),
"ebypass": ("bypass", "warn"),
"ehazard": ("hazard", "fail"),
"rec": ("in recalibration", "warn"),
"test": ("testing", "info"),
"other": ("unknown", "warn"),
}
@dataclass
class UpsReading:
"""One UPS, as reported by one poll."""
name: str
description: str = ""
vars: dict[str, str] = field(default_factory=dict)
commands: list[str] = field(default_factory=list)
writable: list[str] = field(default_factory=list)
ts: float = field(default_factory=time.time)
def get(self, key: str) -> str | None:
for candidate in VAR_MAP.get(key, (key,)):
value = self.vars.get(candidate)
if value not in (None, "", "0.000000"):
return value
return None
def number(self, key: str) -> float | None:
raw = self.get(key)
if raw is None:
return None
try:
return float(raw)
except ValueError:
return None
@property
def status_flags(self) -> list[str]:
return (self.get("status") or "").split()
def states(self) -> list[dict[str, str]]:
"""Human state list. The 2.8 `battery.charger.status` wins over legacy ups.status
when both exist because it distinguishes trim/boost/full/recalibration."""
out: list[dict[str, str]] = []
charger = (self.get("charger_status") or "").strip().lower()
if charger and charger in CHARGER_STATUS:
label, severity = CHARGER_STATUS[charger]
out.append({"id": charger, "label": label, "severity": severity})
raw = " ".join(self.status_flags)
if raw:
if raw in STATE_FLAGS:
label, severity, detail = STATE_FLAGS[raw]
out.append({"id": raw, "label": label, "severity": severity, "detail": detail})
else:
for flag in self.status_flags:
if flag in STATE_FLAGS:
label, severity, detail = STATE_FLAGS[flag]
out.append({"id": flag, "label": label, "severity": severity, "detail": detail})
return out
@property
def on_battery(self) -> bool:
flags = set(self.status_flags)
if "OL" in flags:
return False
if "OB" in flags:
return True
return (self.get("charger_status") or "").lower() in ("dchg", "rec")
@property
def alarms(self) -> list[str]:
raw = self.get("alarm") or ""
return [part.strip() for part in raw.split(",") if part.strip()]
# --------------------------------------------------------------------------- #
# Health assessment
# --------------------------------------------------------------------------- #
SEVERITY_ORDER = {"ok": 0, "info": 0, "unknown": 1, "warn": 2, "fail": 3}
def normalise_celsius(value: float | None) -> float | None:
"""Turn a driver temperature into °C.
NUT drivers disagree: usbhid-ups usually reports °C, while the powerware/mge heritage
and some CyberPower firmware report kelvin. 300 "°C" is not a melting battery, it is
27 °C, and a health check that fails on it is worse than no check. Above 150 the value
can only be kelvin for a UPS battery, so convert; below that, trust the driver.
"""
if value is None:
return None
return value - 273.15 if value > 150 else value
def _check(name: str, status: str, detail: str, *, metric: str | None = None,
value: float | None = None) -> dict[str, Any]:
return {"name": name, "status": status, "detail": detail, "metric": metric, "value": value}
def assess(reading: UpsReading, *, now: float | None = None,
battery_replacement_years: float = 4.0, min_runtime_seconds: float = 180.0) -> dict[str, Any]:
"""A list of explicit checks, not a mystery score.
Every line says what it looked at and what it saw, because "health: 87" is the kind
of number an operator cannot act on. Absent data is `unknown`, never a pass.
"""
now = time.time() if now is None else now
checks: list[dict[str, Any]] = []
states = reading.states()
if not states:
checks.append(_check("Power path", "unknown", "ups.status is not exported"))
else:
worst = min(states, key=lambda s: -SEVERITY_ORDER.get(s["severity"], 0),
default=states[0])
worst_sev = max((s["severity"] for s in states), key=lambda s: SEVERITY_ORDER.get(s, 0))
checks.append(_check("Power path", worst_sev, ", ".join(s["label"] for s in states),
value=None))
if worst_sev == "fail" and worst is not None:
checks[-1]["detail"] = worst.get("detail", worst["label"])
charge = reading.number("charge_pct")
if charge is None:
checks.append(_check("Charge", "unknown", "battery.charge is not exported"))
elif reading.on_battery:
# On battery this number only falls, and `LB` plus the runtime estimate already say
# whether that is urgent. Reporting it as an error here would double-alarm.
checks.append(_check("Charge", "warn" if charge < 30 else "ok",
f"{charge:.0f}% and discharging", value=charge))
elif charge >= 99:
checks.append(_check("Charge", "ok", f"{charge:.0f}%", value=charge))
elif charge >= 90:
# Plenty of drivers float at 90-94 % forever (chemistry, temperature, float voltage).
# Calling that "degraded" keeps the page permanently amber and teaches people to
# ignore the one badge that is supposed to mean something.
checks.append(_check("Charge", "ok", f"{charge:.0f}% — floating", value=charge))
elif charge >= 50:
checks.append(_check("Charge", "warn", f"{charge:.0f}% — not reaching full charge",
value=charge))
else:
checks.append(_check("Charge", "fail", f"{charge:.0f}% — barely charged", value=charge))
runtime = reading.number("battery_runtime")
if runtime is None:
checks.append(_check("Runtime estimate", "unknown",
"battery.runtime is not exported (the driver cannot estimate it)"))
elif runtime >= min_runtime_seconds * 2:
checks.append(_check("Runtime estimate", "ok", f"{_duration(runtime)} at the current load",
value=runtime))
elif runtime >= min_runtime_seconds:
checks.append(_check("Runtime estimate", "warn", f"only {_duration(runtime)} left",
value=runtime))
else:
checks.append(_check("Runtime estimate", "fail",
f"{_duration(runtime)} — too little to shut down cleanly",
value=runtime))
if "RB" in reading.status_flags:
checks.append(_check("Battery replacement", "fail", "The UPS flags RB — replace the battery"))
else:
age = _battery_age_years(reading, now)
if age is None:
checks.append(_check("Battery age", "unknown",
"no battery.mfr.date / battery.date from this driver"))
elif age >= battery_replacement_years:
checks.append(_check("Battery age", "warn",
f"{age:.1f} years — past the {battery_replacement_years:.0f}-year "
f"replacement window", value=age))
else:
checks.append(_check("Battery age", "ok", f"{age:.1f} years", value=age))
result = reading.get("test_result")
if result is None:
checks.append(_check("Last self-test", "unknown",
"no self-test result reported — run a battery test"))
else:
passed = result.strip().lower() in ("ok", "passed", "pass", "completed")
failed = result.strip().lower() in ("failed", "fail", "error", "aborted", "test incomplete")
checks.append(_check("Last self-test", "ok" if passed else "fail" if failed else "warn",
f"{result}" + (f" ({reading.get('test_date')})" if reading.get("test_date") else "")))
raw_temp = reading.number("battery_temperature")
temp = normalise_celsius(raw_temp)
converted = temp is not None and raw_temp is not None and raw_temp > 150
suffix = " (driver reports kelvin)" if converted else ""
if temp is None:
checks.append(_check("Battery temperature", "unknown", "not reported by this driver"))
elif temp >= 50:
checks.append(_check("Battery temperature", "fail", f"{temp:.1f} °C — VRLA damage risk{suffix}",
value=temp))
elif temp >= 40:
checks.append(_check("Battery temperature", "warn",
f"{temp:.1f} °C — every +10 °C roughly halves VRLA life{suffix}",
value=temp))
elif temp < -10:
checks.append(_check("Battery temperature", "warn",
f"{temp:.1f} °C looks wrong for a battery indoors", value=temp))
else:
checks.append(_check("Battery temperature", "ok", f"{temp:.1f} °C{suffix}", value=temp))
load = reading.number("load_pct")
if load is not None:
status = "ok" if load < 70 else "warn" if load < 90 else "fail"
detail = f"{load:.0f}% of rated capacity"
power = reading.number("realpower")
nominal = reading.number("realpower_nominal")
if power is not None and nominal:
detail += f" ({power:.0f} W of {nominal:.0f} W)"
checks.append(_check("Load", status, detail, value=load))
alarms = reading.alarms
if alarms:
blocking = [a for a in alarms if re.search(r"(LB|RB|OVER|NOBAT|TEMP|SC)\b", a)]
checks.append(_check("Alarms", "fail" if blocking else "warn", ", ".join(alarms)))
line = reading.number("line_voltage")
nominal = reading.number("line_voltage_nominal")
if line is not None and nominal:
drift = abs(line - nominal) / nominal * 100.0
status = "ok" if drift <= 10 else "warn" if drift <= 15 else "fail"
checks.append(_check("Input voltage", status,
f"{line:.1f} V against {nominal:.0f} V nominal ({drift:.1f}%)",
value=line))
low, high = reading.number("transfer_low"), reading.number("transfer_high")
if low is not None and high is not None:
checks[-1]["detail"] += f"; transfer at {low:.0f}-{high:.0f} V"
elif line is not None:
checks.append(_check("Input voltage", "ok", f"{line:.1f} V", value=line))
worst = max((SEVERITY_ORDER.get(c["status"], 0) for c in checks), default=0)
overall = {0: "healthy", 1: "unknown", 2: "degraded", 3: "at risk"}[worst]
return {
"overall": overall,
"severity": {0: "ok", 1: "unknown", 2: "warn", 3: "fail"}[worst],
"checks": checks,
"counts": {
name: sum(1 for c in checks if c["status"] == name)
for name in ("ok", "warn", "fail", "unknown")
},
}
def _battery_age_years(reading: UpsReading, now: float) -> float | None:
raw = reading.get("battery_date")
if not raw:
return None
match = re.match(r"(\d{4})[-/](\d{1,2})(?:[-/](\d{1,2}))?", raw.strip())
if not match:
return None
year, month, day = match.groups()
try:
created = int(year) * 365.25 + (int(month or 1) - 1) * 30.44 + (int(day or 1) - 1)
except ValueError:
return None
now_years = time.gmtime(now).tm_year * 365.25 + (time.gmtime(now).tm_yday - 1)
age = (now_years - created) / 365.25
# A clock skew (battery.date defaults to 1970 or the future) is not a 56-year-old battery.
if age < 0 or age > 30:
return None
return age
def _duration(seconds: float) -> str:
seconds = max(0.0, seconds)
hours, remainder = divmod(int(seconds), 3600)
minutes, secs = divmod(remainder, 60)
if hours:
return f"{hours}h {minutes:02d}m"
if minutes:
return f"{minutes}m {secs:02d}s"
return f"{secs}s"
# --------------------------------------------------------------------------- #
# Instant commands (the app's only write path)
# --------------------------------------------------------------------------- #
RISK_SAFE = "safe"
RISK_CAUTION = "caution"
RISK_DANGEROUS = "dangerous"
#: command -> label, risk, what it really does to the hardware behind the screen.
#: Anything not listed here is refused even if the driver advertises it.
COMMANDS: dict[str, dict[str, str]] = {
"test.battery.start.quick": {
"label": "Quick battery test",
"risk": RISK_CAUTION,
"help": "Transfers the load to battery for a few seconds. Safe on a healthy UPS, "
"but connected equipment is on battery for the duration.",
},
"test.battery.start.deep": {
"label": "Deep battery test",
"risk": RISK_DANGEROUS,
"help": "Drains the battery until it is low or a runtime limit is hit. Everything on "
"this UPS runs on battery for minutes, and a worn battery may fail outright. "
"Run it deliberately, with an eye on the runtime number.",
},
"test.battery.stop": {"label": "Stop battery test", "risk": RISK_SAFE,
"help": "Aborts a running battery test and returns to line power."},
"test.panel.start": {"label": "Panel test", "risk": RISK_SAFE,
"help": "Lights and beeps the front panel for a few seconds."},
"test.system.start": {"label": "Full system self-test", "risk": RISK_CAUTION,
"help": "Vendor self-test; behaviour varies by model and may move "
"the load to battery."},
"beeper.enable": {"label": "Beeper on", "risk": RISK_SAFE, "help": "Re-enable audible alarms."},
"beeper.disable": {"label": "Beeper off", "risk": RISK_SAFE,
"help": "Silences audible alarms. Faults are still reported in this dashboard."},
"beeper.mute": {"label": "Beeper mute", "risk": RISK_SAFE, "help": "Silence the current alarm only."},
"calibrate.start": {
"label": "Start calibration",
"risk": RISK_DANGEROUS,
"help": "Discharges the battery to recalibrate the charge gauge. Long and disruptive; "
"only do this after a full charge and when the load can tolerate it.",
},
"calibrate.stop": {"label": "Stop calibration", "risk": RISK_SAFE, "help": "Ends calibration."},
"reset.input.minmax": {"label": "Reset input min/max", "risk": RISK_SAFE,
"help": "Clears the recorded input voltage extremes."},
"load.on": {"label": "Energise output", "risk": RISK_CAUTION,
"help": "Turns the outlets on."},
"load.off": {"label": "Cut output power", "risk": RISK_DANGEROUS,
"help": "Kills power to everything plugged in, immediately, on battery or not."},
"load.reboot": {"label": "Power-cycle outlets", "risk": RISK_DANGEROUS,
"help": "Cuts and restores output power. Everything attached reboots."},
"shutdown.return": {"label": "Graceful shutdown, then restore", "risk": RISK_DANGEROUS,
"help": "Asks attached systems to shut down and powers back on when "
"utility returns."},
"shutdown.standoff": {"label": "Graceful shutdown, stay off", "risk": RISK_DANGEROUS,
"help": "Shuts attached systems down and stays off."},
"shutdown.stop": {"label": "Cancel shutdown", "risk": RISK_SAFE,
"help": "Aborts a pending shutdown sequence."},
}
def command_advice(command: str) -> dict[str, str] | None:
return COMMANDS.get(command)
def available_commands(advertised: list[str], *, allow_actions: bool,
read_only: bool, has_credentials: bool,
on_battery: bool = False) -> list[dict[str, Any]]:
"""What the UI may offer, with the reason when it may not.
Filtering server-side (not just hiding buttons) is the point: the response says
`enabled: false` plus `reason`, so the page never offers a button that 403s.
"""
out: list[dict[str, Any]] = []
for name in advertised:
advice = COMMANDS.get(name)
if advice is None:
continue
reason = None
if read_only:
reason = "the server is in DASH_READ_ONLY mode"
elif not allow_actions:
reason = "set NUT_ALLOW_ACTIONS=1 to enable UPS commands"
elif not has_credentials:
reason = "NUT_USER / NUT_PASSWORD must name a user with `actions` in upsd.users"
elif on_battery and advice["risk"] == RISK_DANGEROUS:
reason = "refusing a destructive command while the UPS is on battery"
out.append({
"command": name,
"label": advice["label"],
"risk": advice["risk"],
"help": advice["help"],
"enabled": reason is None,
"reason": reason,
"confirm_text": None if advice["risk"] != RISK_DANGEROUS else "DANGER",
})
order = {RISK_SAFE: 0, RISK_CAUTION: 1, RISK_DANGEROUS: 2}
out.sort(key=lambda item: (order[item["risk"]], item["label"]))
return out
def payload(reading: UpsReading, *, now: float | None = None, **assess_kwargs: Any) -> dict[str, Any]:
"""The JSON one UPS card renders from."""
now = time.time() if now is None else now
health = assess(reading, now=now, **assess_kwargs)
numbers = {key: reading.number(key) for key in VAR_MAP}
# charts must not plot 300 "°C"; the raw string stays visible in the variable dump
for key in ("battery_temperature", "ups_temperature"):
if numbers.get(key) is not None:
numbers[key] = normalise_celsius(numbers[key])
return {
"name": reading.name,
"description": reading.description,
"identity": {
"mfr": reading.get("mfr"),
"model": reading.get("model"),
"serial": reading.get("serial"),
"firmware": reading.get("firmware"),
"driver": reading.get("driver"),
"battery_type": reading.get("battery_type"),
},
"ts": reading.ts,
"status": reading.get("status"),
"charger_status": reading.get("charger_status"),
"states": reading.states(),
"alarms": reading.alarms,
"on_battery": reading.on_battery,
"numbers": numbers,
"health": health,
"advertised_commands": reading.commands,
"writable": reading.writable,
"extra": {
key: value for key, value in sorted(reading.vars.items())
if key.startswith(EXTRA_VAR_PREFIXES) and key not in {v for vs in VAR_MAP.values() for v in vs}
},
}
def history_samples(reading: UpsReading) -> dict[str, float]:
"""The numeric series worth keeping a history of."""
raw = {
"ups.charge_pct": "charge_pct",
"ups.battery_voltage": "battery_voltage",
"ups.battery_current": "battery_current",
"ups.battery_runtime_seconds": "battery_runtime",
"ups.battery_temperature_c": "battery_temperature",
"ups.line_voltage": "line_voltage",
"ups.output_voltage": "output_voltage",
"ups.output_current": "output_current",
"ups.output_frequency": "output_frequency",
"ups.frequency": "frequency",
"ups.load_pct": "load_pct",
"ups.realpower_w": "realpower",
"ups.temperature_c": "ups_temperature",
}
out: dict[str, float] = {}
for metric, key in raw.items():
value = reading.number(key)
if value is None:
continue
if metric.endswith("_temperature_c") or metric == "ups.temperature_c":
value = normalise_celsius(value)
if value is not None:
out[metric] = value
return out
+214
View File
@@ -0,0 +1,214 @@
"""A *labelled* UPS simulator, for development when there is no upsd to talk to.
Enable with `NUT_SIMULATE=1`. It exists so the UPS page can be built and screenshotted
before the hardware is cabled — and so the battery-test buttons can be exercised without
putting a real load on battery. It is never silent about what it is: the payload carries
`simulated: true` and the page shows a watermark on every simulated number.
The model is deliberately small and physical rather than random:
* charge sits at float (100%) while on line, drains at a Peukert-ish rate when on battery
* runtime estimates scale with charge and fall off steeply with load (as they do for real
VRLA packs, which is why the number jumps around while the load moves)
* a quick battery test moves to OB for ~12s and then reports PASSED, so the whole
command → state change → event-log path is exercised
"""
from __future__ import annotations
import math
import time
from dataclasses import dataclass, field
from typing import Any
from .nut import UpsReading
@dataclass
class SimUps:
"""One fake UPS: nameplate values plus the state a driver would report."""
name: str
model: str
mfr: str
serial: str
nominal_va: float = 1500.0
nominal_w: float = 900.0
battery_cells: int = 2 # 2 x 12V blocks
battery_nominal: float = 24.0
capacity_ah: float = 17.0
load_w: float = 210.0
charge: float = 100.0
on_battery: bool = False
temperature_c: float = 27.0
mfr_date: str = "2022-04-15"
test_result: str = "PASSED"
test_date: str = "2026-09-01"
firmware: str = "UPS 08.3 / IGR 1.00"
driver: str = "usbhid-ups"
_test_until: float = 0.0
_deep_until: float = 0.0
_beeper: bool = True
_alarms: list[str] = field(default_factory=list)
def tick(self, now: float) -> None:
"""Advance the model by wall-clock time since the last tick."""
# a running self-test pulls the load from the wall, whichever test it is
self.on_battery = now < self._test_until or now < self._deep_until
def step(self, dt: float) -> None:
dt = max(0.0, min(dt, 60.0))
if self.on_battery:
# Ah drawn / capacity, with a mild Peukert exponent so a heavy load costs more
hours = dt / 3600.0
load_fraction = max(0.05, self.load_w / self.nominal_w)
drawn = (self.load_w / self.battery_nominal) * hours * (load_fraction ** 0.15)
self.charge = max(0.0, self.charge - (drawn / self.capacity_ah) * 100.0)
elif self.charge < 100.0:
self.charge = min(100.0, self.charge + dt * (100.0 / 14400.0)) # ~4h recharge
self.temperature_c = 27.0 + 0.6 * math.sin(time.time() / 300.0) + self.charge / 400.0
def runtime_seconds(self) -> float:
"""Battery runtime at the present load — the number the UPS itself estimates."""
if not self.on_battery and self.charge >= 99.5:
# Full and on line: what a *fresh* pack would give at this load, which is what
# a UPS reports when it is idle on mains.
usable_ah = self.capacity_ah * 0.9
else:
usable_ah = self.capacity_ah * (self.charge / 100.0)
amps = max(0.5, self.load_w / (self.battery_nominal * 0.9))
load_penalty = max(0.35, 1.0 - (self.load_w / self.nominal_w) ** 2 * 0.45)
return (usable_ah / amps) * 3600.0 * load_penalty
def vars(self, now: float) -> dict[str, str]:
load_pct = self.load_w / self.nominal_w * 100.0
flags = ["OL"] if not self.on_battery else (["OB"] + (["LB"] if self.charge < 20 else []))
if self.charge >= 99.5 and not self.on_battery:
flags.append("CHRG" if self.charge < 99.9 else "")
status = " ".join(part for part in flags if part)
charger = "ol" if not self.on_battery else ("dchg" if self.charge < 99.5 else "ol")
if self.charge >= 99.9 and not self.on_battery:
charger = "full"
volts = (self.battery_nominal * (0.86 + 0.14 * self.charge / 100.0)
if self.on_battery else self.battery_nominal * 1.045)
line = 121.4 + 1.1 * math.sin(now / 90.0)
out = {
"ups.mfr": self.mfr,
"ups.model": self.model,
"ups.serial": self.serial,
"ups.firmware": self.firmware,
"ups.status": status,
"battery.charger.status": charger,
"ups.load": f"{load_pct:.1f}",
"ups.realpower": f"{self.load_w:.1f}",
"ups.realpower.nominal": f"{self.nominal_w:.0f}",
"input.voltage": f"{line:.1f}",
"input.voltage.nominal": "120",
"input.frequency": f"{60.0 + 0.02 * math.sin(now / 40.0):.2f}",
"input.transfer.low": "92",
"input.transfer.high": "139",
"output.voltage": f"{line if not self.on_battery else 119.8:.1f}",
"output.current": f"{self.load_w / 120.0:.2f}",
"output.frequency": f"{60.0 + 0.02 * math.sin(now / 40.0):.2f}",
"battery.charge": f"{self.charge:.1f}",
"battery.voltage": f"{volts:.2f}",
"battery.voltage.nominal": f"{self.battery_nominal:.0f}",
"battery.current": f"{self.load_w / volts:.2f}",
"battery.temperature": f"{self.temperature_c + 273.15:.1f}", # drivers report K
"battery.type": "PbAcid: 12V / 9Ah / 17Ah sealed",
"battery.runtime": f"{self.runtime_seconds():.0f}",
"battery.runtime.low": "120",
"battery.mfr.date": self.mfr_date,
"battery.capacity": f"{self.capacity_ah:.0f}",
"battery.test.result": self.test_result,
"battery.test.date": self.test_date,
"driver.name": self.driver,
"driver.parameter.port": "auto",
"ups.id": self.name,
}
if self._alarms:
out["ups.alarm"] = ",".join(self._alarms)
return out
def commands(self) -> list[str]:
return ["test.battery.start.quick", "test.battery.start.deep", "test.battery.stop",
"test.panel.start", "beeper.enable", "beeper.disable", "beeper.mute",
"calibrate.start", "calibrate.stop", "reset.input.minmax",
"load.on", "load.off", "shutdown.stop"]
def run_command(self, command: str, now: float) -> str:
"""Apply a command the way a driver would, so the UI's whole path is real."""
if command == "test.battery.start.quick":
self._test_until = now + 12.0
self.on_battery = True
self.test_date = time.strftime("%Y-%m-%d", time.gmtime(now))
return "OK Start Quick Battery Test"
if command == "test.battery.start.deep":
self._deep_until = now + 90.0
self.on_battery = True
return "OK Start Deep Battery Test"
if command in ("test.battery.stop", "calibrate.stop"):
self._test_until = self._deep_until = 0.0
self.on_battery = False
return "OK Stop Test"
if command == "test.panel.start":
return "OK Start Panel Test"
if command == "beeper.enable":
self._beeper = True
return "OK Beeper enabled"
if command in ("beeper.disable", "beeper.mute"):
self._beeper = False
return "OK Beeper muted"
if command == "calibrate.start":
self._deep_until = now + 180.0
self.on_battery = True
return "OK Start Calibration"
if command == "reset.input.minmax":
return "OK Input min/max reset"
if command == "load.on":
return "OK Load energised"
if command == "load.off":
return "OK Load switched off (simulated)"
if command == "shutdown.stop":
return "OK Shutdown cancelled"
raise ValueError(f"simulated UPS does not implement {command}")
class NutSimulator:
"""Stands in for upsd: same shapes as NutClient reads, no network."""
def __init__(self, spec: str | None = None) -> None:
self.ups = [
SimUps("cyberpower", "PR1500LCDRT2U", "Cyber Power Systems", "CPS-04213",
nominal_va=1500.0, nominal_w=900.0, load_w=210.0, capacity_ah=17.0,
mfr_date="2022-04-15"),
SimUps("apc", "Smart-UPS 1500", "APC", "AS-11902",
nominal_va=1500.0, nominal_w=865.0, load_w=140.0, capacity_ah=8.0,
mfr_date="2019-11-02", driver="usbhid-ups",
test_result="PASSED", test_date="2026-06-14"),
]
self._last = time.time()
async def read(self) -> list[UpsReading]:
now = time.time()
dt = now - self._last
self._last = now
for ups in self.ups:
ups.tick(now)
ups.step(dt)
return [
UpsReading(name=ups.name, description=f"{ups.mfr} {ups.model} (simulated)",
vars=ups.vars(now), commands=ups.commands(),
writable=["beeper.enable"], ts=now)
for ups in self.ups
]
def command(self, ups_name: str, command: str) -> str:
for ups in self.ups:
if ups.name == ups_name:
return ups.run_command(command, time.time())
raise KeyError(f"unknown simulated UPS {ups_name!r}")
def from_env(value: str | None) -> Any:
"""`NUT_SIMULATE=1` -> a simulator; anything else -> None (talk to real upsd)."""
return NutSimulator() if (value or "").strip().lower() in ("1", "true", "yes", "on") else None
+348
View File
@@ -0,0 +1,348 @@
"""SQLite history: one sample table with three resolutions, plus an event log.
Why SQLite and not Prometheus: the fleet is a handful of boxes, the poll resolution we
need is 1-10s, and a single file the container owns beats running a TSDB. Series are
addressed by `(source, host, metric, res, ts)`:
source the dashboard family — "host", "llm", "ups"
host stable target key ("driveripper", "gufo", "apc-ups1")
metric dotted name, optionally `@stream` for per-device series
(cpu.usage_pct, cpu.usage_pct@core7, disk.used_pct@/srv, gpu.busy_pct@0)
res "raw" | "1m" | "1h"
`maintain()` rolls raw up into 1m, 1m into 1h, and prunes past the retention window, so
the file stays bounded (a 3-host / 3-engine / 2-UPS fleet is a few hundred MB a year,
not a disk emergency).
The connection is guarded by a lock and every method is synchronous: async callers wrap
them in `anyio.to_thread.run_sync`, sync FastAPI handlers run in the threadpool already.
"""
from __future__ import annotations
import json
import math
import sqlite3
import threading
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from pathlib import Path
RES_RAW, RES_MIN, RES_HOUR = "raw", "1m", "1h"
MINUTE = 60.0
HOUR = 3600.0
_SCHEMA = """
CREATE TABLE IF NOT EXISTS sample (
source TEXT NOT NULL,
host TEXT NOT NULL,
metric TEXT NOT NULL,
res TEXT NOT NULL,
ts REAL NOT NULL,
value REAL NOT NULL,
PRIMARY KEY (source, host, metric, res, ts) ON CONFLICT REPLACE
);
CREATE TABLE IF NOT EXISTS event (
id INTEGER PRIMARY KEY,
ts REAL NOT NULL,
source TEXT NOT NULL,
host TEXT NOT NULL,
kind TEXT NOT NULL,
severity TEXT NOT NULL,
message TEXT NOT NULL,
data TEXT
);
CREATE INDEX IF NOT EXISTS event_ts ON event (ts DESC);
CREATE INDEX IF NOT EXISTS event_source_ts ON event (source, ts DESC);
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
"""
@dataclass(frozen=True)
class Sample:
source: str
host: str
metric: str
value: float
ts: float
@dataclass(frozen=True)
class Event:
ts: float
source: str
host: str
kind: str
severity: str # info | warn | error
message: str
data: dict | None = None
def as_json(self) -> dict:
return {
"ts": self.ts,
"source": self.source,
"host": self.host,
"kind": self.kind,
"severity": self.severity,
"message": self.message,
"data": self.data,
}
class SampleStore:
def __init__(self, path: str | Path, *, raw_keep_hours: float = 48,
minute_keep_days: float = 30, hour_keep_days: float = 400) -> None:
self.path = Path(path)
self.raw_keep_seconds = raw_keep_hours * HOUR
self.minute_keep_seconds = minute_keep_days * 24 * HOUR
self.hour_keep_seconds = hour_keep_days * 24 * HOUR
self._lock = threading.Lock()
if str(self.path) != ":memory:":
self.path.parent.mkdir(parents=True, exist_ok=True)
self._db = sqlite3.connect(str(self.path), check_same_thread=False)
self._db.execute("PRAGMA journal_mode=WAL")
self._db.execute("PRAGMA synchronous=NORMAL") # a lost sample is fine, corruption is not
self._db.execute("PRAGMA busy_timeout=5000")
self._db.executescript(_SCHEMA)
self._db.commit()
def close(self) -> None:
with self._lock:
self._db.close()
# -- writing --------------------------------------------------------------- #
def add(self, source: str, host: str, metric: str, value: float, ts: float | None = None) -> None:
self.add_many([Sample(source, host, metric, value, time.time() if ts is None else ts)])
def add_many(self, samples: Iterable[Sample]) -> int:
"""Insert samples; NaN/inf are dropped (SQLite would store them and charts choke).
Returns the number actually written so the caller can report it, never to fail
the poll over one bad value.
"""
rows = [
(s.source, s.host, s.metric, RES_RAW, s.ts, s.value)
for s in samples
if isinstance(s.value, int | float) and math.isfinite(s.value)
]
if not rows:
return 0
with self._lock, self._db:
self._db.executemany("INSERT OR REPLACE INTO sample VALUES (?,?,?,?,?,?)", rows)
return len(rows)
def log_event(self, event: Event) -> None:
with self._lock, self._db:
self._db.execute(
"INSERT INTO event (ts, source, host, kind, severity, message, data) VALUES (?,?,?,?,?,?,?)",
(event.ts, event.source, event.host, event.kind, event.severity, event.message,
json.dumps(event.data) if event.data else None),
)
# -- reading --------------------------------------------------------------- #
def query(self, source: str, host: str, metric: str, *, res: str = RES_RAW,
start: float = 0.0, end: float | None = None, limit: int = 2000,
tail: bool = False) -> list[tuple[float, float]]:
"""(ts, value) ascending by time within [start, end].
`tail=True` keeps the *newest* `limit` rows instead of the oldest. A window query
that truncates at the head draws the left edge of the chart and silently drops
"now", which is the one point a dashboard has to have.
"""
end = time.time() if end is None else end
where = ("source=? AND host=? AND metric=? AND res=? AND ts>=? AND ts<=?")
args: tuple[object, ...] = (source, host, metric, res, start, end)
with self._lock:
if tail:
cur = self._db.execute(
f"SELECT ts, value FROM (SELECT ts, value FROM sample WHERE {where}"
" ORDER BY ts DESC LIMIT ?) ORDER BY ts ASC",
(*args, limit),
)
else:
cur = self._db.execute(
f"SELECT ts, value FROM sample WHERE {where} ORDER BY ts ASC LIMIT ?",
(*args, limit),
)
return cur.fetchall() # type: ignore[no-any-return]
def pick_res(self, source: str, host: str, metric: str, span: float, max_points: int) -> str:
"""Finest resolution that still fits in `max_points` for this window."""
now = time.time()
start = now - span
finest_with_data: str | None = None
for res in (RES_RAW, RES_MIN, RES_HOUR):
with self._lock:
(count,) = self._db.execute(
"SELECT COUNT(*) FROM sample WHERE source=? AND host=? AND metric=? AND res=? AND ts>=?",
(source, host, metric, res, start),
).fetchone()
if not count:
continue
if finest_with_data is None:
# too many points for the budget is fine — history() thins. Returning a
# resolution with *no* rows in the window reads as a dead sensor.
finest_with_data = res
if count <= max_points:
return res
return finest_with_data or RES_HOUR
def history(self, source: str, host: str, metric: str, *, span: float = 3600.0,
max_points: int = 600) -> list[tuple[float, float]]:
"""Windowed series, thinned to `max_points` so the browser never gets 100k rows."""
res = self.pick_res(source, host, metric, span, max_points)
window_start = time.time() - span
points = self.query(source, host, metric, res=res, start=window_start,
limit=max_points * 4, tail=True)
if not points and res != RES_RAW:
# Rollups lag behind (they only cover buckets that are fully in the past, and a
# dashboard that has been up for ten minutes has none at all). Falling back to
# raw turns "no samples yet" into a real chart instead of a lie by omission.
points = self.query(source, host, metric, res=RES_RAW, start=window_start,
limit=max_points * 4, tail=True)
if len(points) <= max_points:
return points
stride = math.ceil(len(points) / max_points)
thinned = points[::stride]
if thinned[-1] != points[-1]:
thinned.append(points[-1]) # never lose "now" to the thinning
return thinned
def series_names(self, source: str | None = None, host: str | None = None) -> list[str]:
sql = "SELECT DISTINCT metric FROM sample"
args: list[object] = []
clauses = []
if source:
clauses.append("source=?")
args.append(source)
if host:
clauses.append("host=?")
args.append(host)
if clauses:
sql += " WHERE " + " AND ".join(clauses)
with self._lock:
return [row[0] for row in self._db.execute(sql + " ORDER BY metric", args)]
def hosts(self, source: str) -> list[str]:
with self._lock:
return [row[0] for row in self._db.execute(
"SELECT DISTINCT host FROM sample WHERE source=? ORDER BY host", (source,))]
def recent_events(self, limit: int = 50, *, source: str | None = None,
since: float | None = None) -> list[Event]:
sql = "SELECT ts, source, host, kind, severity, message, data FROM event"
args: list[object] = []
clauses: list[str] = []
if source:
clauses.append("source=?")
args.append(source)
if since is not None:
clauses.append("ts>=?")
args.append(since)
if clauses:
sql += " WHERE " + " AND ".join(clauses)
sql += " ORDER BY ts DESC LIMIT ?"
args.append(limit)
out: list[Event] = []
with self._lock:
for ts, src, host, kind, sev, msg, data in self._db.execute(sql, args):
out.append(Event(ts, src, host, kind, sev, msg, json.loads(data) if data else None))
return out
def count(self, *, res: str | None = None) -> int:
sql = "SELECT COUNT(*) FROM sample"
args: tuple[object, ...] = ()
if res:
sql, args = "SELECT COUNT(*) FROM sample WHERE res=?", (res,)
with self._lock:
return int(self._db.execute(sql, args).fetchone()[0])
# -- maintenance ----------------------------------------------------------- #
def maintain(self, now: float | None = None) -> dict[str, int]:
"""Roll up and prune. Safe to call every minute; each step is idempotent.
A bucket is only rolled once its whole window is in the past (+60s slack), so a
rollup never averages a partial minute and then gets rewritten by a later one.
"""
now = time.time() if now is None else now
counts = {"to_minute": 0, "to_hour": 0, "pruned": 0}
with self._lock, self._db:
counts["to_minute"] = self._db.execute(
"INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)"
" SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)"
" FROM sample WHERE res=? AND ts < ? AND ts >= ?"
" GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)",
(RES_MIN, MINUTE, MINUTE, RES_RAW, now - MINUTE - 60,
now - self.raw_keep_seconds, MINUTE),
).rowcount
counts["to_hour"] = self._db.execute(
"INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)"
" SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)"
" FROM sample WHERE res=? AND ts < ? AND ts >= ?"
" GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)",
(RES_HOUR, HOUR, HOUR, RES_MIN, now - HOUR - 60,
now - self.minute_keep_seconds, HOUR),
).rowcount
counts["pruned"] = self._db.execute(
"DELETE FROM sample WHERE (res=? AND ts < ?) OR (res=? AND ts < ?) OR (res=? AND ts < ?)",
(RES_RAW, now - self.raw_keep_seconds,
RES_MIN, now - self.minute_keep_seconds,
RES_HOUR, now - self.hour_keep_seconds),
).rowcount
self._db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_maintain', ?)", (str(now),))
return counts
def stats(self) -> dict[str, object]:
with self._lock:
by_res = {row[0]: row[1] for row in self._db.execute(
"SELECT res, COUNT(*) FROM sample GROUP BY res")}
(events,) = self._db.execute("SELECT COUNT(*) FROM event").fetchone()
last = self._db.execute("SELECT value FROM meta WHERE key='last_maintain'").fetchone()
size = 0
if str(self.path) != ":memory:" and self.path.exists():
size = self.path.stat().st_size
return {
"path": str(self.path),
"size_bytes": size,
"samples_by_res": by_res,
"samples_total": sum(by_res.values()),
"events": events,
"last_maintain": float(last[0]) if last else None,
}
def delete_series(self, source: str, host: str, metric: str | None = None) -> int:
"""Drop history for a series (or a whole host). Used by tests and by the
'forget this host' escape hatch when a target is renamed for real."""
sql = "DELETE FROM sample WHERE source=? AND host=?"
args: list[object] = [source, host]
if metric:
sql += " AND metric=?"
args.append(metric)
with self._lock, self._db:
return self._db.execute(sql, args).rowcount
def samples_from_flat(source: str, host: str, values: dict[str, float | None],
ts: float, *, prefix: str = "") -> list[Sample]:
"""Flatten a parse result into samples, skipping absent values.
`n/a` in the UI means "this exporter does not report it" — it must not become a
stored 0, which would look identical on a 30-day chart and lie about history.
"""
out: list[Sample] = []
for key, value in values.items():
if value is None or isinstance(value, dict | list | str | bool):
continue
out.append(Sample(source, host, f"{prefix}{key}", float(value), ts))
return out
def select_window(span: float, allowed: Sequence[float]) -> float:
"""Closest allowed window >= span (used to map ?range=1h onto the presets)."""
for candidate in sorted(allowed):
if span <= candidate:
return candidate
return max(allowed)
+128
View File
@@ -0,0 +1,128 @@
<!doctype html>
<html lang="en" data-theme="dark">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Homelab · overview</title>
<link rel="stylesheet" href="/static/dash.css">
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' rx='22' fill='%23ff8a3d'/></svg>">
</head>
<body>
<header class="masthead">
<a class="brand" href="/"><span class="brand-mark">◆</span> Homelab</a>
<nav class="tabs" aria-label="Dashboards">
<a href="/" aria-current="page">Overview</a>
<a href="/system">System</a>
<a href="/llm">LLM</a>
<a href="/ups">UPS</a>
</nav>
<div class="masthead-right">
<div id="nav-status" class="gap" aria-live="polite"></div>
<span class="clock" id="clock">--:--:--</span>
<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">☾ Dark</button>
</div>
</header>
<main>
<div class="page-head">
<div>
<h1>Overview</h1>
<div class="sub" id="summary-line">reading fleet…</div>
</div>
<div class="spacer"></div>
<span class="chip" id="config-chip" hidden>loading config</span>
</div>
<div id="banners" class="stack" style="margin-bottom:.85rem"></div>
<section class="grid auto" aria-label="Dashboards">
<article class="card">
<header><h2>System resources</h2><span class="spacer"></span><span class="dot idle" id="dot-system"></span></header>
<div class="body">
<div class="metrics">
<div class="metric"><div class="label">Hosts answering</div>
<div class="value"><span id="sys-hosts-live">n/a</span><span class="unit" id="sys-hosts-total"></span></div></div>
<div class="metric"><div class="label">CPU average</div>
<div class="value"><span id="sys-cpu">n/a</span><span class="unit">%</span></div>
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
<div class="metric"><div class="label">Memory</div>
<div class="value"><span id="sys-mem">n/a</span><span class="unit" id="sys-mem-unit"></span></div>
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
<div class="metric"><div class="label">Worst disk</div>
<div class="value"><span id="sys-disk">n/a</span><span class="unit">%</span></div>
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
<div class="metric"><div class="label">GPUs</div>
<div class="value sm" id="sys-gpus">n/a</div></div>
</div>
<div class="gap" style="margin-top:.7rem"><a class="action" href="/system" style="text-decoration:none">Open system dashboard →</a></div>
</div>
</article>
<article class="card">
<header><h2>LLM engines</h2><span class="spacer"></span><span class="dot idle" id="dot-llm"></span></header>
<div class="body">
<div class="metrics">
<div class="metric"><div class="label">Engines answering</div>
<div class="value"><span id="llm-live">n/a</span><span class="unit" id="llm-total"></span></div></div>
<div class="metric"><div class="label">Decode</div>
<div class="value"><span id="llm-decode">n/a</span><span class="unit">tok/s</span></div></div>
<div class="metric"><div class="label">Prompt</div>
<div class="value"><span id="llm-prompt">n/a</span><span class="unit">tok/s</span></div></div>
<div class="metric"><div class="label">Active sessions</div>
<div class="value" id="llm-active">n/a</div></div>
<div class="metric"><div class="label">Queued</div>
<div class="value" id="llm-waiting">n/a</div></div>
<div class="metric"><div class="label" title="highest KV-cache use across engines right now">KV cache now</div>
<div class="value"><span id="llm-kv">n/a</span><span class="unit">%</span></div>
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
</div>
<div class="gap" style="margin-top:.7rem"><a class="action" href="/llm" style="text-decoration:none">Open LLM dashboard →</a></div>
</div>
</article>
<article class="card">
<header><h2>UPS / battery</h2><span class="spacer"></span><span class="dot idle" id="dot-ups"></span></header>
<div class="body">
<div id="ups-list" class="stack">
<div class="muted">loading…</div>
</div>
<div class="gap" style="margin-top:.7rem"><a class="action" href="/ups" style="text-decoration:none">Open UPS dashboard →</a></div>
</div>
</article>
</section>
<section class="grid wide" style="margin-top:.85rem">
<article class="card">
<header><h2>Fleet</h2><span class="spacer"></span><span class="faint mono" id="fleet-note"></span></header>
<div class="body tight">
<table class="data" id="fleet-table">
<thead><tr><th>Host</th><th>CPU</th><th>Load 1</th><th>Mem used</th><th>Worst disk</th><th>GPU busy</th><th>Temp</th><th>Up</th></tr></thead>
<tbody></tbody>
</table>
<div class="empty-state" id="fleet-empty" hidden>
<h3>No hosts configured</h3>
<p>Run <code>agent/host_agent.py</code> on each machine and list it in
<code>HOST_TARGETS</code>.</p>
</div>
</div>
</article>
<article class="card">
<header><h2>Recent events</h2><span class="spacer"></span>
<span class="faint">transitions only</span></header>
<div class="body"><ul class="events" id="events"><li class="muted">loading…</li></ul></div>
</article>
</section>
<footer class="pagefoot">
<span id="foot-versions"></span>
<span class="spacer" style="margin-inline-start:auto"></span>
<span>every number is scraped from the box it belongs to · <code>null</code> renders as <code>n/a</code></span>
</footer>
</main>
<script src="/static/charts.js"></script>
<script src="/static/app.js"></script>
<script src="/static/page-index.js"></script>
</body>
</html>
+121
View File
@@ -0,0 +1,121 @@
<!doctype html>
<html lang="en" data-theme="dark">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Homelab · LLM</title>
<link rel="stylesheet" href="/static/dash.css">
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' rx='22' fill='%23ff8a3d'/></svg>">
</head>
<body>
<header class="masthead">
<a class="brand" href="/"><span class="brand-mark">◆</span> Homelab</a>
<nav class="tabs" aria-label="Dashboards">
<a href="/">Overview</a>
<a href="/system">System</a>
<a href="/llm" aria-current="page">LLM</a>
<a href="/ups">UPS</a>
</nav>
<div class="masthead-right">
<div id="nav-status" class="gap" aria-live="polite"></div>
<span class="clock" id="clock">--:--:--</span>
<button class="ghost" id="pause" type="button" aria-pressed="false">⏸ Pause</button>
<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">☾ Dark</button>
</div>
</header>
<main>
<div class="page-head">
<div>
<h1>LLM engines</h1>
<div class="sub" id="sub">reading engines…</div>
</div>
<div class="spacer"></div>
<div class="gap"><span class="faint">window</span><div id="range"></div></div>
</div>
<div id="banners" class="stack" style="margin-bottom:.85rem"></div>
<section class="card flat" aria-label="Engine totals">
<div class="body">
<div class="metrics">
<div class="metric"><div class="label">Engines answering</div>
<div class="value"><span id="f-live">n/a</span><span class="unit" id="f-total"></span></div></div>
<div class="metric"><div class="label">Decode (sum)</div>
<div class="value"><span id="f-decode">n/a</span><span class="unit">tok/s</span></div></div>
<div class="metric"><div class="label">Prompt (sum)</div>
<div class="value"><span id="f-prompt">n/a</span><span class="unit">tok/s</span></div></div>
<div class="metric"><div class="label">Active sessions</div>
<div class="value" id="f-active">n/a</div></div>
<div class="metric"><div class="label">Queued</div>
<div class="value" id="f-waiting">n/a</div></div>
<div class="metric"><div class="label">Tokens lifetime</div>
<div class="value"><span id="f-tokens">n/a</span><span class="unit"></span></div></div>
<div class="metric"><div class="label" title="highest KV-cache use across engines right now, not a window peak">KV cache now</div>
<div class="value"><span id="f-kv">n/a</span><span class="unit">%</span></div>
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
</div>
<div class="faint" id="seam-note" style="margin-top:.5rem;font-size:11.5px"></div>
</div>
</section>
<section class="grid cols-2" style="margin-top:.85rem" aria-label="Throughput charts">
<article class="card">
<header><h3>Decode throughput</h3><span class="spacer"></span>
<span class="faint">tok/s · every engine, summed slots</span></header>
<div class="body"><div class="chart" id="chart-decode"></div><div class="legend" id="legend-decode"></div></div>
</article>
<article class="card">
<header><h3>Prompt / prefill throughput</h3><span class="spacer"></span>
<span class="faint">tok/s · newly computed tokens</span></header>
<div class="body"><div class="chart" id="chart-prompt"></div><div class="legend" id="legend-prompt"></div></div>
</article>
<article class="card">
<header><h3>KV / prefix cache pressure</h3><span class="spacer"></span><span class="faint">% of cache in use</span></header>
<div class="body"><div class="chart" id="chart-kv"></div><div class="legend" id="legend-kv"></div></div>
</article>
<article class="card">
<header><h3>Sessions &amp; queue</h3><span class="spacer"></span><span class="faint">running vs waiting</span></header>
<div class="body"><div class="chart" id="chart-queue"></div><div class="legend" id="legend-queue"></div></div>
</article>
</section>
<section style="margin-top:.85rem">
<div class="grid cols-2" id="engines"></div>
<div class="empty-state" id="engines-empty" hidden>
<h3>No engines configured</h3>
<p>Set <code>GUFO_BASE_URL</code> (llama.cpp / gufo) and/or <code>VLLM_BASE_URL</code>.
Either half appears only when its variable is set — a server you do not run
should not have a red panel on your dashboard.</p>
</div>
</section>
<section class="grid cols-2" style="margin-top:.85rem">
<article class="card" id="slots-card">
<header><h3>gufo slots</h3><span class="spacer"></span>
<span class="faint">one row per parallel slot</span></header>
<div class="body tight"><table class="data" id="slots-table">
<thead><tr><th>Slot</th><th>State</th><th>Tokens</th><th>N prompt</th><th>N gen</th>
<th>Cache</th><th>Speed</th></tr></thead><tbody></tbody></table>
<div class="empty-state" id="slots-empty" hidden><p>No <code>/slots</code> rows.</p></div>
</div>
</article>
<article class="card">
<header><h3>Events</h3><span class="spacer"></span><span class="faint">engine up / down</span></header>
<div class="body"><ul class="events" id="events"><li class="muted">loading…</li></ul></div>
</article>
</section>
<footer class="pagefoot">
<span>each cell carries its provenance: scraped gauge, <span class="mono">Δ10s</span> measured
counter, or lifetime estimate</span>
<span class="spacer" style="margin-inline-start:auto"></span>
<span class="mono" id="foot-note"></span>
</footer>
</main>
<script src="/static/charts.js"></script>
<script src="/static/app.js"></script>
<script src="/static/page-llm.js"></script>
</body>
</html>
+136
View File
@@ -0,0 +1,136 @@
<!doctype html>
<html lang="en" data-theme="dark">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Homelab · system</title>
<link rel="stylesheet" href="/static/dash.css">
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' rx='22' fill='%23ff8a3d'/></svg>">
</head>
<body>
<header class="masthead">
<a class="brand" href="/"><span class="brand-mark">◆</span> Homelab</a>
<nav class="tabs" aria-label="Dashboards">
<a href="/">Overview</a>
<a href="/system" aria-current="page">System</a>
<a href="/llm">LLM</a>
<a href="/ups">UPS</a>
</nav>
<div class="masthead-right">
<div id="nav-status" class="gap" aria-live="polite"></div>
<span class="clock" id="clock">--:--:--</span>
<button class="ghost" id="pause" type="button" aria-pressed="false">⏸ Pause</button>
<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">☾ Dark</button>
</div>
</header>
<main>
<div class="page-head">
<div>
<h1>System resources</h1>
<div class="sub" id="sub">reading hosts…</div>
</div>
<div class="spacer"></div>
<div class="gap">
<span class="faint">window</span>
<div id="range"></div>
</div>
</div>
<section class="card flat" aria-label="Fleet totals">
<div class="body">
<div class="metrics">
<div class="metric"><div class="label">Hosts answering</div>
<div class="value"><span id="f-hosts">n/a</span><span class="unit" id="f-hosts-of"></span></div></div>
<div class="metric"><div class="label">CPU average</div>
<div class="value"><span id="f-cpu">n/a</span><span class="unit">%</span></div></div>
<div class="metric"><div class="label">Load 1 (sum)</div>
<div class="value" id="f-load">n/a</div></div>
<div class="metric"><div class="label">Memory in use</div>
<div class="value"><span id="f-mem">n/a</span><span class="unit" id="f-mem-of"></span></div></div>
<div class="metric"><div class="label">Worst filesystem</div>
<div class="value"><span id="f-disk">n/a</span><span class="unit">%</span></div></div>
<div class="metric"><div class="label">Network in</div>
<div class="value"><span id="f-rx">n/a</span><span class="unit" id="f-rx-unit"></span></div></div>
<div class="metric"><div class="label">Network out</div>
<div class="value"><span id="f-tx">n/a</span><span class="unit" id="f-tx-unit"></span></div></div>
<div class="metric"><div class="label">GPU busy (avg)</div>
<div class="value"><span id="f-gpu">n/a</span><span class="unit">%</span></div></div>
<div class="metric"><div class="label">Power</div>
<div class="value sm" id="f-power" title="No whole-host power source is wired up yet; this stays n/a rather than guessing.">n/a</div></div>
</div>
</div>
</section>
<section class="grid cols-2" style="margin-top:.85rem" aria-label="Fleet charts">
<article class="card">
<header><h3>CPU utilisation</h3><span class="spacer"></span><span class="faint">% · per host</span></header>
<div class="body"><div class="chart" id="chart-cpu"></div><div class="legend" id="legend-cpu"></div></div>
</article>
<article class="card">
<header><h3>Memory used</h3><span class="spacer"></span><span class="faint">% of MemAvailable basis</span></header>
<div class="body"><div class="chart" id="chart-mem"></div><div class="legend" id="legend-mem"></div></div>
</article>
<article class="card">
<header><h3>Disk I/O throughput</h3><span class="spacer"></span>
<span class="faint">selected host · all physical devices summed</span></header>
<div class="body"><div class="chart" id="chart-disk"></div><div class="legend" id="legend-disk"></div></div>
</article>
<article class="card">
<header><h3>Network throughput</h3><span class="spacer"></span><span class="faint">selected host · physical interfaces</span></header>
<div class="body"><div class="chart" id="chart-net"></div><div class="legend" id="legend-net"></div></div>
</article>
<article class="card">
<header><h3>GPU utilisation</h3><span class="spacer"></span><span class="faint">one series per host GPU</span></header>
<div class="body"><div class="chart" id="chart-gpu"></div><div class="legend" id="legend-gpu"></div></div>
</article>
<article class="card">
<header><h3>Temperature</h3><span class="spacer"></span><span class="faint">CPU package · °C</span></header>
<div class="body"><div class="chart" id="chart-temp"></div><div class="legend" id="legend-temp"></div></div>
</article>
</section>
<section class="card" style="margin-top:.85rem" aria-label="Series detail">
<header>
<h3>Detail</h3>
<span class="spacer"></span>
<label class="field" for="detail-host">host</label>
<select id="detail-host"></select>
<label class="field" for="detail-metric">metric</label>
<select id="detail-metric"></select>
</header>
<div class="body">
<div class="chart" id="chart-detail" style="height:200px"></div>
<div class="legend" id="legend-detail"></div>
<div class="faint" id="detail-note" style="margin-top:.4rem;font-size:11.5px"></div>
</div>
</section>
<section style="margin-top:.85rem">
<div class="page-head"><h2 style="font-size:.95rem;margin:0">Per host</h2>
<span class="sub">agent-reported values; rates measured over the window in each header</span></div>
<div class="grid auto" id="hosts"></div>
<div class="empty-state" id="hosts-empty" hidden>
<h3>No hosts reporting</h3>
<p>Start the agent on a machine and add it to <code>HOST_TARGETS</code>:</p>
<p class="mono">python3 agent/host_agent.py --host 0.0.0.0 --port 9120 --token …</p>
</div>
</section>
<section class="card" style="margin-top:.85rem">
<header><h3>Events</h3><span class="spacer"></span><span class="faint">unreachable / reachable / reboots</span></header>
<div class="body"><ul class="events" id="events"><li class="muted">loading…</li></ul></div>
</section>
<footer class="pagefoot">
<span>numbers are measured, not inferred: rates are Δcounter/Δt over the poll window</span>
<span class="spacer" style="margin-inline-start:auto"></span>
<span class="mono" id="foot-note"></span>
</footer>
</main>
<script src="/static/charts.js"></script>
<script src="/static/app.js"></script>
<script src="/static/page-system.js"></script>
</body>
</html>
+101
View File
@@ -0,0 +1,101 @@
<!doctype html>
<html lang="en" data-theme="dark">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Homelab · UPS</title>
<link rel="stylesheet" href="/static/dash.css">
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><rect width='100' height='100' rx='22' fill='%23ff8a3d'/></svg>">
</head>
<body>
<header class="masthead">
<a class="brand" href="/"><span class="brand-mark">◆</span> Homelab</a>
<nav class="tabs" aria-label="Dashboards">
<a href="/">Overview</a>
<a href="/system">System</a>
<a href="/llm">LLM</a>
<a href="/ups" aria-current="page">UPS</a>
</nav>
<div class="masthead-right">
<div id="nav-status" class="gap" aria-live="polite"></div>
<span class="clock" id="clock">--:--:--</span>
<button class="ghost" id="pause" type="button" aria-pressed="false">⏸ Pause</button>
<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">☾ Dark</button>
</div>
</header>
<main>
<div class="page-head">
<div>
<h1>UPS &amp; battery health</h1>
<div class="sub" id="sub">reading upsd…</div>
</div>
<div class="spacer"></div>
<div class="gap"><span class="faint">window</span><div id="range"></div></div>
</div>
<div id="banners" class="stack" style="margin-bottom:.85rem"></div>
<section class="grid auto" id="ups-cards"></section>
<div class="empty-state" id="ups-empty" hidden>
<h3>No UPS reporting</h3>
<p>The dashboard speaks the NUT protocol directly to <code>upsd</code>. On the NUT host:</p>
<p class="mono">/etc/ups/upsd.conf → <code>LISTEN 0.0.0.0</code> · <code>PORT 3493</code></p>
<p class="mono">/etc/ups/upsd.users → <code>[dashboards]</code> with
<code>password = …</code> and <code>actions = -</code></p>
<p>then <code>systemctl restart nut-server</code> and open the port to this server.</p>
</div>
<section class="grid cols-2" style="margin-top:.85rem" aria-label="Battery charts">
<article class="card">
<header><h3>Battery charge</h3><span class="spacer"></span><span class="faint">%</span></header>
<div class="body"><div class="chart" id="chart-charge"></div><div class="legend" id="legend-charge"></div></div>
</article>
<article class="card">
<header><h3>Runtime estimate</h3><span class="spacer"></span>
<span class="faint">seconds of backup at the current load</span></header>
<div class="body"><div class="chart" id="chart-runtime"></div><div class="legend" id="legend-runtime"></div></div>
</article>
<article class="card">
<header><h3>Voltages</h3><span class="spacer"></span><span class="faint">battery and incoming line</span></header>
<div class="body"><div class="chart" id="chart-voltage"></div><div class="legend" id="legend-voltage"></div></div>
</article>
<article class="card">
<header><h3>Load on the UPS</h3><span class="spacer"></span><span class="faint">% of rated capacity</span></header>
<div class="body"><div class="chart" id="chart-load"></div><div class="legend" id="legend-load"></div></div>
</article>
</section>
<section class="card" style="margin-top:.85rem">
<header><h3>Power events</h3><span class="spacer"></span>
<span class="faint">state changes, self-test results, commands sent</span></header>
<div class="body"><ul class="events" id="events"><li class="muted">loading…</li></ul></div>
</section>
<footer class="pagefoot">
<span>commands are the only write path in this app, and every one of them is confirmed here first</span>
<span class="spacer" style="margin-inline-start:auto"></span>
<span class="mono" id="foot-note"></span>
</footer>
</main>
<dialog id="confirm-dialog">
<header><span id="dialog-risk" class="chip">confirm</span><h2 id="dialog-title">Run this command?</h2></header>
<div class="body">
<p id="dialog-help"></p>
<p id="dialog-target" class="mono" style="font-size:12px"></p>
<p id="dialog-confirm-wrap" hidden>Type <strong>DANGER</strong> to enable the button:</p>
<p hidden id="dialog-input-wrap"><input class="confirm" id="dialog-input" type="text"
autocomplete="off" spellcheck="false" placeholder="DANGER"></p>
</div>
<footer>
<button class="ghost" id="dialog-cancel" type="button">Cancel</button>
<button class="action" id="dialog-run" type="button">Run command</button>
</footer>
</dialog>
<script src="/static/charts.js"></script>
<script src="/static/app.js"></script>
<script src="/static/page-ups.js"></script>
</body>
</html>
+341
View File
@@ -0,0 +1,341 @@
/* app.js — shared behaviour for every page: formatting, polling, theme, status.
*
* Two rules the whole UI follows:
* - `null`/`undefined` renders as `n/a`, never as `0`, `""`, or a leftover number.
* A dash that changes to another dash silently is how dashboards lie.
* - a stale value is shown *and labelled*: the dot goes amber/red, `stale 12s` appears,
* and the card dims. Hiding the failure is worse than the failure.
*/
'use strict';
const Dash = (() => {
const $ = (sel, root) => (root || document).querySelector(sel);
const $$ = (sel, root) => Array.from((root || document).querySelectorAll(sel));
// ---- formatting ---------------------------------------------------------
const BYTE_UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
function fmtBytes(value, digits) {
if (value == null || !isFinite(value)) return null;
let v = Math.abs(value);
let i = 0;
while (v >= 1024 && i < BYTE_UNITS.length - 1) { v /= 1024; i += 1; }
const d = digits != null ? digits : (v >= 100 || i === 0 ? 0 : v >= 10 ? 1 : 2);
return { text: v.toFixed(d), unit: BYTE_UNITS[i] };
}
function bytes(value, digits) {
const parts = fmtBytes(value, digits);
return parts ? `${parts.text} ${parts.unit}` : naText;
}
function rate(value, digits) {
const parts = fmtBytes(value, digits);
return parts ? `${parts.text} ${parts.unit}/s` : naText;
}
function pct(value, digits) {
if (value == null || !isFinite(value)) return naText;
return `${value.toFixed(digits == null ? 1 : digits)}%`;
}
function num(value, digits) {
if (value == null || !isFinite(value)) return naText;
const d = digits == null ? (Math.abs(value) >= 100 ? 0 : Math.abs(value) >= 10 ? 1 : 2) : digits;
return value.toFixed(d);
}
function duration(seconds) {
if (seconds == null || !isFinite(seconds)) return naText;
const s = Math.max(0, seconds);
if (s < 90) return `${Math.round(s)}s`;
const m = s / 60;
if (m < 90) return `${m.toFixed(m < 10 ? 1 : 0)}m`;
const h = m / 60;
if (h < 48) return `${h.toFixed(h < 10 ? 1 : 0)}h`;
const d = h / 24;
if (d < 60) return `${d.toFixed(d < 10 ? 1 : 0)}d`;
return `${(d / 365.25).toFixed(1)}y`;
}
function age(ts) {
if (ts == null) return 'never';
const seconds = Math.max(0, Date.now() / 1000 - ts);
if (seconds < 2) return 'now';
return duration(seconds).replace(/(h|m|s|d|y)$/, '$') + ' ago';
}
const naText = 'n/a';
// ---- DOM helpers --------------------------------------------------------
/** Write text only when it changed: avoids killing transitions and focus. */
function setTxt(el, value) {
if (!el) return;
const text = value == null ? naText : String(value);
if (el.textContent !== text) el.textContent = text;
el.classList.toggle('is-null', value == null);
el.classList.toggle('na', value == null);
}
function setValue(el, value, digits, unit) {
if (!el) return;
const parent = el.parentElement;
if (value == null || !isFinite(value)) {
setTxt(el, naText);
if (parent) parent.classList.add('is-null');
return;
}
if (parent) parent.classList.remove('is-null');
const text = typeof value === 'string' ? value : num(value, digits);
setTxt(el, text);
const unitEl = el.nextElementSibling;
if (unit && unitEl && unitEl.classList.contains('unit')) setTxt(unitEl, unit);
}
function setBar(el, value, kind) {
if (!el) return;
const fill = el.tagName === 'I' ? el : $('i', el);
const pctValue = value == null ? 0 : Math.max(0, Math.min(100, value));
if (fill) fill.style.width = `${pctValue}%`;
const threshold = kind || severityFor(pctValue);
el.classList.remove('ok', 'warn', 'fail');
el.classList.add(threshold);
}
function severityFor(value, warnAt, failAt) {
const warn = warnAt == null ? 75 : warnAt;
const fail = failAt == null ? 90 : failAt;
if (value == null) return '';
if (value >= fail) return 'fail';
if (value >= warn) return 'warn';
return 'ok';
}
function setDot(el, status) {
if (!el) return;
el.classList.remove('ok', 'warn', 'fail', 'idle');
el.classList.add(status || 'idle');
}
function chip(text, kind, title) {
const span = document.createElement('span');
span.className = 'chip' + (kind ? ' ' + kind : '');
span.textContent = text;
if (title) span.title = title;
return span;
}
// ---- networking ---------------------------------------------------------
async function fetchJSON(url, options) {
const response = await fetch(url, Object.assign({ headers: { accept: 'application/json' } },
options || {}));
let body = null;
try { body = await response.json(); } catch (err) { body = null; }
if (!response.ok) {
const detail = body && body.detail
? (typeof body.detail === 'string' ? body.detail : body.detail.detail || JSON.stringify(body.detail))
: `${response.status} ${response.statusText}`;
const error = new Error(detail);
error.status = response.status;
error.body = body;
throw error;
}
return body;
}
/**
* Poll `fn` every `ms`, skipping overlapping runs and pausing when the tab is hidden
* or the user hit pause. Errors are reported, never swallowed silently.
*/
function poll(fn, ms) {
let stopped = false;
let timer = null;
let inFlight = false;
const state = {
get paused() { return state._paused; },
_paused: false,
interval: ms,
setInterval(next) { state.interval = next; if (timer) { clearTimeout(timer); tick(); } },
setPaused(next) {
state._paused = next;
if (next) { if (timer) clearTimeout(timer); timer = null; }
else if (!timer) tick();
},
stop() { stopped = true; if (timer) clearTimeout(timer); },
runNow() { tick(); },
};
async function tick() {
if (stopped || inFlight || state._paused || document.hidden) { schedule(); return; }
inFlight = true;
try { await fn(); } finally { inFlight = false; schedule(); }
}
function schedule() {
if (stopped || state._paused) return;
timer = setTimeout(tick, state.interval);
}
document.addEventListener('visibilitychange', () => { if (!document.hidden && !state._paused) tick(); });
tick();
return state;
}
// ---- chrome -------------------------------------------------------------
function initChrome() {
const root = document.documentElement;
const stored = localStorage.getItem('dash-theme');
const prefersLight = window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches;
root.dataset.theme = stored || (prefersLight ? 'light' : 'dark');
const themeButton = $('#theme-toggle');
if (themeButton) {
const paint = () => {
themeButton.textContent = root.dataset.theme === 'light' ? '☀ Light' : '☾ Dark';
themeButton.setAttribute('aria-pressed', root.dataset.theme === 'light' ? 'true' : 'false');
};
themeButton.addEventListener('click', () => {
root.dataset.theme = root.dataset.theme === 'light' ? 'dark' : 'light';
localStorage.setItem('dash-theme', root.dataset.theme);
paint();
document.dispatchEvent(new CustomEvent('dash:theme'));
});
paint();
}
const clock = $('#clock');
if (clock) {
const tickClock = () => {
const now = new Date();
const target = document.getElementById('sim-flag');
clock.textContent = now.toLocaleTimeString(undefined, { hour12: false });
void target;
};
tickClock();
setInterval(tickClock, 1000);
}
const nav = $('#nav-status');
if (nav) {
const jobs = [
{ key: 'system', label: 'hosts', href: '/system' },
{ key: 'llm', label: 'engines', href: '/llm' },
{ key: 'ups', label: 'UPS', href: '/ups' },
];
nav.innerHTML = jobs.map((job) => `<span class="chip" title="${job.label}">
<span class="dot idle" data-dot="${job.key}"></span>${job.label}</span>`).join('');
const refreshStatus = async () => {
try {
const health = await fetchJSON('/api/health');
for (const job of jobs) {
const dot = nav.querySelector(`[data-dot="${job.key}"]`);
const states = (health.sources || []).filter((s) => s.dashboard === job.key && s.configured);
if (!states.length) { setDot(dot, 'idle'); continue; }
const ok = states.filter((s) => s.ok).length;
setDot(dot, ok === states.length ? 'ok' : ok ? 'warn' : 'fail');
dot.title = `${ok}/${states.length} ${job.label} answering`;
}
} catch (err) {
for (const job of jobs) setDot(nav.querySelector(`[data-dot="${job.key}"]`), 'fail');
}
};
refreshStatus();
setInterval(refreshStatus, 15000);
}
}
/** Range buttons (5m … 30d). Calls onChange(seconds, key). */
function rangeControl(container, current, onChange, choices) {
const options = choices || [['5m', 300], ['15m', 900], ['1h', 3600], ['6h', 21600],
['24h', 86400], ['7d', 604800], ['30d', 2592000]];
let value = current || 3600;
container.className = 'gap';
container.innerHTML = '';
const buttons = options.map(([key, seconds]) => {
const button = document.createElement('button');
button.className = 'ghost';
button.type = 'button';
button.textContent = key;
button.dataset.seconds = String(seconds);
button.setAttribute('aria-pressed', seconds === value ? 'true' : 'false');
button.addEventListener('click', () => {
value = seconds;
buttons.forEach((b) => b.setAttribute('aria-pressed',
Number(b.dataset.seconds) === value ? 'true' : 'false'));
onChange(seconds, key);
});
container.appendChild(button);
return button;
});
return { get value() { return value; } };
}
let toastTimer = null;
function toast(message, kind) {
let host = $('#toasts');
if (!host) {
host = document.createElement('div');
host.id = 'toasts';
host.style.cssText = 'position:fixed;right:1rem;bottom:1rem;z-index:60;display:grid;gap:.4rem';
document.body.appendChild(host);
}
const div = document.createElement('div');
div.className = 'banner ' + (kind || 'info');
div.textContent = message;
host.appendChild(div);
setTimeout(() => div.remove(), kind === 'fail' ? 9000 : 4500);
if (toastTimer) clearTimeout(toastTimer);
toastTimer = setTimeout(() => { host.remove(); host = null; }, 10000);
}
/** Pause button + Space, wired to a poll handle. */
function bindPause(poller) {
const button = $('#pause');
if (!button || !poller) return;
const paint = () => {
button.setAttribute('aria-pressed', poller.paused ? 'true' : 'false');
button.textContent = poller.paused ? '▶ Paused' : '⏸ Pause';
};
button.addEventListener('click', () => { poller.setPaused(!poller.paused); paint(); });
document.addEventListener('keydown', (event) => {
const tag = (event.target && event.target.tagName || '').toLowerCase();
if (tag === 'input' || tag === 'textarea' || event.metaKey || event.ctrlKey) return;
if (event.code === 'Space') { event.preventDefault(); poller.setPaused(!poller.paused); paint(); }
if (event.key === 't' || event.key === 'T') {
const theme = $('#theme-toggle');
if (theme) theme.click();
}
});
paint();
}
/** The "last poll" line every card header shows. */
function staleness(el, state) {
if (!el) return;
if (!state) { setTxt(el, '—'); return; }
if (!state.configured) { setTxt(el, 'not configured'); return; }
if (!state.ok) {
setTxt(el, state.last_ok ? `stale ${duration(ageSeconds(state.last_ok))} · ${state.error || 'down'}`
: (state.error || 'never reached'));
setDot(el.parentElement && el.parentElement.querySelector('.dot'), 'fail');
return;
}
const seconds = ageSeconds(state.last_ok);
setTxt(el, seconds > 30 ? `stale ${duration(seconds)}` : `updated ${duration(seconds)} ago`);
setDot(el.parentElement && el.parentElement.querySelector('.dot'),
seconds > 30 ? 'warn' : 'ok');
}
function ageSeconds(ts) {
return ts == null ? Infinity : Math.max(0, Date.now() / 1000 - ts);
}
/** Cards whose data is older than the poll interval get dimmed + labelled. */
function markStale(card, stale) {
if (!card) return;
card.classList.toggle('stale', !!stale);
}
return { $, $$, fetchJSON, poll, fmtBytes, bytes, rate, pct, num, duration, age, setTxt,
setValue, setBar, setDot, chip, initChrome, rangeControl, toast, bindPause,
staleness, markStale, severityFor, ageSeconds, naText };
})();
document.addEventListener('DOMContentLoaded', () => Dash.initChrome());
+378
View File
@@ -0,0 +1,378 @@
/* charts.js — hand-rolled SVG time-series charts. No CDN, no chart library.
*
* Coordinate space is real pixels (viewBox = the element's measured size, no
* preserveAspectRatio="none"), which is what keeps axis text crisp: a stretched viewBox
* smears every glyph drawn inside it. Redraw happens on ResizeObserver, not on a timer.
*
* Deliberate behaviours:
* - A series with no points draws nothing and the chart says why. It never draws 0,
* because "no data" and "zero" are different claims.
* - The y-range comes from the data actually plotted, and the labels are generated from
* the same numbers the path used, so a label can never drift from the curve.
* - Gaps are gaps: a sample missing in the middle breaks the line rather than drawing a
* straight lie across the hole.
*/
'use strict';
const CHART_COLORS = ['--series-1', '--series-2', '--series-3', '--series-4',
'--series-5', '--series-6', '--series-7', '--series-8'];
function cssVar(name) {
return getComputedStyle(document.documentElement).getPropertyValue(name).trim();
}
/** Round a range to a "nice" tick step: 1, 2, 2.5, 5, 10 x 10^n. */
function niceTicks(min, max, count) {
if (!isFinite(min) || !isFinite(max) || min === max) {
const base = isFinite(min) ? min : 0;
return { lo: base - 1, hi: base + 1, step: 1, values: [base - 1, base, base + 1] };
}
const raw = (max - min) / Math.max(1, count);
const mag = Math.pow(10, Math.floor(Math.log10(Math.abs(raw) || 1)));
const norm = raw / mag;
const step = (norm <= 1 ? 1 : norm <= 2 ? 2 : norm <= 2.5 ? 2.5 : norm <= 5 ? 5 : 10) * mag;
const lo = Math.floor(min / step) * step;
const hi = Math.ceil(max / step) * step;
const values = [];
for (let v = lo; v <= hi + step * 0.5; v += step) values.push(Number(v.toFixed(10)));
return { lo, hi, step, values };
}
const DEFAULT_FORMAT = (v) => (Math.abs(v) >= 100 ? v.toFixed(0) : v.toFixed(1));
class LineChart {
/**
* container: element to fill (its width is used; height comes from opts.height)
* opts: {height, yFormat, xSpan, yMin, yMax, area, legend, showAxis, min, unit}
*/
constructor(container, opts) {
this.el = typeof container === 'string' ? document.querySelector(container) : container;
this.opts = Object.assign({
height: 150, area: true, legend: true, showAxis: true,
yFormat: DEFAULT_FORMAT, xSpan: 3600, yMin: null, yMax: null, unit: '',
}, opts || {});
this.series = [];
this.hoverTs = null;
this._build();
if (typeof ResizeObserver !== 'undefined') {
this._ro = new ResizeObserver(() => this.draw());
this._ro.observe(this.el);
} else {
window.addEventListener('resize', () => this.draw());
}
}
_build() {
this.el.classList.add('chart');
this.el.style.height = this.opts.height + 'px';
this.el.innerHTML = '';
this.svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
this.svg.setAttribute('width', '100%');
this.svg.setAttribute('height', this.opts.height);
this.el.appendChild(this.svg);
this.empty = document.createElement('div');
this.empty.className = 'empty';
this.el.appendChild(this.empty);
this.tooltip = document.createElement('div');
this.tooltip.className = 'tooltip';
this.el.appendChild(this.tooltip);
this.svg.addEventListener('mousemove', (e) => this._onMove(e));
this.svg.addEventListener('mouseleave', () => { this.hoverTs = null; this._hideTip(); this._drawCursor(); });
}
/** series: [{name, points: [{ts, value}], color?, yMax?}] — replaces previous data. */
setData(series) {
this.series = (series || []).map((s, i) => ({
name: s.name,
color: s.color || cssVar(CHART_COLORS[i % CHART_COLORS.length]),
points: (s.points || []).filter((p) => p && Number.isFinite(p.value) && Number.isFinite(p.ts))
.sort((a, b) => a.ts - b.ts),
}));
this.draw();
return this;
}
_layout() {
const w = Math.max(80, this.el.clientWidth);
const h = this.opts.height;
const pad = this.opts.showAxis ? { l: 44, r: 8, t: 8, b: 16 } : { l: 2, r: 2, t: 3, b: 3 };
return { w, h, pad, plotW: Math.max(10, w - pad.l - pad.r), plotH: Math.max(10, h - pad.t - pad.b) };
}
_xExtent() {
const span = this.opts.xSpan;
const end = Date.now() / 1000;
return [end - span, end];
}
_yExtent() {
if (this.opts.yMin !== null && this.opts.yMax !== null) {
return [this.opts.yMin, this.opts.yMax];
}
let lo = Infinity, hi = -Infinity;
for (const s of this.series) {
for (const p of s.points) {
if (p.value < lo) lo = p.value;
if (p.value > hi) hi = p.value;
}
}
if (!isFinite(lo)) return [0, 1];
if (lo === hi) { const pad = Math.abs(lo) * 0.1 || 1; return [lo - pad, hi + pad]; }
const margin = (hi - lo) * 0.12;
lo = this.opts.yMin !== null ? this.opts.yMin : lo - margin;
hi = this.opts.yMax !== null ? this.opts.yMax : hi + margin;
return [lo, hi];
}
draw() {
if (!this.el.clientWidth) return;
const plotted = this.series.filter((s) => s.points.length);
const { w, h, pad, plotW, plotH } = this._layout();
this.svg.setAttribute('viewBox', `0 0 ${w} ${h}`);
this.svg.innerHTML = '';
if (!plotted.length) {
this.empty.textContent = 'no samples yet';
this.empty.style.display = 'grid';
return;
}
this.empty.style.display = 'none';
const [x0, x1] = this._xExtent();
const [ylo, yhi] = this._yExtent();
const ticks = niceTicks(ylo, yhi, 4);
this.scales = {
x: (ts) => pad.l + ((ts - x0) / (x1 - x0)) * plotW,
y: (v) => pad.t + plotH - ((Math.min(Math.max(v, ticks.lo), ticks.hi) - ticks.lo) / (ticks.hi - ticks.lo)) * plotH,
ticks, x0, x1, ylo: ticks.lo, yhi: ticks.hi,
};
const ns = 'http://www.w3.org/2000/svg';
const g = (name) => {
const grp = document.createElementNS(ns, 'g');
grp.setAttribute('class', name);
this.svg.appendChild(grp);
return grp;
};
if (this.opts.showAxis) {
const grid = g('grid');
for (const t of ticks.values) {
const y = this.scales.y(t);
if (y < pad.t - 1 || y > pad.t + plotH + 1) continue;
const line = document.createElementNS(ns, 'line');
line.setAttribute('x1', pad.l); line.setAttribute('x2', pad.l + plotW);
line.setAttribute('y1', y); line.setAttribute('y2', y);
grid.appendChild(line);
const text = document.createElementNS(ns, 'text');
text.setAttribute('x', pad.l - 6); text.setAttribute('y', y + 3.5);
text.setAttribute('text-anchor', 'end');
text.textContent = this.opts.yFormat(t);
grid.appendChild(text);
}
const axis = g('axis');
for (let i = 0; i <= 4; i++) {
const ts = x0 + ((x1 - x0) * i) / 4;
const x = this.scales.x(ts);
const text = document.createElementNS(ns, 'text');
text.setAttribute('x', x); text.setAttribute('y', h - 3);
text.setAttribute('text-anchor', i === 0 ? 'start' : i === 4 ? 'end' : 'middle');
text.textContent = shortTime(ts, x1 - x0);
axis.appendChild(text);
}
}
for (const s of plotted) {
const paths = buildPath(s.points, this.scales, plotH + pad.t);
if (this.opts.area && paths.length) {
for (const seg of paths) {
const area = document.createElementNS(ns, 'path');
area.setAttribute('class', 'area');
area.setAttribute('d', seg.area);
area.setAttribute('fill', s.color);
this.svg.appendChild(area);
}
}
for (const seg of paths) {
const line = document.createElementNS(ns, 'path');
line.setAttribute('class', 'line');
line.setAttribute('d', seg.line);
line.setAttribute('stroke', s.color);
this.svg.appendChild(line);
}
}
const cursor = document.createElementNS(ns, 'g');
cursor.setAttribute('class', 'cursor');
this.svg.appendChild(cursor);
this.cursorEl = cursor;
this._drawCursor();
}
_drawCursor() {
if (!this.cursorEl || !this.scales || this.hoverTs == null) {
if (this.cursorEl) this.cursorEl.innerHTML = '';
return;
}
const { pad } = this._layout();
const x = this.scales.x(this.hoverTs);
this.cursorEl.innerHTML = '';
const line = document.createElementNS('http://www.w3.org/2000/svg', 'line');
line.setAttribute('x1', x); line.setAttribute('x2', x);
line.setAttribute('y1', pad.t); line.setAttribute('y2', this.opts.height - pad.b);
this.cursorEl.appendChild(line);
}
_onMove(event) {
if (!this.scales) return;
const rect = this.svg.getBoundingClientRect();
const box = (this.svg.getAttribute('viewBox') || '0 0 1 1').split(' ');
const px = ((event.clientX - rect.left) / rect.width) * Number(box[2]);
const { pad, plotW } = this._layout();
const { x0, x1 } = this.scales;
const frac = Math.min(1, Math.max(0, (px - pad.l) / plotW));
const at = x0 + frac * (x1 - x0);
this.hoverTs = at;
this._drawCursor();
this._showTip(at, event);
}
_showTip(at, event) {
const rows = this.series.map((s) => {
const near = nearest(s.points, at);
if (!near) return null;
return `<div class="row"><span class="swatch" style="background:${s.color}"></span>` +
`<span>${escapeHtml(s.name)}</span>` +
`<span class="val">${this.opts.yFormat(near.value)}${this.opts.unit ? ' ' + this.opts.unit : ''}</span></div>`;
}).filter(Boolean);
if (!rows.length) { this._hideTip(); return; }
this.tooltip.innerHTML = `<div class="ts">${fullTime(at)}</div>${rows.join('')}`;
this.tooltip.style.opacity = '1';
const rect = this.el.getBoundingClientRect();
const left = Math.min(rect.width - 150, Math.max(4, event.clientX - rect.left + 12));
this.tooltip.style.left = left + 'px';
this.tooltip.style.top = '8px';
}
_hideTip() { this.tooltip.style.opacity = '0'; }
}
/** Split points into contiguous runs; a gap > 3x the median step breaks the line. */
function buildPath(points, scales, baseline) {
if (!points.length) return [];
const steps = [];
for (let i = 1; i < points.length; i++) steps.push(points[i].ts - points[i - 1].ts);
steps.sort((a, b) => a - b);
const median = steps.length ? steps[Math.floor(steps.length / 2)] : 0;
const maxGap = median > 0 ? median * 3 : Infinity;
const runs = [];
let current = [points[0]];
for (let i = 1; i < points.length; i++) {
if (points[i].ts - points[i - 1].ts > maxGap) { runs.push(current); current = [points[i]]; }
else current.push(points[i]);
}
runs.push(current);
return runs.map((run) => {
let line = '';
run.forEach((p, i) => {
const x = scales.x(p.ts).toFixed(1);
const y = scales.y(p.value).toFixed(1);
line += (i === 0 ? `M${x} ${y}` : ` L${x} ${y}`);
});
const first = scales.x(run[0].ts).toFixed(1);
const last = scales.x(run[run.length - 1].ts).toFixed(1);
const area = `${line} L${last} ${baseline.toFixed(1)} L${first} ${baseline.toFixed(1)} Z`;
return { line, area };
});
}
function nearest(points, ts) {
if (!points.length) return null;
let lo = 0, hi = points.length - 1;
while (lo < hi) {
const mid = (lo + hi) >> 1;
if (points[mid].ts < ts) lo = mid + 1; else hi = mid;
}
const a = points[Math.max(0, lo - 1)], b = points[lo];
return Math.abs(a.ts - ts) < Math.abs(b.ts - ts) ? a : b;
}
function shortTime(ts, span) {
const d = new Date(ts * 1000);
if (span > 86400 * 3) return d.toLocaleDateString(undefined, { month: 'short', day: 'numeric' });
if (span > 86400) return d.toLocaleDateString(undefined, { weekday: 'short' }) + ' ' +
d.toLocaleTimeString(undefined, { hour: '2-digit' });
return d.toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' });
}
function fullTime(ts) {
return new Date(ts * 1000).toLocaleString(undefined, {
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit',
});
}
function escapeHtml(value) {
return String(value).replace(/[&<>"']/g, (c) => ({
'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;',
}[c]));
}
/** Legend with the live last value per series, under the chart. */
/**
* Legend with the live last value per series, under the chart.
*
* `showPeak` adds the window maximum too. Throughput is bursty: a legend that reads
* `0.0 tok/s` next to a chart that clearly went to 60 twice is read as a broken dashboard,
* even though "last poll was idle" is the truth. Saying both keeps it honest and legible.
*/
function renderLegend(container, series, format, showPeak) {
container.innerHTML = series.map((s, i) => {
const color = s.color || cssVar(CHART_COLORS[i % CHART_COLORS.length]);
const points = (s.points || []).filter((p) => Number.isFinite(p.value));
const last = points[points.length - 1];
let peakHtml = '';
if (showPeak && points.length) {
const peak = points.reduce((best, p) => (p.value > best.value ? p : best), points[0]);
if (peak.value !== last.value && peak.value > 0) {
peakHtml = `<span class="lv faint" title="highest value in the window">peak ${format(peak.value)}</span>`;
}
}
return `<span class="item"><span class="swatch" style="background:${color}"></span>` +
`<span>${escapeHtml(s.name)}</span>` +
`<span class="lv">${last ? format(last.value) : 'n/a'}</span>${peakHtml}</span>`;
}).join('');
}
/** Column heat bars for per-core usage: 48 cores read as a pattern, not a table. */
function renderCores(container, values, format) {
const f = format || ((v) => (v == null ? 'n/a' : v.toFixed(0) + '%'));
container.innerHTML = (values || []).map((v, i) => {
const pct = v == null ? 0 : Math.max(0, Math.min(100, v));
const hue = pct > 85 ? 'var(--fail)' : pct > 60 ? 'var(--warn)' : 'var(--accent)';
return `<span class="core" title="core ${i}: ${escapeHtml(f(v))}">` +
`<i style="height:${pct.toFixed(1)}%;background:${hue}"></i></span>`;
}).join('');
}
/** Small non-interactive sparkline for cards where a full chart would shout. */
function sparkline(container, points, opts) {
const o = Object.assign({ height: 34, color: cssVar('--series-1'), min: null, max: null }, opts || {});
const pts = (points || []).filter((p) => p && Number.isFinite(p.value));
const w = Math.max(60, container.clientWidth || 160);
const h = o.height;
container.innerHTML = '';
if (pts.length < 2) {
container.innerHTML = '<div class="faint mono" style="font-size:11px">no history</div>';
return;
}
const xs = pts.map((p) => p.ts), ys = pts.map((p) => p.value);
const x0 = Math.min(...xs), x1 = Math.max(...xs);
let lo = o.min !== null ? o.min : Math.min(...ys);
let hi = o.max !== null ? o.max : Math.max(...ys);
if (lo === hi) { const pad = Math.abs(lo) * 0.1 || 1; lo -= pad; hi += pad; }
const sx = (ts) => ((ts - x0) / (x1 - x0 || 1)) * (w - 2) + 1;
const sy = (v) => h - 2 - ((v - lo) / (hi - lo)) * (h - 4);
const d = pts.map((p, i) => `${i ? 'L' : 'M'}${sx(p.ts).toFixed(1)} ${sy(p.value).toFixed(1)}`).join(' ');
const svg = `<svg viewBox="0 0 ${w} ${h}" width="100%" height="${h}" preserveAspectRatio="none"
role="img" aria-label="trend from ${lo.toFixed(2)} to ${hi.toFixed(2)}">
<path d="${d} L${sx(x1).toFixed(1)} ${h} L${sx(x0).toFixed(1)} ${h} Z" fill="${o.color}" opacity=".14"/>
<path d="${d}" fill="none" stroke="${o.color}" stroke-width="1.4"/></svg>`;
container.innerHTML = svg;
}
window.DashCharts = { LineChart, niceTicks, renderLegend, renderCores, sparkline, cssVar,
CHART_COLORS, DEFAULT_FORMAT, escapeHtml };
+442
View File
@@ -0,0 +1,442 @@
/* Homelab dashboards — design system.
*
* No CDN, no webfonts, no framework: this runs on a homelab network that may have no
* route to the internet, and a font that fails to load is a page that reflows in front
* of you. Everything here is system fonts and hand-written SVG.
*
* Two themes via [data-theme] on <html> (persisted in localStorage), defaulting to the
* OS preference. All colours are tokens; nothing in a component hardcodes a colour.
*/
:root {
--bg: #0d0f13;
--bg-raised: #14171e;
--bg-sunken: #0a0c0f;
--border: #232836;
--border-strong: #333b4d;
--text: #e7eaf0;
--text-muted: #8b94a7;
--text-faint: #5c6478;
--accent: #ff8a3d;
--accent-soft: rgba(255, 138, 61, 0.14);
--ok: #3fb950;
--ok-soft: rgba(63, 185, 80, 0.15);
--warn: #d29922;
--warn-soft: rgba(210, 153, 34, 0.16);
--fail: #f85149;
--fail-soft: rgba(248, 81, 73, 0.16);
--info: #58a6ff;
--info-soft: rgba(88, 166, 255, 0.15);
--series-1: #ff8a3d;
--series-2: #58a6ff;
--series-3: #3fb950;
--series-4: #bc8cff;
--series-5: #f778ba;
--series-6: #d29922;
--series-7: #39c5cf;
--series-8: #8b94a7;
--radius: 10px;
--radius-sm: 6px;
--shadow: 0 1px 2px rgba(0, 0, 0, 0.4), 0 8px 24px rgba(0, 0, 0, 0.25);
--mono: ui-monospace, "SF Mono", "JetBrains Mono", "Cascadia Mono", Menlo, Consolas, monospace;
--sans: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
--header-h: 58px;
}
[data-theme="light"] {
--bg: #f4f5f8;
--bg-raised: #ffffff;
--bg-sunken: #e9ebf0;
--border: #dfe3ea;
--border-strong: #c3cad6;
--text: #16191f;
--text-muted: #5b6473;
--text-faint: #8a93a3;
--accent: #d2691e;
--accent-soft: rgba(210, 105, 30, 0.12);
--ok: #1a7f37;
--ok-soft: rgba(26, 127, 55, 0.12);
--warn: #9a6700;
--warn-soft: rgba(154, 103, 0, 0.12);
--fail: #cf222e;
--fail-soft: rgba(207, 34, 46, 0.1);
--info: #0969da;
--info-soft: rgba(9, 105, 218, 0.1);
--series-1: #d2691e;
--series-2: #0969da;
--series-3: #1a7f37;
--series-4: #8250df;
--series-5: #bf3989;
--series-6: #9a6700;
--series-7: #1b7c84;
--series-8: #5b6473;
--shadow: 0 1px 2px rgba(16, 24, 40, 0.06), 0 6px 20px rgba(16, 24, 40, 0.06);
}
* { box-sizing: border-box; }
html, body { height: 100%; }
body {
margin: 0;
background: var(--bg);
color: var(--text);
font: 14px/1.5 var(--sans);
-webkit-font-smoothing: antialiased;
font-variant-numeric: tabular-nums;
}
a { color: var(--accent); text-decoration: none; }
a:hover { text-decoration: underline; }
:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 4px; }
/* ---- header ------------------------------------------------------------- */
.masthead {
position: sticky;
top: 0;
z-index: 40;
display: flex;
align-items: center;
gap: 1.1rem;
height: var(--header-h);
padding: 0 1.25rem;
background: color-mix(in srgb, var(--bg-raised) 92%, transparent);
backdrop-filter: blur(8px);
border-bottom: 1px solid var(--border);
}
.brand {
display: flex;
align-items: center;
gap: .55rem;
font-weight: 650;
letter-spacing: .01em;
color: var(--text);
white-space: nowrap;
}
.brand:hover { text-decoration: none; }
.brand-mark {
width: 22px; height: 22px;
border-radius: 6px;
background: linear-gradient(140deg, var(--accent), color-mix(in srgb, var(--accent) 45%, var(--info)));
box-shadow: inset 0 0 0 1px rgba(255, 255, 255, .12);
display: grid;
place-items: center;
font-size: 12px;
color: #14171e;
font-weight: 800;
}
nav.tabs { display: flex; gap: .15rem; margin-inline-start: .4rem; }
nav.tabs a {
padding: .38rem .7rem;
border-radius: var(--radius-sm);
color: var(--text-muted);
font-weight: 550;
display: flex;
align-items: center;
gap: .4rem;
}
nav.tabs a:hover { background: var(--bg-sunken); color: var(--text); text-decoration: none; }
nav.tabs a[aria-current="page"] { background: var(--accent-soft); color: var(--accent); }
.masthead-right { margin-inline-start: auto; display: flex; align-items: center; gap: .55rem; }
.clock { font-family: var(--mono); font-size: 12.5px; color: var(--text-muted); }
@media (max-width: 900px) { .clock { display: none; } }
button.ghost {
background: transparent;
border: 1px solid var(--border);
color: var(--text-muted);
border-radius: var(--radius-sm);
padding: .32rem .6rem;
font: 550 12.5px var(--sans);
cursor: pointer;
display: inline-flex;
align-items: center;
gap: .35rem;
}
button.ghost:hover { border-color: var(--border-strong); color: var(--text); background: var(--bg-sunken); }
button.ghost[aria-pressed="true"] { border-color: var(--accent); color: var(--accent); background: var(--accent-soft); }
/* ---- layout ------------------------------------------------------------- */
main { padding: 1.15rem 1.25rem 3rem; max-width: 1680px; margin: 0 auto; }
.page-head {
display: flex;
align-items: flex-end;
gap: 1rem;
flex-wrap: wrap;
margin-bottom: 1rem;
}
.page-head h1 { font-size: 1.15rem; margin: 0; font-weight: 650; letter-spacing: .01em; }
.page-head .sub { color: var(--text-muted); font-size: 12.5px; }
.page-head .spacer { margin-inline-start: auto; }
.grid { display: grid; gap: .85rem; }
.grid.auto { grid-template-columns: repeat(auto-fit, minmax(min(360px, 100%), 1fr)); }
.grid.cols-2 { grid-template-columns: repeat(auto-fit, minmax(min(480px, 100%), 1fr)); }
.grid.cols-3 { grid-template-columns: repeat(auto-fit, minmax(min(300px, 100%), 1fr)); }
.grid.wide { grid-template-columns: minmax(0, 2fr) minmax(0, 1fr); }
@media (max-width: 1150px) { .grid.wide { grid-template-columns: 1fr; } }
.stack { display: grid; gap: .85rem; align-content: start; }
/* ---- cards -------------------------------------------------------------- */
.card {
background: var(--bg-raised);
border: 1px solid var(--border);
border-radius: var(--radius);
box-shadow: var(--shadow);
overflow: hidden;
}
.card > header {
display: flex;
align-items: center;
gap: .55rem;
padding: .6rem .8rem;
border-bottom: 1px solid var(--border);
background: color-mix(in srgb, var(--bg-raised) 70%, var(--bg-sunken));
}
.card > header h2, .card > header h3 {
margin: 0; font-size: 12.5px; font-weight: 650; letter-spacing: .04em; text-transform: uppercase;
color: var(--text-muted);
}
.card > header .spacer { margin-inline-start: auto; }
.card > .body { padding: .8rem; }
.card > .body.tight { padding: .45rem .8rem .6rem; }
.card.flat { box-shadow: none; }
.cardset { display: grid; gap: .85rem; }
/* ---- metrics ------------------------------------------------------------ */
.metrics { display: grid; gap: .7rem .9rem; grid-template-columns: repeat(auto-fit, minmax(104px, 1fr)); }
.metric { min-width: 0; }
.metric .label {
font-size: 10.5px; text-transform: uppercase; letter-spacing: .06em; color: var(--text-faint);
white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
}
.metric .value {
font-family: var(--mono); font-size: 1.3rem; font-weight: 600; line-height: 1.25;
display: flex; align-items: baseline; gap: .22rem; row-gap: .1rem;
/* the provenance chip has to be able to drop under the number: with nowrap it painted
over the neighbouring column, which reads as a wrong value on the next metric */
flex-wrap: wrap;
}
.metric .value > span, .metric .value > .unit { white-space: nowrap; }
.metric .value .unit { font-size: .62em; color: var(--text-muted); font-weight: 500; }
.metric .value.sm { font-size: 1.02rem; }
.metric .note { font-size: 11px; color: var(--text-faint); }
.metric .value.is-null { color: var(--text-faint); font-size: 1rem; }
.prov {
font-family: var(--mono); font-size: 9.5px; letter-spacing: .02em;
color: var(--text-faint); border: 1px solid var(--border); border-radius: 999px;
padding: 0 .32rem; margin-inline-start: .28rem; vertical-align: 2px; white-space: nowrap;
}
.na { color: var(--text-faint); }
/* ---- bars, dots, chips -------------------------------------------------- */
.bar {
position: relative; height: 7px; border-radius: 999px; background: var(--bg-sunken);
overflow: hidden; border: 1px solid var(--border);
}
.bar > i {
position: absolute; inset: 0 auto 0 0; width: 0; border-radius: 999px;
background: var(--accent); transition: width .35s ease;
}
.bar.ok > i { background: var(--ok); }
.bar.warn > i { background: var(--warn); }
.bar.fail > i { background: var(--fail); }
.bar.thin { height: 5px; }
.bar.tall { height: 10px; }
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--text-faint); flex: none; }
.dot.ok { background: var(--ok); box-shadow: 0 0 0 3px var(--ok-soft); }
.dot.warn { background: var(--warn); box-shadow: 0 0 0 3px var(--warn-soft); }
.dot.fail { background: var(--fail); box-shadow: 0 0 0 3px var(--fail-soft); }
.dot.idle { background: var(--text-faint); }
.chip {
display: inline-flex; align-items: center; gap: .3rem;
font-size: 11px; font-weight: 600; letter-spacing: .02em;
padding: .16rem .45rem; border-radius: 999px;
background: var(--bg-sunken); border: 1px solid var(--border); color: var(--text-muted);
white-space: nowrap;
}
.chip.ok { color: var(--ok); background: var(--ok-soft); border-color: color-mix(in srgb, var(--ok) 35%, transparent); }
.chip.warn { color: var(--warn); background: var(--warn-soft); border-color: color-mix(in srgb, var(--warn) 35%, transparent); }
.chip.fail { color: var(--fail); background: var(--fail-soft); border-color: color-mix(in srgb, var(--fail) 35%, transparent); }
.chip.info { color: var(--info); background: var(--info-soft); border-color: color-mix(in srgb, var(--info) 35%, transparent); }
.chip.accent { color: var(--accent); background: var(--accent-soft); border-color: color-mix(in srgb, var(--accent) 35%, transparent); }
.banner {
display: flex; gap: .55rem; align-items: center;
padding: .5rem .7rem; border-radius: var(--radius-sm);
background: var(--bg-sunken); border: 1px solid var(--border);
color: var(--text-muted); font-size: 12.5px;
}
.banner.warn { background: var(--warn-soft); border-color: color-mix(in srgb, var(--warn) 35%, transparent); color: var(--warn); }
.banner.fail { background: var(--fail-soft); border-color: color-mix(in srgb, var(--fail) 35%, transparent); color: var(--fail); }
.banner.ok { background: var(--ok-soft); border-color: color-mix(in srgb, var(--ok) 35%, transparent); color: var(--ok); }
.banner.info { background: var(--info-soft); border-color: color-mix(in srgb, var(--info) 30%, transparent); color: var(--info); }
.banner code { font-family: var(--mono); font-size: 11.5px; background: var(--bg-sunken); padding: 0 .25rem; border-radius: 3px; }
/* simulated data is watermarked everywhere it appears */
.sim-badge {
font: 700 10px/1 var(--mono); letter-spacing: .1em; text-transform: uppercase;
color: var(--warn); background: var(--warn-soft);
border: 1px dashed color-mix(in srgb, var(--warn) 55%, transparent);
padding: .2rem .4rem; border-radius: 4px;
}
/* ---- tables ------------------------------------------------------------- */
table.data { width: 100%; border-collapse: collapse; font-size: 12.5px; }
table.data th, table.data td { padding: .34rem .5rem; text-align: right; white-space: nowrap; }
table.data th:first-child, table.data td:first-child { text-align: left; }
table.data thead th {
font-size: 10.5px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint);
border-bottom: 1px solid var(--border); position: sticky; top: 0; background: var(--bg-raised);
}
table.data tbody tr + tr td { border-top: 1px solid color-mix(in srgb, var(--border) 55%, transparent); }
table.data tbody tr:hover td { background: var(--bg-sunken); }
table.data td.name { font-weight: 600; }
table.data .num { font-family: var(--mono); }
table.data .mute { color: var(--text-muted); }
table.data.scroll { display: block; overflow: auto; max-height: 320px; }
/* ---- charts ------------------------------------------------------------- */
.chart { position: relative; width: 100%; }
.chart svg { display: block; width: 100%; height: 100%; overflow: visible; }
.chart .axis text { fill: var(--text-faint); font-family: var(--mono); font-size: 10px; }
.chart .axis line, .chart .grid line { stroke: var(--border); stroke-width: 1; }
.chart .grid line { stroke-dasharray: 2 4; opacity: .75; }
.chart .zero line { stroke: var(--border-strong); }
.chart .line { fill: none; stroke-width: 1.75; stroke-linejoin: round; stroke-linecap: round; }
.chart .area { opacity: .16; }
.chart .cursor line { stroke: var(--text-faint); stroke-width: 1; stroke-dasharray: 3 3; }
.chart .empty {
position: absolute; inset: 0; display: grid; place-items: center;
color: var(--text-faint); font-size: 12px; pointer-events: none;
}
.legend { display: flex; flex-wrap: wrap; gap: .35rem .8rem; font-size: 11.5px; color: var(--text-muted); }
.legend .item { display: inline-flex; align-items: center; gap: .35rem; }
.legend .swatch { width: 9px; height: 9px; border-radius: 2px; }
.legend .lv { font-family: var(--mono); color: var(--text); }
.tooltip {
position: absolute; z-index: 30; pointer-events: none; opacity: 0;
background: color-mix(in srgb, var(--bg-raised) 95%, transparent);
border: 1px solid var(--border-strong); border-radius: var(--radius-sm);
box-shadow: var(--shadow); padding: .35rem .5rem; font-size: 11.5px; min-width: 118px;
transition: opacity .12s ease;
}
.tooltip .ts { color: var(--text-faint); font-family: var(--mono); font-size: 10.5px; margin-bottom: .18rem; }
.tooltip .row { display: flex; align-items: center; gap: .35rem; }
.tooltip .row .val { margin-inline-start: auto; font-family: var(--mono); }
/* per-core usage heat bars */
.cores { display: grid; grid-template-columns: repeat(auto-fill, minmax(9px, 1fr)); gap: 2px; }
.cores .core { height: 22px; border-radius: 2px; background: var(--bg-sunken); position: relative; }
.cores .core > i { position: absolute; bottom: 0; left: 0; right: 0; background: var(--accent); border-radius: 2px; }
/* ---- lists -------------------------------------------------------------- */
ul.events { list-style: none; margin: 0; padding: 0; display: grid; gap: .1rem; }
ul.events li {
display: grid; grid-template-columns: 62px 1fr; gap: .55rem; align-items: baseline;
padding: .3rem .1rem; border-bottom: 1px solid color-mix(in srgb, var(--border) 50%, transparent);
font-size: 12.5px;
}
ul.events li:last-child { border-bottom: 0; }
ul.events time { font-family: var(--mono); font-size: 11px; color: var(--text-faint); }
ul.events .sev { font-weight: 600; }
ul.events .sev.info { color: var(--text-muted); }
ul.events .sev.warn { color: var(--warn); }
ul.events .sev.error { color: var(--fail); }
ul.checks { list-style: none; margin: 0; padding: 0; display: grid; gap: .4rem; }
ul.checks li { display: grid; grid-template-columns: 18px 1fr; gap: .5rem; align-items: start; }
ul.checks .what { font-weight: 600; font-size: 12.5px; }
ul.checks .detail { color: var(--text-muted); font-size: 12px; }
ul.checks .mark { font-family: var(--mono); font-size: 12px; text-align: center; }
ul.checks li.ok .mark { color: var(--ok); }
ul.checks li.warn .mark { color: var(--warn); }
ul.checks li.fail .mark { color: var(--fail); }
ul.checks li.unknown .mark { color: var(--text-faint); }
.kv { display: grid; grid-template-columns: max-content 1fr; gap: .18rem .8rem; font-size: 12.5px; }
.kv dt { color: var(--text-faint); }
.kv dd { margin: 0; font-family: var(--mono); text-align: right; overflow-wrap: anywhere; }
/* ---- buttons ------------------------------------------------------------ */
button.action {
font: 600 12.5px var(--sans);
padding: .42rem .7rem; border-radius: var(--radius-sm);
border: 1px solid var(--border-strong); background: var(--bg-sunken); color: var(--text);
cursor: pointer;
}
button.action:hover:not(:disabled) { border-color: var(--accent); color: var(--accent); }
button.action:disabled { opacity: .5; cursor: not-allowed; }
button.action.caution { border-color: color-mix(in srgb, var(--warn) 45%, var(--border-strong)); }
button.action.dangerous { border-color: color-mix(in srgb, var(--fail) 45%, var(--border-strong)); color: var(--fail); }
button.action.dangerous:hover:not(:disabled) { background: var(--fail-soft); border-color: var(--fail); color: var(--fail); }
fieldset.commands { border: 1px solid var(--border); border-radius: var(--radius-sm); padding: .6rem .7rem; margin: 0; }
fieldset.commands legend { font-size: 11px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint); padding: 0 .3rem; }
fieldset.commands .row { display: flex; flex-wrap: wrap; gap: .4rem; }
input[type="text"].confirm {
font-family: var(--mono); background: var(--bg-sunken); color: var(--text);
border: 1px solid var(--fail); border-radius: var(--radius-sm); padding: .35rem .5rem; width: 11ch;
}
/* ---- dialog ------------------------------------------------------------- */
dialog {
border: 1px solid var(--border-strong); border-radius: var(--radius); background: var(--bg-raised);
color: var(--text); padding: 0; max-width: 34rem; width: calc(100% - 2rem); box-shadow: var(--shadow);
}
dialog::backdrop { background: rgba(0, 0, 0, .55); }
dialog h2 { margin: 0; font-size: 1rem; }
dialog > header { padding: .8rem .9rem; border-bottom: 1px solid var(--border); display: flex; gap: .5rem; align-items: center; }
dialog .body { padding: .9rem; font-size: 13px; color: var(--text-muted); }
dialog footer { padding: .7rem .9rem; border-top: 1px solid var(--border); display: flex; gap: .5rem; justify-content: flex-end; }
/* ---- misc --------------------------------------------------------------- */
.muted { color: var(--text-muted); }
.faint { color: var(--text-faint); }
.mono { font-family: var(--mono); }
.nowrap { white-space: nowrap; }
.right { text-align: right; }
.center { text-align: center; }
.gap { display: flex; gap: .5rem; align-items: center; flex-wrap: wrap; }
/* `[hidden]` must beat any display set on the same element — .banner is display:flex, so a
* hidden banner used to occupy a grey strip above the KPIs on every card. */
[hidden] { display: none !important; }
.vh { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); }
.wrap { overflow-wrap: anywhere; }
.empty-state { padding: 2rem 1rem; text-align: center; color: var(--text-muted); }
.empty-state h3 { margin: 0 0 .4rem; font-size: 1rem; color: var(--text); }
.empty-state code { font-family: var(--mono); font-size: 12px; background: var(--bg-sunken); padding: .1rem .3rem; border-radius: 4px; }
.empty-state ul { text-align: left; display: inline-block; margin: .6rem 0 0; }
.stale { opacity: .62; }
.stale-flag { color: var(--warn); font-size: 11px; }
select, input[type="text"] {
background: var(--bg-sunken); color: var(--text); border: 1px solid var(--border);
border-radius: var(--radius-sm); padding: .3rem .45rem; font: 500 12.5px var(--sans);
}
label.field { font-size: 11px; text-transform: uppercase; letter-spacing: .05em; color: var(--text-faint); }
footer.pagefoot {
margin-top: 1.6rem; padding-top: .8rem; border-top: 1px solid var(--border);
color: var(--text-faint); font-size: 11.5px; display: flex; gap: 1rem; flex-wrap: wrap;
}
@media (prefers-reduced-motion: reduce) {
* { transition: none !important; animation: none !important; }
}
+171
View File
@@ -0,0 +1,171 @@
/* Overview page: one /api/summary call, plus the config-driven setup hints. */
'use strict';
const { $, fetchJSON, setTxt, setValue, setBar, setDot, bytes, pct, num, duration, poll } = Dash;
async function loadConfig() {
try {
const config = await fetchJSON('/api/config');
const banners = [];
if (!config.system.enabled) {
banners.push(['warn', 'System dashboard has no targets — set <code>HOST_TARGETS</code> ' +
'(run <code>agent/host_agent.py</code> on each host first).']);
}
if (config.system.target_problems && config.system.target_problems.length) {
banners.push(['fail', 'Bad HOST_TARGETS entries: ' +
config.system.target_problems.map((p) => `<code>${DashCharts.escapeHtml(p)}</code>`).join(' ')]);
}
if (!config.llm.gufo.enabled && !config.llm.vllm.enabled) {
banners.push(['warn', 'No LLM engines configured — set <code>GUFO_BASE_URL</code> and/or ' +
'<code>VLLM_BASE_URL</code>.']);
}
if (!config.ups.enabled) {
banners.push(['warn', 'No NUT server configured — set <code>NUT_HOST</code> (upsd must ' +
'have a <code>LISTEN</code> line).']);
} else if (!config.ups.actions_enabled) {
banners.push(['info', 'UPS commands are off — set <code>NUT_ALLOW_ACTIONS=1</code> to enable them.']);
} else if (!config.ups.auth && !config.ups.simulated) {
// the simulator needs no credentials: asking for them there sends the operator fixing
// something that is not broken
banners.push(['info', 'UPS commands need credentials: set <code>NUT_USER</code>/' +
'<code>NUT_PASSWORD</code> with <code>actions</code> in <code>upsd.users</code>.']);
}
if (config.ups.simulated) {
banners.push(['warn', '<strong>SIMULATED UPS DATA.</strong> <code>NUT_SIMULATE=1</code> is set: ' +
'the UPS numbers are a development model, not your batteries.']);
}
const host = $('#banners');
host.innerHTML = banners.map(([kind, html]) =>
`<div class="banner ${kind}">${html}</div>`).join('');
$('#foot-versions').textContent =
`dashboards ${config.app.version} · poll system ${config.system.interval_seconds}s · ` +
`llm ${config.llm.interval_seconds}s · ups ${config.ups.interval_seconds}s · ` +
`uptime ${duration(config.app.started_at ? Date.now() / 1000 - config.app.started_at : null)}` +
(config.app.read_only ? ' · read-only' : '');
return config;
} catch (err) {
return null;
}
}
function renderEvents(events) {
const list = $('#events');
if (!events || !events.length) {
list.innerHTML = '<li class="muted" style="grid-template-columns:1fr">Nothing has changed ' +
'since this server started — the log only records transitions.</li>';
return;
}
list.innerHTML = events.map((event) => {
const when = new Date(event.ts * 1000).toLocaleTimeString(undefined,
{ hour: '2-digit', minute: '2-digit', hour12: false });
return `<li><time>${when}</time><span><span class="sev ${event.severity}">${event.host}</span> ` +
`${DashCharts.escapeHtml(event.message.replace(event.host + ':', '').trim() || event.kind)}</span></li>`;
}).join('');
}
function renderSummary(summary) {
const system = summary.system || {};
const fleet = system.fleet || {};
setDot($('#dot-system'), system.enabled ? (fleet.hosts_live === fleet.hosts_total ? 'ok'
: fleet.hosts_live ? 'warn' : 'fail') : 'idle');
setTxt($('#sys-hosts-live'), fleet.hosts_live == null ? 'n/a' : String(fleet.hosts_live));
setTxt($('#sys-hosts-total'), fleet.hosts_total == null ? '' : `of ${fleet.hosts_total}`);
setValue($('#sys-cpu'), fleet.cpu_usage_pct_avg, 0);
setBar($('#sys-cpu').closest('.metric').querySelector('.bar'), fleet.cpu_usage_pct_avg);
if (fleet.memory_used_bytes != null) {
const used = Dash.fmtBytes(fleet.memory_used_bytes, 0);
const total = Dash.fmtBytes(fleet.memory_total_bytes, 0);
// "200 / 253 GiB", not "200 GiB of 253 GiB": the card column is narrow and the long form
// ran into the worst-disk number next to it
setTxt($('#sys-mem'), total ? `${used.text}/${total.text}` : used.text);
setTxt($('#sys-mem-unit'), used.unit);
} else {
setTxt($('#sys-mem'), 'n/a');
setTxt($('#sys-mem-unit'), '');
}
setBar($('#sys-mem').closest('.metric').querySelector('.bar'), fleet.memory_used_pct);
setValue($('#sys-disk'), fleet.disk_worst_used_pct, 0);
setBar($('#sys-disk').closest('.metric').querySelector('.bar'), fleet.disk_worst_used_pct,
Dash.severityFor(fleet.disk_worst_used_pct, 80, 92));
setTxt($('#sys-gpus'), fleet.gpu_count == null ? 'n/a'
: `${fleet.gpu_count}${fleet.gpu_busy_pct_avg != null ? ' · ' + num(fleet.gpu_busy_pct_avg, 0) + '%' : ''}`);
const llm = summary.llm || {};
const engines = llm.fleet || {};
setDot($('#dot-llm'), !llm.enabled ? 'idle'
: (engines.engines_live === engines.engines_total ? 'ok' : engines.engines_live ? 'warn' : 'fail'));
setTxt($('#llm-live'), engines.engines_live == null ? 'n/a' : String(engines.engines_live));
setTxt($('#llm-total'), engines.engines_total == null ? '' : `of ${engines.engines_total}`);
setValue($('#llm-decode'), engines.decode_tokens_per_second, 1);
setValue($('#llm-prompt'), engines.prompt_tokens_per_second, 0);
setTxt($('#llm-active'), engines.requests_active == null ? 'n/a' : num(engines.requests_active, 0));
setTxt($('#llm-waiting'), engines.requests_waiting == null ? 'n/a' : num(engines.requests_waiting, 0));
setValue($('#llm-kv'), engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100, 0);
setBar($('#llm-kv').closest('.metric').querySelector('.bar'),
engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100,
Dash.severityFor(engines.kv_cache_usage_max == null ? null : engines.kv_cache_usage_max * 100, 75, 90));
const ups = summary.ups || {};
setDot($('#dot-ups'), !ups.enabled ? 'idle'
: (ups.ups || []).some((u) => u.overall === 'at risk') ? 'fail'
: (ups.ups || []).some((u) => u.overall !== 'healthy') ? 'warn' : 'ok');
const list = $('#ups-list');
if (!ups.enabled) {
list.innerHTML = '<div class="muted">No NUT server configured.</div>';
} else if (!(ups.ups || []).length) {
list.innerHTML = '<div class="muted">No UPS answering from this upsd yet.</div>';
} else {
list.innerHTML = ups.ups.map((item) => {
const kind = item.overall === 'healthy' ? 'ok' : item.overall === 'at risk' ? 'fail'
: item.overall === 'degraded' ? 'warn' : '';
return `<div class="gap" style="justify-content:space-between">
<span class="gap"><span class="chip ${kind}">${DashCharts.escapeHtml(item.name)}</span>
<span class="mono faint">${DashCharts.escapeHtml(item.status || 'no state')}</span>
${ups.simulated ? '<span class="sim-badge">sim</span>' : ''}</span>
<span class="mono">${item.charge_pct == null ? 'n/a' : num(item.charge_pct, 0) + '%'} ·
${item.load_pct == null ? 'n/a' : num(item.load_pct, 0) + '% load'} ·
${item.runtime_seconds == null ? 'n/a' : duration(item.runtime_seconds) + ' runtime'}</span>
</div>`;
}).join('');
}
const rows = system.hosts || [];
const tbody = $('#fleet-table tbody');
$('#fleet-empty').hidden = rows.length > 0;
if (!rows.length) { tbody.innerHTML = ''; return; }
tbody.innerHTML = rows.map((host) => {
const gpus = (host.gpu_busy_pct || []).filter((v) => v != null);
return `<tr>
<td class="name">${DashCharts.escapeHtml(host.name)}</td>
<td class="num">${host.cpu_usage_pct == null ? 'n/a' : num(host.cpu_usage_pct, 0) + '%'}</td>
<td class="num mute">${host.load1 == null ? 'n/a' : num(host.load1, 2)}</td>
<td class="num">${host.mem_used_pct == null ? 'n/a' : num(host.mem_used_pct, 0) + '%'}</td>
<td class="num">${host.disk_worst_used_pct == null ? 'n/a' : num(host.disk_worst_used_pct, 0) + '%'}</td>
<td class="num">${gpus.length ? num(gpus[0], 0) + '%' : '<span class="mute">—</span>'}</td>
<td class="num">${host.temp_c == null ? '<span class="mute">n/a</span>' : num(host.temp_c, 0) + '°C'}</td>
<td class="num mute">${host.uptime_seconds == null ? 'n/a' : duration(host.uptime_seconds)}</td>
</tr>`;
}).join('');
}
async function refresh() {
try {
const summary = await fetchJSON('/api/summary');
const total = (summary.system.hosts || []).length;
const live = summary.system.fleet ? summary.system.fleet.hosts_live : 0;
const engines = summary.llm.fleet ? `${summary.llm.fleet.engines_live}/${summary.llm.fleet.engines_total} engines` : 'no engines';
const ups = summary.ups.ups ? `${summary.ups.ups.length} UPS` : 'no UPS';
$('#summary-line').textContent = `${live}/${total} hosts · ${engines} · ${ups} · ` +
`${(summary.recent_events || []).length} recent events`;
renderSummary(summary);
renderEvents(summary.recent_events);
} catch (err) {
$('#summary-line').textContent = `summary failed: ${err.message}`;
}
}
document.addEventListener('DOMContentLoaded', async () => {
await loadConfig();
await refresh();
poll(refresh, 10000);
});
+382
View File
@@ -0,0 +1,382 @@
/* LLM dashboard.
*
* Every throughput cell shows where its number came from, because "78 tok/s" means
* different things depending on whether it was scraped from an aggregate gauge, measured as
* Δtokens/Δt across all slots, or estimated from TPOT over the process lifetime. The tag is
* generated from the cell's `source`, never hardcoded in markup, so a parser change cannot
* leave a stale label behind.
*/
'use strict';
const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, pct, duration, poll,
severityFor } = Dash;
const state = { range: 3600, panels: new Map(), charts: {}, poller: null, pollMs: 2000 };
const CELLS = [
{ key: 'decode_tokens_per_second', label: 'Decode', unit: 'tok/s', digits: 1,
tip: 'Aggregate generation speed across every busy slot.' },
{ key: 'prompt_tokens_per_second', label: 'Prompt', unit: 'tok/s', digits: 1,
tip: 'Prefill speed for newly computed tokens.' },
{ key: 'sessions', label: 'Sessions', unit: 'active', digits: 0 },
{ key: 'queued', label: 'Queued', unit: 'waiting', digits: 0 },
{ key: 'cache', label: 'KV cache', unit: '%', digits: 0, bar: true },
{ key: 'ttft_mean', label: 'TTFT mean', unit: 'ms', digits: 0 },
{ key: 'ttft_p95', label: 'TTFT p95', unit: 'ms', digits: 0 },
{ key: 'tpot_mean', label: 'TPOT mean', unit: 'ms', digits: 1 },
{ key: 'queue_mean', label: 'Queue mean', unit: 'ms', digits: 0 },
{ key: 'cache_hit', label: 'Prefix cache hit', unit: '%', digits: 0 },
{ key: 'spec_accept', label: 'Spec accept', unit: '%', digits: 0 },
{ key: 'tokens_total', label: 'Tokens lifetime', unit: '', digits: 0, human: true },
{ key: 'preemptions', label: 'Preemptions', unit: '', digits: 0 },
{ key: 'device_lost', label: 'GPU context lost', unit: '', digits: 0 },
];
function cellValue(engine, metrics, sessions, key) {
const get = (k) => (metrics[k] && metrics[k].value != null ? metrics[k] : null);
switch (key) {
case 'sessions': {
const value = sessions && sessions.active != null ? sessions.active : null;
return { value, source: 'session count', derived: false };
}
case 'queued': {
const cell = get('requests_deferred') || get('num_requests_waiting');
return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null };
}
case 'cache': {
const cell = get('kv_cache_usage_ratio') || get('gpu_cache_usage');
return cell ? { value: cell.value * 100, source: cell.source, derived: false } : { value: null };
}
case 'ttft_mean': {
const cell = get('time_to_first_token_seconds__mean');
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
}
case 'ttft_p95': {
const cell = get('time_to_first_token_seconds__p95');
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
}
case 'tpot_mean': {
const cell = get('time_per_output_token_seconds__mean');
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
}
case 'queue_mean': {
const cell = get('request_queue_time_seconds__mean');
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
}
case 'cache_hit': {
const cell = get('prefix_cache_hit_ratio') || get('prompt_cache_hit_ratio');
return cell ? { value: cell.value * 100, source: cell.source, derived: true } : { value: null };
}
case 'spec_accept': {
const cell = get('spec_accept_rate');
return cell ? { value: cell.value * 100, source: cell.source, derived: true } : { value: null };
}
case 'tokens_total': {
const cell = get('tokens_total');
return cell ? { value: cell.value, source: cell.source, derived: true } : { value: null };
}
case 'preemptions': {
const cell = get('num_preemptions_total');
return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null };
}
case 'device_lost': {
const cell = get('device_lost_total');
return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null };
}
default: {
const cell = get(key);
return cell ? { value: cell.value, source: cell.source, derived: cell.derived,
window: cell.window } : { value: null };
}
}
}
/** The provenance tag: Δ10.0s for measured rates, the formula for estimates, nothing for a
* plain scraped gauge (which would only add noise). */
function provenance(cell) {
if (!cell || !cell.source) return '';
if (!String(cell.source).startsWith('computed:')) return '';
if (cell.window) return `Δ${cell.window.toFixed(1)}s`;
const text = String(cell.source).replace('computed:', '');
return text.length > 16 ? text.slice(0, 15) + '…' : text;
}
function humanNumber(value) {
if (value == null || !isFinite(value)) return null;
if (value >= 1e9) return { text: (value / 1e9).toFixed(2), unit: 'B tok' };
if (value >= 1e6) return { text: (value / 1e6).toFixed(2), unit: 'M tok' };
if (value >= 1e3) return { text: (value / 1e3).toFixed(1), unit: 'k tok' };
return { text: value.toFixed(0), unit: 'tok' };
}
function enginePanel(engine) {
const card = document.createElement('article');
card.className = 'card';
card.dataset.engine = engine;
card.innerHTML = `
<header>
<span class="dot idle"></span>
<h3>${DashCharts.escapeHtml(engine)}</h3>
<span class="chip upstream mono" hidden></span>
<span class="spacer"></span>
<span class="faint mono updated"></span>
</header>
<div class="body stack">
<div class="banner err" hidden></div>
<div class="metrics">${CELLS.map((cell) => `
<div class="metric" data-cell="${cell.key}" ${cell.tip ? `title="${cell.tip}"` : ''}>
<div class="label">${cell.label}</div>
<div class="value${cell.digits === 0 ? '' : ''}"><span data-v>n/a</span><span class="unit">${cell.unit}</span></div>
${cell.bar ? '<div class="bar" style="margin-top:.25rem"><i></i></div>' : ''}
</div>`).join('')}</div>
<div class="faint mono sessions-note" style="font-size:11.5px"></div>
<div class="faint mono debug" style="font-size:11px"></div>
</div>`;
return card;
}
function renderPanel(card, enginePayload, stateRow) {
const metrics = enginePayload.metrics || {};
const sessions = enginePayload.sessions || {};
const q = (sel) => card.querySelector(sel);
const dot = q('.dot');
setDot(dot, stateRow && stateRow.ok ? (Dash.ageSeconds(stateRow.last_ok) > 15 ? 'warn' : 'ok') : 'fail');
setTxt(q('.updated'), stateRow ? (stateRow.last_ok == null
? (stateRow.error || 'never reached')
: (Dash.ageSeconds(stateRow.last_ok) > 15 ? `stale ${duration(Dash.ageSeconds(stateRow.last_ok))}`
: `updated ${duration(Dash.ageSeconds(stateRow.last_ok))}`)) : '—');
Dash.markStale(card, stateRow && (!stateRow.ok || Dash.ageSeconds(stateRow.last_ok) > 15));
const banner = q('.banner.err');
if (stateRow && !stateRow.ok) {
banner.hidden = false;
banner.className = 'banner fail err';
banner.textContent = stateRow.error || 'engine is not answering';
} else {
banner.hidden = true;
}
for (const cell of CELLS) {
const wrapper = q(`[data-cell="${cell.key}"]`);
if (!wrapper) continue;
const value = cellValue(enginePayload.engine, metrics, sessions, cell.key);
const valueEl = wrapper.querySelector('[data-v]');
const unitEl = wrapper.querySelector('.unit');
if (value.value == null) {
setTxt(valueEl, 'n/a');
setTxt(unitEl, ''); // "n/a %" reads like a measurement of nothing
wrapper.classList.add('is-null');
} else {
wrapper.classList.remove('is-null');
if (cell.human) {
const parts = humanNumber(value.value);
setTxt(valueEl, parts ? parts.text : 'n/a');
setTxt(unitEl, parts ? parts.unit : '');
} else {
setTxt(valueEl, num(value.value, cell.digits));
setTxt(unitEl, cell.unit);
}
}
let tag = wrapper.querySelector('.prov');
const label = provenance(value);
if (label) {
if (!tag) {
tag = document.createElement('span');
tag.className = 'prov';
wrapper.querySelector('.value').appendChild(tag);
}
setTxt(tag, label);
tag.title = `${value.source || ''}${value.window ? ` · measured over ${value.window.toFixed(1)}s` : ''}`;
} else if (tag) {
tag.remove();
}
if (cell.bar) setBar(wrapper.querySelector('.bar'), value.value,
severityFor(value.value, 75, 90));
if (cell.key === 'device_lost' && value.value != null && value.value > 0) {
wrapper.querySelector('.value').classList.add('is-null');
wrapper.classList.add('fail');
wrapper.title = `GPU context has been lost ${value.value} times — an incident, not a stat`;
}
}
const note = q('.sessions-note');
setTxt(note, sessions.detail || `${sessions.active == null ? 'n/a' : sessions.active} active sessions`);
const upstream = q('.chip.upstream');
if (enginePayload.upstream) {
upstream.hidden = false;
upstream.textContent = enginePayload.upstream.replace(/^https?:\/\//, '');
upstream.title = enginePayload.upstream;
}
setTxt(q('.debug'), enginePayload.unparsed != null
? `${enginePayload.series_count} series · ${enginePayload.unparsed} unparsed`
: '');
}
function renderSlots(payloads) {
const gufo = payloads.find((p) => p.engine === 'gufo');
const rows = gufo && gufo.sessions ? gufo.sessions.list || [] : [];
const tbody = $('#slots-table tbody');
$('#slots-empty').hidden = rows.length > 0;
$('#slots-card').hidden = !gufo;
tbody.innerHTML = rows.map((slot) => {
const id = slot.id != null ? slot.id : slot.slot != null ? slot.slot : '—';
const stateText = slot.state || (slot.idle === true ? 'idle' : slot.idle === false ? 'busy' : '—');
const withText = (slot.with_messages && slot.with_messages.length) || slot.with_message;
const nPrompt = slot.n_prompt != null ? slot.n_prompt : (slot.prompt_tokens != null ? slot.prompt_tokens : null);
const nPredict = slot.n_predict != null ? slot.n_predict : (slot.generated_tokens != null ? slot.generated_tokens : null);
const cache = slot.cache != null ? slot.cache : slot.cache_usable;
const speed = slot.decode_per_second || slot.tokens_per_second || null;
const cell = (value, digits) => `<td class="num">${value == null ? '<span class="mute">—</span>' : num(value, digits == null ? 0 : digits)}</td>`;
return `<tr>
<td class="name mono" style="font-size:11.5px">${DashCharts.escapeHtml(id)}</td>
<td>${stateText === 'busy' || stateText === 'processing' ? '<span class="chip ok">busy</span>'
: stateText === 'idle' ? '<span class="chip">idle</span>'
: `<span class="chip warn">${DashCharts.escapeHtml(stateText)}</span>`}</td>
<td>${withText ? '<span class="chip info">has msg</span>' : '<span class="mute">—</span>'}</td>
${cell(nPrompt)}${cell(nPredict)}${cell(cache)}${cell(speed, 1)}</tr>`;
}).join('');
}
function makeCharts() {
const common = { height: 165, xSpan: state.range };
state.charts.decode = new DashCharts.LineChart('#chart-decode', Object.assign({}, common, { yMin: 0 }));
state.charts.prompt = new DashCharts.LineChart('#chart-prompt', Object.assign({}, common, { yMin: 0 }));
state.charts.kv = new DashCharts.LineChart('#chart-kv', Object.assign({}, common,
{ yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
state.charts.queue = new DashCharts.LineChart('#chart-queue', Object.assign({}, common,
{ yMin: 0, yFormat: (v) => num(v, 0) }));
}
function pointsFor(response, engine, metric) {
const found = (response.series || []).find((s) => s.host === engine && s.metric === `llm.${metric}`);
return found ? found.points.map((p) => ({ ts: p.ts, value: p.value })) : [];
}
async function refreshCharts(engineNames) {
if (!engineNames.length) return;
const list = engineNames.join(',');
const [decode, prompt, queue] = await Promise.all([
fetchJSON(`/api/llm/history?engines=${list}&metrics=decode_tokens_per_second&range=${state.range}`)
.catch(() => null),
fetchJSON(`/api/llm/history?engines=${list}&metrics=prompt_tokens_per_second&range=${state.range}`)
.catch(() => null),
fetchJSON(`/api/llm/history?engines=${list}&metrics=requests_running,requests_waiting,` +
`kv_cache_usage&range=${state.range}`).catch(() => null),
]);
if (decode) {
const series = engineNames.map((engine) => ({ name: engine,
points: pointsFor(decode, engine, 'decode_tokens_per_second') }));
state.charts.decode.setData(series);
DashCharts.renderLegend($('#legend-decode'), series, (v) => num(v, 1) + ' tok/s', true);
}
if (prompt) {
const series = engineNames.map((engine) => ({ name: engine,
points: pointsFor(prompt, engine, 'prompt_tokens_per_second') }));
state.charts.prompt.setData(series);
DashCharts.renderLegend($('#legend-prompt'), series, (v) => num(v, 0) + ' tok/s', true);
}
if (queue) {
const series = [];
for (const engine of engineNames) {
series.push({ name: `${engine} running`, points: pointsFor(queue, engine, 'requests_running') });
series.push({ name: `${engine} waiting`, points: pointsFor(queue, engine, 'requests_waiting') });
}
state.charts.queue.setData(series);
DashCharts.renderLegend($('#legend-queue'), series, (v) => num(v, 0));
const kv = engineNames.map((engine) => ({ name: engine,
points: pointsFor(queue, engine, 'kv_cache_usage').map((p) => ({ ts: p.ts, value: p.value * 100 })) }));
state.charts.kv.setData(kv);
DashCharts.renderLegend($('#legend-kv'), kv, (v) => num(v, 1) + '%', true);
}
}
function renderFleet(fleet) {
if (!fleet) return;
setTxt($('#f-live'), fleet.engines_live == null ? 'n/a' : String(fleet.engines_live));
setTxt($('#f-total'), fleet.engines_total == null ? '' : `of ${fleet.engines_total}`);
setValue($('#f-decode'), fleet.decode_tokens_per_second, 1);
setValue($('#f-prompt'), fleet.prompt_tokens_per_second, 0);
setTxt($('#f-active'), fleet.requests_active == null ? 'n/a' : num(fleet.requests_active, 0));
setTxt($('#f-waiting'), fleet.requests_waiting == null ? 'n/a' : num(fleet.requests_waiting, 0));
const tokens = humanNumber(fleet.tokens_total);
setTxt($('#f-tokens'), tokens ? tokens.text : 'n/a');
$('#f-tokens').nextElementSibling && setTxt($('#f-tokens').nextElementSibling, tokens ? tokens.unit : '');
const kv = fleet.kv_cache_usage_max == null ? null : fleet.kv_cache_usage_max * 100;
setValue($('#f-kv'), kv, 0);
setBar($('#f-kv').closest('.metric').querySelector('.bar'), kv, severityFor(kv, 75, 90));
$('#seam-note').textContent = fleet.seams || '';
}
async function refresh() {
let snapshot;
try {
snapshot = await fetchJSON('/api/llm/snapshot');
} catch (err) {
$('#sub').textContent = err.message;
return;
}
const rows = {};
(snapshot.states || []).forEach((row) => { rows[row.label.toLowerCase()] = row; });
const payloads = snapshot.engines || [];
const names = payloads.map((p) => p.engine);
payloads.forEach((payload) => {
let card = state.panels.get(payload.engine);
if (!card) {
card = enginePanel(payload.engine);
state.panels.set(payload.engine, card);
$('#engines').appendChild(card);
}
renderPanel(card, payload, rows[payload.engine] ||
(snapshot.states || []).find((row) => row.key.endsWith(payload.engine)));
});
$('#engines-empty').hidden = payloads.length > 0;
renderFleet(snapshot.fleet);
renderSlots(payloads);
const fleet = snapshot.fleet || {};
$('#sub').textContent = `${fleet.engines_live || 0}/${fleet.engines_total || 0} engines · ` +
`${fleet.requests_active == null ? 'n/a' : num(fleet.requests_active, 0)} active · ` +
`window ${duration(state.range)}`;
$('#foot-note').textContent = `poll ${Math.round(state.pollMs / 1000)}s · rates over the ` +
`Δ window in each tag`;
await refreshCharts(names);
}
async function refreshEvents() {
try {
const data = await fetchJSON('/api/events?limit=20&source=llm');
const list = $('#events');
if (!(data.events || []).length) {
list.innerHTML = '<li class="muted" style="grid-template-columns:1fr">No engine transitions yet.</li>';
return;
}
list.innerHTML = data.events.map((event) => `<li><time>${
new Date(event.ts * 1000).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit', hour12: false })
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
`${DashCharts.escapeHtml(event.message)}</span></li>`).join('');
} catch (err) { /* next poll */ }
}
document.addEventListener('DOMContentLoaded', async () => {
makeCharts();
try {
const config = await fetchJSON('/api/config');
state.pollMs = config.llm.poll_ms || 2000;
const banners = [];
if (!config.llm.gufo.enabled && !config.llm.vllm.enabled) {
banners.push(['warn', 'Neither engine is configured.']);
}
if (config.ups.simulated) banners.push(['warn', 'UPS numbers on other pages are simulated.']);
$('#banners').innerHTML = banners.map(([kind, html]) => `<div class="banner ${kind}">${html}</div>`)
.join('');
} catch (err) { /* not fatal */ }
Dash.rangeControl($('#range'), state.range, (seconds) => {
state.range = seconds;
Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = seconds; });
refresh();
});
state.poller = poll(async () => { await refresh(); await refreshEvents(); }, state.pollMs);
Dash.bindPause(state.poller);
});
+503
View File
@@ -0,0 +1,503 @@
/* System dashboard page.
*
* Host cards are created once and patched in place (keyed by host name) — rebuilding the
* DOM every 10s kills hover, transitions, and any scroll position inside the tables.
* Charts get their history from /api/system/history: one request per chart, N hosts each.
*/
'use strict';
const { $, $$, fetchJSON, setTxt, setValue, setBar, setDot, bytes, pct, num, duration, poll,
staleness, markStale, severityFor } = Dash;
const state = {
range: 3600,
detailHost: null,
detailMetric: 'cpu.usage_pct',
hosts: new Map(), // name -> card element
charts: {},
poller: null,
pollMs: 10000,
metricList: [],
};
const DETAIL_METRICS = [
['cpu.usage_pct', 'CPU utilisation', 'pct'],
['cpu.user_pct', 'CPU user', 'pct'],
['cpu.system_pct', 'CPU system', 'pct'],
['cpu.iowait_pct', 'CPU iowait', 'pct'],
['cpu.steal_pct', 'CPU steal', 'pct'],
['cpu.load1', 'Load average (1m)', 'num'],
['cpu.load15', 'Load average (15m)', 'num'],
['cpu.freq_mhz', 'CPU frequency', 'mhz'],
['cpu.context_switches_per_s', 'Context switches', 'num'],
['mem.used_pct', 'Memory used', 'pct'],
['mem.used_bytes', 'Memory used', 'bytes'],
['mem.available_bytes', 'Memory available', 'bytes'],
['mem.swap_used_pct', 'Swap used', 'pct'],
['disk.used_pct', 'Filesystem used (worst)', 'pct'],
['disk.read_bytes_per_s', 'Disk read', 'bytes'],
['disk.write_bytes_per_s', 'Disk write', 'bytes'],
['disk.util_pct', 'Disk busy', 'pct'],
['net.rx_bytes_per_s', 'Network in', 'bytes'],
['net.tx_bytes_per_s', 'Network out', 'bytes'],
['temp.cpu_c', 'CPU temperature', 'temp'],
['temp.nvme_max_c', 'NVMe temperature', 'temp'],
['host.uptime_seconds', 'Uptime', 'dur'],
['host.processes', 'Process count', 'num'],
];
const FORMATS = {
pct: { fn: (v) => num(v, 1), unit: '%', axis: (v) => num(v, 0) },
bytes: { fn: (v) => bytes(v, 1), unit: 'B/s', axis: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text +
((Dash.fmtBytes(v, 0) || { unit: '' }).unit) },
mhz: { fn: (v) => num(v, 0) + ' MHz', unit: 'MHz', axis: (v) => num(v, 0) },
temp: { fn: (v) => num(v, 1) + ' °C', unit: '°C', axis: (v) => num(v, 0) },
dur: { fn: (v) => duration(v), unit: '', axis: (v) => duration(v) },
num: { fn: (v) => num(v, 1), unit: '', axis: (v) => num(v, 0) },
};
function formatFor(kind) { return FORMATS[kind] || FORMATS.num; }
function metricKind(name) {
if (name.endsWith('_pct')) return 'pct';
if (name.endsWith('_bytes') || name.includes('bytes_per_s')) return 'bytes';
if (name.includes('_c') && name.startsWith('temp.')) return 'temp';
if (name === 'host.uptime_seconds') return 'dur';
return 'num';
}
// ---- host cards -----------------------------------------------------------
function hostCard(name) {
const card = document.createElement('article');
card.className = 'card';
card.dataset.host = name;
card.innerHTML = `
<header>
<span class="dot idle"></span>
<h3></h3>
<span class="chip os" hidden></span>
<span class="spacer"></span>
<span class="faint mono updated"></span>
</header>
<div class="body stack">
<div class="metrics">
<div class="metric"><div class="label">CPU</div><div class="value"><span data-k="cpu">n/a</span><span class="unit">%</span></div>
<div class="bar" data-bar="cpu"><i></i></div></div>
<div class="metric"><div class="label">Load 1 / 15</div><div class="value sm" data-k="load">n/a</div></div>
<div class="metric"><div class="label">CPU temp</div><div class="value sm"><span data-k="temp">n/a</span><span class="unit">°C</span></div></div>
<div class="metric"><div class="label">Clock</div><div class="value sm"><span data-k="freq">n/a</span><span class="unit">GHz</span></div></div>
<div class="metric"><div class="label">Memory</div><div class="value sm"><span data-k="mem">n/a</span><span class="unit"></span></div>
<div class="bar" data-bar="mem"><i></i></div></div>
<div class="metric"><div class="label">Worst fs</div><div class="value sm"><span data-k="disk">n/a</span><span class="unit"></span></div>
<div class="bar" data-bar="disk"><i></i></div></div>
</div>
<div>
<div class="label faint" style="font-size:10.5px;letter-spacing:.06em;text-transform:uppercase">Per-core</div>
<div class="cores" data-k="cores" style="margin-top:.25rem"></div>
</div>
<div class="metrics">
<div class="metric"><div class="label">Disk read</div><div class="value sm"><span data-k="dr">n/a</span><span class="unit"></span></div></div>
<div class="metric"><div class="label">Disk write</div><div class="value sm"><span data-k="dw">n/a</span><span class="unit"></span></div></div>
<div class="metric"><div class="label">Disk busy</div><div class="value sm"><span data-k="dutil">n/a</span><span class="unit">%</span></div></div>
<div class="metric"><div class="label">Net in</div><div class="value sm"><span data-k="rx">n/a</span><span class="unit"></span></div></div>
<div class="metric"><div class="label">Net out</div><div class="value sm"><span data-k="tx">n/a</span><span class="unit"></span></div></div>
<div class="metric"><div class="label">Up</div><div class="value sm" data-k="uptime">n/a</div></div>
</div>
<div data-k="gpus"></div>
<details>
<summary class="faint" style="cursor:pointer;font-size:12px">Filesystems &amp; processes</summary>
<div style="margin-top:.6rem" data-k="details"></div>
</details>
</div>`;
return card;
}
function renderHostCard(card, host) {
const q = (key) => card.querySelector(`[data-k="${key}"]`);
const stateRow = host.state || {};
card.querySelector('h3').textContent = host.name;
const osChip = card.querySelector('.os');
const osName = (host.info || {}).os_pretty || (host.info || {}).os_id;
if (osName) { osChip.hidden = false; osChip.textContent = osName; }
setDot(card.querySelector('.dot'), stateRow.ok === false ? 'fail'
: (Dash.ageSeconds(stateRow.last_ok) > 3 * (host.window || 10) ? 'warn' : 'ok'));
setTxt(card.querySelector('.updated'), stalenessText(stateRow));
markStale(card, stateRow.ok === false || Dash.ageSeconds(stateRow.last_ok) > 5 * (host.window || 10));
const cpu = host.cpu || {};
setValue(q('cpu'), cpu.usage_pct, 0);
setBar(card.querySelector('[data-bar="cpu"]'), cpu.usage_pct, severityFor(cpu.usage_pct, 75, 90));
setTxt(q('load'), cpu.load1 == null ? 'n/a' : `${num(cpu.load1, 2)} / ${num(cpu.load15, 2)}`);
setValue(q('temp'), cpu.temp_c, 1);
setValue(q('freq'), cpu.freq_mhz == null ? null : cpu.freq_mhz / 1000, 2);
const mem = host.memory || {};
const memText = Dash.fmtBytes(mem.used_bytes, 1);
const memTotal = Dash.fmtBytes(mem.total_bytes, 0);
setTxt(q('mem'), memText ? memText.text : 'n/a');
setTxt(q('mem').nextElementSibling, memText ? `${memText.unit}/${memTotal ? memTotal.text + memTotal.unit : ''}` : '');
setBar(card.querySelector('[data-bar="mem"]'), mem.used_pct, severityFor(mem.used_pct, 80, 92));
const disks = host.disks || {};
setTxt(q('disk'), disks.worst_used_pct == null ? 'n/a' : num(disks.worst_used_pct, 0) + '%');
setTxt(q('disk').nextElementSibling, disks.worst_mount ? ' ' + disks.worst_mount : '');
setBar(card.querySelector('[data-bar="disk"]'), disks.worst_used_pct, severityFor(disks.worst_used_pct, 80, 92));
DashCharts.renderCores(q('cores'), cpu.core_usage_pct || []);
setValue(q('dr'), disks.read_bytes_per_s == null ? null : disks.read_bytes_per_s, null);
if (disks.read_bytes_per_s != null) {
const parts = Dash.fmtBytes(disks.read_bytes_per_s, 1);
setTxt(q('dr'), parts.text); setTxt(q('dr').nextElementSibling, parts.unit + '/s');
const w = Dash.fmtBytes(disks.write_bytes_per_s, 1);
setTxt(q('dw'), w ? w.text : 'n/a'); setTxt(q('dw').nextElementSibling, w ? w.unit + '/s' : '');
} else { setTxt(q('dr'), 'n/a'); setTxt(q('dw'), 'n/a'); }
setValue(q('dutil'), disks.util_pct, 0);
const rx = Dash.fmtBytes((host.network || {}).rx_bytes_per_s, 1);
const tx = Dash.fmtBytes((host.network || {}).tx_bytes_per_s, 1);
setTxt(q('rx'), rx ? rx.text : 'n/a'); setTxt(q('rx').nextElementSibling, rx ? rx.unit + '/s' : '');
setTxt(q('tx'), tx ? tx.text : 'n/a'); setTxt(q('tx').nextElementSibling, tx ? tx.unit + '/s' : '');
setTxt(q('uptime'), duration((host.info || {}).uptime_seconds));
// GPUs: one row each, with both memory pools named (an APU's model lives in GTT)
const gpus = host.gpus || [];
const gpuHost = q('gpus');
if (!gpus.length) {
gpuHost.innerHTML = '<div class="faint" style="font-size:12px">No GPU reported by this agent</div>';
} else {
gpuHost.innerHTML = gpus.map((gpu) => `
<div class="stack" style="gap:.2rem;margin-top:.35rem">
<div class="gap" style="justify-content:space-between">
<span class="mono" style="font-size:12px">${DashCharts.escapeHtml(gpu.model || 'GPU')}</span>
<span class="mono" style="font-size:12px">
${gpu.busy_pct == null ? 'n/a' : num(gpu.busy_pct, 0) + '% busy'} ·
${gpu.power_watts == null ? 'power n/a' : num(gpu.power_watts, 0) + ' W'} ·
${gpu.temp_c == null ? 'temp n/a' : num(gpu.temp_c, 0) + '°C'}</span>
</div>
<div class="bar thin"><i style="width:${(gpu.busy_pct || 0).toFixed(1)}%"></i></div>
<div class="gap" style="justify-content:space-between;font-size:11.5px" class="faint">
<span class="faint">VRAM ${gpu.vram_used_bytes == null ? 'n/a' : bytes(gpu.vram_used_bytes, 0)}
${gpu.vram_total_bytes ? 'of ' + bytes(gpu.vram_total_bytes, 0) : ''}</span>
<span class="faint">shared (GTT) ${gpu.gtt_used_bytes == null ? 'n/a' : bytes(gpu.gtt_used_bytes, 0)}
${gpu.mem_used_pct == null ? '' : ' · pool ' + num(gpu.mem_used_pct, 0) + '%'}</span>
</div>
<div class="bar thin"><i style="width:${(gpu.mem_used_pct || 0).toFixed(1)}%;background:var(--series-2)"></i></div>
</div>`).join('');
}
const filesystems = (disks.filesystems || []).filter((fs) => fs.used_pct != null);
const procs = host.processes || {};
const details = q('details');
details.innerHTML = `
<table class="data">
<thead><tr><th>Mount</th><th>Type</th><th>Used</th><th>Free</th><th>Total</th></tr></thead>
<tbody>${filesystems.map((fs) => `<tr>
<td class="name mono" style="font-size:11.5px">${DashCharts.escapeHtml(fs.mountpoint)}</td>
<td class="mute">${DashCharts.escapeHtml(fs.fstype)}</td>
<td class="num">${num(fs.used_pct, 1)}%</td>
<td class="num mute">${bytes(fs.free_bytes, 0)}</td>
<td class="num mute">${bytes(fs.total_bytes, 0)}</td></tr>`).join('') ||
'<tr><td colspan="5" class="mute">no filesystems reported</td></tr>'}</tbody>
</table>
<div class="grid cols-2" style="margin-top:.6rem">
<div><div class="faint" style="font-size:11px;margin-bottom:.2rem">Top by CPU</div>
<table class="data"><tbody>${(procs.top_cpu || []).slice(0, 5).map((p) => `<tr>
<td class="mono" style="font-size:11.5px">${DashCharts.escapeHtml(p.name)}</td>
<td class="num">${num(p.cpu_pct, 0)}%</td><td class="num mute">pid ${p.pid}</td></tr>`).join('')
|| '<tr><td class="mute">needs two samples</td></tr>'}</tbody></table></div>
<div><div class="faint" style="font-size:11px;margin-bottom:.2rem">Top by RSS</div>
<table class="data"><tbody>${(procs.top_rss || []).slice(0, 5).map((p) => `<tr>
<td class="mono" style="font-size:11.5px">${DashCharts.escapeHtml(p.name)}</td>
<td class="num">${bytes(p.rss_bytes, 0)}</td><td class="num mute">pid ${p.pid}</td></tr>`).join('')
|| '<tr><td class="mute">—</td></tr>'}</tbody></table></div>
</div>
${(host.agent && host.agent.errors && host.agent.errors.length)
? `<div class="banner warn" style="margin-top:.6rem">agent degraded: ${DashCharts.escapeHtml(host.agent.errors.join('; '))}</div>`
: ''}
${(host.degraded && host.degraded.length)
? `<div class="banner warn" style="margin-top:.4rem">collector degraded: ${DashCharts.escapeHtml(host.degraded.join('; '))}</div>`
: ''}`;
}
function stalenessText(row) {
if (!row || !row.configured) return 'not configured';
if (row.last_ok == null) return row.error || 'never reached';
const seconds = Dash.ageSeconds(row.last_ok);
return seconds > 30 ? `stale ${duration(seconds)}` : `updated ${seconds < 2 ? 'now' : duration(seconds)}`;
}
// ---- charts ---------------------------------------------------------------
function makeCharts() {
const common = { height: 165, xSpan: state.range };
state.charts.cpu = new DashCharts.LineChart('#chart-cpu', Object.assign({}, common, {
yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
state.charts.mem = new DashCharts.LineChart('#chart-mem', Object.assign({}, common, {
yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
state.charts.disk = new DashCharts.LineChart('#chart-disk', Object.assign({}, common, {
yMin: 0, yFormat: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text }));
state.charts.net = new DashCharts.LineChart('#chart-net', Object.assign({}, common, {
yMin: 0, yFormat: (v) => (Dash.fmtBytes(v, 0) || { text: '0' }).text }));
state.charts.gpu = new DashCharts.LineChart('#chart-gpu', Object.assign({}, common, {
yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
state.charts.temp = new DashCharts.LineChart('#chart-temp', Object.assign({}, common, {
yFormat: (v) => num(v, 0) }));
state.charts.detail = new DashCharts.LineChart('#chart-detail', Object.assign({}, common, {
height: 200, yFormat: (v) => num(v, 1) }));
}
function setChartSpan(span) {
Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = span; });
}
function seriesFor(response, metric) {
return (response.series || []).filter((s) => s.metric === metric).map((s) => ({
name: s.host, points: s.points.map((p) => ({ ts: p.ts, value: p.value })),
}));
}
async function refreshCharts(snapshot) {
const hostNames = (snapshot.hosts || []).map((h) => h.name);
const detailHost = state.detailHost || hostNames[0] || null;
const span = state.range;
const requests = {
cpu: fetchJSON(`/api/system/history?metrics=cpu.usage_pct&range=${span}`).catch(() => null),
mem: fetchJSON(`/api/system/history?metrics=mem.used_pct&range=${span}`).catch(() => null),
temp: fetchJSON(`/api/system/history?metrics=temp.cpu_c&range=${span}`).catch(() => null),
disk: detailHost ? fetchJSON(`/api/system/history?hosts=${encodeURIComponent(detailHost)}&` +
`metrics=disk.read_bytes_per_s,disk.write_bytes_per_s&range=${span}`).catch(() => null) : null,
net: detailHost ? fetchJSON(`/api/system/history?hosts=${encodeURIComponent(detailHost)}&` +
`metrics=net.rx_bytes_per_s,net.tx_bytes_per_s&range=${span}`).catch(() => null) : null,
};
const [cpu, mem, temp, disk, net] = await Promise.all([requests.cpu, requests.mem,
requests.temp, requests.disk, requests.net]);
if (cpu) {
const series = seriesFor(cpu, 'cpu.usage_pct');
state.charts.cpu.setData(series);
DashCharts.renderLegend($('#legend-cpu'), series, (v) => num(v, 1) + '%', true);
}
if (mem) {
const series = seriesFor(mem, 'mem.used_pct');
state.charts.mem.setData(series);
DashCharts.renderLegend($('#legend-mem'), series, (v) => num(v, 1) + '%');
}
if (temp) {
const series = seriesFor(temp, 'temp.cpu_c');
state.charts.temp.setData(series);
DashCharts.renderLegend($('#legend-temp'), series, (v) => num(v, 1) + '°C', true);
}
if (disk) {
const read = (disk.series.find((s) => s.metric === 'disk.read_bytes_per_s') || { points: [] })
.points.map((p) => ({ ts: p.ts, value: p.value }));
const write = (disk.series.find((s) => s.metric === 'disk.write_bytes_per_s') || { points: [] })
.points.map((p) => ({ ts: p.ts, value: p.value }));
const series = [{ name: 'read', points: read }, { name: 'write', points: write }];
state.charts.disk.setData(series);
DashCharts.renderLegend($('#legend-disk'), series, (v) => bytes(v, 1) + '/s', true);
}
if (net) {
const rx = (net.series.find((s) => s.metric === 'net.rx_bytes_per_s') || { points: [] })
.points.map((p) => ({ ts: p.ts, value: p.value }));
const tx = (net.series.find((s) => s.metric === 'net.tx_bytes_per_s') || { points: [] })
.points.map((p) => ({ ts: p.ts, value: p.value }));
const series = [{ name: 'in', points: rx }, { name: 'out', points: tx }];
state.charts.net.setData(series);
DashCharts.renderLegend($('#legend-net'), series, (v) => bytes(v, 1) + '/s', true);
}
await refreshGpuChart(hostNames);
await refreshDetailChart();
}
async function refreshGpuChart(hostNames) {
if (!hostNames.length) {
state.charts.gpu.setData([]);
$('#legend-gpu').innerHTML = '<span class="faint">no hosts</span>';
return;
}
const series = [];
for (const host of hostNames) {
// per-stream names are discovered, not guessed: gpu.busy_pct@0, @1, ...
const found = await fetchJSON(`/api/metrics?source=host&host=${encodeURIComponent(host)}`)
.catch(() => ({ metrics: [] }));
const names = (found.metrics || []).filter((m) => m.startsWith('gpu.busy_pct@'));
for (const name of names) {
const data = await fetchJSON(`/api/history?source=host&host=${encodeURIComponent(host)}` +
`&metric=${encodeURIComponent(name)}&range=${state.range}`).catch(() => null);
if (data) series.push({ name: `${host} ${name.split('@')[1]}`,
points: data.points.map((p) => ({ ts: p.ts, value: p.value })) });
}
}
state.charts.gpu.setData(series);
DashCharts.renderLegend($('#legend-gpu'), series, (v) => num(v, 0) + '%', true);
}
async function refreshDetailChart() {
const host = state.detailHost;
if (!host) { state.charts.detail.setData([]); return; }
const kind = metricKind(state.detailMetric);
const format = formatFor(kind);
const data = await fetchJSON(`/api/history?source=host&host=${encodeURIComponent(host)}` +
`&metric=${encodeURIComponent(state.detailMetric)}&range=${state.range}`).catch(() => null);
if (!data) return;
const series = [{ name: state.detailMetric,
points: data.points.map((p) => ({ ts: p.ts, value: p.value })) }];
state.charts.detail.opts.yFormat = format.axis;
state.charts.detail.opts.yMin = state.detailMetric.endsWith('_pct') ? 0 : null;
state.charts.detail.opts.yMax = state.detailMetric.endsWith('_pct') ? 100 : null;
state.charts.detail.setData(series);
DashCharts.renderLegend($('#legend-detail'), series, format.fn);
const last = series[0].points.slice(-1)[0];
$('#detail-note').textContent = `${series[0].points.length} samples over ${duration(state.range)} · ` +
`last ${last ? format.fn(last.value) : 'n/a'} · ${state.range <= 86400 ? 'raw resolution' : 'rolled up'}`;
}
async function populateDetailMetrics(host) {
const select = $('#detail-metric');
const found = await fetchJSON(`/api/metrics?source=host&host=${encodeURIComponent(host)}`)
.catch(() => ({ metrics: [] }));
const curated = new Set(DETAIL_METRICS.map(([name]) => name));
const extras = (found.metrics || []).filter((m) => !curated.has(m)).sort();
select.innerHTML = '';
const groupMain = document.createElement('optgroup');
groupMain.label = 'Common';
DETAIL_METRICS.forEach(([name, label]) => {
const option = document.createElement('option');
option.value = name; option.textContent = label;
groupMain.appendChild(option);
});
select.appendChild(groupMain);
if (extras.length) {
const groupExtra = document.createElement('optgroup');
groupExtra.label = `Per-device (${extras.length})`;
extras.forEach((name) => {
const option = document.createElement('option');
option.value = name; option.textContent = name;
groupExtra.appendChild(option);
});
select.appendChild(groupExtra);
}
select.value = DETAIL_METRICS.some(([name]) => name === state.detailMetric)
? state.detailMetric : (extras[0] || DETAIL_METRICS[0][0]);
state.detailMetric = select.value || DETAIL_METRICS[0][0];
}
// ---- snapshot -------------------------------------------------------------
function renderFleet(fleet) {
if (!fleet) return;
setTxt($('#f-hosts'), fleet.hosts_live == null ? 'n/a' : String(fleet.hosts_live));
setTxt($('#f-hosts-of'), fleet.hosts_total == null ? '' : `of ${fleet.hosts_total}`);
setValue($('#f-cpu'), fleet.cpu_usage_pct_avg, 0);
setValue($('#f-load'), fleet.load1_total, 2);
const used = Dash.fmtBytes(fleet.memory_used_bytes, 0);
const total = Dash.fmtBytes(fleet.memory_total_bytes, 0);
setTxt($('#f-mem'), used ? used.text : 'n/a');
setTxt($('#f-mem-of'), used && total ? `${used.unit} of ${total.text} ${total.unit}` : '');
setValue($('#f-disk'), fleet.disk_worst_used_pct, 0);
const rx = Dash.fmtBytes(fleet.net_rx_bytes_per_s, 1);
const tx = Dash.fmtBytes(fleet.net_tx_bytes_per_s, 1);
setTxt($('#f-rx'), rx ? rx.text : 'n/a'); setTxt($('#f-rx-unit'), rx ? rx.unit + '/s' : '');
setTxt($('#f-tx'), tx ? tx.text : 'n/a'); setTxt($('#f-tx-unit'), tx ? tx.unit + '/s' : '');
setValue($('#f-gpu'), fleet.gpu_busy_pct_avg, 0);
}
async function refresh() {
let snapshot;
try {
snapshot = await fetchJSON('/api/system/snapshot');
} catch (err) {
$('#sub').textContent = err.message;
return;
}
const states = {};
(snapshot.states || []).forEach((row) => { states[row.key.split(':')[1]] = row; });
const hostNames = [];
(snapshot.hosts || []).forEach((host) => {
hostNames.push(host.name);
host.state = states[host.name];
host.window = snapshot.window_seconds;
let card = state.hosts.get(host.name);
if (!card) {
card = hostCard(host.name);
state.hosts.set(host.name, card);
$('#hosts').appendChild(card);
card.addEventListener('click', () => {
if (state.detailHost !== host.name) {
state.detailHost = host.name;
$('#detail-host').value = host.name;
populateDetailMetrics(host.name).then(refreshDetailChart);
}
});
}
renderHostCard(card, host);
});
$('#hosts-empty').hidden = hostNames.length > 0;
const hostSelect = $('#detail-host');
if (hostSelect.dataset.names !== hostNames.join(',')) {
hostSelect.dataset.names = hostNames.join(',');
hostSelect.innerHTML = hostNames.map((name) =>
`<option value="${DashCharts.escapeHtml(name)}">${DashCharts.escapeHtml(name)}</option>`).join('');
if (!state.detailHost || !hostNames.includes(state.detailHost)) {
state.detailHost = hostNames[0] || null;
}
if (state.detailHost) hostSelect.value = state.detailHost;
if (state.detailHost) await populateDetailMetrics(state.detailHost);
}
renderFleet(snapshot.fleet);
const fleet = snapshot.fleet || {};
$('#sub').textContent = `${fleet.hosts_live || 0}/${fleet.hosts_total || 0} hosts answering · ` +
`${fleet.cpu_cores || 0} cores · ${fleet.gpu_count || 0} GPUs · ` +
`window ${duration(state.range)}`;
$('#foot-note').textContent = `rates over Δ${snapshot.window_seconds}s · ` +
`poll ${Math.round(state.pollMs / 1000)}s`;
await refreshCharts(snapshot);
}
async function refreshEvents() {
try {
const data = await fetchJSON('/api/events?limit=25&source=host');
const list = $('#events');
if (!(data.events || []).length) {
list.innerHTML = '<li class="muted" style="grid-template-columns:1fr">No host transitions ' +
'recorded yet — this log fills when a host goes away, comes back, or reboots.</li>';
return;
}
list.innerHTML = data.events.map((event) => `<li><time>${
new Date(event.ts * 1000).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit', hour12: false })
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
`${DashCharts.escapeHtml(event.message)}</span></li>`).join('');
} catch (err) { /* the next poll will try again */ }
}
document.addEventListener('DOMContentLoaded', async () => {
makeCharts();
let pollMs = 10000;
try {
const config = await fetchJSON('/api/config');
pollMs = config.system.poll_ms || 10000;
if (!config.system.enabled) $('#sub').textContent = config.system.target_problems.length
? `bad HOST_TARGETS: ${config.system.target_problems.join('; ')}`
: 'no hosts configured — set HOST_TARGETS';
} catch (err) { /* config is nice-to-have */ }
state.pollMs = pollMs;
Dash.rangeControl($('#range'), state.range, (seconds) => {
state.range = seconds;
setChartSpan(seconds);
refresh();
});
$('#detail-host').addEventListener('change', async (event) => {
state.detailHost = event.target.value;
await populateDetailMetrics(state.detailHost);
await refreshDetailChart();
});
$('#detail-metric').addEventListener('change', async (event) => {
state.detailMetric = event.target.value;
await refreshDetailChart();
});
state.poller = poll(async () => { await refresh(); await refreshEvents(); }, state.pollMs);
Dash.bindPause(state.poller);
});
+358
View File
@@ -0,0 +1,358 @@
/* UPS dashboard: health assessment, NUT instant commands, history.
*
* The health list is deliberately explicit checks rather than a score: every row names the
* variable it read and what it saw, because "health: 87" is not actionable. The backend
* owns that logic (sources/nut.py::assess) so the same rules apply in tests and API clients.
*
* Commands are the only write path in the product. They are offered only when the backend
* says `enabled`, and anything that can cut power requires typing DANGER here *and* the
* server re-checks the same gate — a hidden button is not a safety control.
*/
'use strict';
const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, duration, poll, severityFor } = Dash;
const state = { range: 86400, cards: new Map(), charts: {}, poller: null, pollMs: 5000,
simulated: false, pending: null };
const MARKS = { ok: '✓', warn: '!', fail: '✕', unknown: '?' };
/**
* Runtime left as "1h 32m", not "1.5h". The health check next to it spells the same number
* out in h+m; two formats for one value invites a misread while the UPS is discharging.
*/
function runtimeText(seconds) {
if (seconds == null || !isFinite(seconds)) return 'n/a';
if (seconds < 0) return 'on battery';
const total = Math.round(seconds);
if (total < 120) return `${total}s`;
const minutes = Math.round(total / 60);
if (minutes < 60) return `${minutes}m`;
return `${Math.floor(minutes / 60)}h ${String(minutes % 60).padStart(2, '0')}m`;
}
function upsCard(name) {
const card = document.createElement('article');
card.className = 'card';
card.dataset.ups = name;
card.innerHTML = `
<header>
<span class="dot idle"></span>
<h3></h3>
<span class="chip model" hidden></span>
<span class="sim-badge" hidden>simulated</span>
<span class="spacer"></span>
<span class="chip health">unknown</span>
<span class="faint mono updated"></span>
</header>
<div class="body stack">
<div class="gap states"></div>
<div class="metrics">
<div class="metric"><div class="label">Charge</div>
<div class="value"><span data-k="charge">n/a</span><span class="unit">%</span></div>
<div class="bar" data-bar="charge"><i></i></div></div>
<div class="metric"><div class="label">Runtime left</div>
<div class="value sm" data-k="runtime">n/a</div></div>
<div class="metric"><div class="label">Load</div>
<div class="value sm"><span data-k="load">n/a</span><span class="unit">%</span></div>
<div class="bar" data-bar="load"><i></i></div></div>
<div class="metric"><div class="label">Line in</div>
<div class="value sm"><span data-k="line">n/a</span><span class="unit">V</span></div></div>
<div class="metric"><div class="label">Out</div>
<div class="value sm"><span data-k="out">n/a</span><span class="unit">V</span></div></div>
<div class="metric"><div class="label">Battery</div>
<div class="value sm"><span data-k="batt">n/a</span><span class="unit">V</span></div></div>
<div class="metric"><div class="label">Battery A</div>
<div class="value sm"><span data-k="amps">n/a</span><span class="unit">A</span></div></div>
<div class="metric"><div class="label">Battery °C</div>
<div class="value sm"><span data-k="temp">n/a</span><span class="unit">°C</span></div></div>
</div>
<div>
<div class="faint" style="font-size:10.5px;letter-spacing:.06em;text-transform:uppercase;margin-bottom:.3rem">
Health checks</div>
<ul class="checks" data-k="checks"></ul>
</div>
<div data-k="commands"></div>
<details>
<summary class="faint" style="cursor:pointer;font-size:12px">NUT variables</summary>
<div class="body tight" data-k="raw" style="padding:0;margin-top:.5rem"></div>
</details>
</div>`;
return card;
}
function renderCard(card, item, stateRow) {
const q = (key) => card.querySelector(`[data-k="${key}"]`);
card.querySelector('h3').textContent = item.name;
const model = [item.identity && item.identity.mfr, item.identity && item.identity.model]
.filter(Boolean).join(' ');
const modelChip = card.querySelector('.model');
if (model) { modelChip.hidden = false; modelChip.textContent = model; }
card.querySelector('.sim-badge').hidden = !state.simulated;
const overall = (item.health && item.health.overall) || 'unknown';
const healthChip = card.querySelector('.health');
healthChip.textContent = overall;
healthChip.className = 'chip health ' + ({ healthy: 'ok', degraded: 'warn', 'at risk': 'fail',
unknown: '' }[overall] || '');
const severity = stateRow && !stateRow.ok ? 'fail'
: (item.health && item.health.severity) || 'idle';
setDot(card.querySelector('.dot'), severity === 'ok' ? 'ok' : severity === 'warn' ? 'warn'
: severity === 'fail' ? 'fail' : 'idle');
setTxt(card.querySelector('.updated'), stateRow && stateRow.last_ok != null
? (Dash.ageSeconds(stateRow.last_ok) > 30 ? `stale ${duration(Dash.ageSeconds(stateRow.last_ok))}`
: `updated ${duration(Dash.ageSeconds(stateRow.last_ok))}`)
: (stateRow && stateRow.error) || 'never reached');
const states = card.querySelector('.states');
states.innerHTML = (item.states || []).map((s) => {
const kind = s.severity === 'ok' ? 'ok' : s.severity === 'warn' ? 'warn'
: s.severity === 'fail' ? 'fail' : 'info';
return `<span class="chip ${kind}" title="${DashCharts.escapeHtml(s.detail || s.label)}">${
DashCharts.escapeHtml(s.label)}</span>`;
}).join('') + (item.status ? `<span class="chip mono" title="raw ups.status">${
DashCharts.escapeHtml(item.status)}</span>` : '') +
(item.alarms || []).map((alarm) => `<span class="chip warn">alarm: ${
DashCharts.escapeHtml(alarm)}</span>`).join('');
const n = item.numbers || {};
const cell = (key) => q(key);
setValue(cell('charge'), n.charge_pct, 0);
setBar(card.querySelector('[data-bar="charge"]'), n.charge_pct,
n.charge_pct == null ? '' : n.charge_pct >= 95 ? 'ok' : n.charge_pct >= 60 ? 'warn' : 'fail');
setTxt(cell('runtime'), runtimeText(n.battery_runtime));
setValue(cell('load'), n.load_pct, 1);
setBar(card.querySelector('[data-bar="load"]'), n.load_pct, severityFor(n.load_pct, 70, 90));
setValue(cell('line'), n.line_voltage, 1);
setValue(cell('out'), n.output_voltage, 1);
setValue(cell('batt'), n.battery_voltage, 2);
setValue(cell('amps'), n.battery_current, 2);
setValue(cell('temp'), n.battery_temperature, 1);
const health = item.health || { checks: [] };
q('checks').innerHTML = (health.checks || []).map((check) => `
<li class="${check.status}">
<span class="mark" title="${check.status}">${MARKS[check.status] || '?'}</span>
<span><span class="what">${DashCharts.escapeHtml(check.name)}</span><br>
<span class="detail">${DashCharts.escapeHtml(check.detail)}</span></span>
</li>`).join('') || '<li class="unknown"><span class="mark">?</span><span>No checks available</span></li>';
const commands = item.commands || [];
const groups = [['safe', 'Diagnostics'], ['caution', 'Affects the load briefly'],
['dangerous', 'Can cut power']];
const host = q('commands');
host.innerHTML = groups.map(([risk, legend]) => {
const inGroup = commands.filter((command) => command.risk === risk);
if (!inGroup.length) return '';
return `<fieldset class="commands" style="margin-top:.5rem"><legend>${legend}</legend>
<div class="row">${inGroup.map((command) => `
<button class="action ${risk}" type="button" data-command="${command.command}"
${command.enabled ? '' : 'disabled'}
title="${DashCharts.escapeHtml(command.enabled ? command.help : command.reason)}">${
DashCharts.escapeHtml(command.label)}</button>`).join('')}</div></fieldset>`;
}).join('') || '<div class="faint" style="font-size:12px;margin-top:.4rem">' +
'The driver advertises no commands we know how to send.</div>';
host.querySelectorAll('button[data-command]').forEach((button) => {
button.addEventListener('click', () => openCommandDialog(item.name, button.dataset.command,
commands));
});
const extra = item.extra || {};
const entries = Object.entries(extra);
q('raw').innerHTML = `<table class="data"><tbody>${entries.map(([key, value]) =>
`<tr><td class="mono" style="font-size:11px">${DashCharts.escapeHtml(key)}</td>
<td class="mono" style="font-size:11px">${DashCharts.escapeHtml(value)}</td></tr>`).join('')
|| '<tr><td class="mute">no extra variables</td></tr>'}</tbody></table>` +
`<div class="faint mono" style="font-size:11px;margin-top:.4rem">driver ${
DashCharts.escapeHtml((item.identity || {}).driver || '?')} · firmware ${
DashCharts.escapeHtml((item.identity || {}).firmware || '?')} · serial ${
DashCharts.escapeHtml((item.identity || {}).serial || '?')} · ${
(item.advertised_commands || []).length} commands advertised</div>`;
}
// ---- command dialog -------------------------------------------------------
function openCommandDialog(upsName, commandName, commands) {
const command = commands.find((c) => c.command === commandName);
if (!command) return;
const dialog = $('#confirm-dialog');
const dangerous = command.risk === 'dangerous';
$('#dialog-title').textContent = command.label;
$('#dialog-help').textContent = command.help;
$('#dialog-target').textContent = `UPS ${upsName} · ${command.command} · risk: ${command.risk}` +
(state.simulated ? ' · this UPS is simulated' : '');
const riskChip = $('#dialog-risk');
riskChip.textContent = command.risk;
riskChip.className = 'chip ' + (command.risk === 'dangerous' ? 'fail'
: command.risk === 'caution' ? 'warn' : 'ok');
$('#dialog-confirm-wrap').hidden = !dangerous;
$('#dialog-input-wrap').hidden = !dangerous;
$('#dialog-input').value = '';
const run = $('#dialog-run');
run.classList.toggle('dangerous', dangerous);
run.disabled = dangerous;
state.pending = { ups: upsName, command: commandName, dangerous };
dialog.showModal();
if (dangerous) $('#dialog-input').focus();
}
async function runPending() {
const pending = state.pending;
if (!pending) return;
const body = { ups: pending.ups, command: pending.command };
if (pending.dangerous) body.confirm = $('#dialog-input').value.trim();
$('#confirm-dialog').close();
try {
const result = await fetchJSON('/api/ups/command', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(body),
});
Dash.toast(`sent ${result.command} to ${result.ups}: ${result.detail}`, 'ok');
} catch (err) {
Dash.toast(`refused: ${err.message}`, 'fail');
}
state.pending = null;
refresh();
}
// ---- charts ---------------------------------------------------------------
function makeCharts() {
const common = { height: 160, xSpan: state.range };
state.charts.charge = new DashCharts.LineChart('#chart-charge', Object.assign({}, common,
{ yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
state.charts.runtime = new DashCharts.LineChart('#chart-runtime', Object.assign({}, common,
{ yMin: 0, yFormat: (v) => duration(v) }));
state.charts.voltage = new DashCharts.LineChart('#chart-voltage', Object.assign({}, common,
{ yFormat: (v) => num(v, 1) }));
state.charts.load = new DashCharts.LineChart('#chart-load', Object.assign({}, common,
{ yMin: 0, yFormat: (v) => num(v, 0) }));
}
function points(response, upsName, metric) {
const found = (response.series || []).find((s) => s.host === upsName && s.metric === metric);
return found ? found.points.map((p) => ({ ts: p.ts, value: p.value })) : [];
}
async function refreshCharts(names) {
if (!names.length) return;
const list = names.join(',');
const metrics = 'ups.charge_pct,ups.battery_runtime_seconds,ups.battery_voltage,' +
'ups.line_voltage,ups.load_pct';
const data = await fetchJSON(`/api/ups/history?ups=${encodeURIComponent(list)}&metrics=${metrics}` +
`&range=${state.range}`).catch(() => null);
if (!data) return;
const build = (metric, scale) => names.map((name) => ({
name, points: points(data, name, metric).map((p) => (scale ? { ts: p.ts, value: scale(p.value) } : p)),
}));
const charge = build('ups.charge_pct');
state.charts.charge.setData(charge);
DashCharts.renderLegend($('#legend-charge'), charge, (v) => num(v, 0) + '%');
const runtime = build('ups.battery_runtime_seconds');
state.charts.runtime.setData(runtime);
DashCharts.renderLegend($('#legend-runtime'), runtime, (v) => duration(v), true);
const battery = build('ups.battery_voltage');
const line = build('ups.line_voltage');
const voltage = battery.map((series, index) => ({ name: `${series.name} battery`,
points: series.points })).concat(line.map((series, index) => ({ name: `${series.name} line`,
points: series.points, color: undefined })));
state.charts.voltage.setData(voltage);
DashCharts.renderLegend($('#legend-voltage'), voltage, (v) => num(v, 1) + ' V');
const load = build('ups.load_pct');
state.charts.load.setData(load);
DashCharts.renderLegend($('#legend-load'), load, (v) => num(v, 0) + '%', true);
}
async function refresh() {
let snapshot;
try {
snapshot = await fetchJSON('/api/ups/snapshot');
} catch (err) {
$('#sub').textContent = err.message;
return;
}
state.simulated = !!snapshot.simulated;
const stateRow = (snapshot.states || [])[0];
const items = snapshot.ups || [];
items.forEach((item) => {
let card = state.cards.get(item.name);
if (!card) {
card = upsCard(item.name);
state.cards.set(item.name, card);
$('#ups-cards').appendChild(card);
}
renderCard(card, item, stateRow);
});
$('#ups-empty').hidden = items.length > 0;
const banners = [];
if (state.simulated) {
banners.push(['warn', '<strong>SIMULATED DATA.</strong> <code>NUT_SIMULATE=1</code> — these are ' +
'modelled UPSes for development, not your batteries. Unset it to read real upsd.']);
}
if (!snapshot.actions_enabled) {
banners.push(['info', 'Command buttons are disabled. Set <code>NUT_ALLOW_ACTIONS=1</code> plus ' +
'<code>NUT_USER</code>/<code>NUT_PASSWORD</code> for a user with <code>actions</code> in ' +
'<code>upsd.users</code>.']);
}
if (stateRow && !stateRow.ok && stateRow.error) {
banners.push(['fail', `upsd: ${DashCharts.escapeHtml(stateRow.error)}`]);
}
$('#banners').innerHTML = banners.map(([kind, html]) => `<div class="banner ${kind}">${html}</div>`)
.join('');
// "0 healthy" reads like an outage when every UPS is merely past its replacement window;
// name the states that are actually present instead of counting the absence of one.
const counts = { healthy: 0, degraded: 0, 'at risk': 0, unknown: 0 };
items.forEach((item) => { counts[(item.health && item.health.overall) || 'unknown'] += 1; });
const breakdown = Object.entries(counts).filter(([, n]) => n > 0)
.map(([name, n]) => `${n} ${name}`).join(', ') || 'none reporting';
$('#sub').textContent = `${items.length} UPS${items.length === 1 ? '' : 'es'} · ${breakdown} · ` +
(snapshot.server ? `upsd ${snapshot.server} · ` : '') + `window ${duration(state.range)}`;
$('#foot-note').textContent = `poll ${Math.round(state.pollMs / 1000)}s · ` +
`${items.reduce((total, item) => total + (item.advertised_commands || []).length, 0)} commands advertised`;
await refreshCharts(items.map((item) => item.name));
await refreshEvents();
}
async function refreshEvents() {
try {
const data = await fetchJSON('/api/events?limit=40&source=ups');
const list = $('#events');
if (!(data.events || []).length) {
list.innerHTML = '<li class="muted" style="grid-template-columns:1fr">No power events yet — ' +
'state changes, self-test results, and commands you send all land here.</li>';
return;
}
list.innerHTML = data.events.map((event) => `<li><time>${
new Date(event.ts * 1000).toLocaleString(undefined,
{ month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', hour12: false })
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
`${DashCharts.escapeHtml(event.message.replace(event.host + ':', '').trim())}` +
`${event.data && event.data.runtime_seconds ? ` <span class="faint mono">(${
duration(event.data.runtime_seconds)} runtime)</span>` : ''}</span></li>`).join('');
} catch (err) { /* next poll */ }
}
document.addEventListener('DOMContentLoaded', async () => {
makeCharts();
try {
const config = await fetchJSON('/api/config');
state.pollMs = config.ups.poll_ms || 5000;
} catch (err) { /* default poll rate */ }
Dash.rangeControl($('#range'), state.range, (seconds) => {
state.range = seconds;
Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = seconds; });
refresh();
}, [['1h', 3600], ['6h', 21600], ['24h', 86400], ['7d', 604800], ['30d', 2592000],
['90d', 7776000]]);
$('#dialog-cancel').addEventListener('click', () => { $('#confirm-dialog').close(); state.pending = null; });
$('#dialog-run').addEventListener('click', runPending);
$('#dialog-input').addEventListener('input', (event) => {
$('#dialog-run').disabled = event.target.value.trim() !== 'DANGER';
});
state.poller = poll(refresh, state.pollMs);
Dash.bindPause(state.poller);
});