phase: 04_llm_chart_readability
**Phase 04 final verification pass — all tasks already in `complete/`; verified + fixed 3 defects found in verification.**
- Verified `/llm` live (both engines, gufo-only instance on :8099, `llm_monitor` on :8095) in a real browser; dev server left running/healthy.
- Completed task-04's truncated layout fix: `#engines` now carries `.items-start` (the CSS rule existed but nothing used it) — gufo 544 px vs vLLM 715 px, no blank card; pinned by `test_llm_engine_panels_are_not_stretched_to_their_tallest_sibling`.
- Fixed: event rows said the subject twice ("gufo gufo is answering again"), and `/ups` mangled mid-message names ("sent to apc: OK" → "sent to OK") → shared `Dash.eventText()` (strips `UpstreamError: ` + a *leading* name only, mirroring `alerts._clean_error`) used by `/system`, `/llm`, `/ups`; new test.
- Fixed: count sparklines autoscaled rollup averages (queue max 0.1379) into full-height spikes → `sparkline({minSpan})`, `minSpan: 1` on sessions/queued (spike 30 px → 4 px); assertions added.
- Tests: `bash .agents/validate.sh` → **EXIT=0** (`uv run pytest --cov --cov-fail-under=94`: **485 passed, 7 skipped**, coverage **96.87%**; `ruff check src tests agent`: All checks passed; `pyright`: 0 errors; `node --check` all static JS clean; repo hygiene clean; `uv pip list --outdated` empty). `uv run pytest -m live --live` → **7 passed**.
- Completion criteria: heroes largest per engine ✔ (38.4 px vs 16.8 px row values); sparkline per cell + selected window ✔ (5/5 gufo, 7/7 vLLM; 6 h chip moved KV max 53 %→58 %); mean+p95 without hover ✔ (243 / 48.8 / 285 ms feet); provenance tag ✔ (`Δ10.0s`/`Δ8.0s` + `title`=exact source); single-engine ✔ (1 panel, totals hidden, zero red); screenshots ✔ `data/logs/llm-verify-{both-engines,charts-and-slots,single-engine,monitor-reference}.png` + `NOTES-chart-set.md`; pytest/cov/ruff/pyright ✔; earlier phases unchanged-behaviour ✔ (all 01–03 tests pass).
- Deviation (documented, not code): throughput heroes stay **neutral ink** — `severity.py` has no throughput row and 0 tok/s idle is healthy; colour comes from rated cells (KV) + reachability. Kept per `NOTES-chart-set`.
- Deviation: could not append the task-04 written verdict to `NOTES-chart-set.md` (it lives in `todo/`, which I must not edit). Verdict: ours reads better than `:8095` — bigger throughput numbers, per-cell shape over the selected window, percentiles inline; `llm_monitor` only sparklines decode+KV and has no window control.
- Docs: AGENTS.md updated (485 test count, `sparkline` `minSpan` rule, `Dash.eventText` event-row rule, `/llm` panel `items-start`); README reviewed in full — commands, config table (all `config.py` vars present), architecture bullets, page descriptions verified accurate; no other inconsistencies found.
- Note for the harness: `.agents/phases/todo/03…/NOTES-verify.md` and `todo/04…/NOTES-chart-set.md` are untracked notes left behind by their phases (task files moved, notes did not).
- Next pending phase: `05_ups_hero_battery_glyph`.
This commit is contained in:
@@ -118,8 +118,10 @@ HISTORY_METRICS: dict[str, tuple[str, ...]] = {
|
||||
"requests_waiting": ("requests_deferred", "num_requests_waiting"),
|
||||
"ttft_seconds": ("time_to_first_token_seconds__mean",),
|
||||
"ttft_p95_seconds": ("time_to_first_token_seconds__p95",),
|
||||
"tpot_seconds": ("time_per_output_token_seconds__mean",), # vLLM only
|
||||
"queue_seconds": ("request_queue_time_seconds__mean",),
|
||||
"prefix_cache_hit_ratio": ("prefix_cache_hit_ratio", "prompt_cache_hit_ratio"),
|
||||
"spec_accept_rate": ("spec_accept_rate",), # gufo only
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -36,52 +36,29 @@
|
||||
|
||||
<div id="banners" class="stack" style="margin-bottom:.85rem"></div>
|
||||
|
||||
<section class="card flat" aria-label="Engine totals">
|
||||
<!-- Engine totals, hidden until a second engine exists: a cross-engine sum of one
|
||||
engine only duplicates that engine's own panel. What stays here is what no panel
|
||||
says — how many engines are answering, and the session/queue counts across them.
|
||||
Summed decode/prompt/KV left with phase 04: per-engine is the honest view, and
|
||||
averaging a Strix Halo iGPU with a Threadripper is not one number. -->
|
||||
<section class="card flat" id="totals" aria-label="Engine totals" hidden>
|
||||
<div class="body">
|
||||
<div class="metrics">
|
||||
<div class="metric"><div class="label">Engines answering</div>
|
||||
<div class="value"><span id="f-live">n/a</span><span class="unit" id="f-total"></span></div></div>
|
||||
<div class="metric"><div class="label">Decode (sum)</div>
|
||||
<div class="value"><span id="f-decode">n/a</span><span class="unit">tok/s</span></div></div>
|
||||
<div class="metric"><div class="label">Prompt (sum)</div>
|
||||
<div class="value"><span id="f-prompt">n/a</span><span class="unit">tok/s</span></div></div>
|
||||
<div class="metric"><div class="label">Active sessions</div>
|
||||
<div class="value" id="f-active">n/a</div></div>
|
||||
<div class="metric"><div class="label">Queued</div>
|
||||
<div class="value" id="f-waiting">n/a</div></div>
|
||||
<div class="metric"><div class="label">Tokens lifetime</div>
|
||||
<div class="value"><span id="f-tokens">n/a</span><span class="unit"></span></div></div>
|
||||
<div class="metric"><div class="label" title="highest KV-cache use across engines right now, not a window peak">KV cache now</div>
|
||||
<div class="value"><span id="f-kv">n/a</span><span class="unit">%</span></div>
|
||||
<div class="bar" style="margin-top:.3rem"><i></i></div></div>
|
||||
</div>
|
||||
<div class="faint" id="seam-note" style="margin-top:.5rem;font-size:11.5px"></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="grid cols-2" style="margin-top:.85rem" aria-label="Throughput charts">
|
||||
<article class="card">
|
||||
<header><h3>Decode throughput</h3><span class="spacer"></span>
|
||||
<span class="faint">tok/s · every engine, summed slots</span></header>
|
||||
<div class="body"><div class="chart" id="chart-decode"></div><div class="legend" id="legend-decode"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Prompt / prefill throughput</h3><span class="spacer"></span>
|
||||
<span class="faint">tok/s · newly computed tokens</span></header>
|
||||
<div class="body"><div class="chart" id="chart-prompt"></div><div class="legend" id="legend-prompt"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>KV / prefix cache pressure</h3><span class="spacer"></span><span class="faint">% of cache in use</span></header>
|
||||
<div class="body"><div class="chart" id="chart-kv"></div><div class="legend" id="legend-kv"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Sessions & queue</h3><span class="spacer"></span><span class="faint">running vs waiting</span></header>
|
||||
<div class="body"><div class="chart" id="chart-queue"></div><div class="legend" id="legend-queue"></div></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<!-- The page leads with the engine panels: two throughput heroes per engine, a compact
|
||||
sparkline cell list, and the gufo slot table below. The four trend charts follow,
|
||||
half-height — supporting evidence, not the main event. -->
|
||||
<section style="margin-top:.85rem">
|
||||
<div class="grid cols-2" id="engines"></div>
|
||||
<div class="grid cols-2 items-start" id="engines"></div>
|
||||
<div class="empty-state" id="engines-empty" hidden>
|
||||
<h3>No engines configured</h3>
|
||||
<p>Set <code>GUFO_BASE_URL</code> (llama.cpp / gufo) and/or <code>VLLM_BASE_URL</code>.
|
||||
@@ -90,6 +67,27 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="grid cols-2" style="margin-top:.85rem" aria-label="Trend charts">
|
||||
<article class="card">
|
||||
<header><h3>Decode throughput</h3><span class="spacer"></span>
|
||||
<span class="faint">tok/s</span></header>
|
||||
<div class="body"><div class="chart" id="chart-decode"></div><div class="legend" id="legend-decode"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Prompt / prefill throughput</h3><span class="spacer"></span>
|
||||
<span class="faint">tok/s</span></header>
|
||||
<div class="body"><div class="chart" id="chart-prompt"></div><div class="legend" id="legend-prompt"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>KV / prefix cache pressure</h3><span class="spacer"></span><span class="faint">% in use</span></header>
|
||||
<div class="body"><div class="chart" id="chart-kv"></div><div class="legend" id="legend-kv"></div></div>
|
||||
</article>
|
||||
<article class="card">
|
||||
<header><h3>Sessions & queue</h3><span class="spacer"></span><span class="faint">running vs waiting</span></header>
|
||||
<div class="body"><div class="chart" id="chart-queue"></div><div class="legend" id="legend-queue"></div></div>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section class="grid cols-2" style="margin-top:.85rem">
|
||||
<article class="card" id="slots-card">
|
||||
<header><h3>gufo slots</h3><span class="spacer"></span>
|
||||
@@ -107,8 +105,8 @@
|
||||
</section>
|
||||
|
||||
<footer class="pagefoot">
|
||||
<span>each cell carries its provenance: scraped gauge, <span class="mono">Δ10s</span> measured
|
||||
counter, or lifetime estimate</span>
|
||||
<span>each throughput number carries its provenance: scraped gauge, <span class="mono">Δ10s</span>
|
||||
measured counter, or lifetime estimate</span>
|
||||
<span class="spacer" style="margin-inline-start:auto"></span>
|
||||
<span class="mono" id="foot-note"></span>
|
||||
</footer>
|
||||
|
||||
@@ -70,6 +70,28 @@ const Dash = (() => {
|
||||
|
||||
const naText = 'n/a';
|
||||
|
||||
/**
|
||||
* The text of an event-feed row: the subject said **once**.
|
||||
*
|
||||
* `poller.py` writes messages that name their own subject ("gufo is answering again
|
||||
* after 1 failure"), and every feed prints the host as its own chip beside the message,
|
||||
* so printing the stored words verbatim reads as two problems — the same reasoning
|
||||
* `alerts._clean_error()` applies to the overview's alert list, including the drop of
|
||||
* `UpstreamError: ` (this app has one upstream error type, so its name is never a fact).
|
||||
* A leading `host:` goes too, but only at the front: the UPS rows used to `replace()`
|
||||
* the name anywhere, which mangled "Quick battery test sent to apc: OK". Presentation
|
||||
* only — the stored event and `/api/events` keep every word.
|
||||
*/
|
||||
function eventText(event) {
|
||||
const text = String((event && event.message) || '');
|
||||
const host = String((event && event.host) || '');
|
||||
const out = text.replace(/^UpstreamError:\s+/, '');
|
||||
if (!host) return out;
|
||||
const esc = host.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const stripped = out.replace(new RegExp('^' + esc + '\\s*[:\\-]?\\s*'), '');
|
||||
return stripped.trim() ? stripped : out;
|
||||
}
|
||||
|
||||
// ---- DOM helpers --------------------------------------------------------
|
||||
/** Write text only when it changed: avoids killing transitions and focus. */
|
||||
function setTxt(el, value) {
|
||||
@@ -110,9 +132,9 @@ const Dash = (() => {
|
||||
*
|
||||
* Thresholds live in `src/dashboards/severity.py` and travel in the payload as
|
||||
* `severity`; `severityFrom()` below is how a page reads them. This function survives
|
||||
* only because `page-llm.js` and `page-ups.js` still call it with their own numbers
|
||||
* (phases 04–05 convert them page by page, so each page's change is reviewable on its
|
||||
* own — `/` was converted in phase 02, `/system` in phase 03). Do not add a new call
|
||||
* only because `page-ups.js` still calls it with its own numbers (phase 05 converts it,
|
||||
* so each page's change is reviewable on its own — `/` was converted in phase 02,
|
||||
* `/system` in phase 03, `/llm` in phase 04). Do not add a new call
|
||||
* site: a threshold written in JS is a threshold that can disagree with the other pages.
|
||||
*/
|
||||
function severityFor(value, warnAt, failAt) {
|
||||
@@ -444,7 +466,8 @@ const Dash = (() => {
|
||||
|
||||
return { $, $$, fetchJSON, poll, fmtBytes, bytes, rate, pct, num, duration, age, setTxt,
|
||||
setValue, setBar, setDot, chip, initChrome, rangeControl, toast, bindPause,
|
||||
staleness, markStale, severityFor, severityFrom, hero, ageSeconds, naText };
|
||||
staleness, markStale, severityFor, severityFrom, hero, ageSeconds, naText,
|
||||
eventText };
|
||||
})();
|
||||
|
||||
document.addEventListener('DOMContentLoaded', () => Dash.initChrome());
|
||||
|
||||
@@ -344,30 +344,131 @@ function renderLegend(container, series, format, showPeak) {
|
||||
}).join('');
|
||||
}
|
||||
|
||||
/** Small non-interactive sparkline for cards where a full chart would shout. */
|
||||
/**
|
||||
* Small non-interactive sparkline for cards where a full chart would shout.
|
||||
*
|
||||
* Same honesty rules as `LineChart`, at 160×34: fewer than `minPoints` points say
|
||||
* "no history" (never a flat zero line, never a confident line through one point), and
|
||||
* labels quote the data, not the padded axis — a flat window reports the one real value.
|
||||
*
|
||||
* opts (all optional; omitting them keeps the autoscaled, full-width line this function
|
||||
* has always drawn):
|
||||
* - height: plot height in px (default 34; the box is the point of a sparkline)
|
||||
* - color: explicit stroke/fill colour; wins over `severity`
|
||||
* - severity: payload verdict ('ok'|'warn'|'fail', or a `{severity}` cell) → stroke from
|
||||
* that theme token (`cssVar('--ok')` etc.). Anything else — null, '', an
|
||||
* unknown word — draws **no colour**: a faint grey stroke and no fill,
|
||||
* because an unrated series is a shape, not a verdict (an unknown number
|
||||
* must never look green)
|
||||
* - min / max: explicit y bounds (today's behaviour)
|
||||
* - minSpan: smallest y span an autoscaled axis may have, in the series' own units.
|
||||
* For counts (sessions, queued) pass 1: a 1-minute rollup averages 8 seconds
|
||||
* of one waiting request down to 0.14, and an axis that shrinks to fit turns
|
||||
* that into a full-height spike — the twin of the rescale fixedMax fixes. An
|
||||
* all-zero window still pads to the mid-line, so "nothing queued" reads flat
|
||||
* - fixedMax: known ceiling for percent-like metrics — plot against [min ?? 0, fixedMax]
|
||||
* instead of rescaling to the data, so an 8 % KV window that once held a
|
||||
* 92 % spike reads the same as one that did not. Adds top/bottom scale marks
|
||||
* so "the line is at a third" says a third *of what*; values clamp to the
|
||||
* axis rather than drawing off-box
|
||||
* - align: 'right' pins the newest sample to the right edge and grows the line
|
||||
* leftwards as history fills in (llm_monitor's drawSpark). With `span` a
|
||||
* partly-filled window occupies only the right of the plot — and x stays
|
||||
* time, not sample index, so the shape keeps meaning *when*. Default 'left'
|
||||
* spans the data domain (today's behaviour)
|
||||
* - span: x window in seconds for align:'right' (default: the oldest sample)
|
||||
* - labels: `{current, max}` → the small `now / max` foot drawSpark prints, formatted
|
||||
* with `format`; `max` is the window maximum of the data, never the fixed
|
||||
* axis
|
||||
* - format: value → string for the labels and scale marks (default DEFAULT_FORMAT)
|
||||
* - minPoints: finite points required before a line is drawn (default 2)
|
||||
* - ariaLabel: names the spark for assistive tech; the generated fallback quotes values
|
||||
*
|
||||
* No animation and no CSS transition at all — `prefers-reduced-motion` is honoured by
|
||||
* construction, and a moving squiggle competes with the number it annotates.
|
||||
*/
|
||||
function sparkline(container, points, opts) {
|
||||
const o = Object.assign({ height: 34, color: cssVar('--series-1'), min: null, max: null }, opts || {});
|
||||
const pts = (points || []).filter((p) => p && Number.isFinite(p.value));
|
||||
const o = Object.assign({
|
||||
height: 34, color: null, severity: null, min: null, max: null, fixedMax: null,
|
||||
minSpan: null, align: 'left', span: null, labels: null, format: DEFAULT_FORMAT,
|
||||
minPoints: 2, ariaLabel: null,
|
||||
}, opts || {});
|
||||
const fmt = typeof o.format === 'function' ? o.format : DEFAULT_FORMAT;
|
||||
const pts = (points || [])
|
||||
.map((p, i) => (p && Number.isFinite(p.value)
|
||||
? { ts: Number.isFinite(p.ts) ? p.ts : i, value: p.value }
|
||||
: null))
|
||||
.filter(Boolean)
|
||||
.sort((a, b) => a.ts - b.ts);
|
||||
const w = Math.max(60, container.clientWidth || 160);
|
||||
const h = o.height;
|
||||
container.innerHTML = '';
|
||||
if (pts.length < 2) {
|
||||
if (pts.length < Math.max(1, o.minPoints)) {
|
||||
container.innerHTML = '<div class="faint mono" style="font-size:11px">no history</div>';
|
||||
return;
|
||||
}
|
||||
const xs = pts.map((p) => p.ts), ys = pts.map((p) => p.value);
|
||||
const x0 = Math.min(...xs), x1 = Math.max(...xs);
|
||||
let lo = o.min !== null ? o.min : Math.min(...ys);
|
||||
let hi = o.max !== null ? o.max : Math.max(...ys);
|
||||
if (lo === hi) { const pad = Math.abs(lo) * 0.1 || 1; lo -= pad; hi += pad; }
|
||||
const sx = (ts) => ((ts - x0) / (x1 - x0 || 1)) * (w - 2) + 1;
|
||||
const sy = (v) => h - 2 - ((v - lo) / (hi - lo)) * (h - 4);
|
||||
const ys = pts.map((p) => p.value);
|
||||
const dataMax = Math.max(...ys);
|
||||
let lo, hi;
|
||||
if (o.fixedMax != null) {
|
||||
lo = o.min != null ? o.min : 0;
|
||||
hi = Math.max(o.fixedMax, lo); // never an inverted axis
|
||||
} else {
|
||||
lo = o.min !== null ? o.min : Math.min(...ys);
|
||||
hi = o.max !== null ? o.max : Math.max(...ys);
|
||||
if (lo === hi) { const pad = Math.abs(lo) * 0.1 || 1; lo -= pad; hi += pad; }
|
||||
// the axis never gets finer than one thing (see `minSpan` above)
|
||||
if (o.minSpan > 0 && hi - lo < o.minSpan) hi = lo + o.minSpan;
|
||||
}
|
||||
const first = pts[0];
|
||||
const last = pts[pts.length - 1];
|
||||
let x0 = first.ts;
|
||||
const x1 = last.ts;
|
||||
if (o.align === 'right' && o.span > 0) x0 = x1 - o.span;
|
||||
// Marks live in a left gutter so they never sit on the line; a 160px box has the pixels.
|
||||
const marksOn = o.fixedMax != null;
|
||||
const padL = marksOn ? 24 : 1;
|
||||
const inner = { l: padL, r: 1, t: 2, b: 2 };
|
||||
const plotW = Math.max(10, w - inner.l - inner.r);
|
||||
const plotH = Math.max(6, h - inner.t - inner.b);
|
||||
const sx = (ts) => {
|
||||
const span = (x1 - x0) || 1;
|
||||
const clamped = Math.min(Math.max(ts, x0), x1);
|
||||
return inner.l + ((clamped - x0) / span) * plotW;
|
||||
};
|
||||
const sy = (v) => {
|
||||
const c = o.fixedMax != null ? Math.min(Math.max(v, lo), hi) : v;
|
||||
return inner.t + plotH - ((c - lo) / ((hi - lo) || 1)) * plotH;
|
||||
};
|
||||
let sev = o.severity;
|
||||
if (sev && typeof sev === 'object') sev = sev.severity;
|
||||
const sevColor = sev === 'ok' ? cssVar('--ok')
|
||||
: sev === 'warn' ? cssVar('--warn')
|
||||
: sev === 'fail' ? cssVar('--fail') : null;
|
||||
const stroke = o.color || sevColor || cssVar('--text-faint');
|
||||
const filled = Boolean(o.color || sevColor);
|
||||
const d = pts.map((p, i) => `${i ? 'L' : 'M'}${sx(p.ts).toFixed(1)} ${sy(p.value).toFixed(1)}`).join(' ');
|
||||
const svg = `<svg viewBox="0 0 ${w} ${h}" width="100%" height="${h}" preserveAspectRatio="none"
|
||||
role="img" aria-label="trend from ${lo.toFixed(2)} to ${hi.toFixed(2)}">
|
||||
<path d="${d} L${sx(x1).toFixed(1)} ${h} L${sx(x0).toFixed(1)} ${h} Z" fill="${o.color}" opacity=".14"/>
|
||||
<path d="${d}" fill="none" stroke="${o.color}" stroke-width="1.4"/></svg>`;
|
||||
container.innerHTML = svg;
|
||||
const baseY = (inner.t + plotH).toFixed(1);
|
||||
const area = `<path d="${d} L${sx(last.ts).toFixed(1)} ${baseY} L${sx(first.ts).toFixed(1)} ${baseY} Z" fill="${stroke}" opacity=".14"/>`;
|
||||
const baseline = `<line x1="${inner.l}" x2="${(inner.l + plotW).toFixed(1)}" y1="${baseY}" y2="${baseY}" stroke="${stroke}" opacity=".22"/>`;
|
||||
let marks = '';
|
||||
if (marksOn) {
|
||||
// top/bottom of the *axis* only — enough to read the line as a fraction of the ceiling
|
||||
const mk = (v, y) => `<text x="2" y="${y}" font-size="8" fill="${cssVar('--text-faint')}" dominant-baseline="${y < h / 2 ? 'hanging' : 'alphabetic'}">${escapeHtml(fmt(v))}</text>`;
|
||||
marks = mk(hi, inner.t + 7) + mk(lo, h - inner.b);
|
||||
}
|
||||
const aria = o.ariaLabel ||
|
||||
`trend from ${fmt(first.value)} to ${fmt(last.value)}, max ${fmt(dataMax)}`;
|
||||
let foot = '';
|
||||
if (o.labels && (o.labels.current || o.labels.max)) {
|
||||
const bits = [];
|
||||
if (o.labels.current) bits.push(`now ${fmt(last.value)}`);
|
||||
if (o.labels.max) bits.push(`max ${fmt(dataMax)}`);
|
||||
foot = `<div class="faint mono" style="font-size:10px;line-height:1.25">${escapeHtml(bits.join(' / '))}</div>`;
|
||||
}
|
||||
container.innerHTML = `<svg viewBox="0 0 ${w} ${h}" width="100%" height="${h}" preserveAspectRatio="none"
|
||||
role="img" aria-label="${escapeHtml(aria)}">${filled ? area : ''}${baseline}
|
||||
<path d="${d}" fill="none" stroke="${stroke}" stroke-width="1.4"/>${marks}</svg>${foot}`;
|
||||
}
|
||||
|
||||
window.DashCharts = { LineChart, niceTicks, renderLegend, sparkline, cssVar,
|
||||
|
||||
@@ -179,6 +179,11 @@ main { padding: 1.15rem 1.25rem 3rem; max-width: 1680px; margin: 0 auto; }
|
||||
.grid.auto { grid-template-columns: repeat(auto-fit, minmax(min(360px, 100%), 1fr)); }
|
||||
.grid.cols-2 { grid-template-columns: repeat(auto-fit, minmax(min(480px, 100%), 1fr)); }
|
||||
.grid.cols-3 { grid-template-columns: repeat(auto-fit, minmax(min(300px, 100%), 1fr)); }
|
||||
/* Cards that carry different amounts of content sit at their own height instead of being
|
||||
stretched to the tallest sibling: a 5-row engine panel next to a 6-row one otherwise
|
||||
grows a blank bottom, and a blank panel reads as "no data" (verified on `/llm`, phase
|
||||
04 — 172px of empty card under the shorter panel). */
|
||||
.grid.items-start { align-items: start; }
|
||||
.stack { display: grid; gap: .85rem; align-content: start; }
|
||||
|
||||
/* ---- cards -------------------------------------------------------------- */
|
||||
@@ -234,6 +239,45 @@ main { padding: 1.15rem 1.25rem 3rem; max-width: 1680px; margin: 0 auto; }
|
||||
|
||||
.na { color: var(--text-faint); }
|
||||
|
||||
/* ---- engine cell list -----------------------------------------------------
|
||||
* `/llm`'s secondary rows: label left, value right-aligned in tabular numerals, sparkline
|
||||
* in a fixed-width third column so rows line up and growth stays right-aligned the way
|
||||
* `llm_monitor`'s drawSpark did. Percentiles ride under the value as a mono foot line —
|
||||
* visible without a hover, because a mean latency without its p95 is half the truth.
|
||||
* The heroes above are the loudest thing here on purpose; nothing in this block competes
|
||||
* with them, and only payload verdicts colour it (KV through `.cell.warn/.cell.fail`).
|
||||
*/
|
||||
.cells { display: grid; gap: .45rem; }
|
||||
.cell {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) max-content 160px;
|
||||
gap: .1rem .7rem;
|
||||
align-items: center;
|
||||
}
|
||||
.cell .cell-label {
|
||||
font-size: 10.5px; text-transform: uppercase; letter-spacing: .06em; color: var(--text-faint);
|
||||
white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||||
}
|
||||
.cell .cell-value {
|
||||
font-family: var(--mono); font-size: 1.05rem; font-weight: 600; line-height: 1.3;
|
||||
font-variant-numeric: tabular-nums;
|
||||
text-align: right; white-space: nowrap;
|
||||
}
|
||||
.cell .cell-value .unit { font-size: .62em; color: var(--text-muted); font-weight: 500; margin-inline-start: .18rem; }
|
||||
.cell .cell-value.is-null { color: var(--text-faint); }
|
||||
.cell .cell-foot { grid-column: 2; text-align: right; font-family: var(--mono); font-size: 10.5px; color: var(--text-faint); }
|
||||
.cell .cell-spark { display: block; width: 160px; }
|
||||
/* the KV verdict, taken from the API's `severity` exactly as the hero cards take theirs */
|
||||
.cell.warn .cell-value { color: var(--warn); }
|
||||
.cell.fail .cell-value { color: var(--fail); }
|
||||
/* alarm chips reuse the one fail chip vocabulary — the row itself only exists while the
|
||||
counter is non-zero, so this is the presence of an incident, never a graded threshold */
|
||||
.alarms { display: flex; flex-wrap: wrap; gap: .4rem; }
|
||||
@media (max-width: 640px) {
|
||||
.cell { grid-template-columns: minmax(0, 1fr) max-content; }
|
||||
.cell .cell-spark { grid-column: 1 / -1; width: 100%; }
|
||||
}
|
||||
|
||||
/* ---- hero numbers --------------------------------------------------------
|
||||
* The big glanceable number: label, value + unit, and optional bar and footnote.
|
||||
* Colour comes from the *same* `ok|warn|fail` classes the rest of the stylesheet uses
|
||||
|
||||
@@ -1,40 +1,116 @@
|
||||
/* LLM dashboard.
|
||||
*
|
||||
* Layout (phase 04, per NOTES-chart-set): per engine, two big throughput numbers you can
|
||||
* read across the room, then a compact list of metric cells that each carry their own
|
||||
* sparkline over the *selected* window, percentile foot lines under the latency means, and
|
||||
* alarm chips that exist only while there is an incident. The four trend charts stay, but
|
||||
* half-height and below the panels: they are supporting evidence, not the main event.
|
||||
*
|
||||
* Every throughput cell shows where its number came from, because "78 tok/s" means
|
||||
* different things depending on whether it was scraped from an aggregate gauge, measured as
|
||||
* Δtokens/Δt across all slots, or estimated from TPOT over the process lifetime. The tag is
|
||||
* generated from the cell's `source`, never hardcoded in markup, so a parser change cannot
|
||||
* leave a stale label behind.
|
||||
*
|
||||
* Colour comes from payload verdicts only (`Dash.severityFrom` / `Dash.hero`). Throughput,
|
||||
* sessions, queue depth and latency carry **no** severity — 0 tok/s on an idle server is
|
||||
* not red, and a page-invented threshold is what `severity.py` exists to prevent.
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, pct, duration, poll,
|
||||
severityFor } = Dash;
|
||||
const { $, fetchJSON, setTxt, setDot, num, duration, poll, markStale, severityFrom, hero,
|
||||
eventText } = Dash;
|
||||
|
||||
const state = { range: 3600, panels: new Map(), charts: {}, poller: null, pollMs: 2000 };
|
||||
const state = {
|
||||
range: 3600,
|
||||
panels: new Map(), // engine -> card element
|
||||
charts: {},
|
||||
poller: null,
|
||||
pollMs: 2000,
|
||||
};
|
||||
|
||||
const fmtCount = (v) => num(v, 0);
|
||||
const fmtPct = (v) => `${num(v, 0)}%`;
|
||||
const fmtMs = (v) => `${num(v, 0)} ms`;
|
||||
|
||||
/**
|
||||
* What each engine panel shows, driven by flags instead of fourteen equal-size cells:
|
||||
* `hero` is the read-across-the-room row, `spark` is one compact row with a fixed-width
|
||||
* sparkline, `alarm` renders only as a red chip while the counter is non-zero, and
|
||||
* `footnote` is the quiet lifetime strip.
|
||||
*
|
||||
* `minSpan: 1` on the count rows keeps their axis at least one request tall: history is
|
||||
* stored as rollups, so eight seconds of a single queued request averages to 0.14 and an
|
||||
* axis fitted to that draws a full-height spike for a queue nobody was in.
|
||||
*
|
||||
* `spark` is the `/api/llm/history` series (without the `llm.` prefix); `scale` converts
|
||||
* stored units to printed ones (a 0-1 ratio to %, seconds to ms) for the plot as well as
|
||||
* the number, so the line and the digits agree. `fixedMax` pins percent-like metrics to a
|
||||
* 0-100 axis — an 8 % window that once held a 92 % spike must not read flat. `only` scopes
|
||||
* a row to the engines that publish it, so a gufo panel never shows a latency mystery and a
|
||||
* vLLM panel never shows a speculative-decoding row it cannot fill.
|
||||
*/
|
||||
const CELLS = [
|
||||
{ key: 'decode_tokens_per_second', label: 'Decode', unit: 'tok/s', digits: 1,
|
||||
{ key: 'decode', label: 'Decode', unit: 'tok/s', digits: 1, hero: true,
|
||||
tip: 'Aggregate generation speed across every busy slot.' },
|
||||
{ key: 'prompt_tokens_per_second', label: 'Prompt', unit: 'tok/s', digits: 1,
|
||||
{ key: 'prompt', label: 'Prompt', unit: 'tok/s', digits: 1, hero: true,
|
||||
tip: 'Prefill speed for newly computed tokens.' },
|
||||
{ key: 'sessions', label: 'Sessions', unit: 'active', digits: 0 },
|
||||
{ key: 'queued', label: 'Queued', unit: 'waiting', digits: 0 },
|
||||
{ key: 'cache', label: 'KV cache', unit: '%', digits: 0, bar: true },
|
||||
{ key: 'ttft_mean', label: 'TTFT mean', unit: 'ms', digits: 0 },
|
||||
{ key: 'ttft_p95', label: 'TTFT p95', unit: 'ms', digits: 0 },
|
||||
{ key: 'tpot_mean', label: 'TPOT mean', unit: 'ms', digits: 1 },
|
||||
{ key: 'queue_mean', label: 'Queue mean', unit: 'ms', digits: 0 },
|
||||
{ key: 'cache_hit', label: 'Prefix cache hit', unit: '%', digits: 0 },
|
||||
{ key: 'spec_accept', label: 'Spec accept', unit: '%', digits: 0 },
|
||||
{ key: 'tokens_total', label: 'Tokens lifetime', unit: '', digits: 0, human: true },
|
||||
{ key: 'preemptions', label: 'Preemptions', unit: '', digits: 0 },
|
||||
{ key: 'device_lost', label: 'GPU context lost', unit: '', digits: 0 },
|
||||
{ key: 'sessions', label: 'Sessions', unit: 'active', digits: 0,
|
||||
spark: 'requests_running', format: fmtCount, minSpan: 1 },
|
||||
{ key: 'queued', label: 'Queued', unit: 'waiting', digits: 0,
|
||||
spark: 'requests_waiting', format: fmtCount, minSpan: 1 },
|
||||
{ key: 'cache', label: 'KV cache', unit: '%', digits: 0, fixedMax: 100,
|
||||
spark: 'kv_cache_usage', scale: 100, format: fmtPct,
|
||||
tip: 'Share of the KV cache in use — the only engine-pressure verdict on this page, '
|
||||
+ 'and it comes from the API.' },
|
||||
{ key: 'ttft', label: 'TTFT', unit: 'ms', digits: 0, p95: true, only: ['vllm'],
|
||||
spark: 'ttft_seconds', scale: 1000, format: fmtMs },
|
||||
{ key: 'tpot', label: 'TPOT', unit: 'ms', digits: 1, p95: true, only: ['vllm'],
|
||||
spark: 'tpot_seconds', scale: 1000, format: fmtMs },
|
||||
{ key: 'queue_mean', label: 'Queue time', unit: 'ms', digits: 0, p95: true, only: ['vllm'],
|
||||
spark: 'queue_seconds', scale: 1000, format: fmtMs },
|
||||
{ key: 'cache_hit', label: 'Prefix cache hit', unit: '%', digits: 0, fixedMax: 100,
|
||||
spark: 'prefix_cache_hit_ratio', scale: 100, format: fmtPct },
|
||||
{ key: 'spec_accept', label: 'Spec accept', unit: '%', digits: 0, fixedMax: 100,
|
||||
only: ['gufo'], spark: 'spec_accept_rate', scale: 100, format: fmtPct },
|
||||
{ key: 'preemptions', label: 'Preemptions', alarm: true,
|
||||
tip: 'Running requests kicked back to the queue — an incident, not a stat.' },
|
||||
{ key: 'device_lost', label: 'GPU context lost', alarm: true,
|
||||
tip: 'The engine lost its GPU context — an incident, not a stat.' },
|
||||
{ key: 'tokens_total', footnote: true },
|
||||
];
|
||||
|
||||
const HERO_CELLS = CELLS.filter((cell) => cell.hero);
|
||||
const SPARK_CELLS = CELLS.filter((cell) => cell.spark);
|
||||
const ALARM_CELLS = CELLS.filter((cell) => cell.alarm);
|
||||
|
||||
/** One batched history request feeds every chart *and* every cell sparkline. */
|
||||
const HISTORY_SERIES = Array.from(new Set([
|
||||
'decode_tokens_per_second', 'prompt_tokens_per_second',
|
||||
...SPARK_CELLS.map((cell) => cell.spark),
|
||||
]));
|
||||
|
||||
const WINDOW_NAMES = { 300: '5 minutes', 900: '15 minutes', 3600: '1 hour',
|
||||
21600: '6 hours', 86400: '24 hours', 604800: '7 days',
|
||||
2592000: '30 days' };
|
||||
|
||||
function windowName(seconds) {
|
||||
return WINDOW_NAMES[seconds] || duration(seconds);
|
||||
}
|
||||
|
||||
/** The payload cell behind a UI key. A missing number stays `null` — never a 0. */
|
||||
function cellValue(engine, metrics, sessions, key) {
|
||||
const get = (k) => (metrics[k] && metrics[k].value != null ? metrics[k] : null);
|
||||
const scaled = (cell, factor) => ({ value: cell.value * factor, source: cell.source,
|
||||
derived: cell.derived });
|
||||
switch (key) {
|
||||
case 'decode':
|
||||
case 'prompt': {
|
||||
// the hero keys are short for the flags; the payload keeps the public metric name
|
||||
const cell = get(`${key}_tokens_per_second`);
|
||||
return cell ? { value: cell.value, source: cell.source, derived: cell.derived,
|
||||
window: cell.window } : { value: null };
|
||||
}
|
||||
case 'sessions': {
|
||||
const value = sessions && sessions.active != null ? sessions.active : null;
|
||||
return { value, source: 'session count', derived: false };
|
||||
@@ -44,32 +120,33 @@ function cellValue(engine, metrics, sessions, key) {
|
||||
return cell ? { value: cell.value, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'cache': {
|
||||
const cell = get('kv_cache_usage_ratio') || get('gpu_cache_usage');
|
||||
return cell ? { value: cell.value * 100, source: cell.source, derived: false } : { value: null };
|
||||
// the ratio cell carries the API verdict (`sources/llm.py` SEVERITY_CELLS ->
|
||||
// severity.py); it travels with the converted value so the row and its sparkline
|
||||
// paint exactly what the overview's alerts paint
|
||||
const cell = metrics.kv_cache_usage_ratio || metrics.gpu_cache_usage;
|
||||
if (!cell || cell.value == null) return { value: null };
|
||||
return { value: cell.value * 100, source: cell.source, derived: cell.derived,
|
||||
severity: cell.severity };
|
||||
}
|
||||
case 'ttft_mean': {
|
||||
case 'ttft': {
|
||||
const cell = get('time_to_first_token_seconds__mean');
|
||||
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
|
||||
return cell ? scaled(cell, 1000) : { value: null };
|
||||
}
|
||||
case 'ttft_p95': {
|
||||
const cell = get('time_to_first_token_seconds__p95');
|
||||
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
|
||||
}
|
||||
case 'tpot_mean': {
|
||||
case 'tpot': {
|
||||
const cell = get('time_per_output_token_seconds__mean');
|
||||
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
|
||||
return cell ? scaled(cell, 1000) : { value: null };
|
||||
}
|
||||
case 'queue_mean': {
|
||||
const cell = get('request_queue_time_seconds__mean');
|
||||
return cell ? { value: cell.value * 1000, source: cell.source, derived: false } : { value: null };
|
||||
return cell ? scaled(cell, 1000) : { value: null };
|
||||
}
|
||||
case 'cache_hit': {
|
||||
const cell = get('prefix_cache_hit_ratio') || get('prompt_cache_hit_ratio');
|
||||
return cell ? { value: cell.value * 100, source: cell.source, derived: true } : { value: null };
|
||||
return cell ? scaled(cell, 100) : { value: null };
|
||||
}
|
||||
case 'spec_accept': {
|
||||
const cell = get('spec_accept_rate');
|
||||
return cell ? { value: cell.value * 100, source: cell.source, derived: true } : { value: null };
|
||||
return cell ? scaled(cell, 100) : { value: null };
|
||||
}
|
||||
case 'tokens_total': {
|
||||
const cell = get('tokens_total');
|
||||
@@ -91,6 +168,15 @@ function cellValue(engine, metrics, sessions, key) {
|
||||
}
|
||||
}
|
||||
|
||||
/** The histogram p95 beside a latency mean — printed as a foot line, never behind a hover. */
|
||||
function p95Value(metrics, key) {
|
||||
const source = { ttft: 'time_to_first_token_seconds__p95',
|
||||
tpot: 'time_per_output_token_seconds__p95',
|
||||
queue_mean: 'request_queue_time_seconds__p95' }[key];
|
||||
const cell = source && metrics[source];
|
||||
return cell && cell.value != null ? cell.value * 1000 : null;
|
||||
}
|
||||
|
||||
/** The provenance tag: Δ10.0s for measured rates, the formula for estimates, nothing for a
|
||||
* plain scraped gauge (which would only add noise). */
|
||||
function provenance(cell) {
|
||||
@@ -109,10 +195,25 @@ function humanNumber(value) {
|
||||
return { text: value.toFixed(0), unit: 'tok' };
|
||||
}
|
||||
|
||||
/** The rows this engine publishes. A row it can never fill is not shown at all — a
|
||||
* permanent `n/a` is the dead-cell anti-pattern NOTES-chart-set exists to prevent. */
|
||||
function sparkCellsFor(engine) {
|
||||
return SPARK_CELLS.filter((cell) => !cell.only || cell.only.indexOf(engine) >= 0);
|
||||
}
|
||||
|
||||
function enginePanel(engine) {
|
||||
const card = document.createElement('article');
|
||||
card.className = 'card';
|
||||
card.dataset.engine = engine;
|
||||
const heroes = HERO_CELLS.map((cell) =>
|
||||
`<div class="hero" data-hero="${cell.key}"></div>`).join('');
|
||||
const rows = sparkCellsFor(engine).map((cell) => `
|
||||
<div class="cell" data-cell="${cell.key}"${cell.tip ? ` title="${cell.tip}"` : ''}>
|
||||
<span class="cell-label">${cell.label}</span>
|
||||
<span class="cell-value"><span data-v>n/a</span><span class="unit">${cell.unit}</span></span>
|
||||
<span class="cell-spark" data-spark data-metric="${cell.spark}"></span>
|
||||
${cell.p95 ? '<span class="cell-foot" data-foot></span>' : ''}
|
||||
</div>`).join('');
|
||||
card.innerHTML = `
|
||||
<header>
|
||||
<span class="dot idle"></span>
|
||||
@@ -123,13 +224,11 @@ function enginePanel(engine) {
|
||||
</header>
|
||||
<div class="body stack">
|
||||
<div class="banner err" hidden></div>
|
||||
<div class="metrics">${CELLS.map((cell) => `
|
||||
<div class="metric" data-cell="${cell.key}" ${cell.tip ? `title="${cell.tip}"` : ''}>
|
||||
<div class="label">${cell.label}</div>
|
||||
<div class="value${cell.digits === 0 ? '' : ''}"><span data-v>n/a</span><span class="unit">${cell.unit}</span></div>
|
||||
${cell.bar ? '<div class="bar" style="margin-top:.25rem"><i></i></div>' : ''}
|
||||
</div>`).join('')}</div>
|
||||
<div class="heroes">${heroes}</div>
|
||||
<div class="alarms" data-k="alarms" hidden></div>
|
||||
<div class="cells">${rows}</div>
|
||||
<div class="faint mono sessions-note" style="font-size:11.5px"></div>
|
||||
<div class="faint mono token-note" style="font-size:11px"></div>
|
||||
<div class="faint mono debug" style="font-size:11px"></div>
|
||||
</div>`;
|
||||
return card;
|
||||
@@ -146,7 +245,7 @@ function renderPanel(card, enginePayload, stateRow) {
|
||||
? (stateRow.error || 'never reached')
|
||||
: (Dash.ageSeconds(stateRow.last_ok) > 15 ? `stale ${duration(Dash.ageSeconds(stateRow.last_ok))}`
|
||||
: `updated ${duration(Dash.ageSeconds(stateRow.last_ok))}`)) : '—');
|
||||
Dash.markStale(card, stateRow && (!stateRow.ok || Dash.ageSeconds(stateRow.last_ok) > 15));
|
||||
markStale(card, stateRow && (!stateRow.ok || Dash.ageSeconds(stateRow.last_ok) > 15));
|
||||
|
||||
const banner = q('.banner.err');
|
||||
if (stateRow && !stateRow.ok) {
|
||||
@@ -157,51 +256,57 @@ function renderPanel(card, enginePayload, stateRow) {
|
||||
banner.hidden = true;
|
||||
}
|
||||
|
||||
for (const cell of CELLS) {
|
||||
const wrapper = q(`[data-cell="${cell.key}"]`);
|
||||
if (!wrapper) continue;
|
||||
// The two heroes. No severity, by design: an idle server at 0 tok/s is healthy, so the
|
||||
// digits stay neutral ink and reachability colour lives on the dot and the stale dimming
|
||||
// (the `/system` NET precedent). `Dash.hero()` still owns the markup and the n/a rule.
|
||||
for (const cell of HERO_CELLS) {
|
||||
const value = cellValue(enginePayload.engine, metrics, sessions, cell.key);
|
||||
const valueEl = wrapper.querySelector('[data-v]');
|
||||
const unitEl = wrapper.querySelector('.unit');
|
||||
const node = hero(q(`[data-hero="${cell.key}"]`), {
|
||||
label: cell.label, value: value.value, unit: cell.unit, digits: cell.digits,
|
||||
sub: provenance(value),
|
||||
});
|
||||
if (node) node.title = value.source || ''; // the exact source, as llm_monitor's tag does
|
||||
}
|
||||
|
||||
// Compact value rows; their sparklines are drawn once per window per refresh from the
|
||||
// one batched history response (`refreshHistory`).
|
||||
for (const cell of sparkCellsFor(enginePayload.engine)) {
|
||||
const row = q(`[data-cell="${cell.key}"]`);
|
||||
if (!row) continue;
|
||||
const value = cellValue(enginePayload.engine, metrics, sessions, cell.key);
|
||||
const valueEl = row.querySelector('[data-v]');
|
||||
const unitEl = row.querySelector('.unit');
|
||||
// the row carries the payload verdict (KV cache is the only rated cell) so the
|
||||
// sparkline, drawn from the history response, paints the same verdict the number does.
|
||||
// Like the fleet table, only a breach paints the *text* — a healthy row stays quiet
|
||||
// ink; the spark still strokes green through the same verdict.
|
||||
const sev = severityFrom(value.severity);
|
||||
row.dataset.sev = sev;
|
||||
['ok', 'warn', 'fail'].forEach((name) => row.classList.toggle(name, name === sev));
|
||||
if (value.value == null) {
|
||||
setTxt(valueEl, 'n/a');
|
||||
setTxt(unitEl, ''); // "n/a %" reads like a measurement of nothing
|
||||
wrapper.classList.add('is-null');
|
||||
} else {
|
||||
wrapper.classList.remove('is-null');
|
||||
if (cell.human) {
|
||||
const parts = humanNumber(value.value);
|
||||
setTxt(valueEl, parts ? parts.text : 'n/a');
|
||||
setTxt(unitEl, parts ? parts.unit : '');
|
||||
} else {
|
||||
setTxt(valueEl, num(value.value, cell.digits));
|
||||
setTxt(unitEl, cell.unit);
|
||||
}
|
||||
setTxt(valueEl, num(value.value, cell.digits));
|
||||
setTxt(unitEl, cell.unit);
|
||||
}
|
||||
let tag = wrapper.querySelector('.prov');
|
||||
const label = provenance(value);
|
||||
if (label) {
|
||||
if (!tag) {
|
||||
tag = document.createElement('span');
|
||||
tag.className = 'prov';
|
||||
wrapper.querySelector('.value').appendChild(tag);
|
||||
}
|
||||
setTxt(tag, label);
|
||||
tag.title = `${value.source || ''}${value.window ? ` · measured over ${value.window.toFixed(1)}s` : ''}`;
|
||||
} else if (tag) {
|
||||
tag.remove();
|
||||
}
|
||||
if (cell.bar) setBar(wrapper.querySelector('.bar'), value.value,
|
||||
severityFor(value.value, 75, 90));
|
||||
if (cell.key === 'device_lost' && value.value != null && value.value > 0) {
|
||||
wrapper.querySelector('.value').classList.add('is-null');
|
||||
wrapper.classList.add('fail');
|
||||
wrapper.title = `GPU context has been lost ${value.value} times — an incident, not a stat`;
|
||||
if (cell.p95) {
|
||||
const p95 = p95Value(metrics, cell.key);
|
||||
setTxt(row.querySelector('[data-foot]'),
|
||||
p95 == null ? 'p95 n/a' : `p95 ${num(p95, cell.digits)} ${cell.unit}`);
|
||||
}
|
||||
}
|
||||
|
||||
renderAlarms(card, enginePayload);
|
||||
|
||||
const note = q('.sessions-note');
|
||||
setTxt(note, sessions.detail || `${sessions.active == null ? 'n/a' : sessions.active} active sessions`);
|
||||
// a lifetime counter is a footnote, not a glance metric (NOTES-chart-set, demoted)
|
||||
const tokens = cellValue(enginePayload.engine, metrics, sessions, 'tokens_total');
|
||||
const parts = tokens.value == null ? null : humanNumber(tokens.value);
|
||||
setTxt(q('.token-note'), parts
|
||||
? `lifetime ${parts.text} ${parts.unit} · ${tokens.source || 'unknown source'}`
|
||||
: 'lifetime tokens n/a');
|
||||
const upstream = q('.chip.upstream');
|
||||
if (enginePayload.upstream) {
|
||||
upstream.hidden = false;
|
||||
@@ -213,6 +318,39 @@ function renderPanel(card, enginePayload, stateRow) {
|
||||
: '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Alarm chips exist only while there is an incident.
|
||||
*
|
||||
* A permanent cell for a counter that is always 0 trains the eye to ignore the row, so at
|
||||
* zero the row does not exist: the container is emptied and hidden, and only a non-zero —
|
||||
* an *incident*, not a graded threshold — puts a `.chip.fail` back. `sources/llm.py`
|
||||
* already treats any non-zero gufo_device_lost_total this way.
|
||||
*/
|
||||
function renderAlarms(card, enginePayload) {
|
||||
const host = card.querySelector('[data-k="alarms"]');
|
||||
if (!host) return;
|
||||
const chips = [];
|
||||
for (const cell of ALARM_CELLS) {
|
||||
const value = cellValue(enginePayload.engine, enginePayload.metrics || {},
|
||||
enginePayload.sessions || {}, cell.key);
|
||||
if (value.value != null && value.value > 0) {
|
||||
chips.push(`<span class="chip fail" data-alarm="${cell.key}">${cell.label} ${num(value.value, 0)}</span>`);
|
||||
}
|
||||
}
|
||||
const signature = chips.join('');
|
||||
if (host.dataset.sig !== signature) {
|
||||
host.innerHTML = signature;
|
||||
host.dataset.sig = signature;
|
||||
}
|
||||
for (const chip of host.querySelectorAll('[data-alarm]')) {
|
||||
const cell = ALARM_CELLS.find((c) => c.key === chip.dataset.alarm);
|
||||
const value = cellValue(enginePayload.engine, enginePayload.metrics || {},
|
||||
enginePayload.sessions || {}, cell.key);
|
||||
chip.title = `${cell.tip || cell.label} · ${value.source || 'source unknown'}`;
|
||||
}
|
||||
host.hidden = chips.length === 0;
|
||||
}
|
||||
|
||||
function renderSlots(payloads) {
|
||||
const gufo = payloads.find((p) => p.engine === 'gufo');
|
||||
const rows = gufo && gufo.sessions ? gufo.sessions.list || [] : [];
|
||||
@@ -239,13 +377,19 @@ function renderSlots(payloads) {
|
||||
}
|
||||
|
||||
function makeCharts() {
|
||||
const common = { height: 165, xSpan: state.range };
|
||||
state.charts.decode = new DashCharts.LineChart('#chart-decode', Object.assign({}, common, { yMin: 0 }));
|
||||
state.charts.prompt = new DashCharts.LineChart('#chart-prompt', Object.assign({}, common, { yMin: 0 }));
|
||||
state.charts.kv = new DashCharts.LineChart('#chart-kv', Object.assign({}, common,
|
||||
{ yMin: 0, yMax: 100, yFormat: (v) => num(v, 0) }));
|
||||
state.charts.queue = new DashCharts.LineChart('#chart-queue', Object.assign({}, common,
|
||||
{ yMin: 0, yFormat: (v) => num(v, 0) }));
|
||||
// Half of the old 165px: the cells read first, the charts back them up. Every plot
|
||||
// names itself for assistive tech — the legend is a sibling that labels nothing.
|
||||
const common = { height: 110, xSpan: state.range };
|
||||
state.charts.decode = new DashCharts.LineChart('#chart-decode', Object.assign({}, common, {
|
||||
yMin: 0, ariaLabel: 'Decode throughput in tokens per second over the selected window, one line per engine' }));
|
||||
state.charts.prompt = new DashCharts.LineChart('#chart-prompt', Object.assign({}, common, {
|
||||
yMin: 0, ariaLabel: 'Prompt / prefill throughput in tokens per second over the selected window, one line per engine' }));
|
||||
state.charts.kv = new DashCharts.LineChart('#chart-kv', Object.assign({}, common, {
|
||||
yMin: 0, yMax: 100, yFormat: (v) => num(v, 0),
|
||||
ariaLabel: 'KV / prefix cache pressure in per cent over the selected window, one line per engine' }));
|
||||
state.charts.queue = new DashCharts.LineChart('#chart-queue', Object.assign({}, common, {
|
||||
yMin: 0, yFormat: (v) => num(v, 0),
|
||||
ariaLabel: 'Requests running and requests waiting over the selected window, one pair of lines per engine' }));
|
||||
}
|
||||
|
||||
function pointsFor(response, engine, metric) {
|
||||
@@ -253,59 +397,89 @@ function pointsFor(response, engine, metric) {
|
||||
return found ? found.points.map((p) => ({ ts: p.ts, value: p.value })) : [];
|
||||
}
|
||||
|
||||
async function refreshCharts(engineNames) {
|
||||
function drawChart(chartEl, legendEl, chart, series, format, showPeak) {
|
||||
chart.opts.xSpan = state.range;
|
||||
chart.setData(series);
|
||||
DashCharts.renderLegend(legendEl, series, format, showPeak);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every chart and every cell sparkline, from ONE `/api/llm/history` request.
|
||||
*
|
||||
* The endpoint answers a metric list for every engine, and unknown names come back as
|
||||
* empty series on purpose — so the page asks for the full cell list once per refresh
|
||||
* instead of one XHR per cell, and a partly-filled window stays partly filled rather than
|
||||
* rescaling to whatever samples exist.
|
||||
*/
|
||||
async function refreshHistory(engineNames) {
|
||||
if (!engineNames.length) return;
|
||||
const list = engineNames.join(',');
|
||||
const [decode, prompt, queue] = await Promise.all([
|
||||
fetchJSON(`/api/llm/history?engines=${list}&metrics=decode_tokens_per_second&range=${state.range}`)
|
||||
.catch(() => null),
|
||||
fetchJSON(`/api/llm/history?engines=${list}&metrics=prompt_tokens_per_second&range=${state.range}`)
|
||||
.catch(() => null),
|
||||
fetchJSON(`/api/llm/history?engines=${list}&metrics=requests_running,requests_waiting,` +
|
||||
`kv_cache_usage&range=${state.range}`).catch(() => null),
|
||||
]);
|
||||
if (decode) {
|
||||
const series = engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(decode, engine, 'decode_tokens_per_second') }));
|
||||
state.charts.decode.setData(series);
|
||||
DashCharts.renderLegend($('#legend-decode'), series, (v) => num(v, 1) + ' tok/s', true);
|
||||
const response = await fetchJSON(
|
||||
`/api/llm/history?engines=${engineNames.join(',')}` +
|
||||
`&metrics=${HISTORY_SERIES.join(',')}&range=${state.range}`).catch(() => null);
|
||||
if (!response) return;
|
||||
drawChart($('#chart-decode'), $('#legend-decode'), state.charts.decode,
|
||||
engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(response, engine, 'decode_tokens_per_second') })),
|
||||
(v) => num(v, 1) + ' tok/s', true);
|
||||
drawChart($('#chart-prompt'), $('#legend-prompt'), state.charts.prompt,
|
||||
engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(response, engine, 'prompt_tokens_per_second') })),
|
||||
(v) => num(v, 0) + ' tok/s', true);
|
||||
drawChart($('#chart-kv'), $('#legend-kv'), state.charts.kv,
|
||||
engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(response, engine, 'kv_cache_usage').map((p) => ({ ts: p.ts, value: p.value * 100 })) })),
|
||||
(v) => num(v, 1) + '%', true);
|
||||
const sessionsSeries = [];
|
||||
for (const engine of engineNames) {
|
||||
sessionsSeries.push({ name: `${engine} running`, points: pointsFor(response, engine, 'requests_running') });
|
||||
sessionsSeries.push({ name: `${engine} waiting`, points: pointsFor(response, engine, 'requests_waiting') });
|
||||
}
|
||||
if (prompt) {
|
||||
const series = engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(prompt, engine, 'prompt_tokens_per_second') }));
|
||||
state.charts.prompt.setData(series);
|
||||
DashCharts.renderLegend($('#legend-prompt'), series, (v) => num(v, 0) + ' tok/s', true);
|
||||
}
|
||||
if (queue) {
|
||||
const series = [];
|
||||
for (const engine of engineNames) {
|
||||
series.push({ name: `${engine} running`, points: pointsFor(queue, engine, 'requests_running') });
|
||||
series.push({ name: `${engine} waiting`, points: pointsFor(queue, engine, 'requests_waiting') });
|
||||
drawChart($('#chart-queue'), $('#legend-queue'), state.charts.queue, sessionsSeries,
|
||||
(v) => num(v, 0), false);
|
||||
drawSparks(response);
|
||||
}
|
||||
|
||||
/** One sparkline per cell, over the *selected* window, x mapped to real time. */
|
||||
function drawSparks(response) {
|
||||
for (const [engine, card] of state.panels) {
|
||||
for (const cell of sparkCellsFor(engine)) {
|
||||
const row = card.querySelector(`[data-cell="${cell.key}"]`);
|
||||
const slot = row && row.querySelector('[data-spark]');
|
||||
if (!slot) continue;
|
||||
const points = pointsFor(response, engine, cell.spark)
|
||||
.map((p) => ({ ts: p.ts, value: p.value * (cell.scale || 1) }));
|
||||
DashCharts.sparkline(slot, points, {
|
||||
align: 'right', // newest pinned right; x stays time, not sample index
|
||||
span: state.range,
|
||||
fixedMax: cell.fixedMax != null ? cell.fixedMax : null,
|
||||
minSpan: cell.minSpan != null ? cell.minSpan : null,
|
||||
// the only rated spark is KV, and its colour is the API's verdict; everything else
|
||||
// draws the faint unrated stroke, because a shape is not a verdict
|
||||
severity: row.dataset.sev || '',
|
||||
labels: { max: true },
|
||||
format: cell.format,
|
||||
ariaLabel: `${engine} ${cell.label} over the last ${windowName(state.range)}`,
|
||||
});
|
||||
}
|
||||
state.charts.queue.setData(series);
|
||||
DashCharts.renderLegend($('#legend-queue'), series, (v) => num(v, 0));
|
||||
const kv = engineNames.map((engine) => ({ name: engine,
|
||||
points: pointsFor(queue, engine, 'kv_cache_usage').map((p) => ({ ts: p.ts, value: p.value * 100 })) }));
|
||||
state.charts.kv.setData(kv);
|
||||
DashCharts.renderLegend($('#legend-kv'), kv, (v) => num(v, 1) + '%', true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The engine-totals card: only what no single panel can say.
|
||||
*
|
||||
* Summed decode/prompt/KV left with phase 04 (per-engine is the honest view), and the card
|
||||
* hides entirely below two engines — a fleet of one does not need a fleet row.
|
||||
*/
|
||||
function renderFleet(fleet) {
|
||||
if (!fleet) return;
|
||||
const card = $('#totals');
|
||||
if (!card) return;
|
||||
const total = fleet && fleet.engines_total != null ? fleet.engines_total : 0;
|
||||
card.hidden = total < 2;
|
||||
if (!total) return;
|
||||
setTxt($('#f-live'), fleet.engines_live == null ? 'n/a' : String(fleet.engines_live));
|
||||
setTxt($('#f-total'), fleet.engines_total == null ? '' : `of ${fleet.engines_total}`);
|
||||
setValue($('#f-decode'), fleet.decode_tokens_per_second, 1);
|
||||
setValue($('#f-prompt'), fleet.prompt_tokens_per_second, 0);
|
||||
setTxt($('#f-active'), fleet.requests_active == null ? 'n/a' : num(fleet.requests_active, 0));
|
||||
setTxt($('#f-waiting'), fleet.requests_waiting == null ? 'n/a' : num(fleet.requests_waiting, 0));
|
||||
const tokens = humanNumber(fleet.tokens_total);
|
||||
setTxt($('#f-tokens'), tokens ? tokens.text : 'n/a');
|
||||
$('#f-tokens').nextElementSibling && setTxt($('#f-tokens').nextElementSibling, tokens ? tokens.unit : '');
|
||||
const kv = fleet.kv_cache_usage_max == null ? null : fleet.kv_cache_usage_max * 100;
|
||||
setValue($('#f-kv'), kv, 0);
|
||||
setBar($('#f-kv').closest('.metric').querySelector('.bar'), kv, severityFor(kv, 75, 90));
|
||||
$('#seam-note').textContent = fleet.seams || '';
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
@@ -339,7 +513,7 @@ async function refresh() {
|
||||
`window ${duration(state.range)}`;
|
||||
$('#foot-note').textContent = `poll ${Math.round(state.pollMs / 1000)}s · rates over the ` +
|
||||
`Δ window in each tag`;
|
||||
await refreshCharts(names);
|
||||
await refreshHistory(names);
|
||||
}
|
||||
|
||||
async function refreshEvents() {
|
||||
@@ -353,7 +527,7 @@ async function refreshEvents() {
|
||||
list.innerHTML = data.events.map((event) => `<li><time>${
|
||||
new Date(event.ts * 1000).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit', hour12: false })
|
||||
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
|
||||
`${DashCharts.escapeHtml(event.message)}</span></li>`).join('');
|
||||
`${DashCharts.escapeHtml(eventText(event))}</span></li>`).join('');
|
||||
} catch (err) { /* next poll */ }
|
||||
}
|
||||
|
||||
@@ -373,8 +547,7 @@ document.addEventListener('DOMContentLoaded', async () => {
|
||||
|
||||
Dash.rangeControl($('#range'), state.range, (seconds) => {
|
||||
state.range = seconds;
|
||||
Object.values(state.charts).forEach((chart) => { chart.opts.xSpan = seconds; });
|
||||
refresh();
|
||||
refresh(); // charts and sparklines both re-read the selected window
|
||||
});
|
||||
|
||||
state.poller = poll(async () => { await refresh(); await refreshEvents(); }, state.pollMs);
|
||||
|
||||
@@ -472,7 +472,7 @@ async function refreshEvents() {
|
||||
list.innerHTML = data.events.map((event) => `<li><time>${
|
||||
new Date(event.ts * 1000).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit', hour12: false })
|
||||
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
|
||||
`${DashCharts.escapeHtml(event.message)}</span></li>`).join('');
|
||||
`${DashCharts.escapeHtml(Dash.eventText(event))}</span></li>`).join('');
|
||||
} catch (err) { /* the next poll will try again */ }
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,8 @@
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, duration, poll, severityFor } = Dash;
|
||||
const { $, fetchJSON, setTxt, setValue, setBar, setDot, num, duration, poll, severityFor,
|
||||
eventText } = Dash;
|
||||
|
||||
const state = { range: 86400, cards: new Map(), charts: {}, poller: null, pollMs: 5000,
|
||||
simulated: false, pending: null };
|
||||
@@ -327,7 +328,7 @@ async function refreshEvents() {
|
||||
new Date(event.ts * 1000).toLocaleString(undefined,
|
||||
{ month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', hour12: false })
|
||||
}</time><span><span class="sev ${event.severity}">${DashCharts.escapeHtml(event.host)}</span> ` +
|
||||
`${DashCharts.escapeHtml(event.message.replace(event.host + ':', '').trim())}` +
|
||||
`${DashCharts.escapeHtml(eventText(event))}` +
|
||||
`${event.data && event.data.runtime_seconds ? ` <span class="faint mono">(${
|
||||
duration(event.data.runtime_seconds)} runtime)</span>` : ''}</span></li>`).join('');
|
||||
} catch (err) { /* next poll */ }
|
||||
|
||||
Reference in New Issue
Block a user