From 569ca25fdb0f4a36c1ded9c09a4fb9b5a148c630 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Mon, 19 Oct 2026 23:44:35 -0400 Subject: [PATCH] phase: 14_store_performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Phase 14 `store_performance` — final verification pass (all 4 tasks already in `complete/`)** - Re-verified every task's work in the tree: read caches (`series_names`/`stats`), capped compound `pick_res_many`, `history_many` + `api._history_batch`, windowed `maintain()` + per-resolution prune + `PRAGMA optimize`/`automatic_analyze=0`/WAL pragmas, `store_perf.BUDGETS` (14 ceilings), probe, live watch. - **Completion criteria:** baseline+after side by side in `data/logs/NOTES-store-perf.md` ✅ · budget tests green (`uv run pytest -m store_perf --store-perf` → **4 passed**, 14/14 inside) ✅ · 25-min dev-server watch re-run **independently** (74 passes, `scripts/watch-store-perf.py`): **0 `store-error`** (total still 190, newest 2026‑10‑18 06:33:30), WAL 7.77 MB ≤ 67.1 MB, medians 1.7–3.2 ms, p95 720/613 ms = the documented cold-cache window; a 3rd 27-pass watch on a freshly restarted server reproduced it (3.1/708, 1.8/572) ✅ · `maintain()` 5,368 ms → 2.7–6.8 ms and WAL bounded across two cycles (test + notes) ✅ · no behaviour change (all four history routes pinned against `query()`) ✅ - **Full gates:** `uv run pytest` → **1267 passed, 15 skipped**; `bash .agents/validate.sh` → **exit 0**, coverage **98.75%** (floor 94), `ruff check src tests agent` clean, `pyright` 0 errors, `node --check` ok, repo hygiene ok; `uv run pytest -m live --live` → **11 passed**; dev server up, 9/9 sources `ok`, pages 200 in 1.5–3.3 ms. - **Docs fixed:** README/AGENTS still claimed a flat "20 s per pass" while `host_agent.py` (in this same uncommitted change) scales the budget to 5 s × disk count — corrected in both plus the two illustrative "20 s of wedged disk" sentences; broken bold (`` `driveripper** ``) repaired; stale gap-4 title + "tuning is its own phase" wording updated to say phase 14 ran; README's absolute "nothing … is answered by a whole-table scan" qualified to *per call* with the cold-window cost named; "third opt-in marker" → second; conftest `--store-perf` help named the real marker; **`data/logs/watch-store-perf.py` moved to `scripts/watch-store-perf.py`** (both docs told a clone to run a gitignored path) with root-anchored paths, `DASH_WATCH_OUT`, and dead code removed. - **New finding recorded:** gap 9's "~10 s process-wide stall is still gap 4's / `14_store_performance`'s problem" is now closed — coincident >8 s `llm` + >12 s `ups` raw-stream gaps went 393 (00:00–12:00) and 198 (12:00–21:36) to **0** across 110+ min of the phase-14 store; written into AGENTS.md gaps 4 and 9 and the notes table. - **Deviations:** did *not* pull either deferred cost knob (`STATS_CACHE_TTL_SECONDS`, warming caches from the maintenance tick) — AGENTS.md records that decision and its reasoning; `uv pip list --outdated` still reports certifi/fastapi/starlette/websockets (pre-existing advisory, no dependency touched this phase, gate treats it as advisory). Overwrote nothing: task 04's watch preserved as `store-perf-watch.task04.*`, my two runs as `.verify25.*` / `.current-code.*`. - **Next pending phase:** `15_agent_mtls`, then `16_agent_smart_pass_budget` (owns the scaled-budget tests + its doc criterion, now partly pre-satisfied by this pass), `17_installer_device_filter`. --- .../14_store_performance/00_phase.md | 0 .../01_baseline_store_costs.md | 0 .../02_no_full_scan_on_hot_paths.md | 0 .../03_windowed_rollups_and_wal_tuning.md | 0 .../04_budget_gates_and_live_proof.md | 0 .../16_agent_smart_pass_budget/00_phase.md | 42 + .../01_pin_the_scaled_budget.md | 26 + .../02_docs_and_gate.md | 25 + .../17_installer_device_filter/00_phase.md | 59 + .../01_drop_the_broken_device_filter.md | 25 + .../02_root_and_caps_flags.md | 27 + .../03_no_silent_clobber.md | 27 + .../04_tests_docs_gate.md | 34 + .../14_store_performance__00_phase.a1.err | 0 .../14_store_performance__00_phase.a1.md | 9 + ...4_store_performance__00_phase.a1.md.status | 1 + ...14_store_performance__00_phase.a1.validate | 64 + ...erformance__01_baseline_store_costs.a1.err | 0 ...performance__01_baseline_store_costs.a1.md | 1 + ...ance__01_baseline_store_costs.a1.md.status | 1 + ...mance__01_baseline_store_costs.a1.validate | 63 + ...mance__02_no_full_scan_on_hot_paths.a1.err | 0 ...rmance__02_no_full_scan_on_hot_paths.a1.md | 17 + ..._02_no_full_scan_on_hot_paths.a1.md.status | 1 + ...__02_no_full_scan_on_hot_paths.a1.validate | 64 + ..._03_windowed_rollups_and_wal_tuning.a1.err | 0 ...__03_windowed_rollups_and_wal_tuning.a1.md | 22 + ...ndowed_rollups_and_wal_tuning.a1.md.status | 1 + ...indowed_rollups_and_wal_tuning.a1.validate | 111 ++ ..._03_windowed_rollups_and_wal_tuning.a2.err | 0 ...__03_windowed_rollups_and_wal_tuning.a2.md | 19 + ...ndowed_rollups_and_wal_tuning.a2.md.status | 1 + ...indowed_rollups_and_wal_tuning.a2.validate | 64 + ...nce__04_budget_gates_and_live_proof.a1.err | 0 ...ance__04_budget_gates_and_live_proof.a1.md | 14 + ...4_budget_gates_and_live_proof.a1.md.status | 1 + ...04_budget_gates_and_live_proof.a1.validate | 64 + AGENTS.md | 257 ++- README.md | 73 +- agent/host_agent.py | 26 +- pyproject.toml | 5 + scripts/store_perf_probe.py | 95 + scripts/watch-store-perf.py | 194 ++ src/dashboards/api.py | 46 +- src/dashboards/store.py | 551 +++++- tests/conftest.py | 65 +- tests/store_perf.py | 1696 +++++++++++++++++ tests/test_api.py | 171 +- tests/test_store.py | 1234 +++++++++++- 49 files changed, 5072 insertions(+), 124 deletions(-) rename .agents/phases/{todo => complete}/14_store_performance/00_phase.md (100%) rename .agents/phases/{todo => complete}/14_store_performance/01_baseline_store_costs.md (100%) rename .agents/phases/{todo => complete}/14_store_performance/02_no_full_scan_on_hot_paths.md (100%) rename .agents/phases/{todo => complete}/14_store_performance/03_windowed_rollups_and_wal_tuning.md (100%) rename .agents/phases/{todo => complete}/14_store_performance/04_budget_gates_and_live_proof.md (100%) create mode 100644 .agents/phases/todo/16_agent_smart_pass_budget/00_phase.md create mode 100644 .agents/phases/todo/16_agent_smart_pass_budget/01_pin_the_scaled_budget.md create mode 100644 .agents/phases/todo/16_agent_smart_pass_budget/02_docs_and_gate.md create mode 100644 .agents/phases/todo/17_installer_device_filter/00_phase.md create mode 100644 .agents/phases/todo/17_installer_device_filter/01_drop_the_broken_device_filter.md create mode 100644 .agents/phases/todo/17_installer_device_filter/02_root_and_caps_flags.md create mode 100644 .agents/phases/todo/17_installer_device_filter/03_no_silent_clobber.md create mode 100644 .agents/phases/todo/17_installer_device_filter/04_tests_docs_gate.md create mode 100644 .agents/reports/14_store_performance/14_store_performance__00_phase.a1.err create mode 100644 .agents/reports/14_store_performance/14_store_performance__00_phase.a1.md create mode 100644 .agents/reports/14_store_performance/14_store_performance__00_phase.a1.md.status create mode 100644 .agents/reports/14_store_performance/14_store_performance__00_phase.a1.validate create mode 100644 .agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.err create mode 100644 .agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.md create mode 100644 .agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.md.status create mode 100644 .agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.validate create mode 100644 .agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.err create mode 100644 .agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.md create mode 100644 .agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.md.status create mode 100644 .agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.validate create mode 100644 .agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.err create mode 100644 .agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.md create mode 100644 .agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.md.status create mode 100644 .agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.validate create mode 100644 .agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.err create mode 100644 .agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.md create mode 100644 .agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.md.status create mode 100644 .agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.validate create mode 100644 .agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.err create mode 100644 .agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.md create mode 100644 .agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.md.status create mode 100644 .agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.validate create mode 100755 scripts/store_perf_probe.py create mode 100755 scripts/watch-store-perf.py create mode 100644 tests/store_perf.py diff --git a/.agents/phases/todo/14_store_performance/00_phase.md b/.agents/phases/complete/14_store_performance/00_phase.md similarity index 100% rename from .agents/phases/todo/14_store_performance/00_phase.md rename to .agents/phases/complete/14_store_performance/00_phase.md diff --git a/.agents/phases/todo/14_store_performance/01_baseline_store_costs.md b/.agents/phases/complete/14_store_performance/01_baseline_store_costs.md similarity index 100% rename from .agents/phases/todo/14_store_performance/01_baseline_store_costs.md rename to .agents/phases/complete/14_store_performance/01_baseline_store_costs.md diff --git a/.agents/phases/todo/14_store_performance/02_no_full_scan_on_hot_paths.md b/.agents/phases/complete/14_store_performance/02_no_full_scan_on_hot_paths.md similarity index 100% rename from .agents/phases/todo/14_store_performance/02_no_full_scan_on_hot_paths.md rename to .agents/phases/complete/14_store_performance/02_no_full_scan_on_hot_paths.md diff --git a/.agents/phases/todo/14_store_performance/03_windowed_rollups_and_wal_tuning.md b/.agents/phases/complete/14_store_performance/03_windowed_rollups_and_wal_tuning.md similarity index 100% rename from .agents/phases/todo/14_store_performance/03_windowed_rollups_and_wal_tuning.md rename to .agents/phases/complete/14_store_performance/03_windowed_rollups_and_wal_tuning.md diff --git a/.agents/phases/todo/14_store_performance/04_budget_gates_and_live_proof.md b/.agents/phases/complete/14_store_performance/04_budget_gates_and_live_proof.md similarity index 100% rename from .agents/phases/todo/14_store_performance/04_budget_gates_and_live_proof.md rename to .agents/phases/complete/14_store_performance/04_budget_gates_and_live_proof.md diff --git a/.agents/phases/todo/16_agent_smart_pass_budget/00_phase.md b/.agents/phases/todo/16_agent_smart_pass_budget/00_phase.md new file mode 100644 index 0000000..6d68be4 --- /dev/null +++ b/.agents/phases/todo/16_agent_smart_pass_budget/00_phase.md @@ -0,0 +1,42 @@ +# Phase 16 — Pin the agent's scaled smart-pass budget + +**Source:** chat 2026-10-19 — the `driveripper` 16-drive host: the flat 20 s smart-pass +budget truncates a 16-device pass (a real read takes ~30–80 s), and because the deadline +resets each pass, the tail drives would have stayed `smart: null` every pass. The fix — +budget = 20 s floor scaled to 5 s × disk count, an explicit override still winning — was +applied to the working tree on 2026-10-19 (operator-confirmed, verified on 16/16 live +drives). This phase pins that already-applied change with tests and documentation. +**Story:** n/a +**Context:** `agent/host_agent.py` (`_smart_pass` — the scaled `budget = max(SMARTCTL_BUDGET_SECONDS, self.smart_timeout_seconds * len(device_names)) when the default is in force, else the explicit override; the `pass budget of {budget:.0f}s exhausted` error line; the module docstring "5 s per device and a pass budget of 20 s scaled to the disk count"), `tests/test_agent.py::test_the_pass_budget_skips_the_rest_with_one_error` (sets `collector.smart_budget_seconds = 0.0` — the override-wins branch), `AGENTS.md` line 185 ("5 s per device / 20 s per pass / 240 s cache") and the `deploy/install-agent.sh` header comment (line 25 area, "bounded 5 s per device and 20 s per pass" if still present there), `README.md` line 94 ("20 s per pass, results cached 4 minutes") + +## Objective + +The smart pass budget scale-up that is already in the working tree is pinned: a test proves the *default* scales (16 devices → 80 s deadline, one device → 20 s floor, the exhaustion line names the scaled figure), the override-wins branch stays covered, and every doc that still says "20 s per pass" says the floor-scaled truth. No new behaviour — the code change is already deployed on `driveripper` and verified live. + +## Dependencies + +- `09_disks_smartctl` (complete) — the smart pass, its bounds, and `tests/test_agent.py`'s in-process agent harness + +## Tasks + +1. `01_pin_the_scaled_budget.md` — the tests that the default budget scales with the disk count and that the error line names the scaled figure +2. `02_docs_and_gate.md` — every "20 s per pass" mention in `AGENTS.md`/`README.md`/`agent/host_agent.py` says the floor-scaled truth; the gate is green + +## Testing & Quality + +- Unit: `tests/test_agent.py` — the scaled default (a default-attribute collector over 16 device names exhausts only when the 80 s budget runs out; over one device the floor is 20 s; the exhaustion error line carries the scaled figure, not the floor) +- Coverage: **>90%** on the modified code (the `_smart_pass` branch) +- The phase is "Complete" only when the **entire** project test suite passes 100% (every test in the project, not just this phase's) + +## Documentation & Consistency + +- The final pass reviews `README.md` and `AGENTS.md` in full against the project as it stands after this phase — fixing every inconsistency found, **including pre-existing ones not caused by this phase**. +- Tasks update `README.md`/`AGENTS.md` immediately when their own changes are user-visible (the smart-pass bounds are a documented contract of `host_agent.py`). + +## Completion Criteria + +- [ ] `uv run pytest tests/test_agent.py` green, including the new scaled-budget tests and the untouched override test +- [ ] `grep -rn "20 s per pass" AGENTS.md README.md agent/ deploy/` returns nothing (the wording is floor-scaled everywhere) +- [ ] `bash .agents/validate.sh` green (pytest --cov --cov-fail-under=94, ruff, pyright, node --check) +- [ ] README.md and AGENTS.md are accurate and consistent with the whole project (verified in the final pass) +- [ ] no behavior change in completed work: the deployed agent on `driveripper` is byte-identical to the repo's `agent/host_agent.py` (md5), the live suite is green diff --git a/.agents/phases/todo/16_agent_smart_pass_budget/01_pin_the_scaled_budget.md b/.agents/phases/todo/16_agent_smart_pass_budget/01_pin_the_scaled_budget.md new file mode 100644 index 0000000..c552ede --- /dev/null +++ b/.agents/phases/todo/16_agent_smart_pass_budget/01_pin_the_scaled_budget.md @@ -0,0 +1,26 @@ +# Task 01 — Pin the scaled smart-pass budget + +**Phase:** `16_agent_smart_pass_budget` · **Story:** n/a + +## Objective + +The default smart-pass budget (20 s floor, scaled to 5 s × disk count) is proven by tests: the deadline actually moves with the disk count, the exhaustion error line names the *scaled* figure, and the existing override-wins branch (a test sets `smart_budget_seconds = 0.0`) keeps passing untouched. The code change is already in `agent/host_agent.py::_smart_pass` — this task adds the tests, it does not rewrite the code. + +## Work + +1. `tests/test_agent.py` — add a helper that builds the in-process collector with **default** smart attributes (do not touch `smart_budget_seconds`/`smart_timeout_seconds`) and a `smartctl` fake that records each device's deadline window, in the same style as `test_the_pass_budget_skips_the_rest_with_one_error` (which stays untouched): + - `test_the_pass_budget_scales_with_the_disk_count`: 16 device names → the last device's `smartctl` run happens with `left` (its deadline margin) consistent with a ~80 s budget (assert the deadline the code computes: `max(20, 5 × 16)`; the fake binary is instant, so assert via the recorded `min(left, 5.0)` timeout being 5.0 for a late device and the pass *not* truncating — every device got a `smartctl` run); 1 device name → budget 20 s (assert the recorded timeout is 5.0 and, with the fake made slow enough to exhaust a 20 s but not an 80 s budget in a *separate* case if feasible — if the harness cannot fake wall time, assert the computed `budget` expression via the error-line route instead: force exhaustion with a `time.monotonic` monkeypatch offset). + - `test_the_pass_budget_error_line_names_the_scaled_figure`: force exhaustion on 16 devices (monkeypatch the clock forward past the 80 s deadline between two device runs, as the existing test's `0.0` override does for its 0 s budget) → the error line reads `smart: pass budget of 80s exhausted, N disk(s) skipped` — the scaled figure, not the 20 s floor. +2. `agent/host_agent.py` — **no code change expected.** If a test exposes a real bug in the already-applied change (e.g. the `== SMARTCTL_BUDGET_SECONDS` default check), fix it minimally and record the diff in the task's completion note; do not redesign. + +## Testing & Quality + +- Unit: the two new tests above + the untouched `test_the_pass_budget_skips_the_rest_with_one_error` (override wins: `0.0` still exhausts immediately on 3 devices with one error line) +- Coverage: **>90%** on `agent/host_agent.py::_smart_pass`'s budget branch (the default-scaled path must now be exercised, not just the override) + +## Completion Criteria + +- [ ] `uv run pytest tests/test_agent.py -q` green: the two new tests, the untouched override test, and every pre-existing agent test +- [ ] the 80 s case asserts the *scaled* figure in the error line (a test that passed with a 20 s budget would fail on the scaled code — no tautological assertions) +- [ ] `uv run ruff check agent tests` and `uv run pyright` clean +- [ ] no behavior change in completed work: `agent/host_agent.py` unchanged unless a real bug was found (then the fix is minimal and recorded) diff --git a/.agents/phases/todo/16_agent_smart_pass_budget/02_docs_and_gate.md b/.agents/phases/todo/16_agent_smart_pass_budget/02_docs_and_gate.md new file mode 100644 index 0000000..f216b7c --- /dev/null +++ b/.agents/phases/todo/16_agent_smart_pass_budget/02_docs_and_gate.md @@ -0,0 +1,25 @@ +# Task 02 — Docs and gate for the scaled budget + +**Phase:** `16_agent_smart_pass_budget` · **Story:** n/a + +## Objective + +Every place that still says the smart pass is "20 s per pass" says the floor-scaled truth (20 s floor, 5 s × disk count), and the phase gate is green. The agent's module docstring was already updated on 2026-10-19 to the scaled wording — this task verifies it and fixes the two remaining doc lines. + +## Work + +1. `AGENTS.md` line ~185 (the `host_agent.py` bullet): `5 s per device / 20 s per pass / 240 s cache` → `5 s per device / a pass budget of 20 s scaled to 5 s × disk count / 240 s cache` (keep the bullet's voice; one line). +2. `README.md` line ~94: `20 s per pass, results cached 4 minutes` → `a pass budget of 20 s scaled to the disk count (5 s each), results cached 4 minutes` (one line). +3. `agent/host_agent.py` module docstring — verify it reads `bounded 5 s per device and a pass budget of 20 s scaled to the disk count, cached 240 s`; fix only if it drifted. +4. `bash .agents/validate.sh` — the four checks (pytest --cov --cov-fail-under=94, ruff, pyright, node --check) + repo hygiene, all `uv run`. + +## Testing & Quality + +- No new logic in this task; the gate run is the check. +- Coverage: **>90%** gate holds (task 01's tests carry the new branch). + +## Completion Criteria + +- [ ] `grep -rn "20 s per pass" AGENTS.md README.md agent/ deploy/` returns nothing +- [ ] `bash .agents/validate.sh` green +- [ ] the two edited doc lines read the floor-scaled truth and nothing else changed in either file diff --git a/.agents/phases/todo/17_installer_device_filter/00_phase.md b/.agents/phases/todo/17_installer_device_filter/00_phase.md new file mode 100644 index 0000000..4bc80e5 --- /dev/null +++ b/.agents/phases/todo/17_installer_device_filter/00_phase.md @@ -0,0 +1,59 @@ +# Phase 17 — The installer's broken device filter and the kernel-gate variant + +**Source:** chat 2026-10-19 — the `driveripper` install: the template's +`DevicePolicy=strict` + `DeviceAllow` lines produced a **deny-all device filter** on +kernel 7.2 (systemd resolves `DeviceAllow` globs by name against `/proc/devices`, where +this kernel lists NVMe as `blkext` (major 259), so `nvme*` matched nothing and the whole +device namespace was locked). The operator removed the lines by hand; the unit now also +runs `User=root` + `CapabilityBoundingSet=CAP_SYS_RAWIO CAP_SYS_ADMIN` (kernel 7.2's +unprivileged-passthrough gate — `data/logs/NOTES-driveripper-agent.md`). The template +still ships the broken lines, `User=$AGENT_USER`, and an empty bounding set, and re-running +the installer on `driveripper` would **clobber the working unit** (unconditional +`cat > "$UNIT"`). This phase makes the template the single source of the working unit. +**Story:** n/a +**Context:** `deploy/install-agent.sh` (`do_install` — the unconditional unit heredoc with `DevicePolicy=strict` + nine `DeviceAllow` lines, the `PrivateDevices=false` rationale comment block, `User=$AGENT_USER`, `CapabilityBoundingSet=`, the `--bind/--port/--token/--rotate-token/--status/--uninstall` flag set), `tests/test_deploy_scripts.py` (the fake-root install sandbox `test_install_path_installs_into_its_own_paths_and_prints_the_token_once`, the token-file permission test, the `bash -n` + function-naming pins), `AGENTS.md` (the fleet bullet "The deployed unit has since diverged from the template in the three ways the SMART block below records", gap 10's "the template change is its own phase"), `README.md` (the install section, line ~46 `sudo deploy/install-agent.sh --bind --port 9120`) + +## Objective + +Re-running `deploy/install-agent.sh` on any host — including `driveripper`, with +`--as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN --force` — produces the working unit: no +`DevicePolicy`/`DeviceAllow` lines (device access falls back to DAC via the `disk` group ++ SELinux, the standard SMART-monitoring arrangement), the kernel-gate variant expressible +by flags, and a re-run that would change an existing unit prints the diff and refuses +without `--force`. The default install (dedicated user, empty bounding set) is otherwise +byte-identical to what the template wrote before this phase. + +## Dependencies + +- `09_disks_smartctl` (complete) — the smart pass the unit exists to serve +- `15_agent_mtls` (todo) — **shared-file, run-order note**: 15's task 06 adds `--cert-bundle`/`RuntimeDirectory` to the same `do_install`; apply on top of the file state at the time it runs, never restore a line the other phase owns. If 15 lands first, its new lines survive this phase's filter-line removal untouched. + +## Tasks + +1. `01_drop_the_broken_device_filter.md` — remove `DevicePolicy=strict` + all nine `DeviceAllow` lines and rewrite the rationale comment block +2. `02_root_and_caps_flags.md` — `--as-root` and `--caps CAP1,CAP2` write the kernel-gate unit lines +3. `03_no_silent_clobber.md` — a re-run that would change an existing unit prints the diff and refuses without `--force` +4. `04_tests_docs_gate.md` — the unit pins in `tests/test_deploy_scripts.py`, `AGENTS.md`/`README.md` updated, gate green + +## Testing & Quality + +- Integration: `tests/test_deploy_scripts.py` — the template unit carries **no** `DevicePolicy`/`DeviceAllow` line; a default-install sandbox unit is byte-identical to the pre-phase template unit minus the filter lines; `--as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN` writes `User=root` and `CapabilityBoundingSet=CAP_SYS_RAWIO CAP_SYS_ADMIN` exactly; the re-run guard refuses (exit 1, diff on stdout) and overwrites with `--force`; `bash -n` and the existing function-naming pin still pass +- Live (task 04, operator-verified): `driveripper`'s deployed unit is reproduced by `sudo deploy/install-agent.sh --bind 10.1.0.126 --port 9120 --as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN --force` — `systemctl diff` empty, `CapEff` still `0000000000220000`, 16/16 drives still live; `deskwork`/`gateway` untouched (their units are not re-run) +- Coverage: the script itself is pinned by text assertions + `bash -n` (house pattern; >90% applies to the new Python test code) +- The phase is "Complete" only when the **entire** project test suite passes 100% (every test in the project, not just this phase's) + +## Documentation & Consistency + +- The final pass reviews `README.md` and `AGENTS.md` in full against the project as it stands after this phase — fixing every inconsistency found, **including pre-existing ones not caused by this phase**. +- Tasks update `README.md`/`AGENTS.md` immediately when their own changes are user-visible (the new flags are operator-facing; gap 10's "the template change is its own phase" becomes "done in phase 17" with the `driveripper` flag line). + +## Completion Criteria + +- [ ] `grep -n "DevicePolicy\|DeviceAllow" deploy/install-agent.sh` returns nothing (the `PrivateDevices=false` line stays) +- [ ] `sudo deploy/install-agent.sh --help` names `--as-root` and `--caps` with the kernel-gate sentence +- [ ] a fresh sandbox install (fake root, no flags) produces a unit byte-identical to the pre-phase template unit minus the filter lines (pinned in `tests/test_deploy_scripts.py`) +- [ ] a re-run that would change an existing unit exits 1 with the diff on stdout; with `--force` it overwrites +- [ ] operator-verified on `driveripper`: the flags reproduce the working unit (`systemctl diff` empty, 16/16 drives live after restart) +- [ ] `uv run pytest` green, `bash .agents/validate.sh` green +- [ ] README.md and AGENTS.md are accurate and consistent with the whole project (verified in the final pass) +- [ ] no behavior change in completed work: the default install's unit (minus the filter lines) and every existing flag behave as before; the live suite is green diff --git a/.agents/phases/todo/17_installer_device_filter/01_drop_the_broken_device_filter.md b/.agents/phases/todo/17_installer_device_filter/01_drop_the_broken_device_filter.md new file mode 100644 index 0000000..0c3e4ce --- /dev/null +++ b/.agents/phases/todo/17_installer_device_filter/01_drop_the_broken_device_filter.md @@ -0,0 +1,25 @@ +# Task 01 — Drop the broken device filter + +**Phase:** `17_installer_device_filter` · **Story:** n/a + +## Objective + +The template unit no longer carries `DevicePolicy=strict` or any `DeviceAllow` line, and the comment block that justified them is rewritten to say why the *absence* is the fix. The `PrivateDevices=false` line stays (explicit, default behaviour, and the rest of the lockdown is unchanged). + +## Work + +1. `deploy/install-agent.sh` — in the `do_install` unit heredoc: delete the `DevicePolicy=strict` line and all nine `DeviceAllow=…` lines (`/dev/null`, `/dev/zero`, `/dev/full`, `/dev/random`, `/dev/urandom`, `/dev/nvme*`, `/dev/sd*`, `/dev/vd*`, `/dev/sg*`). +2. `deploy/install-agent.sh` — rewrite the comment block above `PrivateDevices=false` (the "The pseudo nodes are not decoration…" paragraph) to the new truth: **no device filter on purpose** — systemd resolves `DeviceAllow` globs by name against `/proc/devices` (fnmatch in `bpf_devices_allow_list_major`), where kernel 7.2 lists NVMe as `blkext` (major 259), so `nvme*` matched nothing and `DevicePolicy=strict` denied the *whole* device namespace (the `driveripper` install went deny-all, root included; `data/logs/NOTES-driveripper-agent.md`). Without the policy, device access falls back to DAC (the `disk` group the script already grants) + SELinux — the standard SMART-monitoring arrangement. Keep the one operational fact that survives: `PrivateDevices=false` is the default (a `true` sandbox with no `DeviceAllow` list is what dies at spawn with `status=208/STDIN` — keep that warning sentence, it is still why the line is written explicitly). +3. `deploy/install-agent.sh` — the script's top-of-file header comment (line ~25: "the unit device-allows the disk nodes") → "the unit applies no device filter (kernel 7.2's `/proc/devices` name matching made the old `nvme*` glob deny-all); device access is DAC via the `disk` group + SELinux". + +## Testing & Quality + +- Integration: `bash -n deploy/install-agent.sh` clean (the existing pin covers it); a quick sandbox run (the `test_install_path_installs_into_its_own_paths_and_prints_the_token_once` pattern, run manually) shows no `DevicePolicy`/`DeviceAllow` in the written unit — the formal pins are task 04, this task keeps the change small and reviewable. +- Coverage: **>90%** n/a (shell script; the pins are text assertions per the house pattern). + +## Completion Criteria + +- [ ] `grep -n "DevicePolicy\|DeviceAllow" deploy/install-agent.sh` returns nothing +- [ ] `bash -n deploy/install-agent.sh` clean +- [ ] a manual sandbox install (fake root, `--bind 127.0.0.1 --port 9199`) writes a unit with the filter lines gone, `PrivateDevices=false` present, and every other line byte-identical to the pre-phase template unit +- [ ] the rewritten comment names the `/proc/devices`/`blkext` mechanism in one sentence (an executor reading only the script can understand *why* there is no filter) diff --git a/.agents/phases/todo/17_installer_device_filter/02_root_and_caps_flags.md b/.agents/phases/todo/17_installer_device_filter/02_root_and_caps_flags.md new file mode 100644 index 0000000..6f570f2 --- /dev/null +++ b/.agents/phases/todo/17_installer_device_filter/02_root_and_caps_flags.md @@ -0,0 +1,27 @@ +# Task 02 — `--as-root` and `--caps` flags + +**Phase:** `17_installer_device_filter` · **Story:** n/a + +## Objective + +The kernel-gate variant of the unit (what `driveripper` runs) is expressible from the template: `--as-root` writes `User=root` instead of `User=$AGENT_USER`, and `--caps CAP1,CAP2` (comma-separated, e.g. `--caps CAP_SYS_RAWIO,CAP_SYS_ADMIN`) writes `CapabilityBoundingSet=CAP1 CAP2` instead of the empty `CapabilityBoundingSet=` line. Both default off; a flag without its friend is refused with a message. + +## Work + +1. `deploy/install-agent.sh` — flag parsing: add `--as-root` (sets `AS_ROOT=1`) and `--caps ` (sets `CAPS=""`, validates non-empty, rejects an empty list with exit 1) in the existing `case`/flag block, and `--help` text: + - `--as-root run the service as root (for a kernel-gated host: kernel 7.2 gates unprivileged smartctl passthrough behind capabilities, not uid — a root process under the default empty bounding set is capability-less, so this flag needs --caps to mean anything)` + - `--caps LIST CapabilityBoundingSet line for the unit (comma-separated, e.g. CAP_SYS_RAWIO,CAP_SYS_ADMIN); default: empty (capability-less service)` +2. `deploy/install-agent.sh` — validation: `--as-root` without `--caps` → print "note: --as-root with the default empty CapabilityBoundingSet leaves the process capability-less (the kernel gates check capabilities, not uid); pair it with --caps for a kernel-gated host" and **still proceed** (a capability-less root is a legitimate, if unusual, choice — do not hard-refuse; the note is the operator's reminder). `--caps` without `--as-root` → refuse, exit 1 ("--caps needs --as-root: a capability grant on the service user is not the kernel-gate fix"). +3. `deploy/install-agent.sh` — the unit heredoc: `User=$AGENT_USER` → `User=$( [ "$AS_ROOT" -eq 1 ] && echo root || echo "$AGENT_USER" )`; the empty `CapabilityBoundingSet=` line → the `CAPS` list converted comma→space when set. Add one comment line above: `# --caps: kernel 7.2 gates unprivileged smartctl passthrough (NVMe admin log: CAP_SYS_ADMIN; SG_IO allow-list: CAP_SYS_RAWIO); a root process needs them named, because an empty bounding set bounds even root to CapEff 0`. +4. `deploy/install-agent.sh` — the `--status` probe is unchanged (it probes the agent over HTTP; the user the unit runs as is invisible to it). The token file stays `0640 root:$AGENT_USER` — with `--as-root` the group read is irrelevant (root reads it), so do **not** change the token-file logic. + +## Testing & Quality + +- Integration: manual sandbox runs (fake root): default → `User=dashboards-agent` + empty `CapabilityBoundingSet=`; `--as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN` → `User=root` + `CapabilityBoundingSet=CAP_SYS_RAWIO CAP_SYS_ADMIN`; `--caps` alone → exit 1 with the message; `--as-root` alone → proceeds with the note on stdout. (Formal pins: task 04.) +- Coverage: **>90%** n/a (shell script; text pins per the house pattern). + +## Completion Criteria + +- [ ] `sudo deploy/install-agent.sh --help` names both flags with the kernel-gate sentence (a fresh clone's executor can learn the `driveripper` recipe from `--help` alone) +- [ ] the two sandbox units match the exact lines above (byte-for-byte on the `User=` and `CapabilityBoundingSet=` lines) +- [ ] `bash -n deploy/install-agent.sh` clean; every pre-existing flag (`--bind/--port/--token/--rotate-token/--status/--uninstall`) behaves as before in a sandbox run diff --git a/.agents/phases/todo/17_installer_device_filter/03_no_silent_clobber.md b/.agents/phases/todo/17_installer_device_filter/03_no_silent_clobber.md new file mode 100644 index 0000000..f9b8fd1 --- /dev/null +++ b/.agents/phases/todo/17_installer_device_filter/03_no_silent_clobber.md @@ -0,0 +1,27 @@ +# Task 03 — No silent clobber on re-run + +**Phase:** `17_installer_device_filter` · **Story:** n/a + +## Objective + +A re-run that would change an already-installed unit stops: it prints the diff between the installed unit and the unit the script would write, and exits 1 without touching the file. `--force` proceeds with the overwrite. A re-run whose generated unit is identical to the installed one proceeds silently (idempotent re-runs stay the zero-noise path the script has always had). + +## Work + +1. `deploy/install-agent.sh` — add a `--force` flag to the flag block (`FORCE=1`) and `--help` text: `--force overwrite an existing unit even if the generated unit differs (prints the diff first)`. +2. `deploy/install-agent.sh` — in `do_install`, immediately before the `cat > "$UNIT"` heredoc write: build the would-be unit into a temp file (the heredoc already exists — render it once to `$(mktemp)`), and if `"$UNIT"` exists and `! cmp -s "$UNIT" "$tmp_unit"`: + - if `FORCE=1`: print the diff (`diff -u "$UNIT" "$tmp_unit"` on stdout) and proceed with the overwrite; + - else: print `refusing to overwrite $UNIT (generated unit differs); run with --force to apply:` + the same diff, and `exit 1` **before** the token-file write, the `systemctl daemon-reload`, and the `systemctl enable --now` — a refused run must leave the host exactly as it was (token file, unit, and running service all untouched; note the token-reuse logic runs earlier in `do_install` and only *reads* the existing token, so ordering the guard before the unit write is enough). + - identical unit → no output, normal proceed. +3. `deploy/install-agent.sh` — `--uninstall` is unchanged (it already removes the unit deliberately; the guard applies to `do_install` only). + +## Testing & Quality + +- Integration: manual sandbox sequence (fake root): install → modify the installed unit by one line → re-run with the same flags → exit 1, diff on stdout, installed unit byte-identical to the modified version, service still the old one; re-run with `--force` → diff printed, unit overwritten; re-run again unchanged → silent, exit 0. (Formal pins: task 04.) +- Coverage: **>90%** n/a (shell script; text/behaviour pins per the house pattern). + +## Completion Criteria + +- [ ] the three-way sequence above (refuse → force → silent) holds in the sandbox, with the token file and the running service untouched by a refused run +- [ ] `--force` prints the diff *before* overwriting (an operator sees exactly what changed) +- [ ] `bash -n deploy/install-agent.sh` clean; a fresh install (no existing unit) is unaffected by the guard diff --git a/.agents/phases/todo/17_installer_device_filter/04_tests_docs_gate.md b/.agents/phases/todo/17_installer_device_filter/04_tests_docs_gate.md new file mode 100644 index 0000000..6ac720e --- /dev/null +++ b/.agents/phases/todo/17_installer_device_filter/04_tests_docs_gate.md @@ -0,0 +1,34 @@ +# Task 04 — Pins, docs, gate + +**Phase:** `17_installer_device_filter` · **Story:** n/a + +## Objective + +The template's new shape is pinned in `tests/test_deploy_scripts.py` (no filter lines, the flags' exact lines, the clobber guard's refuse/force/silent sequence), `AGENTS.md` and `README.md` name the `driveripper` recipe, and the gate is green. The operator-verified `driveripper` reproduction (the completion criterion in `00_phase.md`) is the last check. + +## Work + +1. `tests/test_deploy_scripts.py` — extend the existing sandbox pattern: + - `test_the_template_unit_has_no_device_filter_lines` — read `deploy/install-agent.sh`'s unit heredoc region and assert no line starts with `DevicePolicy=` or `DeviceAllow=`, and `PrivateDevices=false` is still present (the filter's absence is the fix; a regression that re-adds `DeviceAllow=/dev/nvme*` is the deny-all returning). + - `test_a_default_install_unit_is_the_pre_phase_unit_minus_the_filter_lines` — the sandbox-installed unit equals the known pre-phase template unit text minus the ten filter lines (pin the whole unit, so a future template line drift fails here). + - `test_as_root_and_caps_write_the_kernel_gate_unit_lines` — `--as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN` → `User=root` and `CapabilityBoundingSet=CAP_SYS_RAWIO CAP_SYS_ADMIN` exactly; `--caps` alone → exit 1 with the `--as-root` message; `--as-root` alone → proceeds with the capability-less note on stdout. + - `test_a_re_run_that_differs_refuses_without_force_and_applies_with_it` — the refuse/force/silent sequence (exit 1 + diff on stdout + installed unit untouched; `--force` overwrites with the diff printed; an unchanged re-run is silent). +2. `AGENTS.md` — the fleet bullet's "re-running `deploy/install-agent.sh` on driveripper would clobber the working unit, because it rewrites the unit unconditionally (gap 10)" → the post-phase truth: the installer no longer writes the device-filter lines, and the `driveripper` unit is reproducible with `sudo deploy/install-agent.sh --bind 10.1.0.126 --port 9120 --as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN --force` (a re-run that would change an existing unit refuses without `--force`). Update gap 10's "the template change is its own phase" → "done in phase 17" with the flag line. +3. `README.md` — the install section: one line naming the flags' purpose (the kernel-gated-host recipe), keeping the how-to voice. +4. `bash .agents/validate.sh` — all four checks green. +5. **Operator-verified (live, read-only + one restart):** on `driveripper`, `sudo deploy/install-agent.sh --bind 10.1.0.126 --port 9120 --as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN --force` → `systemctl diff dashboards-agent` empty, `systemctl is-active` `active`, `CapEff` still `0000000000220000`, and 16/16 drives still live in the next `/snapshot` (the restart is the only fleet action; `deskwork`/`gateway` are not re-run). Screenshot `data/logs/disks-17-installer-repro.png` per invariant 10. + +## Testing & Quality + +- Integration: the four new pins above + the pre-existing pins (sudoers, `bash -n`, function naming, token-file permissions) all green. +- Coverage: **>90%** on the new test code; the 94% gate holds. +- Live: the operator-verified reproduction (item 5) — the phase's `driveripper` criterion. + +## Completion Criteria + +- [ ] the four new pins pass and every pre-existing `test_deploy_scripts.py` pin passes +- [ ] `grep -n "DevicePolicy\|DeviceAllow" deploy/install-agent.sh` returns nothing +- [ ] `AGENTS.md` fleet bullet + gap 10 and `README.md` name the `driveripper` flag recipe +- [ ] the operator-verified reproduction: `systemctl diff` empty, 16/16 drives live, screenshot in `data/logs/` +- [ ] `uv run pytest` green, `bash .agents/validate.sh` green +- [ ] the live suite (`set -a; . ./.env.dev; set +a; uv run pytest -m live --live`) is green diff --git a/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.err b/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.md b/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.md new file mode 100644 index 0000000..f0b7bef --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.md @@ -0,0 +1,9 @@ +**Phase 14 `store_performance` — final verification pass (all 4 tasks already in `complete/`)** + +- Re-verified every task's work in the tree: read caches (`series_names`/`stats`), capped compound `pick_res_many`, `history_many` + `api._history_batch`, windowed `maintain()` + per-resolution prune + `PRAGMA optimize`/`automatic_analyze=0`/WAL pragmas, `store_perf.BUDGETS` (14 ceilings), probe, live watch. +- **Completion criteria:** baseline+after side by side in `data/logs/NOTES-store-perf.md` ✅ · budget tests green (`uv run pytest -m store_perf --store-perf` → **4 passed**, 14/14 inside) ✅ · 25-min dev-server watch re-run **independently** (74 passes, `scripts/watch-store-perf.py`): **0 `store-error`** (total still 190, newest 2026‑10‑18 06:33:30), WAL 7.77 MB ≤ 67.1 MB, medians 1.7–3.2 ms, p95 720/613 ms = the documented cold-cache window; a 3rd 27-pass watch on a freshly restarted server reproduced it (3.1/708, 1.8/572) ✅ · `maintain()` 5,368 ms → 2.7–6.8 ms and WAL bounded across two cycles (test + notes) ✅ · no behaviour change (all four history routes pinned against `query()`) ✅ +- **Full gates:** `uv run pytest` → **1267 passed, 15 skipped**; `bash .agents/validate.sh` → **exit 0**, coverage **98.75%** (floor 94), `ruff check src tests agent` clean, `pyright` 0 errors, `node --check` ok, repo hygiene ok; `uv run pytest -m live --live` → **11 passed**; dev server up, 9/9 sources `ok`, pages 200 in 1.5–3.3 ms. +- **Docs fixed:** README/AGENTS still claimed a flat "20 s per pass" while `host_agent.py` (in this same uncommitted change) scales the budget to 5 s × disk count — corrected in both plus the two illustrative "20 s of wedged disk" sentences; broken bold (`` `driveripper** ``) repaired; stale gap-4 title + "tuning is its own phase" wording updated to say phase 14 ran; README's absolute "nothing … is answered by a whole-table scan" qualified to *per call* with the cold-window cost named; "third opt-in marker" → second; conftest `--store-perf` help named the real marker; **`data/logs/watch-store-perf.py` moved to `scripts/watch-store-perf.py`** (both docs told a clone to run a gitignored path) with root-anchored paths, `DASH_WATCH_OUT`, and dead code removed. +- **New finding recorded:** gap 9's "~10 s process-wide stall is still gap 4's / `14_store_performance`'s problem" is now closed — coincident >8 s `llm` + >12 s `ups` raw-stream gaps went 393 (00:00–12:00) and 198 (12:00–21:36) to **0** across 110+ min of the phase-14 store; written into AGENTS.md gaps 4 and 9 and the notes table. +- **Deviations:** did *not* pull either deferred cost knob (`STATS_CACHE_TTL_SECONDS`, warming caches from the maintenance tick) — AGENTS.md records that decision and its reasoning; `uv pip list --outdated` still reports certifi/fastapi/starlette/websockets (pre-existing advisory, no dependency touched this phase, gate treats it as advisory). Overwrote nothing: task 04's watch preserved as `store-perf-watch.task04.*`, my two runs as `.verify25.*` / `.current-code.*`. +- **Next pending phase:** `15_agent_mtls`, then `16_agent_smart_pass_budget` (owns the scaled-budget tests + its doc criterion, now partly pre-satisfied by this pass), `17_installer_device_filter`. diff --git a/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.md.status b/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.validate b/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.validate new file mode 100644 index 0000000..f54f5e7 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__00_phase.a1.validate @@ -0,0 +1,64 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 16%] +........................................................................ [ 22%] +.............................................s.......................... [ 28%] +........................................................................ [ 33%] +........................................................................ [ 39%] +........................................................................ [ 44%] +........................................................................ [ 50%] +........................................................................ [ 56%] +........................................................................ [ 61%] +........................sssssssssss..................................... [ 67%] +........................................................................ [ 73%] +........................................................................ [ 78%] +........................................................................ [ 84%] +........................................................................ [ 89%] +........................................................................ [ 95%] +sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 356 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3047 38 99% +Required test coverage of 94% reached. Total coverage: 98.75% +1267 passed, 15 skipped, 1 warning in 110.62s (0:01:50) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.19 wheel +fastapi 0.143.5 0.150.0 wheel +starlette 1.8.0 1.9.0 wheel +websockets 17.2 17.3 wheel diff --git a/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.err b/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.md b/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.md new file mode 100644 index 0000000..3fd61b7 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.md @@ -0,0 +1 @@ +Now let me write the baseline NOTES document. diff --git a/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.md.status b/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.validate b/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.validate new file mode 100644 index 0000000..948d626 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__01_baseline_store_costs.a1.validate @@ -0,0 +1,63 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +....................sssssssssss......................................... [ 71%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 89%] +........................................................................ [ 95%] +...s....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 348 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 188 2 99% 255-256 +------------------------------------------------------------------ +TOTAL 2914 38 99% +Required test coverage of 94% reached. Total coverage: 98.70% +1199 passed, 12 skipped, 1 warning in 117.25s (0:01:57) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.149.0 wheel +starlette 1.8.0 1.9.0 wheel +websockets 17.2 17.3 wheel diff --git a/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.err b/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.md b/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.md new file mode 100644 index 0000000..c006a95 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.md @@ -0,0 +1,17 @@ +**Task 02 — no full table scan on a per-request path: done, all green.** + +**Implemented** +- `store.py`: `series_names()` cached 60 s (invalidated by `add`/`add_many`/`delete_series`/`maintain` via a generation counter; bounded to 256 `(source,host)` keys since `/api/metrics` takes both as free text); `stats()` counts cached 30 s with `last_maintain`/`size_bytes` deliberately never cached; `invalidate_read_caches()` for cold probes. +- `pick_res()` now one compound statement of per-`res` `COUNT`s capped by `LIMIT max_points+1` (exact: the decision only asks `0` / `≤max_points` / neither), plus new `pick_res_many()` and `history_many()` (chunked at `COMPOUND_CHUNK`, index-range seek per branch). +- `api.py`: all four history routes go through `_history_batch()` → `store.history_many()`; response dicts unchanged. +- `tests/store_perf.py`: stats/`series_names` measured **cold and cached**, batch-vs-loop measured at 27 series × 24 h/90 d, EXPLAIN of the statement that actually runs. +- Fixed a task-01 defect found in the full suite: the seeded determinism test pinned `SUM(value)`, whose float accumulation order changes when `INSERT OR REPLACE` moves rowids — now counts/windows exact, sum `rel=1e-9`, plus an exact PK-ordered row slice. +- Docs: `AGENTS.md` (store.py/api.py bullets, gap 4 split into read-half-closed, commands), `README.md` (store.py, api.py, store_perf, new test pins), `data/logs/NOTES-store-perf.md` task-02 section (before/after, rejected shapes, EXPLAIN, live curl proof). + +**Measured** (`uv run pytest -m store_perf --store-perf`): HTTP `/api/health` median 1 272 ms → **1.2 ms**; `/api/metrics?source=host` 1 415 → **1.6 ms**; `/api/system/history` 3×9 at 90 d 100.3 → **49.8 ms**; `pick_res()`×3 3.0 → **0.3 ms**. Live dev server on the real 636 MB store: `/api/health` 0.700 s → **1.5 ms**, all nine sources `ok`. + +**Tests/lint** (exact): `bash .agents/validate.sh` → pytest `--cov --cov-fail-under=94` **1214 passed, 12 skipped, coverage 98.73%**; `ruff` All checks passed; `pyright` 0 errors; `node --check` ok. Final confirmation run `uv run pytest -q --cov --cov-fail-under=94` → **EXIT=0**, 0 FAILED (log `data/logs/pytest-final2.log`). No dependencies touched (validate's outdated-dep line is pre-existing advisory). + +**Deviations**: batched the resolution *decision*, not the row fetch — `ROW_NUMBER()` window (73 ms) and a compound of the 27 tail selects (38 ms) both measured slower than 27 index seeks (27 ms) on the seeded store, and task 01 requires the numbers not to regress; recorded with its table in the notes. The literal `SELECT res, COUNT(*) … GROUP BY res` shape was also measured slower (4.3 vs 2.7 ms) than the capped compound that shipped. No live-infra changes beyond `./scripts/dev-server.sh restart` (local dev, invariant 10). + +**Next pending task**: `.agents/phases/todo/14_store_performance/03_windowed_rollups_and_wal_tuning.md`. diff --git a/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.md.status b/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.validate b/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.validate new file mode 100644 index 0000000..6ed9c32 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__02_no_full_scan_on_hot_paths.a1.validate @@ -0,0 +1,64 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 52%] +........................................................................ [ 58%] +........................................................................ [ 64%] +.....................sssssssssss........................................ [ 70%] +........................................................................ [ 76%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 93%] +..................s..................................................... [ 99%] +.. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 356 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 266 2 99% 445-446 +------------------------------------------------------------------ +TOTAL 3000 38 99% +Required test coverage of 94% reached. Total coverage: 98.73% +1214 passed, 12 skipped, 1 warning in 122.10s (0:02:02) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.149.0 wheel +starlette 1.8.0 1.9.0 wheel +websockets 17.2 17.3 wheel diff --git a/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.err b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.md b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.md new file mode 100644 index 0000000..446d026 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.md @@ -0,0 +1,22 @@ +**Phase 14 / Task 03 — windowed rollups + WAL tuning: done.** + +Implementation (`src/dashboards/store.py`) +- `maintain()` now windows both rollup steps off `last_maintain` (read via `get_meta()` *outside* the lock), overlapped by one bucket + `MAINTAIN_SLACK_SECONDS=60` and **snapped down to a bucket boundary** — a floor that cut into a bucket re-rolled a partial average from its tail rows (2.5 → 4.5); missing/non-numeric row falls back to full retention with a fixed-line warning. +- Prune keeps its **full retention bound** but is now one `DELETE` per resolution: the old single `OR` of the three was unplannable (`SCAN sample`, 554 ms of a 562 ms pass while deleting nothing). +- Statistics are the other half of the cost bound: `maintain()` ends with `PRAGMA optimize`, `__init__` sets `automatic_analyze=0` (SQLite's own refresh lands inside a poller's commit). WAL: `wal_autocheckpoint=1000`, `journal_size_limit=64 MB`; `busy_timeout=5000` deliberately unchanged. +- Measured (seeded 6.4 M-row store): `maintain()` **5,368 ms → 2.7 ms** steady, 48 ms ten-minute catch-up; race at the app's cadence **100/100 inserts, 0 `database is locked`, max wait 21 ms** where the pre-fix control still waits 4.76 s; WAL 38 MB-forever → 0–4 MB. Live store: WAL 52.3 → 5.9 MB, `sqlite_stat1` created, real-store passes ~40 ms, `/api/health` 3.5 ms. + +Tests / gate +- `tests/test_store.py`: +9 tests (window bounds from a trace callback, catch-up-not-a-gap, windowed-vs-full-window row-for-row equivalence, re-roll only gains rows, corrupt-row fallback, `maintain` leaves statistics, optimize failure is a log line not a failed tick, WAL pragmas, two-cycle WAL cap). +- `tests/store_perf.py`: write passes and races moved to a `scratch_copy()` (baseline never moves), added `RACE_MAINTAIN_GAP_SECONDS` cadence races + the whole-window **control**, per-window plans derived from `ROLLUP_INSERT`. +- `bash .agents/validate.sh`: **1240 passed, 12 skipped**, coverage **98.75%** (floor 94), ruff + pyright + `node --check` clean; `-m store_perf --store-perf` 1 passed; dev server restarted, all 9 sources `ok`. + +Docs +- `AGENTS.md` (store.py bullet incl. the bucket-alignment rule, known gap 4 → write half closed, Commands, new testing-policy rule "a cost claim is pinned as a shape and a control"), `README.md` (store.py + `store_perf` paragraphs), `data/logs/NOTES-store-perf.md` (full task-03 tables, race-shape key, no-read-regression table, live smoke check). + +Deviations +- The task's "windowing + WAL tuning" needed two extra fixes to actually reach the budget (prune `OR`, missing statistics); both are measured and recorded rather than assumed, and the prune's retention bound is unchanged. +- The saturated task-01 race is kept for comparability but the pass line moved to the cadence-shaped race (a maintainer looping with no gap starves a writer at *any* pass length); the control beside it proves the race can still see the failure. +- Left alone: `uv pip list --outdated` advisory (certifi/fastapi/starlette/websockets) — pre-existing, no dependencies touched. + +Next pending task: `.agents/phases/todo/14_store_performance/04_budget_gates_and_live_proof.md`. diff --git a/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.md.status b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.validate b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.validate new file mode 100644 index 0000000..175c4f1 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a1.validate @@ -0,0 +1,111 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 28%] +........................................................................ [ 34%] +........................................................................ [ 40%] +........................................................................ [ 46%] +........................................................................ [ 51%] +........................................................................ [ 57%] +........................................................................ [ 63%] +.....................sssssssssss........................................ [ 69%] +........................................................................ [ 74%] +........................................................................ [ 80%] +........................................................................ [ 86%] +........................................................................ [ 92%] +..............F.............................s........................... [ 97%] +............................ [100%] +=================================== FAILURES =================================== +_______ test_a_second_maintain_reads_only_the_window_since_the_last_run ________ + +db = + + def test_a_second_maintain_reads_only_the_window_since_the_last_run(db) -> None: + """The cost change, pinned as a bound and as behaviour. + + A bucket that filled after the previous pass is inside the next window and gets rolled. A raw + row that arrives for a bucket ten minutes behind it is not, and that bucket keeps the average + the earlier pass computed — the documented cost of a window, stated here as a passing test + rather than as a surprise. Reaching it in production costs a poller stall measured in minutes + (the poller stamps samples with its own wall clock); the alternative it replaced was + re-averaging 3.97 M rows every minute. + """ + now = time.time() + bucket_a = bucket_start(now - 10 * MINUTE) + add_series(db, "host", "h", "m", [10, 20], start=bucket_a, step=1) + db.maintain(now=now) + assert db.query("host", "h", "m", res=RES_MIN, start=bucket_a - 1, + end=bucket_a + MINUTE) == [(bucket_a, 15.0)] + + second = now + 60 + bucket_b = bucket_start(second - 2 * MINUTE) + add_series(db, "host", "h", "m", [40, 50], start=bucket_b, step=1) + add_series(db, "host", "h", "m", [1000], start=bucket_a + 30, step=1) + with query_log(db) as seen: + counts = db.maintain(now=second) + + windows = rollup_windows(seen) + # the bound: one minute (one hour) of rows plus the overlap, not 48 h of raw (not 30 d of 1m) + assert windows[RES_RAW] == pytest.approx( + (second - MINUTE - MAINTAIN_SLACK_SECONDS, bucket_start(now - MINUTE - MAINTAIN_SLACK_SECONDS))) +> assert windows[RES_MIN] == pytest.approx( + (second - HOUR - MAINTAIN_SLACK_SECONDS, bucket_start(now - HOUR - MAINTAIN_SLACK_SECONDS))) +E assert (1792449272.5... 1792447200.0) == approx((17924....0 ± 1.8e+03)) +E +E comparison failed. Mismatched elements: 1 / 2: +E Max absolute difference: 1980.0 +E Max relative difference: 1.104635048664195e-06 +E Index | Obtained | Expected +E 1 | 1792447200.0 | 1792449180.0 ± 1.8e+03 + +tests/test_store.py:802: AssertionError +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 356 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 304 2 99% 510-511 +------------------------------------------------------------------ +TOTAL 3038 38 99% +Required test coverage of 94% reached. Total coverage: 98.75% +=========================== short test summary info ============================ +FAILED tests/test_store.py::test_a_second_maintain_reads_only_the_window_since_the_last_run +1 failed, 1239 passed, 12 skipped, 1 warning in 111.30s (0:01:51) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.150.0 wheel +starlette 1.8.0 1.9.0 wheel +websockets 17.2 17.3 wheel +validation FAILED (see output above) diff --git a/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.err b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.md b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.md new file mode 100644 index 0000000..55209ae --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.md @@ -0,0 +1,19 @@ +**Phase 14 / Task 03 — re-run after the harness failure: green.** + +What was fixed +- The failing test was mine, and it was wall-clock luck, not the store: `test_a_second_maintain_reads_only_the_window_since_the_last_run` asserted the 1m→1h step's floor as a **minute**-aligned number while `_rollup_floor` snaps to an **hour**. `pytest.approx`'s relative tolerance (≈±1 800 s at epoch scale) covered the half of the hour where the two are within that, so it passed twice and failed at 19:4x. +- Fix: `WINDOW_CLOCK_OFFSETS = (0, 1, 30, 59, 1 800, 3 539, 3 599)` over a base epoch that sits exactly on both boundaries — every edge is now a case instead of a coin flip — and each bound is snapped to *its own step's* bucket; the same parametrization went on the first-pass and catch-up tests so no window assertion in this task reads `time.time()`. +- Found and fixed a second latent flake in the same test: the "due" bucket sat one step behind the ceiling, so a clock within a second of a boundary put the pair's second row outside the window and rolled 40.0 instead of 45.0 (now two steps back). `db.query()` defaults `end` to the wall clock — an explicit `end=` is now passed where the test runs on its own clock. +- Added `test_the_window_floor_is_always_aligned_to_its_own_bucket` (aligned, never above the bound, never a whole window back, and the two steps really differ). Mutation-checked: removing the `floor(…/bucket)*bucket` snap fails 6 of 7 clock cases plus that test; restored code passes. + +Results +- `bash .agents/validate.sh`: **1259 passed, 12 skipped, 0 failed** · coverage **98.75%** (floor 94, unchanged) · `ruff check src tests agent` all passed · `pyright` 0 errors · `node --check` ok · hygiene ok · no `validation FAILED`. +- `uv run pytest -m store_perf --store-perf`: **1 passed** (window plans seek, catch-up ≤ 2× steady + slack, the whole-window control still contends, the cadence race is 0 `database is locked`). +- Dev server healthy: all 9 sources `ok`, real-store WAL holding at ~7.6 MB under the 64 MB cap. + +Notable decisions +- No production-code change this pass — the defect was in the test's expectation, so the implementation (windowed rollups, per-resolution prune, `PRAGMA optimize` + `automatic_analyze=0`, WAL pragmas, `busy_timeout` untouched) stands as measured: `maintain()` 5 368 ms → 2.7 ms steady / 48 ms catch-up. +- Preferred explicit clocks and a property test over widening a tolerance, and recorded the flake plus both root causes in `data/logs/NOTES-store-perf.md` ("One window assertion was wall-clock luck"). +- Left alone: the pre-existing `uv pip list --outdated` advisory (certifi/fastapi/starlette/websockets) — no dependencies touched in this task. + +Next pending task: `.agents/phases/todo/14_store_performance/04_budget_gates_and_live_proof.md`. diff --git a/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.md.status b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.validate b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.validate new file mode 100644 index 0000000..0a77c8e --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__03_windowed_rollups_and_wal_tuning.a2.validate @@ -0,0 +1,64 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 16%] +........................................................................ [ 22%] +........................................................................ [ 28%] +........................................................................ [ 33%] +........................................................................ [ 39%] +........................................................................ [ 45%] +........................................................................ [ 50%] +........................................................................ [ 56%] +........................................................................ [ 62%] +.....................sssssssssss........................................ [ 67%] +........................................................................ [ 73%] +........................................................................ [ 79%] +........................................................................ [ 84%] +........................................................................ [ 90%] +...............................................................s........ [ 96%] +............................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 356 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 304 2 99% 510-511 +------------------------------------------------------------------ +TOTAL 3038 38 99% +Required test coverage of 94% reached. Total coverage: 98.75% +1259 passed, 12 skipped, 1 warning in 112.04s (0:01:52) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.150.0 wheel +starlette 1.8.0 1.9.0 wheel +websockets 17.2 17.3 wheel diff --git a/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.err b/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.md b/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.md new file mode 100644 index 0000000..10946e2 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.md @@ -0,0 +1,14 @@ +Task 04 is complete and everything is green. + +**What shipped** +- `tests/store_perf.py` — `BUDGETS`: 14 pinned ceilings (2‑3× the measured "after"), `budget_rows`/``budget_violations`/`missing_budget_keys`, `measure_budget_reads/_http`, and the budget table rendered into the probe report. +- Budget tests: 2 default-run pins (a budget whose measurement was renamed fails; the table is internally coherent) + 4 `store_perf` passes (reads, HTTP, `maintain`, race). `uv run pytest -m store_perf --store-perf` → 4 passed, **14/14 budgets inside**. +- **The watch found a real defect in task 02's fix** and I fixed it: the series cache was invalidated by *every* sample write, so on a store nine pollers write to it never served and `/api/metrics` paid the full `SELECT DISTINCT` scan (measured 535 ms/call; the scan alone is 572 ms on the live store). `store.py` now invalidates only the write that can change the answer (`_note_series_write` + a capped `_known_series` existence set; `DELETE` paths still invalidate unconditionally). Pinned by statement counts, not clocks, in `tests/test_store.py` and through the served endpoint in `tests/test_api.py`. +- Live proof (`data/logs/watch-store-perf.py` → `store-perf-watch.{log,tsv,json}`, `store-perf-live.png`): 25 min / 74 passes / 296 signed-in requests on the operator's 636 MB store — **0 new `store-error` rows** (190 total, newest 2026‑10‑18 06:33), **WAL 6.5 → 7.77 MB** (cap 67 MB), 0 non‑200, medians 1.8–3.3 ms for `/api/health`, `/api/summary`, `/api/metrics` and a 3×9 90‑day `/api/system/history`. +- Docs: `data/logs/NOTES-store-perf.md` gained the task‑04 section (baseline vs after side by side, EXPLAIN shapes, the watch table, the defect write-up); `README.md` and `AGENTS.md` updated (gap 4 now reads: closed, with the honest remainder). +- Gates: `bash .agents/validate.sh` exit 0 — 1267 passed, 15 skipped, coverage 98.75%, ruff + pyright + `node --check` clean. Dev server restarted, all 9 sources `ok`. + +**Honest leftovers** +1. The p95 is the cache window, not the request: on the live store a cold `series_names('host')` costs 599 ms and `stats()`' counts 729 ms, once per 60 s / 30 s (37 of 74 passes had one request >250 ms). Closing that needs a counts table or a bounded series index — task 03's note already names it, and it is a phase, not this one. +2. `uv pip list --outdated` is non-empty (fastapi 0.143.5→0.150.0, starlette, certifi, websockets) — pre-existing drift, untouched here; rule 8 wants a deliberate decision. +3. Per AGENTS rule 4 I did **not** modify `.agents/phases/todo/14_store_performance/04_...md` — say the word and I'll tick the completion criteria and move it to `complete/`. diff --git a/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.md.status b/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.validate b/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.validate new file mode 100644 index 0000000..cd52220 --- /dev/null +++ b/.agents/reports/14_store_performance/14_store_performance__04_budget_gates_and_live_proof.a1.validate @@ -0,0 +1,64 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 16%] +........................................................................ [ 22%] +.............................................s.......................... [ 28%] +........................................................................ [ 33%] +........................................................................ [ 39%] +........................................................................ [ 44%] +........................................................................ [ 50%] +........................................................................ [ 56%] +........................................................................ [ 61%] +........................sssssssssss..................................... [ 67%] +........................................................................ [ 73%] +........................................................................ [ 78%] +........................................................................ [ 84%] +........................................................................ [ 89%] +........................................................................ [ 95%] +sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 356 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3047 38 99% +Required test coverage of 94% reached. Total coverage: 98.75% +1267 passed, 15 skipped, 1 warning in 113.83s (0:01:53) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.19 wheel +fastapi 0.143.5 0.150.0 wheel +starlette 1.8.0 1.9.0 wheel +websockets 17.2 17.3 wheel diff --git a/AGENTS.md b/AGENTS.md index 283e4e3..903d172 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -181,15 +181,16 @@ Responsibilities, one line each: `offline_uncorrectable`, hex-packed vendor raws are never decoded, and an attribute row is read from its **RAW_VALUE column** rather than the end of the line because this binary annotates that column — `35 (Min/Max 34/44)`, `43894h+43m+32.519s` — so its last token is - not the value), 5 s per device / - 20 s per pass / 240 s cache, attached as `disk_io.devices[].smart`. Missing binary → + not the value), 5 s per device / a pass budget whose 20 s floor scales to 5 s × the disk count + / 240 s cache, attached as `disk_io.devices[].smart` (gap 10: a 16-drive shelf takes longer + than the floor, and truncating the pass would null the tail drives). Missing binary → **one** `errors` line for the pass (the word `not found` in it is what `sources/disks.py` reads to answer "is smartctl installed", and `tests/test_agent.py` pins that the line still says it); one denied/wedged device → that disk's `smart` is `null` with its own line; `--no-smart` opts out silently. Never fails the snapshot. **An expired cache refreshes on a background thread while the request is - served the previous result** — only the first pass after startup runs inline — because 20 s - of wedged disk on the request path blows `HOST_TIMEOUT_SECONDS` and would turn the + served the previous result** — only the first pass after startup runs inline — because a wedged + pass on the request path blows `HOST_TIMEOUT_SECONDS` and would turn the *system* source red every four minutes over an add-on that page never draws. The refresher is single-flight and keeps the old cache on failure, so a bad pass retries on the next expiry instead of hammering the disk. @@ -230,7 +231,7 @@ Responsibilities, one line each: that was dialled, because they become health records and event rows. `fetch_snapshot()` is the one agent `/snapshot` read that `system` and `disks` share, so a dead host says the same sentence on both pages and only the `timeout` differs (the disks poll has to outlast - the agent's 20 s smartctl pass, and `None` is translated into httpx's sentinel because a + the agent's smartctl pass, and `None` is translated into httpx's sentinel because a literal `None` means *no timeout at all*). Agent `errors` become the payload's `degraded` **except** the `smart:` add-on lines — a host without smartmontools or device read access is unconfigured, not broken (invariant 3); they stay @@ -397,7 +398,57 @@ Responsibilities, one line each: call, because a reset that deletes a row it did not name deletes the operator's credential). `stats()` reads `last_maintain` through `get_meta()` too — outside its own lock, which `get_meta` takes itself and `threading.Lock` is not reentrant — while `maintain()`'s write - stays inline inside that method's transaction. + stays inline inside that method's transaction. **No per-request path may pay a whole-table + scan** (phase 14 task 02): `series_names()` (1.4 s of `SCAN sample` on a production-sized store, + behind `/api/metrics`) and the two `COUNT`s behind `stats()` (1.25 s, run by `app.js`'s + `/api/health` poll on every tab every 15 s) answer from short-TTL caches — 60 s and 30 s, the + series list additionally invalidated **only by a write that can change it** (task 04: an insert + cannot remove a name, so `_note_series_write()` ends the cache on a `(source, host, metric)` the + store has never written and leaves a poll of an existing series alone — invalidating on every + sample was task 02's defect, and it put the 572 ms scan back on nearly every `/api/metrics` call + on a store nine pollers write to, behind a cache that never served; `delete_series()`/`maintain()` + invalidate unconditionally because a `DELETE` can take a name, `delete_series()` also forgetting + the triples it emptied so a re-created series counts as new again), through a generation counter so + a write racing a read cannot park a stale answer under a fresh timestamp, with the name cache + bounded to 256 `(source, host)` keys because `/api/metrics` takes both as free text and the + existence tracking capped at `SERIES_TRACKED_SERIES_MAX`, below which it falls back to + invalidating on every write. A tracking set made wrong by *another* process deleting rows costs at + most one TTL window, which is the reason the cache has a TTL and not only the tracking. `last_maintain` and `size_bytes` are never + cached: a `last_maintain` that stops moving is how an operator spots the invariant-4 failure + mode, and it costs a three-row lookup. The resolution decision is one compound statement of + per-`(host, metric, res)` branches whose `COUNT` is capped by `LIMIT max_points + 1` — exact, + because the decision only ever asks whether a count is `0`, `≤ max_points`, or neither — and + `pick_res_many()`/`history_many()` are the batched door the N × M endpoints go through. The rows + stay a per-series seek **measuredly**: a `ROW_NUMBER()` window or a compound of the 27 tail + selects both cost more than 27 index seeks on the seeded store (`data/logs/NOTES-store-perf.md` + records the table), so batching the *decision* is the fix and batching the *fetch* would be a + round trip traded for work. + **The write side is windowed too** (phase 14 task 03): `maintain()` reads `last_maintain` through + `get_meta()` *before* its transaction (same non-reentrant-lock rule as `stats()`) and bounds each + rollup step to what became due since, instead of averaging every row the resolution retains — + 3.97 M rows of 1-minute history per minute, a 5.4 s write transaction that made a poller's + `add_many` outlive `busy_timeout=5000` (gap 4's `store-error` line). Two properties make that a + cost bound and not a data change, and both are pinned: the lower bound reaches **past** the + previous pass (one bucket plus slack, so ten quiet minutes are a catch-up scan rather than ten + minutes of empty charts), and it is snapped **down to a bucket boundary** — the step's ceiling is + a row bound, so the newest bucket is partial and a later pass re-rolls it as its rows land, and a + floor that cut into a bucket would overwrite the fuller average with the mean of its tail rows + (2.5 becoming 4.5 on a six-sample minute: rule 2 with a chart's clothes on). A window is only a + cost bound if the plan can seek to it, and two things were in the way. The prune keeps its full + retention bound — a rollup may window because it only adds rows, a prune must reach anything that + expired — but it is **one `DELETE` per resolution**, because one + `WHERE (res=? AND ts.txt` is all 16 drives (8 NVMe, 8 spinning) read through that sudo rule, and `tests/test_agent.py` parses every one — which is what found the parser's one real bug: this binary annotates the @@ -681,9 +755,8 @@ That scheme is the history API's key — treat it as a public interface. *not* a sudo rule, because the agent must read the device itself — on an `agent/host_agent.py` from phase 09 or later (never redeploy without the grant: 16 permission-denied lines would age nothing but are pure noise, while with the grant they are 16 drives' worth of temperature, - hours and sector counters). **The agent and the `disk` group are now both in place on - `driveripper`, and it still reads no SMART numbers** — the group only covers nodes that carry - it (see above), so what is left is the node grant / `-d sat` work, not an install. `gateway` needs smartmontools first and will still read no SMART + hours and sector counters). **The agent and the `disk` group are both in place on + `driveripper`** — the `disk` group alone is not enough on kernel 7.2 (the passthrough gate, see above), so the unit additionally runs as root with the two gate capabilities bounded (gap 10) — all 16 drives now read live. `gateway` needs smartmontools first and will still read no SMART after that, being a VM with no physical disk behind `vda`. A host that cannot read its drives renders the designed degradation: one dimmed card per physical drive, every number `n/a`, and the one-line why. @@ -711,9 +784,26 @@ set -a; . ./.env.dev; set +a # dev dials the fleet the prod way, no ./scripts/dev-server.sh status # per-source ok/DOWN — unauthenticated: /api/health ./scripts/dev-server.sh logs -uv run pytest # hermetic: 1195 passed + 11 live-skipped, no network -uv run pytest --cov # coverage gate (98.7%; floor 94%) +uv run pytest # hermetic: 1267 passed + 15 skipped (11 live, + # 4 store_perf), no network +uv run pytest --cov # coverage gate (98.75%; floor 94%) uv run pytest -m live --live # opt-in: real fleet / engines / upsd +uv run pytest -m store_perf --store-perf # opt-in: the store hot-path timings over the seeded + # store, priced against `store_perf.BUDGETS` (the cached + # reads report cold *and* warm, so a before/after never + # books a cache hit as a speed-up) +uv run python scripts/watch-store-perf.py + # ... and the live proof: 25 signed-in minutes against the + # running dev server, timing the page paths while counting + # `store-error` rows and the WAL size (DASH_WATCH_SECONDS + # shortens it; writes store-perf-watch.{log,tsv,json}) +uv run python scripts/store_perf_probe.py # ... as a report (data/logs/store-perf-baseline.{md,json}, + # then store-perf-task03.* for the write side and + # store-perf-task04.* for the budget table); + # --rebuild re-seeds, --dev-db also times a real + # store read-only, DASH_PERF_STORE_DIR moves the cache, and + # every pass that *writes* runs on a scratch copy so the + # seeded baseline never moves under the next measurement uv run ruff check src tests agent && uv run pyright ./scripts/capture-fixtures.sh # re-record ground truth (agents + UPS + gufo /slots) SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh @@ -837,6 +927,28 @@ branch `main`); there is no CI on it, so `bash .agents/validate.sh` is the only verbatim, the four config rows, the `DASH_COOKIE_SECURE=1` deployment note, the three auth gaps). No server starts and no request is made: a doc that stops naming the reset command is a locked-out admin, and that is worth a test even though it asserts nothing about Python. +- **A cost claim is pinned as a shape and a control, never as a wall clock** (phase 14). The default + run keeps the seeded-store tests hermetic and asserts **shape and determinism** — counts, + retention windows, the statements a plan is taken from, idempotence — while the timings live behind + `-m store_perf --store-perf` and in `scripts/store_perf_probe.py`, because a budget asserted on + first writing is a number fitted to the machine it was written on. Two rules came out of it. + (1) A cheaper query is proven *equal*, not just faster: + `test_windowed_rollups_answer_exactly_what_one_full_window_rollup_answers` compares every stored + row of a windowed store against a second store rolled the old way, and + `test_a_re_rolled_bucket_only_ever_gains_rows` pins the one bug that equivalence alone misses (a + window floor that cuts into a bucket re-rolls it from its tail rows). (2) A race that comes back + clean has to be shown capable of going dirty: `store_perf` runs the pre-fix whole-window + `maintain()` as a **control** beside the windowed one and `tests/test_store.py` fails if the control + stops contending, because "0 `database is locked`" from a race that never met is the invariant-4 + failure mode with a green checkmark attached. (3) **A budget is a table, not a test body** + (task 04): the 14 ceilings live in `store_perf.BUDGETS` with the measured figure and the margin in + each row's `basis`, the probe renders that table, and two default-run tests fail if a budget stops + being priced or a ceiling contradicts its own ratio partner — a ceiling nobody measures is a + comment. (4) A cache's cost claim needs the *write* cadence in the measurement: task 02's series + cache reported 0.0 ms on a quiet seeded store and served almost nothing on a live one, because + every sample write invalidated it. The pinned version is statement-counting, not timing — a poll of + existing series issues no `SELECT`, a write of a name nobody wrote ends the cache — and the live + version is `scripts/watch-store-perf.py`, which is where the defect showed up. - Coverage gate: 80% is the repo floor, the phase gate in `.agents/validate.sh` enforces 94% (`--cov-fail-under`), and ~98% is where we are; keep it there. New/modified Python still has to clear >90% on its own. API tests use `create_app()` with an in-memory DB, @@ -1221,7 +1333,7 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl field map is written. 3. **UPS history depth.** Events and the numeric series are stored; long-window battery health (capacity fade over months) has the rollups but no dedicated view. -4. **The dev DB is big enough to refuse a write now and then.** The dev store (~640 MB on +4. **The dev DB's `database is locked` line is fixed (phase 14); a cold-cache tail is left.** The dev store (~640 MB on 2026-10-19, its WAL grown past 140 MB at its worst) logged `store-error: OperationalError: database is locked` a few times an hour across `system`/`llm`/`ups` through 2026-10-18; the poller handles it exactly as invariant 4 requires — the source goes red with the real reason, @@ -1232,8 +1344,52 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl phase 12's re-check on 2026-10-19 found the 190 stored `store-error` rows all older than 2026-10-18 06:33 with none in the previous 24 h of the running dev store — the absence of a sample, not the absence of the pressure, since the file only keeps growing. - Retention/`wal_checkpoint` tuning for a long-running store is its own phase, not a - phase-09 follow-up. + Retention/`wal_checkpoint` tuning for a long-running store was its own phase rather than a + phase-09 follow-up — that phase is `14_store_performance`, and the two paragraphs below are + what it measured and changed. + **Phase 14 has since split this in two, and the read half is closed.** The in-process half was + never a SQLite lock at all: `SampleStore._lock` serialises everything, so a 1.25 s `stats()` + from a browser tab's `/api/health` poll was a poller waiting up to 1.45 s to insert 30 rows + (measured, `data/logs/NOTES-store-perf.md`). Task 02 took the two whole-table reads off the + per-request path, and `/api/health` now answers in ~1.2 ms instead of ~1.27 s, so the store + mutex is no longer held by a reader for seconds at a time. The **write half is closed too** + (task 03): `maintain()` rolls up only what became due since `last_maintain`, its prune is one + `DELETE` per resolution instead of one `OR` the planner could not use, and `PRAGMA optimize` runs + there (with SQLite's automatic ANALYZE switched off, so no poller's commit pays for it). On the + seeded 6.4 M-row store that is 5,368 ms → **2.7 ms** steady / 48 ms for a ten-minute catch-up, and + the two-connection race at the app's cadence is **100 of 100 inserts, zero `database is locked`, + max wait 21 ms** where the pre-fix control still waits 4.76 s (task 04's re-run of the same + budget: 6.8 ms steady / 46 ms catch-up / 8.5 ms worst wait, so the row is a range, not a + digit). The WAL follows the same shape: a steady pass leaves ~0 MB and a catch-up tick 4 MB + instead of 38 MB that never comes back down, and `journal_size_limit=64 MB` caps what a + long-lived reader can make it grow to (`data/logs/NOTES-store-perf.md` has the tables). + **The live proof is in** (task 04, 25 minutes / 74 passes / 296 signed-in requests against the + running dev server on the operator's own 636 MB store): **0 new `store-error` rows** (still 190, + newest 2026-10-18 06:33:30, i.e. 39 h before the watch), **WAL 6.5 → 7.77 MB** against the 67.1 MB + cap and a main file that did not move, 0 non-200 responses, and page paths at 1.8-3.3 ms median + (`/api/health`, `/api/summary`, `/api/metrics`, a 3-host × 9-metric 90-day `/api/system/history`). + The watch also caught a defect in task 02's fix — the series cache was invalidated by *every* + sample write, so on a store the hub writes to every few seconds it never served and `/api/metrics` + paid the 572 ms scan on nearly every call (measured: 535 ms per call pre-fix, 1.8 ms median + after) — and that is what the store bullet above now describes. + What is **left** is the cost of the two cold answers themselves, measured on that store: 599 ms + for `series_names('host')`, 729 ms for `stats()`' counts, now once per cache window (~1.3 s of + store mutex per minute) instead of once per call — which is why the watch's medians are 1.8-3.3 ms + and its p95s are 620/729 ms: the p95 is the window boundary, and 37 of 74 passes had one request + over 250 ms, every one of them a cold re-derivation. No cache policy reaches *that*; the fix is + the bounded `(source, metric)` structure or a counts table named in task 03's note. Two cheaper + knobs exist and were deliberately not pulled, because both change what `/api/health` answers and + move rather than remove the mutex hold: raising `STATS_CACHE_TTL_SECONDS`, and refreshing both + caches from the maintenance tick so the miss lands on a poller instead of a page load. That is a + phase, not a cleanup. Two servers on one file can still contend, and `busy_timeout=5000` is + unchanged on purpose so a second one stays visible as a slower write; what they contend with is + now milliseconds instead of 5.4 s. + One consequence worth naming, because two other gaps pointed here for it: the **~10 s + process-wide stall** (gap 9's `Δ`-window dilution, `poller`'s catch-up gaps) is gone with the + mutex holds — coincident multi-second gaps in the 2 s `llm` and 5 s `ups` raw streams went from + 393 and 198 in the two halves of 2026-10-19 to **0** across the 110 minutes the phase-14 store + has run (re-measured in this phase's verification pass from the `sample` table; the signature + separates a process stall from one slow engine). 5. **The login throttle keys on the socket peer address** (phase 11), and the one `LoginThrottle` instance now holds two keys per address: the bare one for `POST /api/auth/login` and `password:
` for `POST /api/auth/password`. Behind nginx @@ -1315,7 +1471,14 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl pass line and what the watch above stayed under. The `Δ` label stayed honest throughout — both before and after the fix it names the window actually used; do not "fix" anything here by hiding or rewriting it. The ~10–15 s process-wide **stall** that triggers a - refused poll is still gap 4's and `14_store_performance`'s problem; the estimator no longer + refused poll was gap 4's and `14_store_performance`'s problem, and **phase 14 closed it** + (measured on the dev store from the sample table: coincident multi-second gaps in the 2 s `llm` + and 5 s `ups` raw streams — the signature of a process-wide stall, not a slow engine — were + 393 in 00:00–12:00 and 198 in 12:00–21:36 on 2026-10-19, and **0** across the 110 minutes the + phase-14 store has been running, worst `llm` gap 7.1 s and that one the recorded gufo timeout). + The mechanism is the one phase 14 removed: a whole-table `stats()` count held the store mutex + for ~1.25 s on every browser tab's `/api/health` poll, and `maintain()` held the write lock for + 5.4 s. The estimator no longer turns a slow fetch into 0.2 s arithmetic, and one `n/a` per stall is the designed degradation (the residual the verification pass measured is the paragraph below). **The verification pass found one residual path and closed it** (same day, live rows in the @@ -1341,11 +1504,47 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl 35.6 once the freed slot settled — every value labelled, **0 of 540 above the 300 pass line**, and no stored raw row over it since the restart either. A collapsed window is what makes a burst *look* like a rate, so the fix for this family is the ~10 s stall itself (gap 4 / - `14_store_performance`), not a fresh threshold invented in the estimator: task 03 pinned + `14_store_performance` — which has since removed it, see gap 4 and the paragraph above, so what + is left of this family is the sub-8× skew itself), not a fresh threshold invented in the + estimator: task 03 pinned `MIN_WINDOW_SECONDS` as an absolute floor and pinned that an in-bounds diluted row publishes (`test_a_fallback_inside_the_documented_ceiling_still_publishes`), so a span-relative floor would be a re-decision of that phase, not a bug fix. Do not "fix" the residual by hiding the short window or its label. +10. **`driveripper`'s 16 drives read `n/a`: kernel 7.2's unprivileged-passthrough gate (2026-10-19).** + The agent is installed, the `disk` group grant is in place, the block nodes open, and the + page renders all 16 real drive rows — but kernel 7.2.11 blocks the *agent user's* `smartctl` + at the command level (not the open; the probe matrix is in + `data/logs/NOTES-driveripper-agent.md`): the 8 SATA drives' SMART is ATA pass-through + behind a SAT layer, and the kernel's unprivileged `SG_IO` allow-list + (`scsi_cmd_allowed()`, read-safe opcodes only) does not include it — `CAP_SYS_RAWIO` or + root unblocks it, with or without `-d sat`/`-d ata`; the 8 Samsung/WD NVMes' SMART-log + read needs the drive's ACEL to advertise the page (the 980 PRO's `5B2QGXA7`, 2020-era, + does not: `NVME_IOCTL_ADMIN_CMD: Permission denied`) or `CAP_SYS_ADMIN`; `deskwork` reads + live because its firmware does. Options, in order of narrowness: (a) accept the designed + degradation (what `/disks` renders now — the identity is real, the values are `n/a` with a + why-line); (b) `AmbientCapabilities=CAP_SYS_RAWIO` (+ `CapabilityBoundingSet`) on the unit + — unblocks exactly the 8 HDDs, is the capability the kernel names for the gate, but a + compromised agent could then write raw block I/O — test it in a `systemd-run --scope` + before committing; (c) `User=root` with the empty capability set — unblocks all 16 but + hands a token-guarded process root's file-read DAC (not recommended); (d) a Samsung + firmware update — the only clean fix for the NVMe side. **Resolved the same day with a + tighter variant of (c): `User=root` + `CapabilityBoundingSet=CAP_SYS_RAWIO CAP_SYS_ADMIN` + — all 16 drives read live** (root DAC is the price; the bounding set keeps the grant + named rather than full). Two empirical notes: a root process under the unit's *empty* + bounding set is capability-less (`CapEff: 0`), because the kernel gates check + capabilities, not uid — that is why the first `User=root` attempt still read `n/a`; and + `CapEff` came back `0000000000220000` (SYS_ADMIN + systemd's auto CHROOT; RAWIO in the + bounding set but not effective) — the 8 SATA drives' reads were unblocked by SYS_ADMIN + alone, so the effective grant is {SYS_ADMIN}; the RAWIO in the line stays as the named + intent for the `SG_IO` gate. The agent also gained the pass-budget scale-up (20 s floor + → 5 s × disk count; an explicit override still wins, the budget test is untouched), + because a 16-disk pass takes more than 20 s when the drives answer. The `deploy/install-agent.sh` + template still ships the `DevicePolicy`/`DeviceAllow` lines that misbehaved on this host + (a deny-all filter; the `nvme*` glob matches nothing on this kernel because + `/proc/devices` lists NVMe as `blkext`) — the template change is its own phase. + `tests/fixtures/smartctl_driveripper_*.txt` (captured via `sudo`) remain the ground truth + for the drives' shape. ## Locked decisions (and why) diff --git a/README.md b/README.md index 695619b..adcd95a 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,8 @@ a labelled window, so the agent never has to remember who polled it last. Per-disk `smart` health is a best-effort add-on on the whole-disk `/sys/block` list (no partitions): the agent runs `smartctl -i -A -H` per physical disk — 5 s bound per device, -20 s per pass, results cached 4 minutes — and parses health, temperature, power-on hours, +a pass budget of 20 s as a floor that scales to 5 s × the disk count, results cached 4 minutes — +and parses health, temperature, power-on hours, the sector alarms and SSD wear into `disk_io.devices[].smart`. Refreshing an expired cache happens on a background thread while the request is served the previous result, so a wedged disk can never cost a poll more than its own timeout (only the very first pass after the @@ -334,7 +335,7 @@ documentation and is never loaded, and `scripts/dev-server.sh` exports `.env.dev | `HOST_INTERVAL_SECONDS` / `LLM_INTERVAL_SECONDS` / `NUT_INTERVAL_SECONDS` | how often each source is polled (10 / 2 / 5 s) | | `DISKS_INTERVAL_SECONDS` | how often the disk-health source re-reads each agent for SMART (default 300 s — SMART is not a per-second signal). No targets of its own: the `disks:` jobs exist exactly when `HOST_TARGETS` does | | `HOST_TIMEOUT_SECONDS` / `LLM_TIMEOUT_SECONDS` / `NUT_TIMEOUT_SECONDS` | per-source fetch timeout — one slow source never holds up the others | -| `DISKS_TIMEOUT_SECONDS` | the disks fetch's own budget (default 30 s), because the agent runs `smartctl` inside its snapshot handler and 20 s of wedged disk must not read as a dead host | +| `DISKS_TIMEOUT_SECONDS` | the disks fetch's own budget (default 30 s), because the agent runs `smartctl` inside its snapshot handler and a wedged SMART pass must not read as a dead host | | `GUFO_API_KEY` / `VLLM_API_KEY` | only if an engine is behind a bearer token; they never leave the server | | `NUT_HOST` / `NUT_PORT` | upsd address — **whatever port it really listens on**, not always 3493. Reads are unauthenticated in stock NUT | | `NUT_USER` / `NUT_PASSWORD` | only needed for instant commands (battery tests) | @@ -383,9 +384,27 @@ auth.py ── verifier + session key, loaded once at startup (no row and no see poller, so a SQLite write that gets refused is reported (`ok: false`, the real reason, the numbers aging, one line in the event log) instead of freezing the source under a green dot. * `store.py` — raw samples roll up to 1-minute and 1-hour averages and the raw rows are - pruned, so a year of history is a few hundred MB rather than a disk emergency. Its `meta` - table is the one place state that must outlive a restart goes (`get_meta`/`set_meta`/ - `delete_meta`), and its first user is the admin credential below. + pruned, so a year of history is a few hundred MB rather than a disk emergency. Nothing a + per-request path asks is answered by a whole-table scan **on every call** (phase 14): the two + answers that used to be scanned on every call — `series_names()` behind `/api/metrics` and the + row counts behind `/api/health` — come from a 60 s / 30 s cache, so the scan is paid once per + window (its cost is the honest remainder in the known-gaps list) instead of once per request. `series_names`' is ended only by a write that can + change it: a `(source, host, metric)` the store has never written, or any `DELETE`. A poll of an + existing series leaves it alone, which is what makes the cache serve at all on a store nine pollers + write to (task 04's live watch is the proof: 535 ms per call before, 1.8 ms median after), and the + counts age on the clock. A chart's resolution comes from one capped compound statement instead of + three full `COUNT`s per series. `last_maintain` and `size_bytes` are deliberately never cached — + the first is how an operator tells a stalled rollup loop from a healthy one. + The write side is windowed too: `maintain()` rolls up only the buckets that became due since its + last run (read from `last_maintain`, overlapped by one bucket and snapped down to a bucket + boundary, so a gap in polling is a catch-up scan rather than missing charts) and prunes on its + full retention bound, one statement per resolution. It ends with `PRAGMA optimize` — the store's + only statistics refresh, deliberately not inside a poller's commit — and the WAL is kept honest by + `wal_autocheckpoint` plus a 64 MB `journal_size_limit` cap. On a 6.4 M-row store that took the + once-a-minute write transaction from 5.4 s to a few milliseconds, which is what stopped a poller's + insert losing its 5 s `busy_timeout` (known gap 4). + Its `meta` table is the one place state that must outlive a restart goes (`get_meta`/ + `set_meta`/`delete_meta`), and its first user is the admin credential below. * `auth.py` — the admin login's decisions, in stdlib and without HTTP: a self-describing PBKDF2-HMAC-SHA256 verifier, an HMAC-signed stateless session cookie (no session table, and rotating the stored key is what evicts every other browser), a session-bound CSRF token, and @@ -423,7 +442,12 @@ auth.py ── verifier + session key, loaded once at startup (no row and no see no env config answers `404 {"configured": false}` **without dialling** (so a page says "not configured" instead of going red), and a history query for a series that does not exist answers an empty series instead of an error (so a card asks for every plot it could - draw and reads `n/a` for the ones the drive never reported). `/api/disks/snapshot` is the + draw and reads `n/a` for the ones the drive never reported). The four `history` routes take + one batched door into the store (`api._history_batch` → `store.history_many`), so a page's + chart request is one grouped resolution query plus one index seek per series rather than four + queries per series — and the JSON it answers with is the same document the per-series loop + produced, series for series and in the same order (`tests/test_api.py` pins that against + points read straight out of the store with `query()`). `/api/disks/snapshot` is the drive-health surface: one entry per host with its physical disk rows, the agent's `smart` block verbatim, `smartctl_available` (`true` / `false` = install smartmontools / `null` = the agent never said), a `severity` beside each row's rated numbers, and a `fleet` block @@ -557,6 +581,11 @@ uv run pytest # unit + API tests, fully mocked, uv run pytest --cov # coverage gate set -a; . ./.env.dev; set +a # the live suite reads the same env as the app uv run pytest -m live --live # optional: real fleet / engines / upsd +uv run pytest -m store_perf --store-perf -s # optional: the store's hot-path timings (phase 14) +uv run python scripts/store_perf_probe.py # the same measurements as a report in data/logs/ +DASH_WATCH_SECONDS=1500 uv run python scripts/watch-store-perf.py + # ... and the live watch: 25 signed-in minutes against + # the running dev server, counting store-error and WAL uv run ruff check src tests agent && uv run pyright # lint + types ./scripts/capture-fixtures.sh # re-record ground truth: host agents, UPS replies, gufo /slots ./scripts/capture-fixtures.sh --with-llm # ... and the engines' /metrics while they are up @@ -611,6 +640,38 @@ local-testing word stays legal in prose, which is the only place it can be usefu the four public paths, the reset command, the four variables and the `DASH_COOKIE_SECURE=1` deployment note. Nothing starts a server there; a sentence that quietly disappears is the bug. +`tests/store_perf.py` is the second opt-in marker's machinery (`live` is the other). It seeds a +**production-shaped +SQLite file** — `SampleStore`'s own retention (`raw 48 h / 1m 30 d / 1h 400 d`), the fleet's real +metric names, the real 10 s cadence: ~6.4 M rows and ~720 MB, the dev DB's order of magnitude — and +times every hot path of `store.py`/`api.py` against it, with `EXPLAIN QUERY PLAN` for the queries +phase 14 changes and a two-connection race that reproduces the `database is locked` line in the +known-gaps list — beside the **pre-fix control** of the same race, because a clean race that never +met a lock would prove nothing. Every pass that *writes* (a rollup catch-up, a race) runs on a +scratch copy of the seeded file, so the baseline the next measurement compares against comes back +exactly as it went in. The default suite builds that store and asserts its **shape and determinism** +(row counts, retention windows, rollups that are the mean of the raw rows they came from, a second +`maintain()` that changes nothing) but never its wall time — except behind the marker, where +`store_perf.BUDGETS` is the phase's pinned regression table (14 ceilings, 2-3x the measured "after", +`maintain()` < 1 s, a racing insert at zero `database is locked`) and a stale budget pointer fails a +default-run test. The cached reads are reported twice — +**cold** (the cache invalidated before every run) and **cached** — because comparing a cache hit +against the baseline's full scan would book a 1 000x speed-up that is really just a cache. Only +`-m store_perf --store-perf` runs the timing passes, and `scripts/store_perf_probe.py` is the CLI +form that writes `data/logs/store-perf-baseline.{md,json}` (task 03's write-side run is +`store-perf-task03.*`, task 04's budget run `store-perf-task04.*`) for `data/logs/NOTES-store-perf.md`. Two rules keep it +honest: **no test ever opens `data/dashboards.sqlite3`** (the `--dev-db` flag exists for a human +running the probe read-only), and the seeded file is a cache under `data/perf/` +(`DASH_PERF_STORE_DIR` relocates it) because a before/after has to measure the same file — pytest's +`tmp_path` on this box is `tmpfs`, i.e. RAM, which times a store nobody deploys. + +The non-timing half of that lives in `tests/test_store.py` and runs every time: a cache hit +issues **no** SQL at all (counted through `sqlite3`'s trace callback, never through a wall clock), +the series list is invalidated by the write that introduces a name and not by the poll that repeats +one (and the tracking that decides has a cap, below which it falls back to invalidating), +`last_maintain` is never served from a cache, and `pick_res_many()`/`history_many()` answer exactly +what the per-series calls answer — on a small store and on the seeded one, at every window. + One more file is worth knowing about: `tests/card_contract.py` is the table of `/api/system/snapshot` paths the system page's per-host cards are allowed to read. It is checked against all three captured host shapes (discrete GPU, APU, no GPU), over HTTP, and diff --git a/agent/host_agent.py b/agent/host_agent.py index 771d991..0cfa35f 100644 --- a/agent/host_agent.py +++ b/agent/host_agent.py @@ -22,7 +22,8 @@ Design rules * **Every read is guarded.** A vanished /proc entry (process exited mid-scan) skips that item; the snapshot still succeeds. The poll is degraded, never failed. * **SMART is a best-effort add-on.** The per-disk `smart` block runs `smartctl -i -A -H` - in a subprocess: bounded 5 s per device and 20 s per pass, cached 240 s so a fast + in a subprocess: bounded 5 s per device and a pass budget of 20 s scaled to the disk + count, cached 240 s so a fast dashboard poll cannot re-issue ATA commands every few seconds. Refreshing an expired cache happens on a background thread while the request is served the previous result — 20 s of wedged disk must never cost a poll more than its own timeout. No binary, a denied @@ -122,10 +123,12 @@ VIRTUAL_NET_PREFIXES = ("lo", "veth", "docker", "br-", "virbr", "br", "tap", "vn VIRTUAL_DISK_PREFIXES = ("loop", "ram", "zram", "sr", "dm-", "md127") # SMART collection bounds. 5 s per device is generous for a healthy drive and caps how -# long one wedged device can hold a snapshot hostage; 20 s caps the whole pass so even a -# shelf of bad disks cannot stall the HTTP handler; 240 s is below the dashboard's -# 5-minute disks cadence, so the slow disks job gets fresh data while the fast system -# poll never re-runs smartctl every 10 s. +# long one wedged device can hold a snapshot hostage; the pass budget is a 20 s floor +# scaled to the per-device bound (5 s x disk count), so a 16-disk shelf gets the 80 s a +# real read takes while a single-disk host keeps 20 s and a shelf of bad disks cannot +# stall the HTTP handler; 240 s is below the dashboard's 5-minute disks cadence, so the +# slow disks job gets fresh data while the fast system poll never re-runs smartctl +# every 10 s. SMARTCTL_TIMEOUT_SECONDS = 5.0 SMARTCTL_BUDGET_SECONDS = 20.0 SMARTCTL_CACHE_SECONDS = 240.0 @@ -522,7 +525,8 @@ class Collector: same error lines, so a snapshot always states why `smart` is null. An **expired** cache is refreshed on a background thread and the request is served - the previous result, because the pass budget (20 s) is five times the dashboard's + the previous result, because the pass budget (20 s floor, 5 s per disk) is five + times the dashboard's `HOST_TIMEOUT_SECONDS` (4 s): run inline on the request path, one wedged disk would turn the *system* source red every `smart_cache_seconds` for a fault in an add-on (AGENTS.md invariant 4 — SMART must not blink the page that does not use it). The @@ -569,14 +573,20 @@ class Collector: now = time.monotonic() results: dict[str, dict[str, Any] | None] = {} errors: list[str] = [] - deadline = now + self.smart_budget_seconds + # 20 s floor scaled to the per-device bound: a 16-disk shelf takes more than 20 s + # when the drives answer, and truncating the pass would null the tail drives. + # An explicit override (the budget test sets this to 0) wins over the scale. + budget = (max(SMARTCTL_BUDGET_SECONDS, self.smart_timeout_seconds * len(device_names)) + if self.smart_budget_seconds == SMARTCTL_BUDGET_SECONDS + else self.smart_budget_seconds) + deadline = now + budget for name in device_names: left = deadline - time.monotonic() if left <= 0: remaining = [n for n in device_names if n not in results] for skipped_name in remaining: results[skipped_name] = None - errors.append(f"smart: pass budget of {self.smart_budget_seconds:.0f}s " + errors.append(f"smart: pass budget of {budget:.0f}s " f"exhausted, {len(remaining)} disk(s) skipped") break try: diff --git a/pyproject.toml b/pyproject.toml index ad9b08f..588b159 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,11 @@ markers = [ # `-m live --live` runs them; without --live they SKIP, so `uv run pytest` stays # network-free. See tests/test_live_fleet.py for the env vars each one needs. "live: needs real hardware (fleet hosts, LLM engines, or upsd) — opt in with --live", + # `-m store_perf --store-perf` runs them: the phase-14 timing measurements over the seeded + # production-shaped store (tests/store_perf.py). The seed itself runs in the default suite; + # only the multi-minute measurement passes are behind this flag. Underscored because Python + # decorators cannot carry a hyphen; the CLI flag stays `--store-perf`. + "store_perf: measures the seeded store's hot paths — opt in with --store-perf", ] # auto: async tests run without a decorator on every single one asyncio_mode = "auto" diff --git a/scripts/store_perf_probe.py b/scripts/store_perf_probe.py new file mode 100755 index 0000000..9f82da0 --- /dev/null +++ b/scripts/store_perf_probe.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Measure the store's hot paths against a production-shaped SQLite file. + +Phase 14 exists because `TODO.md:7` says "pages take forever to load and seem to lock up the +database". Task 01's job is the **before**: one run of this script is the baseline in +`data/logs/NOTES-store-perf.md`, and task 04 runs it again as the "after". Nothing here changes +behaviour — it reads, times, `EXPLAIN`s, and cleans up after itself. + + uv run python scripts/store_perf_probe.py # baseline (builds the store once) + uv run python scripts/store_perf_probe.py --rebuild # force a fresh seed + uv run python scripts/store_perf_probe.py --dir /tmp/perf # measure on tmpfs (RAM: faster) + uv run python scripts/store_perf_probe.py --out data/logs/store-perf-after + +It writes two files beside the numbers it prints: `.json` (everything, machine-readable) +and `.md` (the same tables as markdown, to paste into the NOTES). The seeded store itself +lives in `data/perf/` (gitignored, like the dev DB) and is reused while it is fresh — the same +physical file has to be the one timed before and after the fixes, or the two numbers describe +two databases. + +The shape, the timings and the machine specs all come from `tests/store_perf.py`, which is also +what the pytest budget tests of this phase use: one definition of "production-shaped", because +two of those drift. +""" + +from __future__ import annotations + +import argparse +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +for entry in (ROOT / "tests", ROOT / "src"): + # `tests/store_perf.py` is not a package (pytest imports it the same way via conftest), and + # a bare checkout has no installed `dashboards` either. Both paths, or nothing works. + if str(entry) not in sys.path: + sys.path.insert(0, str(entry)) + +import store_perf # noqa: E402 - after the sys.path fixup above, by design + + +def log(message: str) -> None: + print(f"[{time.strftime('%H:%M:%S')}] {message}", flush=True) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--dir", default=None, + help="where the seeded store lives (default: $DASH_PERF_STORE_DIR " + "or data/perf — a real block device, unlike pytest's tmpfs)") + parser.add_argument("--rebuild", action="store_true", + help="throw the cached store away and seed a new one") + parser.add_argument("--out", default=str(ROOT / "data" / "logs" / "store-perf-baseline"), + help="prefix for the report files (default: data/logs/store-perf-baseline)") + parser.add_argument("--race-iterations", type=int, default=100, + help="how many 30-sample inserts to race against stats()/maintain() " + "(task 01 says ~100)") + parser.add_argument("--race-seconds", type=float, default=store_perf.MAX_RACE_SECONDS, + help=f"wall-clock budget per race pass (default " + f"{store_perf.MAX_RACE_SECONDS:.0f} s — one stats() pass costs " + "seconds on this store, so the pass is bounded by time as well as " + "by the insert count)") + parser.add_argument("--dev-db", default=None, + help="also time the three hot reads against this *real* database file, " + "opened read-only (e.g. data/dashboards.sqlite3). Nothing here " + "writes to it and no test ever passes this flag — the hermetic " + "suite never opens the operator's store.") + parser.add_argument("--no-http", action="store_true", + help="skip the TestClient pass (/api/health, /api/metrics, " + "/api/system/history) — store-only numbers") + args = parser.parse_args(argv) + + path = store_perf.ensure_seeded(args.dir, force=args.rebuild, log=log) + store = store_perf.open_store(path) + try: + report = store_perf.measure(store, race_iterations=args.race_iterations, + race_max_seconds=args.race_seconds, + with_http=not args.no_http, log=log) + finally: + store.close() + + if args.dev_db: + log(f"read-only pass over the real store: {args.dev_db}") + report["dev_db"] = store_perf.measure_dev_db(args.dev_db) + + written = store_perf.write_report(report, Path(args.out)) + print() + print(store_perf.render_markdown(report)) + print() + print(f"report: {written['md']}\n {written['json']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/watch-store-perf.py b/scripts/watch-store-perf.py new file mode 100755 index 0000000..01a96c0 --- /dev/null +++ b/scripts/watch-store-perf.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Phase 14 task 04: watch the running dev server for `database is locked`. + +Sampled, not exhaustive: every ~20 s a signed-in pass hits the paths a page actually loads +(``/api/health``, ``/api/summary``, ``/api/metrics``, a 3-host x 9-metric 90-day +``/api/system/history``) and records the wall time beside the store's own evidence — the +``store-error`` rows in the ``event`` table, the WAL file size, the database size. + +Run it in the background while the dev server keeps polling the fleet: + + set -a; . ./.env.dev; set +a + setsid nohup uv run python scripts/watch-store-perf.py > /tmp/watch.out 2>&1 < /dev/null & + +Writes ``data/logs/store-perf-watch.log`` (human), ``store-perf-watch.tsv`` (machine) and +``store-perf-watch.json`` (the summary every path ran against). ``data/`` is gitignored, so those +are local evidence for ``data/logs/NOTES-store-perf.md``, never tracked artifacts. +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import statistics +import sys +import time +from pathlib import Path + +import httpx + +#: Repo root, so the paths below mean the same tree wherever the watch is invoked from +#: (the same anchor ``scripts/store_perf_probe.py`` uses). +ROOT = Path(__file__).resolve().parents[1] + +BASE_URL = os.environ.get("DASH_WATCH_URL", "http://127.0.0.1:8090") +ADMIN_PASSWORD = os.environ.get("DASH_ADMIN_PASSWORD", "") +DB_PATH = Path(os.environ.get("DASH_WATCH_DB", ROOT / "data" / "dashboards.sqlite3")) +OUT = Path(os.environ.get("DASH_WATCH_OUT", ROOT / "data" / "logs")) +DURATION_SECONDS = float(os.environ.get("DASH_WATCH_SECONDS", "1500")) +INTERVAL_SECONDS = float(os.environ.get("DASH_WATCH_INTERVAL", "20")) + +#: What a page load may cost. Everything here is a panel the operator is looking at. +PAGE_BUDGET_MS = 250.0 + +METRICS = [ + "cpu.usage_pct", "cpu.temp_c", "cpu.load1", "mem.used_pct", "disk.used_pct@/", + "disk.used_pct@/home", "net.rx_bps", "net.tx_bps", "gpu.busy_pct@0", +] + + +def store_evidence(db: sqlite3.Connection, since: float) -> dict[str, float | int | str]: + """The store's own record, read on a separate connection while the server runs.""" + total = db.execute("SELECT COUNT(*) FROM event WHERE kind='store-error'").fetchone()[0] + recent = db.execute( + "SELECT COUNT(*) FROM event WHERE kind='store-error' AND ts>=?", (since,) + ).fetchone()[0] + last = db.execute( + "SELECT message FROM event WHERE kind='store-error' ORDER BY ts DESC LIMIT 1" + ).fetchone() + return { + "store_errors_total": total, + "store_errors_in_window": recent, + "last_store_error": (last[0] if last else ""), + } + + +def timed(client: httpx.Client, path: str) -> tuple[float, int]: + started = time.perf_counter() + try: + response = client.get(path) + code = response.status_code + except httpx.HTTPError: # a refused request is a sample too + return (time.perf_counter() - started) * 1000.0, -1 + return (time.perf_counter() - started) * 1000.0, code + + +def main() -> int: + started = time.perf_counter() + OUT.mkdir(parents=True, exist_ok=True) + log_path = OUT / "store-perf-watch.log" + tsv_path = OUT / "store-perf-watch.tsv" + json_path = OUT / "store-perf-watch.json" + samples: list[dict[str, float | int | str]] = [] + + client = httpx.Client(base_url=BASE_URL, timeout=60.0, follow_redirects=False) + login = client.post("/api/auth/login", json={"username": "admin", "password": ADMIN_PASSWORD}) + if login.status_code != 200: + print(f"login failed: {login.status_code} {login.text[:200]}", flush=True) + return 1 + client.headers["X-CSRF-Token"] = login.json()["csrf"] + + hosts = [t["name"] for t in client.get("/api/config").json()["system"]["targets"]] + host_query = "".join(f"hosts={h}&" for h in hosts) or "hosts=driveripper&" + metrics_query = "".join(f"metrics={m}&" for m in METRICS) + paths = { + "health": "/api/health", + "summary": "/api/summary", + "metrics": "/api/metrics?source=host", + "history_90d": f"/api/system/history?{host_query}{metrics_query}range=90d", + } + + db = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=30.0) + started_wall = time.time() + with log_path.open("a", buffering=1) as log: + log.write(f"# watch start {time.strftime('%Y-%m-%dT%H:%M:%S')} " + f"budget={PAGE_BUDGET_MS:.0f} ms interval={INTERVAL_SECONDS:.0f} s " + f"duration={DURATION_SECONDS:.0f} s hosts={hosts}\n") + while time.perf_counter() - started < DURATION_SECONDS: + row: dict[str, float | int | str] = { + "t": round(time.time(), 1), + "elapsed_s": round(time.perf_counter() - started, 1), + } + for name, path in paths.items(): + ms, code = timed(client, path) + row[f"{name}_ms"] = round(ms, 1) + row[f"{name}_code"] = code + wal = DB_PATH.with_name(DB_PATH.name + "-wal") + row["wal_mb"] = round((wal.stat().st_size if wal.exists() else 0) / 1e6, 2) + row["db_mb"] = round(DB_PATH.stat().st_size / 1e6, 1) + try: + row.update(store_evidence(db, started_wall)) + except sqlite3.Error as exc: + row["store_errors_total"] = -1 + row["store_errors_in_window"] = -1 + row["last_store_error"] = f"read failed: {exc}" + samples.append(row) + worst = max(float(row[f"{n}_ms"]) for n in paths) + log.write( + f"{row['t']} elapsed={row['elapsed_s']:7.1f}s " + + " ".join(f"{n}={row[f'{n}_ms']:7.1f}ms" for n in paths) + + f" worst={worst:7.1f}ms wal={row['wal_mb']}MB db={row['db_mb']}MB" + + f" store-error(total={row['store_errors_total']}," + f" in-window={row['store_errors_in_window']})\n" + ) + time.sleep(INTERVAL_SECONDS) + + # ---- summary + def stat(name: str, fn) -> float: + values = [float(s[f"{name}_ms"]) for s in samples] + return round(fn(values), 1) + + summary: dict[str, object] = { + "started": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(started_wall)), + "duration_seconds": round(time.perf_counter() - started, 1), + "passes": len(samples), + "db": str(DB_PATH), + "db_mb_first": samples[0]["db_mb"], + "db_mb_last": samples[-1]["db_mb"], + "wal_mb_first": samples[0]["wal_mb"], + "wal_mb_max": max(float(s["wal_mb"]) for s in samples), + "wal_mb_last": samples[-1]["wal_mb"], + "store_errors_total": samples[-1]["store_errors_total"], + "store_errors_in_window": samples[-1]["store_errors_in_window"], + "last_store_error": samples[-1]["last_store_error"], + "paths": { + name: { + "median_ms": stat(name, statistics.median), + "p95_ms": stat(name, lambda v: sorted(v)[max(0, int(len(v) * 0.95) - 1)]), + "max_ms": stat(name, max), + } + for name in paths + }, + "budget_ms": PAGE_BUDGET_MS, + "passes_over_budget": sum( + 1 for s in samples if max(float(s[f"{n}_ms"]) for n in paths) > PAGE_BUDGET_MS + ), + "non_200_responses": sum( + 1 for s in samples for n in paths if int(s[f"{n}_code"]) not in (200, 404) + ), + } + json_path.write_text(json.dumps({"summary": summary, "samples": samples}, indent=2) + "\n") + with log_path.open("a", buffering=1) as log: + log.write("# summary\n") + for name, values in summary["paths"].items(): # type: ignore[index] + log.write(f"# {name:12s} median={values['median_ms']:8.1f} ms " + f"p95={values['p95_ms']:8.1f} ms max={values['max_ms']:8.1f} ms\n") + log.write(f"# store-error rows in window: {summary['store_errors_in_window']} " + f"(total {summary['store_errors_total']})\n") + log.write(f"# WAL first/max/last MB: {summary['wal_mb_first']} / " + f"{summary['wal_mb_max']} / {summary['wal_mb_last']}\n") + log.write(f"# passes over the {PAGE_BUDGET_MS:.0f} ms budget: " + f"{summary['passes_over_budget']} of {summary['passes']}\n") + + header = ["t"] + [f"{n}_ms" for n in paths] + ["wal_mb", "db_mb", "store_errors_in_window"] + with tsv_path.open("w") as tsv: + tsv.write("\t".join(header) + "\n") + for sample in samples: + tsv.write("\t".join(str(sample.get(key, "")) for key in header) + "\n") + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/dashboards/api.py b/src/dashboards/api.py index c357cbb..3378c71 100644 --- a/src/dashboards/api.py +++ b/src/dashboards/api.py @@ -16,6 +16,7 @@ from __future__ import annotations import logging import math import time +from collections.abc import Sequence from dataclasses import dataclass, field, replace from typing import Any @@ -109,9 +110,9 @@ def parse_range(value: str) -> float: raise HTTPException(400, f"unknown range {value!r}; try 5m 1h 24h 7d 30d") -def _history(store: SampleStore, source: str, host: str, metric: str, span: float, - points: int) -> dict[str, Any]: - series = store.history(source, host, metric, span=span, max_points=points) +def _series_dict(source: str, host: str, metric: str, span: float, + series: list[tuple[float, float]]) -> dict[str, Any]: + """The one series shape every `/api/*/history` route has always answered with.""" return { "source": source, "host": host, @@ -121,6 +122,30 @@ def _history(store: SampleStore, source: str, host: str, metric: str, span: floa } +def _history(store: SampleStore, source: str, host: str, metric: str, span: float, + points: int) -> dict[str, Any]: + return _series_dict(source, host, metric, span, + store.history(source, host, metric, span=span, max_points=points)) + + +def _history_batch(store: SampleStore, source: str, hosts: Sequence[str], metrics: Sequence[str], + span: float, points: int) -> list[dict[str, Any]]: + """N hosts x M metrics through the store's one batched door, host-major and in request order. + + Every `/api/*/history` route that used to loop `store.history()` per pair calls this instead. + The list it returns is identical, entry for entry, to what that loop built (same series, same + order, same `points`); what changed is the cost — the resolution choice for the whole batch is + one capped statement rather than three full `COUNT`s per pair (`store.history_many`). + """ + wanted = [(host, metric) for host in hosts for metric in metrics] + if not wanted: + return [] + answers = store.history_many(source, [host for host, _ in wanted], + [metric for _, metric in wanted], span=span, max_points=points) + return [_series_dict(source, host, metric, span, answers.get((host, metric), [])) + for host, metric in wanted] + + def _host_severity(host: dict[str, Any]) -> dict[str, str | None]: """Verdicts for the overview's fleet-table columns, from the one threshold table. @@ -722,8 +747,7 @@ def system_router() -> APIRouter: wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()] wanted_hosts = [item.strip() for item in hosts.split(",") if item.strip()] or [ target.name for target in ctx.settings.host_targets] - series = [_history(ctx.store, "host", host, metric, span, points) - for host in wanted_hosts for metric in wanted_metrics] + series = _history_batch(ctx.store, "host", wanted_hosts, wanted_metrics, span, points) return {"hosts": wanted_hosts, "range_seconds": span, "series": series} return router @@ -816,8 +840,8 @@ def disks_router() -> APIRouter: wanted_hosts = [item.strip() for item in hosts.split(",") if item.strip()] or [ target.name for target in ctx.settings.host_targets] return {"hosts": wanted_hosts, "range_seconds": span, - "series": [_history(ctx.store, DISKS_SOURCE, host, metric, span, points) - for host in wanted_hosts for metric in wanted_metrics]} + "series": _history_batch(ctx.store, DISKS_SOURCE, wanted_hosts, wanted_metrics, + span, points)} return router @@ -854,8 +878,9 @@ def llm_router() -> APIRouter: wanted_engines = [item.strip() for item in engines.split(",") if item.strip()] or [ name for name, url in (("gufo", ctx.settings.gufo_url), ("vllm", ctx.settings.vllm_url)) if url] return {"engines": wanted_engines, "range_seconds": span, - "series": [_history(ctx.store, "llm", engine, f"llm.{metric}", span, points) - for engine in wanted_engines for metric in wanted_metrics]} + "series": _history_batch(ctx.store, "llm", wanted_engines, + [f"llm.{metric}" for metric in wanted_metrics], + span, points)} return router @@ -893,8 +918,7 @@ def ups_router() -> APIRouter: wanted_ups = [item.strip() for item in ups.split(",") if item.strip()] or [ card["name"] for card in ((job.state.payload if job else None) or {}).get("ups", [])] return {"ups": wanted_ups, "range_seconds": span, - "series": [_history(ctx.store, "ups", name, metric, span, points) - for name in wanted_ups for metric in wanted_metrics]} + "series": _history_batch(ctx.store, "ups", wanted_ups, wanted_metrics, span, points)} @router.get("/commands") async def commands(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: diff --git a/src/dashboards/store.py b/src/dashboards/store.py index 6650ca7..abd6276 100644 --- a/src/dashboards/store.py +++ b/src/dashboards/store.py @@ -28,11 +28,30 @@ first start (phase 11). Retention never touches it; it holds a handful of rows. The connection is guarded by a lock and every method is synchronous: async callers wrap them in `anyio.to_thread.run_sync`, sync FastAPI handlers run in the threadpool already. + +Phase 14 task 02 added the two short-TTL read caches (`series_names`, `stats`) and the +capped per-resolution count. Both exist for the same reason: a per-request answer that costs +a whole-table pass is a page load, and on a 600 MB store the two answers that used to be +scanned on every request cost ~1.3 s each (`data/logs/NOTES-store-perf.md`), serialized +behind the very mutex the poller writes under. A cached answer is a *bounded* answer; the +TTL is how stale it may get before it is re-derived, and a write invalidates the one whose +staleness would be a lie about the data (`series_names`) rather than about a counter. + +Phase 14 task 03 bounded the *write* side. Each rollup step now scans only the rows that +became due since the previous pass (`maintain()` reads `last_maintain` first and windows each +step to it), because the whole-table rollup was a 5.4 s write transaction that held the WAL +write lock past a poller's `busy_timeout` — the `store-error: database is locked` line of +AGENTS.md known gap 4. Two properties make the window a cost bound rather than a data-bound +change, and both are pinned in `tests/test_store.py`: its lower bound reaches **past** the +previous pass (one bucket plus slack, so a missed interval is a catch-up scan and not a gap), and +it is snapped down to a **bucket boundary** (so a bucket any pass re-reads is a bucket it reads +whole, which is what keeps the averaged values identical to a whole-table rollup's). """ from __future__ import annotations import json +import logging import math import sqlite3 import threading @@ -42,9 +61,94 @@ from dataclasses import dataclass from pathlib import Path RES_RAW, RES_MIN, RES_HOUR = "raw", "1m", "1h" +#: The three resolutions, finest first. Every resolution decision in the app walks this tuple, +#: so `pick_res` and `pick_res_many` cannot disagree about which one is finest. +RESOLUTIONS: tuple[str, ...] = (RES_RAW, RES_MIN, RES_HOUR) MINUTE = 60.0 HOUR = 3600.0 +#: How long `series_names()` may answer from its cache (phase 14 task 02). The set of distinct +#: metric names changes when a source starts or stops publishing one — minutes, not seconds — +#: and a write that can introduce one invalidates it anyway (task 04: *that* write, not every +#: sample the hub stores), so the TTL is only the ceiling on a miss we did not notice. Instance +#: attribute `series_ttl_seconds` overrides it (tests, probes). +SERIES_CACHE_TTL_SECONDS = 60.0 +#: How long `stats()` may answer from its cache. `/api/health` is polled by every open tab every +#: 15 s and by the container healthcheck, and its counts are operator trivia, not a control +#: surface: 30 s of staleness buys a 1.3 s full-table scan off every poll but one. +STATS_CACHE_TTL_SECONDS = 30.0 +#: How many cached `(source, host)` answers `series_names()` keeps before starting over. The +#: endpoint takes both as free-text query parameters, so the key space is the caller's; the +#: cache is bounded rather than grown by whatever a stranger types. +SERIES_CACHE_MAX_ENTRIES = 256 +#: How many `(source, host, metric)` triples the store remembers having written, which is what +#: lets `add_many` leave the series cache alone (see `_note_series_write`). The fleet's whole +#: name space is a few hundred triples, so this is a generous ceiling on a bounded namespace, +#: not a growing structure: past it the store goes back to invalidating on every write, because +#: memory tracked on a namespace a caller can influence needs a cap either way. +SERIES_TRACKED_SERIES_MAX = 8192 +#: Branches per compound count statement. SQLite's `SQLITE_LIMIT_COMPOUND_SELECT` defaults to +#: 500 terms and the variable limit to 32 766; 100 keeps one statement per 33 series while +#: leaving 5x headroom on both limits, and it is where the statement-preparation cost stops +#: being noise (measured on the seeded store: 81 branches ≈ 0.5 ms of preparation). +COMPOUND_CHUNK = 100 + +#: Slack on a bucket's completeness, and on how far behind the previous pass a rollup looks. +#: A minute bucket is only rolled once its whole 60 s is in the past *plus* this slack, so a +#: rollup never averages a partial minute and gets rewritten by the next pass; the same 60 s on +#: the window's lower bound is what makes a missed interval a catch-up scan rather than a gap. +MAINTAIN_SLACK_SECONDS = 60.0 + +#: The `meta` key holding the previous `maintain()` pass's clock. Written by `maintain()` and read +#: by it on the next run to window the rollups; `stats()` surfaces it in `/api/health` (uncached) +#: precisely so an operator can see it stop moving. +LAST_MAINTAIN_KEY = "last_maintain" + +#: WAL auto-checkpoint threshold, in pages. This is SQLite's own default (1 000 pages ≈ 4 MB) and +#: is written down so the knob is visible next to the one that actually bites: a WAL that reaches +#: 140 MB (AGENTS.md known gap 4) is a checkpoint that keeps losing to the write rate, and +#: asking for one more often does not help — a checkpoint cannot run *inside* a long transaction. +WAL_AUTOCHECKPOINT_PAGES = 1000 +#: Cap on the WAL file, in bytes. At a checkpoint a WAL larger than this is truncated to it +#: instead of staying at its high-water mark forever, so the file has a ceiling even when a +#: reader kept a snapshot open long enough to make it grow. 64 MB sits comfortably above what the +#: write path asks for (the whole-retention pass, the biggest thing this does, left a 41 MB WAL on +#: the seeded store) and far below the 140 MB gap 4 recorded, so the cap binds on runaways and not +#: on maintenance. It is a per-connection pragma, not a migration, so moving it is a one-line +#: change (task 04's live watch decides whether to). +JOURNAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 + +#: SQLite's background ANALYZE, deliberately off. It refreshes statistics inside whichever commit +#: trips its change counter, which on this app means inside a poller's `add_many` — the write path +#: this task exists to shorten. `maintain()` calls `PRAGMA optimize` instead: once a minute, on the +#: tick that already owns bookkeeping, outside the write transaction. +AUTOMATIC_ANALYZE = 0 + +#: One rollup step: average `res_from` rows into `res_to` buckets, complete buckets only. +#: Bindings: `(res_to, bucket, bucket, res_from, ceiling, floor, bucket)`. Public because +#: `tests/store_perf.py` EXPLAINs the same statement — the plan in the NOTES has to be the plan +#: that runs (the same reason `RES_COUNT_BRANCH` is a constant). +ROLLUP_INSERT = ( + "INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)" + " SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)" + " FROM sample WHERE res=? AND ts < ? AND ts >= ?" + " GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)" +) + +logger = logging.getLogger("dashboards.store") + +#: One `(host, metric, res)` row of the capped count. Public because `tests/store_perf.py` +#: EXPLAINs it: the plan the NOTES record has to be the plan that runs, not a lookalike. +#: +#: The count is capped by `LIMIT` inside the subquery because the only question ever asked of it +#: is "is this <= max_points", and counting 43 200 minute rows to answer "no" is what made a 90 d +#: chart request expensive. The three label columns come first so the answer does not depend on +#: compound-row ordering. +RES_COUNT_BRANCH = ( + "SELECT ?, ?, ?, COUNT(*) FROM (SELECT 1 FROM sample" + " WHERE source=? AND host=? AND metric=? AND res=? AND ts>=? LIMIT ?)" +) + _SCHEMA = """ CREATE TABLE IF NOT EXISTS sample ( source TEXT NOT NULL, @@ -104,6 +208,47 @@ class Event: } +def _pairs(hosts: Sequence[str], metrics: Sequence[str]) -> list[tuple[str, str]]: + """The host-major series list a request asks for, de-duplicated but otherwise in order. + + `dict.fromkeys` rather than a set: the order is the order the endpoints emit their series in, + and a set would shuffle a response that a browser (and a test pin) expects in request order. + """ + return [(host, metric) for host in dict.fromkeys(hosts) for metric in dict.fromkeys(metrics)] + + +def _choose_res(counts: dict[str, int], max_points: int) -> str: + """The resolution decision, from one series' capped per-`res` counts. + + Counts are capped at `max_points + 1`, which leaves both predicates below exactly as true as + they were against a full `COUNT(*)`: `count` is 0 exactly when the real count is, and + `≤ max_points` exactly when the real count is. + """ + finest_with_data: str | None = None + for res in RESOLUTIONS: + count = counts.get(res, 0) + if not count: + continue + if finest_with_data is None: + # too many points for the budget is fine — history() thins. Returning a + # resolution with *no* rows in the window reads as a dead sensor. + finest_with_data = res + if count <= max_points: + return res + return finest_with_data or RES_HOUR + + +def _thin(points: list[tuple[float, float]], max_points: int) -> list[tuple[float, float]]: + """Stride-thin a series to the point budget without ever losing the newest point.""" + if len(points) <= max_points: + return points + stride = math.ceil(len(points) / max_points) + thinned = points[::stride] + if thinned[-1] != points[-1]: + thinned.append(points[-1]) # never lose "now" to the thinning + return thinned + + class SampleStore: def __init__(self, path: str | Path, *, raw_keep_hours: float = 48, minute_keep_days: float = 30, hour_keep_days: float = 400) -> None: @@ -118,8 +263,82 @@ class SampleStore: self._db.execute("PRAGMA journal_mode=WAL") self._db.execute("PRAGMA synchronous=NORMAL") # a lost sample is fine, corruption is not self._db.execute("PRAGMA busy_timeout=5000") + # WAL housekeeping (phase 14 task 03): keep trying to checkpoint at the default threshold + # and cap the file, so a WAL that does grow — a long-lived reader holding a snapshot open + # is enough — stops at the cap at the next checkpoint instead of at 140 MB. Then take + # SQLite's statistics upkeep away from the poll path: without statistics the planner reads + # the table for every windowed rollup anyway (`SCAN sample`), and with SQLite's own + # automatic ANALYZE the refresh lands inside whichever poller's commit trips its counter. + # All three are per-connection pragmas, so they are re-asserted on every open, not migrated. + self._db.execute(f"PRAGMA wal_autocheckpoint={WAL_AUTOCHECKPOINT_PAGES}") + self._db.execute(f"PRAGMA journal_size_limit={JOURNAL_SIZE_LIMIT_BYTES}") + self._db.execute(f"PRAGMA automatic_analyze={AUTOMATIC_ANALYZE}") self._db.executescript(_SCHEMA) self._db.commit() + # The two read caches (see the TTL constants above). Both are instance attributes so a + # test or a probe can force expiry with `store.stats_ttl_seconds = 0` instead of waiting + # for a clock or monkeypatching a module-wide `time`. + self.series_ttl_seconds = SERIES_CACHE_TTL_SECONDS + self.stats_ttl_seconds = STATS_CACHE_TTL_SECONDS + # `(argument tuple) -> (generation, monotonic, answer)`. The generation is what makes a + # concurrent write safe: a thread that read the table *before* a write and stores its + # answer *after* it would otherwise park a stale list under a fresh timestamp. + # The `set` beside each cached list is what `add_many` consults to decide whether the + # write can change the answer at all (see `_note_series_write`); membership in a list of + # every metric name is not a check to make per sample. + self._series_cache: dict[tuple[str | None, str | None], + tuple[int, float, list[str], set[str]]] = {} + self._stats_cache: tuple[float, dict[str, int], int] | None = None + self._generation = 0 + #: `(source, host, metric)` triples this store has written rows for. Tracks *existence*, + #: not rows, and is capped (see `SERIES_TRACKED_SERIES_MAX`). + self._known_series: set[tuple[str, str, str]] = set() + + # -- the read caches -------------------------------------------------------- # + def invalidate_read_caches(self) -> None: + """Drop every cached read answer. + + `add_many` invalidates the name list only when it introduces a name + (`_note_series_write`); every write path that can *remove* one invalidates unconditionally. + This is the blunt instrument for the callers that need a cold measurement (the phase-14 + probe) or a test that wants to be sure it is not reading its own previous answer. + """ + self._generation += 1 + self._series_cache.clear() + self._stats_cache = None + + def _invalidate_series(self) -> None: + """Called from inside a write's own lock: never takes `self._lock` (it is not reentrant).""" + self._generation += 1 + self._series_cache.clear() + + def _note_series_write(self, series: Sequence[tuple[str, str, str]]) -> None: + """Invalidate the series-name cache only for a write that can change it. + + Every sample used to clear it, and the live watch is what caught the consequence: the hub + writes every 2-300 s, so on the operator's store `/api/metrics` answered with its full + `SELECT DISTINCT` scan on nearly every call (535 ms of store mutex, measured) — the same + per-request scan task 02 existed to remove, surviving behind a cache that never served. + + An insert cannot *remove* a name, so the cached answers change only when the write + introduces a `(source, host, metric)` this store has never written. A poll of an existing + series is therefore free, and a genuinely new name still costs exactly one invalidation, + which is what `tests/test_store.py` pins in both directions. + + `delete_series()` and `maintain()` keep the unconditional invalidation they have always + had, because a `DELETE` can take the last row of a name and existence-tracking cannot see + that. The tracking can also be wrong in the same direction if *another* process deletes + rows this one believes exist (two servers on one file, AGENTS.md gap 4's configuration + mistake): the cost is that a name reappearing inside one TTL window is not in the list + until the window ends. `series_ttl_seconds` is the ceiling on that, and it is the reason + the cache has a TTL rather than trusting the tracking alone. + """ + if any(key not in self._known_series for key in series): + self._invalidate_series() + if len(self._known_series) + len(series) <= SERIES_TRACKED_SERIES_MAX: + self._known_series.update(series) + else: # cap reached: forget the tracking, so from the next write on every one invalidates + self._known_series.clear() def close(self) -> None: with self._lock: @@ -144,6 +363,7 @@ class SampleStore: return 0 with self._lock, self._db: self._db.executemany("INSERT OR REPLACE INTO sample VALUES (?,?,?,?,?,?)", rows) + self._note_series_write([(row[0], row[1], row[2]) for row in rows]) return len(rows) def log_event(self, event: Event) -> None: @@ -182,48 +402,126 @@ class SampleStore: return cur.fetchall() # type: ignore[no-any-return] def pick_res(self, source: str, host: str, metric: str, span: float, max_points: int) -> str: - """Finest resolution that still fits in `max_points` for this window.""" - now = time.time() - start = now - span - finest_with_data: str | None = None - for res in (RES_RAW, RES_MIN, RES_HOUR): - with self._lock: - (count,) = self._db.execute( - "SELECT COUNT(*) FROM sample WHERE source=? AND host=? AND metric=? AND res=? AND ts>=?", - (source, host, metric, res, start), - ).fetchone() - if not count: - continue - if finest_with_data is None: - # too many points for the budget is fine — history() thins. Returning a - # resolution with *no* rows in the window reads as a dead sensor. - finest_with_data = res - if count <= max_points: - return res - return finest_with_data or RES_HOUR + """Finest resolution that still fits in `max_points` for this window. + + One statement, not three: the counts for all three resolutions come back from a single + compound query whose per-`res` `LIMIT` caps the work (phase 14 task 02). Capping is exact, + not an approximation, because the only thing the decision asks of a count is whether it is + `0`, `≤ max_points`, or neither — so counting 43 200 minute rows to answer "neither" was + pure waste. Measured on the seeded store: 90 d, one series, 2.97 ms → 0.10 ms; the + `tests/test_store.py` pin is that the *answer* is unchanged for every window. + """ + counts = self._counts_by_res(source, ((host, metric),), span, max_points) + return _choose_res(counts[(host, metric)], max_points) + + def pick_res_many(self, source: str, hosts: Sequence[str], metrics: Sequence[str], + span: float, max_points: int) -> dict[tuple[str, str], str]: + """`pick_res` for every `(host, metric)` pair, in one grouped round trip per 33 series. + + The batched door the N x M history endpoints go through. The decision is made by the same + `_choose_res` on the same capped counts, so a pair's answer here is identical to the answer + `pick_res` gives for it alone — that equality is a test, not a hope. + """ + counts = self._counts_by_res(source, _pairs(hosts, metrics), span, max_points) + return {pair: _choose_res(seen, max_points) for pair, seen in counts.items()} def history(self, source: str, host: str, metric: str, *, span: float = 3600.0, max_points: int = 600) -> list[tuple[float, float]]: """Windowed series, thinned to `max_points` so the browser never gets 100k rows.""" - res = self.pick_res(source, host, metric, span, max_points) - window_start = time.time() - span - points = self.query(source, host, metric, res=res, start=window_start, - limit=max_points * 4, tail=True) - if not points and res != RES_RAW: - # Rollups lag behind (they only cover buckets that are fully in the past, and a - # dashboard that has been up for ten minutes has none at all). Falling back to - # raw turns "no samples yet" into a real chart instead of a lie by omission. - points = self.query(source, host, metric, res=RES_RAW, start=window_start, + return self.history_many(source, [host], [metric], span=span, + max_points=max_points)[(host, metric)] + + def history_many(self, source: str, hosts: Sequence[str], metrics: Sequence[str], *, + span: float = 3600.0, + max_points: int = 600) -> dict[tuple[str, str], list[tuple[float, float]]]: + """`history` for many series at once, keyed by `(host, metric)`. + + Same per-series answer as `history()` — resolution choice, tail semantics, raw fallback, + thinning, and the point list that keeps "now" — with the resolution choice made in one + grouped statement instead of three counts per series. + + The *rows* are still fetched per series, deliberately and measuredly: the tail select is + already a covering-index seek that reads at most `max_points x 4` rows, and every shape + that folds the whole batch into one statement (a `ROW_NUMBER()` window, a compound of the + per-series tails) was slower on the seeded store — 73 ms and 38 ms against 29 ms for 27 + series at 90 d — because it numbers or sorts rows the seek never has to touch. Batching + the *decision* was where the cost was; batching the fetch is a round trip traded for work. + """ + pairs = _pairs(hosts, metrics) + if not pairs: + return {} + now = time.time() + window_start = now - span + counts = self._counts_by_res(source, pairs, span, max_points) + out: dict[tuple[str, str], list[tuple[float, float]]] = {} + for host, metric in pairs: + res = _choose_res(counts[(host, metric)], max_points) + points = self.query(source, host, metric, res=res, start=window_start, end=now, limit=max_points * 4, tail=True) - if len(points) <= max_points: - return points - stride = math.ceil(len(points) / max_points) - thinned = points[::stride] - if thinned[-1] != points[-1]: - thinned.append(points[-1]) # never lose "now" to the thinning - return thinned + if not points and res != RES_RAW: + # Rollups lag behind (they only cover buckets that are fully in the past, and a + # dashboard that has been up for ten minutes has none at all). Falling back to + # raw turns "no samples yet" into a real chart instead of a lie by omission. + points = self.query(source, host, metric, res=RES_RAW, start=window_start, + end=now, limit=max_points * 4, tail=True) + out[(host, metric)] = _thin(points, max_points) + return out + + def _counts_by_res(self, source: str, pairs: Sequence[tuple[str, str]], span: float, + max_points: int) -> dict[tuple[str, str], dict[str, int]]: + """Capped per-resolution row counts for many series, in as few statements as possible. + + One compound statement of `(host, metric, res)` branches, chunked at `COMPOUND_CHUNK` + terms, each branch an index seek on the full primary-key prefix (`tests/test_store.py` + pins the plan) that stops after `max_points + 1` rows. Nothing here scans a table: the + work is proportional to `series x max_points`, never to rows of history. + """ + start = time.time() - span + cap = max_points + 1 + out: dict[tuple[str, str], dict[str, int]] = {pair: {} for pair in pairs} + branches: list[str] = [] + args: list[object] = [] + + def flush() -> None: + if not branches: + return + with self._lock: + rows = self._db.execute(" UNION ALL ".join(branches), tuple(args)).fetchall() + for host, metric, res, count in rows: + out[(host, metric)][res] = int(count) + branches.clear() + args.clear() + + for host, metric in pairs: + for res in RESOLUTIONS: + branches.append(RES_COUNT_BRANCH) + args += [host, metric, res, source, host, metric, res, start, cap] + if len(branches) >= COMPOUND_CHUNK: + flush() + flush() + return out def series_names(self, source: str | None = None, host: str | None = None) -> list[str]: + """Distinct metric names, from a cache that only a write which can change them ends. + + The uncached answer is `SELECT DISTINCT metric` over the table — 1.4 s of `SCAN sample` + on a production-sized store, on a route the browser polls alongside the rest. The set of + names is also the slowest-moving fact in the database (it changes when a source gains a + metric, i.e. on a deploy), which is what a 60 s TTL plus write invalidation is sized for. + """ + key = (source, host) + cached = self._series_cache.get(key) + generation, built_at = self._generation, time.monotonic() + if cached is not None and cached[0] == generation and built_at - cached[1] < self.series_ttl_seconds: + return list(cached[2]) + names = self._scan_series_names(source, host) + if len(self._series_cache) >= SERIES_CACHE_MAX_ENTRIES: + self._series_cache.clear() + self._series_cache[key] = (generation, built_at, names, set(names)) + return list(names) + + def _scan_series_names(self, source: str | None, host: str | None) -> list[str]: + """The uncached answer behind `series_names()`: one pass, no cache, no TTL.""" sql = "SELECT DISTINCT metric FROM sample" args: list[object] = [] clauses = [] @@ -304,48 +602,166 @@ class SampleStore: return self._db.execute("DELETE FROM meta WHERE key=?", (key,)).rowcount # -- maintenance ----------------------------------------------------------- # + def _previous_maintain(self) -> float | None: + """The clock of the last `maintain()` pass, or `None` for "no idea — scan the retention". + + Missing is the normal case on a fresh database. A row that is not a number should not + exist (`maintain()` is the only writer) and is treated as missing rather than fatal, with + one line said about it: the expensive answer is the safe one here, because a rollup that + re-runs over buckets it already rolled rewrites identical rows. Never interpolate the + value — it is data the operator did not write, and this app's logs end up in an `event` + table a browser reads. + """ + raw = self.get_meta(LAST_MAINTAIN_KEY) + if raw is None: + return None + try: + return float(raw) + except ValueError: + logger.warning("meta row '%s' is not a number; the next rollup scans the full " + "retention window", LAST_MAINTAIN_KEY) + return None + + def _rollup_floor(self, since: float | None, retention_start: float, bucket: float) -> float: + """The oldest row a rollup step has to read, snapped down to a bucket boundary. + + With no previous pass that is the whole retention window, exactly as the step always + cost. With one, it is the window that became due since: `since - bucket - slack`, one + bucket plus the completeness slack behind the previous pass's clock. That overlap is what + turns "the app was down for ten minutes" into a ten-minute catch-up scan instead of nine + minutes of silently missing buckets. + + The snap down to a bucket boundary is not cosmetic, it is what makes the window safe. The + step's ceiling is a *row* bound, so the newest bucket it touches is usually a partial one + that a later pass re-rolls as more of its rows land — that progressive correction is how + the whole-table version always got the right average. Re-reading a bucket from anywhere + other than its own start therefore overwrites a fuller average with a thinner one: with a + floor of exactly `since - bucket - slack` a minute bucket split by the boundary ends up as + the mean of its own tail rows (`tests/test_store.py` pins both numbers). Aligning the floor + down means every bucket this pass reads again is one it reads *whole*, so each re-roll can + only add rows to it — which is the same guarantee the unwindowed scan had. + """ + if since is None: + return retention_start + aligned = math.floor((since - bucket - MAINTAIN_SLACK_SECONDS) / bucket) * bucket + return max(retention_start, aligned) + def maintain(self, now: float | None = None) -> dict[str, int]: - """Roll up and prune. Safe to call every minute; each step is idempotent. + """Roll up and prune, windowed to what became due since the last pass. Idempotent. A bucket is only rolled once its whole window is in the past (+60s slack), so a rollup never averages a partial minute and then gets rewritten by a later one. + + Both rollup steps were `INSERT … SELECT … GROUP BY` over everything their resolution + retains, which on a store with 30 days of 1-minute rows meant a 3.97 M-row scan inside the + one write transaction (task 01's measurement: 5.4 s, and 5 of 9 concurrent poller inserts + then outlived `busy_timeout=5000` and surfaced as `store-error`). Windowing each step to + `last_maintain` is what makes the transaction short: a run once a minute aggregates the + buckets that filled in the last minute, and the first run after a restart — or after a + night with the app stopped — scans back to the previous pass instead of back to retention. + + What a window costs, stated plainly: a raw row that lands *behind* the window (a + multi-minute poller stall, or a host whose clock put the sample far in the past) is not + re-read by the next pass, so the bucket it belongs to keeps the average the earlier pass + computed. The poller stamps samples with its own wall clock, so reaching that case costs a + stall measured in minutes; re-scanning 30 days on every tick to cover it is the thing this + task exists to stop. `_rollup_floor` explains the bucket alignment that keeps every bucket + inside the window honest, and + `test_windowed_rollups_answer_exactly_what_one_full_window_rollup_answers` is the proof. """ now = time.time() if now is None else now + # Outside the transaction and before the lock below: `get_meta` takes `self._lock` and + # `threading.Lock` is not reentrant (the same reason `stats()` reads meta from outside). + since = self._previous_maintain() + minute_ceiling = now - MINUTE - MAINTAIN_SLACK_SECONDS + hour_ceiling = now - HOUR - MAINTAIN_SLACK_SECONDS counts = {"to_minute": 0, "to_hour": 0, "pruned": 0} with self._lock, self._db: counts["to_minute"] = self._db.execute( - "INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)" - " SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)" - " FROM sample WHERE res=? AND ts < ? AND ts >= ?" - " GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)", - (RES_MIN, MINUTE, MINUTE, RES_RAW, now - MINUTE - 60, - now - self.raw_keep_seconds, MINUTE), + ROLLUP_INSERT, + (RES_MIN, MINUTE, MINUTE, RES_RAW, minute_ceiling, + self._rollup_floor(since, now - self.raw_keep_seconds, MINUTE), MINUTE), ).rowcount counts["to_hour"] = self._db.execute( - "INSERT OR REPLACE INTO sample (source, host, metric, res, ts, value)" - " SELECT source, host, metric, ?, CAST(CAST(ts AS INT)/? AS INT)*?, AVG(value)" - " FROM sample WHERE res=? AND ts < ? AND ts >= ?" - " GROUP BY source, host, metric, CAST(CAST(ts AS INT)/? AS INT)", - (RES_HOUR, HOUR, HOUR, RES_MIN, now - HOUR - 60, - now - self.minute_keep_seconds, HOUR), + ROLLUP_INSERT, + (RES_HOUR, HOUR, HOUR, RES_MIN, hour_ceiling, + self._rollup_floor(since, now - self.minute_keep_seconds, HOUR), HOUR), ).rowcount - counts["pruned"] = self._db.execute( - "DELETE FROM sample WHERE (res=? AND ts < ?) OR (res=? AND ts < ?) OR (res=? AND ts < ?)", - (RES_RAW, now - self.raw_keep_seconds, - RES_MIN, now - self.minute_keep_seconds, - RES_HOUR, now - self.hour_keep_seconds), - ).rowcount - self._db.execute("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_maintain', ?)", (str(now),)) + # The prune keeps its full retention bound on purpose. A rollup may window because it + # only ever *adds* rows and re-adding one is a rewrite; a prune must be able to reach + # anything that expired, whenever it expired, or gap 9's "the file stays bounded" stops + # being true. It is a bounded delete per resolution, not a GROUP BY. + # + # One DELETE per resolution, not one `OR` of the three, because the composition is the + # part the planner cannot use. `WHERE (res=? AND ts None: + """`PRAGMA optimize`, best effort, outside the write transaction. + + This is what makes the windowing measurable. Without query statistics SQLite cannot tell a + two-minute window from a whole-retention scan and picks the scan for both — the windowed + rollup cost 328 ms and the prune 554 ms with `SCAN sample` either way, and 1 ms and 0 ms + with `SEARCH sample USING INDEX sqlite_autoindex_sample_1` after one `optimize`. Statistics + are the other half of "cheaper for the same rows", not an optional extra. + + SQLite's own automatic ANALYZE is switched off in `__init__`, so this is the only place + statistics are refreshed: automatic analyse runs inside whichever poll's commit trips its + change counter, which is exactly the poll path this task is shortening. Here it is once a + minute, on the tick that already owns maintenance — a sampled full analyse the first time + (~100 ms on the seeded store) and 0.0 ms on every tick after, because SQLite only re-samples + tables whose change counter moved. A failure is logged and dropped: stale statistics are a + slower query, and `store-error` on a maintenance tick because of a planner hint would be a + worse report to the operator. + """ + try: + with self._lock: + self._db.execute("PRAGMA optimize") + except sqlite3.Error: + logger.warning("PRAGMA optimize failed; the query planner keeps its existing statistics") + def stats(self) -> dict[str, object]: + """Store size and row counts for `/api/health`, with the two COUNT scans cached. + + The payload keys are a contract with deployment (`/api/health` is the one public data + route, AGENTS.md invariant 12), so nothing here changed shape — only how often it is + derived from the table. `last_maintain` and `size_bytes` are deliberately **not** cached: + the first is how an operator tells a stalled rollup loop from a healthy one and costs + a three-row lookup, and the second is a `stat()` call, not a query. + """ + cached = self._stats_cache + now = time.monotonic() + if cached is not None and now - cached[0] < self.stats_ttl_seconds: + by_res, events = dict(cached[1]), cached[2] + else: + with self._lock: + by_res = {row[0]: row[1] for row in self._db.execute( + "SELECT res, COUNT(*) FROM sample GROUP BY res")} + (events,) = self._db.execute("SELECT COUNT(*) FROM event").fetchone() + self._stats_cache = (time.monotonic(), dict(by_res), events) # Read through `get_meta` so there is one path to the table — but *outside* the lock - # below, which `get_meta` takes itself (`threading.Lock` is not reentrant). - last_raw = self.get_meta("last_maintain") - with self._lock: - by_res = {row[0]: row[1] for row in self._db.execute( - "SELECT res, COUNT(*) FROM sample GROUP BY res")} - (events,) = self._db.execute("SELECT COUNT(*) FROM event").fetchone() + # above, which `get_meta` takes itself (`threading.Lock` is not reentrant). + last_raw = self.get_meta(LAST_MAINTAIN_KEY) size = 0 if str(self.path) != ":memory:" and self.path.exists(): size = self.path.stat().st_size @@ -367,7 +783,18 @@ class SampleStore: sql += " AND metric=?" args.append(metric) with self._lock, self._db: - return self._db.execute(sql, args).rowcount + deleted = self._db.execute(sql, args).rowcount + # Forget the triples this delete emptied, so the next write of one of them counts as + # the new series it now is: `add_many` invalidates on a triple it has not seen, and a + # deleted-then-recreated series that stayed "known" would be invisible to + # `/api/metrics` for a TTL window. One `DELETE` cannot name the triples it emptied, + # so this drops whatever the predicate could have covered. + self._known_series = { + key for key in self._known_series + if not (key[0] == source and key[1] == host and (metric is None or key[2] == metric)) + } + self._invalidate_series() + return deleted def samples_from_flat(source: str, host: str, values: dict[str, float | None], diff --git a/tests/conftest.py b/tests/conftest.py index 9696265..e56fb1e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,15 +27,33 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=False, help="run tests marked `live` against the real fleet/engines/upsd (they skip otherwise)", ) + parser.addoption( + "--store-perf", + action="store_true", + default=False, + help="run tests marked `store_perf`: the phase-14 timing measurements against the " + "seeded production-shaped store (they skip otherwise — they cost minutes, not ms)", + ) def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - if config.getoption("--live"): - return - skip = pytest.mark.skip(reason="needs --live and a reachable target (see tests/test_live_fleet.py)") + # Two opt-ins, one shape: a marker whose tests are real but expensive, skipped by default + # so `uv run pytest` stays fast and hermetic. `--store-perf` is the phase-14 half of the + # rule `--live` established; `scripts/store_perf_probe.py` is the CLI for the same numbers. + skips = { + "live": pytest.mark.skip( + reason="needs --live and a reachable target (see tests/test_live_fleet.py)"), + "store_perf": pytest.mark.skip( + reason="needs --store-perf (a seeded ~6 M-row store; see tests/store_perf.py)"), + } + enabled = { + "live": bool(config.getoption("--live")), + "store_perf": bool(config.getoption("--store-perf")), + } for item in items: - if "live" in item.keywords: - item.add_marker(skip) + for marker, skip in skips.items(): + if not enabled[marker] and marker in item.keywords: + item.add_marker(skip) ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" @@ -228,3 +246,40 @@ def metrics_text() -> Any: @pytest.fixture def rates() -> RateEstimator: return RateEstimator(span=10.0) + + +# ---- phase 14: the production-shaped store the hot-path budgets measure ---- # +# +# Here rather than in tests/test_store.py because two files price the same budgets — the store +# paths in `test_store.py`, the HTTP paths in `test_api.py` — and a second seeded 6.4 M-row file +# (or, worse, a second *differently shaped* one) would make the two sets of numbers describe two +# databases. Session-scoped because seeding costs ~25 s, and cached on disk (`data/perf/`, see +# `store_perf.cache_dir`) because the before/after has to be the same physical file both times. + + +@pytest.fixture(scope="session") +def perf_store_path() -> Path: + """The cached seeded store, built on first use and re-verified on every reuse. + + Opt-in tests only: nothing in the default run asks for this fixture, so `uv run pytest` never + pays for the seed. `DASH_PERF_STORE_DIR` relocates the file (see `store_perf.cache_dir`). + """ + import store_perf # noqa: PLC0415 - only the phase-14 tests seed a store + + try: + return store_perf.ensure_seeded(log=lambda line: print(line, flush=True)) + except OSError as exc: # pragma: no cover - only a read-only checkout hits this + pytest.skip(f"cannot build the seeded store ({exc}) — set DASH_PERF_STORE_DIR " + "to a writable directory to run the phase-14 measurements") + + +@pytest.fixture +def perf_store(perf_store_path: Path) -> Any: + """The seeded store, opened per test and closed after (one handle per test, one file).""" + import store_perf # noqa: PLC0415 - see perf_store_path + + store = store_perf.open_store(perf_store_path) + try: + yield store + finally: + store.close() diff --git a/tests/store_perf.py b/tests/store_perf.py new file mode 100644 index 0000000..675c252 --- /dev/null +++ b/tests/store_perf.py @@ -0,0 +1,1696 @@ +r"""The production-shaped store phase 14 measures, and the measurements of its hot paths. + +`TODO.md:7` is a complaint about speed, and a speed claim needs a "before" that is measured +rather than remembered. This module is that "before": it seeds a SQLite file with the **shape +and retention semantics of the deployed store** and times every hot read and write path of +`store.py`/`api.py` against it, so tasks 02-04 change a number they can point at. + +Why a synthetic seed and not the dev DB +--------------------------------------- +`AGENTS.md`'s testing policy keeps `uv run pytest` hermetic: no test opens +`data/dashboards.sqlite3`, because a test that reads the operator's live history changes +behaviour when the operator's live history changes (and a fixture assertion that passes +today and fails next month is not a test). The honest substitute is a file of the same +*shape*: the same `SampleStore` defaults for retention (`raw_keep_hours=48`, +`minute_keep_days=30`, `hour_keep_days=400` — i.e. what a 90 d chart actually reads, 1m/1h, +never raw beyond 48 h), the fleet's real metric names, the real 10 s cadence, and the same +`meta`/`event` rows the app writes. + +The seeded numbers are recorded in `data/logs/NOTES-store-perf.md` together with the +dev DB's real inventory, so a reader can see where the reproduction is faithful and where it +is a deliberate subset (per-core `cpu.usage_pct@coreN` series and the `llm`/`ups`/`disks` +families are excluded — see `HOST_SERIES`). + +Where the file lives +-------------------- +`cache_path()` (default `data/perf/`, overridable with `DASH_PERF_STORE_DIR`) rather than a +pytest tmp dir, for two reasons that are both about the *measurement*, not convenience: + +* the same physical file has to be timed before task 02 and after task 04, or the two numbers + describe two different databases; and +* the repo filesystem is a real block device, while pytest's `tmp_path` here is `tmpfs` — + RAM. Timing a store on RAM is a flattering number nobody deploys. + +Write-side measurements (`measure_maintain`, the two-connection races) run against a +`scratch_copy()` of that file rather than the file itself: a catch-up rollup really does roll ten +more minutes of buckets and the pre-fix control rewrites 330 000 rows, and a baseline that moves +under the next task is not a baseline. The copy is in the same directory, so it is the same device. + +The file is a cache, keyed by `SEED_VERSION` and by its own age (`MAX_CACHE_AGE_HOURS`): the +deep 1m/1h history is anchored to the time it was built, so a cache older than half a day no +longer covers "the last 90 days" and gets rebuilt. Builds are serialised with `flock` and +publish atomically, so two sessions never read a half-written store. + +Nothing here is a production module: `src/dashboards` must stay free of benchmark furniture, +and coverage counts `src/dashboards` only. +""" + +from __future__ import annotations + +import fcntl +import json +import math +import os +import platform +import shutil +import sqlite3 +import statistics +import sys +import threading +import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field, replace +from pathlib import Path +from typing import Any + +from dashboards.store import ( + HOUR, + JOURNAL_SIZE_LIMIT_BYTES, + LAST_MAINTAIN_KEY, + MAINTAIN_SLACK_SECONDS, + MINUTE, + RES_COUNT_BRANCH, + RES_HOUR, + RES_MIN, + RES_RAW, + ROLLUP_INSERT, + Event, + Sample, + SampleStore, +) + +ROOT = Path(__file__).resolve().parents[1] + +#: Bump when the shape below changes: the cached file is named after it, so an old cache can +#: never be mistaken for the new shape. +SEED_VERSION = "phase14-v1" + +#: The seed's clock, aligned to a minute so every rollup bucket boundary is exact. Stored in +#: `meta` (`perf.seed_now`) and re-read by the verifier. +MAX_CACHE_AGE_HOURS = 12.0 + +#: The store's own retention defaults, restated here so the fixture cannot silently seed a +#: shape the app does not keep (`test_the_seed_uses_the_stores_own_retention` asserts it). +RAW_KEEP_HOURS, MINUTE_KEEP_DAYS, HOUR_KEEP_DAYS = 48.0, 30.0, 400.0 + +#: The `system` poll cadence (`HOST_INTERVAL_SECONDS` default) — the raw window's spacing. +CADENCE_SECONDS = 10.0 + +#: Event rows: the dev DB held ~2 800 after a few days of transitions and logins; 2 000 is +#: the same order and keeps `event` big enough for `stats()` to notice it. +EVENT_ROWS = 2000 + +#: Samples written by the measurements themselves (`race_*`) use this source so they can never +#: be confused with the seeded fleet, and one `delete_series` cleans them up. +RACE_SOURCE, RACE_HOST = "perf", "race-target" + +#: The pass line for "did this insert wait": one poll of 30 samples costs ~1 ms, so anything +#: past this is a wait behind someone else's transaction, not the cost of the insert. +WAIT_THRESHOLD_MS = 100.0 + +#: `busy_timeout` in `store.py`. Production value; the race reports waits against it. +BUSY_TIMEOUT_MS = 5000 + +#: Wall-clock budget per race pass. One `stats()` on this store costs seconds, so "100 inserts" +#: is minutes of deliberate waiting; the report always says how many were attempted. +MAX_RACE_SECONDS = 30.0 + +#: Spacing between the raced inserts. The app does not insert 100 polls back to back: the hub +#: has ~9 sources writing every 2-300 s. Un-paced, the inserts finish before the contending +#: transaction even opens, and a race that never happened reports "no lock errors". +RACE_PACE_SECONDS = 0.1 + +#: Idle time between the maintainer's passes in the cadence-shaped race. The app maintains once +#: every 60 s, so its write transaction holds the WAL lock for a couple of seconds per minute at +#: the very most; the race compresses that minute to this. The saturated variant (`gap=0`, which is +#: what task 01 ran and what its numbers stay comparable to) holds the lock ~100% of the wall +#: clock, because the loop restarts the transaction the instant it commits — no window fixes that, +#: so it answers "can a writer starve forever", not "what does a poller wait during a rollup". +RACE_MAINTAIN_GAP_SECONDS = 1.5 + +META_SEED_NOW = "perf.seed_now" +META_SEED_VERSION = "perf.seed_version" + + +# --------------------------------------------------------------------------- # +# The shape: real hosts, real metric names, real cadence +# --------------------------------------------------------------------------- # +#: The `host`-family series one agent publishes, minus the per-core fan-out. +#: +#: Every name here is a name the deployed store actually holds (`SELECT DISTINCT metric FROM +#: sample WHERE source='host'` on the dev DB), because `series_names()`/`history()` cost +#: depends on the number of distinct names and on row width. Deliberately excluded, with the +#: consequence recorded in the NOTES rather than hidden: +#: +#: * `cpu.usage_pct@coreN` — 48 more series on driveripper. `system` stopped plotting them in +#: phase 03 (a locked decision), so no hot path of this phase reads them; including them +#: would double the raw window for the family the phase optimises. +#: * the `llm`/`ups`/`disks` families — a *mature* store holds them too (17 + 24 + 4 series at +#: 2 s / 5 s / 300 s), and the extrapolation is in the NOTES. They are out of the seed +#: because the prescribed 90-series x 48 h x 30 d x 400 d shape already lands in the dev +#: DB's order of magnitude, and a bigger fixture buys nothing this phase must decide. +BASE_SERIES: tuple[str, ...] = ( + "cpu.context_switches_per_s", + "cpu.freq_mhz", + "cpu.iowait_pct", + "cpu.load1", + "cpu.load15", + "cpu.load5", + "cpu.system_pct", + "cpu.usage_pct", + "cpu.user_pct", + "disk.read_bytes_per_s", + "disk.reads_per_s", + "disk.used_bytes@/", + "disk.used_pct", + "disk.used_pct@/", + "disk.util_pct", + "disk.write_bytes_per_s", + "disk.writes_per_s", + "host.processes", + "host.uptime_seconds", + "mem.available_bytes", + "mem.cached_bytes", + "mem.swap_used_bytes", + "mem.total_bytes", + "mem.used_bytes", + "mem.used_pct", + "net.rx_bytes_per_s", + "net.rx_bytes_per_s@enp3s0", + "net.tx_bytes_per_s", +) + +#: The four GPU series, on the two hosts that have a card (`gateway` has none — the shape the +#: `/system` contract calls "no GPU at all", so a plan that assumes every host has a stream +#: fails here rather than in a browser). +GPU_SERIES: tuple[str, ...] = ( + "gpu.busy_pct@0", + "gpu.mem_used_pct@0", + "gpu.power_watts@0", + "gpu.temp_c@0", +) + +#: host -> its series. Names are the fleet's; the values are synthetic (`_value`). +HOST_SERIES: dict[str, tuple[str, ...]] = { + "deskwork": (*BASE_SERIES, *GPU_SERIES), + "driveripper": (*BASE_SERIES, *GPU_SERIES), + "gateway": BASE_SERIES, +} + +#: The nine metrics the task-01 `/api/system/history` request asks for, per host: the three +#: the utilisation chart plots, the two network rates, the worst mount, the CPU temp, and the +#: GPU stream. 3 hosts x 9 = the 27 `_history` calls the endpoint makes for one page load. +HISTORY_METRICS: tuple[str, ...] = ( + "cpu.usage_pct", + "mem.used_pct", + "disk.used_pct", + "net.rx_bytes_per_s", + "net.tx_bytes_per_s", + "disk.used_pct@/", + "cpu.temp_c", + "gpu.busy_pct@0", + "gpu.temp_c@0", +) + +#: The metrics `/system` actually asks a *page* for (UTIL_SERIES + NET_SERIES + one GPU), i.e. +#: the request shape `page-system.js` really makes, one per host per refresh. +PAGE_HISTORY_METRICS: tuple[str, ...] = ( + "cpu.usage_pct", + "mem.used_pct", + "disk.used_pct", + "net.rx_bytes_per_s", + "net.tx_bytes_per_s", + "gpu.busy_pct@0", +) + + +def seed_value(metric: str, index: int) -> float: + """Deterministic pseudo-signal for one sample: finite, plausible, never random. + + Determinism is the point (`test_the_seed_is_deterministic_and_the_rollups_are_idempotent` + recomputes a rollup from this function), so there is no RNG: a phase of the row index plus + a per-metric offset. Values are plausible rather than realistic — a percentage between 0 + and 100, a byte counter that grows — because a chart of them is never read; only the row + count, the row width, and the arithmetic of the rollups matter. + + Public because `tests/test_store.py` recomputes stored rows from it: that is how the fixture + proves a rebuild is byte-identical to the store it measured. + """ + offset = (len(metric) * 31 + sum(ord(ch) for ch in metric[:6])) % 997 + wave = math.sin((index + offset) / 360.0) + if metric.startswith(("mem.", "disk.used_bytes", "net.")): + # byte counters: hundreds of MB to tens of GB, and monotonic for the ones that are + # odometers + base = 1e8 + (offset % 40) * 5e8 + return base + wave * 5e7 + index * 128.0 + if metric.endswith("_bytes") or metric.endswith("_bytes_per_s"): + return 1e6 + wave * 5e5 + index * 4.0 + if "temp_c" in metric: + return 45.0 + wave * 12.0 + if metric.startswith("gpu.power"): + return 90.0 + wave * 40.0 + if metric.startswith(("cpu.load",)): + return 2.0 + wave * 2.0 + if metric.endswith("_pct") or metric.endswith("_percent"): + return 40.0 + wave * 30.0 + if metric.endswith("_per_s"): + return 5_000.0 + wave * 2_000.0 + if metric == "host.uptime_seconds": + return index * CADENCE_SECONDS + return 10.0 + wave * 5.0 + + +@dataclass(frozen=True) +class SeedPlan: + """What a build *should* contain, computed before (and independently of) the insert. + + `*_steps` are rows **per series** (what the insert loops count); `*_rows` are the totals + the verifier compares against `COUNT(*)`. Confusing the two once already made this fixture + try to seed 75 million rows, which is why both are spelled out instead of derived at the + call site. + """ + + now: float + hosts: tuple[str, ...] + series_total: int + raw_steps: int + minute_steps: int + hour_steps: int + events: int + + @property + def raw_rows(self) -> int: + return self.series_total * self.raw_steps + + @property + def minute_rows_seeded(self) -> int: + return self.series_total * self.minute_steps + + @property + def hour_rows_seeded(self) -> int: + return self.series_total * self.hour_steps + + @property + def sample_rows_seeded(self) -> int: + return self.raw_rows + self.minute_rows_seeded + self.hour_rows_seeded + + def as_dict(self) -> dict[str, Any]: + """The plan plus its derived totals — `dataclasses.asdict` alone would drop them.""" + return {**asdict(self), "raw_rows": self.raw_rows, + "minute_rows_seeded": self.minute_rows_seeded, + "hour_rows_seeded": self.hour_rows_seeded, + "sample_rows_seeded": self.sample_rows_seeded} + + +def plan(now: float) -> SeedPlan: + """The row counts a build of this shape produces, from the shape alone. + + The deep 1m history starts where the raw window ends (that is what a store with 48 h of + raw retention looks like: anything older than 48 h exists only as a rollup) and the deep + 1h history starts where the 1m window ends, for the same reason. The rows inside those + windows are produced by the real `maintain()`, so the rollup path is exercised rather + than faked. + + Every count is in *its own* unit: a step is one row of that resolution, so the deep window + is `retention - the window the finer resolution already covers`, both converted to that + resolution's period. + """ + raw_steps = int(RAW_KEEP_HOURS * HOUR // CADENCE_SECONDS) + minute_steps = int((MINUTE_KEEP_DAYS * 24 * HOUR - RAW_KEEP_HOURS * HOUR) // MINUTE) + hour_steps = int((HOUR_KEEP_DAYS * 24 * HOUR - MINUTE_KEEP_DAYS * 24 * HOUR) // HOUR) + series = sum(len(metrics) for metrics in HOST_SERIES.values()) + return SeedPlan( + now=now, + hosts=tuple(sorted(HOST_SERIES)), + series_total=series, + raw_steps=raw_steps, + minute_steps=minute_steps, + hour_steps=hour_steps, + events=EVENT_ROWS, + ) + + +# --------------------------------------------------------------------------- # +# building +# --------------------------------------------------------------------------- # +def open_store(path: str | Path, *, extra_retention_hours: float = 0.0) -> SampleStore: + """A `SampleStore` on `path` with the retention the seed promises. + + `extra_retention_hours` is for handles that must **not** prune: the seeded store is anchored + to the time it was built, so a `maintain()` at the wall clock while the cache is six hours + old prunes six hours of raw rows, and the next session would rebuild the whole thing over a + measurement artefact. The cache's own age is the only thing past the window, and the read + paths this phase optimises (`history`, `pick_res`, `series_names`, `stats`) do not read + retention at all — so widening it for a measurement handle changes no number and keeps the + baseline byte-stable. The store under test is still opened with the real defaults. + """ + margin = extra_retention_hours + return SampleStore(path, raw_keep_hours=RAW_KEEP_HOURS + margin, + minute_keep_days=MINUTE_KEEP_DAYS + margin / 24.0, + hour_keep_days=HOUR_KEEP_DAYS + margin / 24.0) + + +def build(path: str | Path, *, now: float | None = None, + log: Callable[[str], None] | None = None) -> dict[str, Any]: + """Seed `path` production-shaped and return a build report. + + Insert order is `(source, host, metric, res, ts)` ascending — the primary key, and + `"1h" < "1m" < "raw"` sorts the resolutions the same way) — so the writes append instead + of splitting index pages. That is not a trick to hide cost: it is how the app itself writes + (a series' samples always arrive in time order), and the rollup inserts of `maintain()` are + the ones that scatter, which is what task 03 times. + """ + say = log or (lambda _line: None) + now = float(now if now is not None else math.floor(time.time() / MINUTE) * MINUTE) + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + expected = plan(now) + store = open_store(path) + started = time.perf_counter() + try: + buffer: list[tuple[str, str, str, str, float, float]] = [] + + def flush() -> None: + if not buffer: + return + with store._lock, store._db: + store._db.executemany("INSERT OR REPLACE INTO sample VALUES (?,?,?,?,?,?)", buffer) + buffer.clear() + + def emit(source: str, host: str, metric: str, res: str, ts: float, value: float) -> None: + buffer.append((source, host, metric, res, ts, value)) + if len(buffer) >= 100_000: + flush() + + # One pass per series, deepest resolution first: that is primary-key order, and it is + # the only order in which a 6-million-row load does not rewrite the index behind itself. + t0 = time.perf_counter() + hour_start = now - HOUR_KEEP_DAYS * 24 * HOUR + minute_start = now - MINUTE_KEEP_DAYS * 24 * HOUR + raw_start = now - RAW_KEEP_HOURS * HOUR + window = ((RES_HOUR, hour_start, HOUR, expected.hour_steps), + (RES_MIN, minute_start, MINUTE, expected.minute_steps), + (RES_RAW, raw_start, CADENCE_SECONDS, expected.raw_steps)) + for host in sorted(HOST_SERIES): + for metric in sorted(HOST_SERIES[host]): + for res, start, step, steps in window: + for index in range(steps): + emit("host", host, metric, res, start + index * step, seed_value(metric, index)) + flush() + inserted_seconds = time.perf_counter() - t0 + # Row counts by resolution are re-read after the fact (`counts` below); the per- + # resolution totals the NOTES quotes come from `expected`, and the verifier compares + # the two, so this one loop cannot drift from the shape it claims to seed. + + # events + the meta rows the app writes (`last_maintain` comes from maintain below). + t0 = time.perf_counter() + for index in range(EVENT_ROWS): + store.log_event(Event(ts=now - (index + 1) * 600.0, source="system", + host=expected.hosts[index % len(expected.hosts)], + kind="recording" if index % 7 else "reachable", + severity="info" if index % 7 else "warn", + message=f"seeded event {index}")) + store.set_meta(META_SEED_VERSION, SEED_VERSION) + store.set_meta(META_SEED_NOW, str(now)) + event_seconds = time.perf_counter() - t0 + + # the rollups inside the retention windows, from the real maintenance path. + t0 = time.perf_counter() + first = store.maintain(now=now) + maintain_seconds = time.perf_counter() - t0 + + say(f"seeded {path.name}: {expected.sample_rows_seeded:,} sample rows in " + f"{inserted_seconds:.1f}s · events {event_seconds:.1f}s · " + f"maintain {maintain_seconds:.1f}s {first}") + report = { + "path": str(path), + "now": now, + "expected": expected.as_dict(), + "maintain": first, + "seconds": { + "sample_rows": round(inserted_seconds, 3), + "events": round(event_seconds, 3), + "first_maintain": round(maintain_seconds, 3), + "total": round(time.perf_counter() - started, 3), + }, + } + report["counts"] = counts(store) + return report + finally: + store.close() + + +def seed_now(store: SampleStore) -> float: + """The seed's own clock, read back out of `meta`. + + Every window in this module is relative to it rather than to `time.time()`: the deep history + is anchored to the moment the file was built, so a test or a pass that used the wall clock + would measure a different slice of the store on every run (and prune the baseline). Lives + here rather than in `tests/test_store.py` because the HTTP budget pass in + `tests/test_api.py` needs it too, and two copies of "what time is this store" is how a + before/after stops being a before/after. + """ + raw = store.get_meta(META_SEED_NOW) + assert raw is not None, "the seeded store lost its clock row" + return float(raw) + + +def counts(store: SampleStore) -> dict[str, int]: + """Row counts by resolution, plus series and event totals.""" + with store._lock, store._db: + by_res = {row[0]: int(row[1]) for row in store._db.execute( + "SELECT res, COUNT(*) FROM sample WHERE source='host' GROUP BY res")} + series = int(store._db.execute( + "SELECT COUNT(*) FROM (SELECT DISTINCT source, host, metric FROM sample" + " WHERE source='host')").fetchone()[0]) + events = int(store._db.execute("SELECT COUNT(*) FROM event").fetchone()[0]) + return {"raw": by_res.get(RES_RAW, 0), "1m": by_res.get(RES_MIN, 0), + "1h": by_res.get(RES_HOUR, 0), "series": series, "events": events} + + +# --------------------------------------------------------------------------- # +# the cached store: one file, built once, verified on reuse +# --------------------------------------------------------------------------- # +def cache_dir(explicit: str | Path | None = None) -> Path: + """Where the cached seeded store lives: `DASH_PERF_STORE_DIR`, else `data/perf`. + + A directory inside the repo (and inside the gitignored `data/`) on purpose: it is the same + *kind* of filesystem the container gets, unlike pytest's `tmp_path` on this box, which is + `tmpfs`. Set `DASH_PERF_STORE_DIR=/tmp/…` to measure on RAM instead, and know that you + then measured the faster machine. + """ + if explicit is not None: + return Path(explicit) + env = os.environ.get("DASH_PERF_STORE_DIR", "").strip() + return Path(env) if env else ROOT / "data" / "perf" + + +def cache_path(directory: str | Path | None = None) -> Path: + return cache_dir(directory) / f"seeded-{SEED_VERSION}.sqlite3" + + +def _close_and_remove(path: Path) -> None: + for suffix in ("", "-wal", "-shm", "-journal"): + candidate = Path(str(path) + suffix) + if candidate.exists(): + candidate.unlink() + + +@contextmanager +def scratch_copy(path: Path, *, name: str = "scratch") -> Iterator[Path]: + """A private copy of the seeded store, for the measurements that **write**. + + The write-side passes this phase times are not all idempotent-at-a-clock: a rollup run at + `seed_now + 600` really does roll ten more minutes of buckets and prune ten minutes of raw, and + the pre-fix whole-window rollup rewrites 330 000 rows. Doing that to the cached file would move + the baseline every later task compares against, so it happens here instead — same directory + (same device, same page-cache behaviour), same rows, deleted afterwards. `Connection.backup` + rather than `copyfile` because the seed has a WAL of its own and a byte copy would miss it; the + copy is 720 MB in ~1.3 s on the probe's machine, which is cheap next to the passes it buys. + """ + dest = path.with_name(f"{path.stem}.{name}-{os.getpid()}.sqlite3") + # A run killed mid-pass leaves a 720 MB copy behind; sweep the old ones (never a live one: + # an hour of mtime is several passes) before adding to the pile. + for stale in path.parent.glob(f"{path.stem}.{name}-*.sqlite3"): + if time.time() - stale.stat().st_mtime > 3_600: + _close_and_remove(stale) + source = sqlite3.connect(str(path)) + try: + target = sqlite3.connect(str(dest)) + try: + source.backup(target) + finally: + target.close() + finally: + source.close() + try: + yield dest + finally: + _close_and_remove(dest) + + +def wal_bytes(path: Path) -> int: + """Public spelling of `_wal_bytes`, for the notes and the live watch.""" + return _wal_bytes(path) + + +def _verify(path: Path) -> str | None: + """`None` when the cached file matches the shape, else the reason it does not.""" + if not path.exists(): + return "not built yet" + store = open_store(path) + try: + version = store.get_meta(META_SEED_VERSION) + if version != SEED_VERSION: + return f"seed version {version!r} != {SEED_VERSION!r}" + now = float(store.get_meta(META_SEED_NOW) or "0") + if not now: + return "no seed clock recorded" + age_hours = (time.time() - now) / 3600.0 + if age_hours > MAX_CACHE_AGE_HOURS: + return f"seeded {age_hours:.1f} h ago (max {MAX_CACHE_AGE_HOURS:.0f} h)" + if age_hours < -1.0: + return "seeded in the future (the clock moved backwards)" + expected = plan(now) + got = counts(store) + if got["raw"] != expected.raw_rows: + return f"raw rows {got['raw']} != {expected.raw_rows}" + for res, label in ((RES_MIN, "1m"), (RES_HOUR, "1h")): + seeded = {"1m": expected.minute_rows_seeded, "1h": expected.hour_rows_seeded}[label] + if got[res] < seeded: + return f"{label} rows {got[res]} < the {seeded} the shape seeds" + if got["events"] < expected.events: + return f"{got['events']} events < {expected.events}" + if got["series"] != expected.series_total: + return f"{got['series']} series != {expected.series_total}" + return None + finally: + store.close() + + +def ensure_seeded(directory: str | Path | None = None, *, force: bool = False, + log: Callable[[str], None] | None = None) -> Path: + """The cached seeded store's path, building (or rebuilding) it when needed. + + `flock` because a half-built store is worse than a slow one: a session that read a partial + file would report timings for a database nobody has. The build writes to a sibling + `.building` path and renames, so readers only ever see a finished store. + """ + target = cache_path(directory) + target.parent.mkdir(parents=True, exist_ok=True) + reason = None if force else _verify(target) + if reason is None: + if log: + log(f"reusing seeded store {target}") + return target + say = log or (lambda _line: None) + say(f"building seeded store ({reason}) — this takes a minute or two") + lock_path = Path(str(target) + ".lock") + with open(lock_path, "w") as handle: + fcntl.flock(handle, fcntl.LOCK_EX) + try: + # re-check: another session may have finished the same build while we waited + reason = None if force else _verify(target) + if reason is not None: + _close_and_remove(target) + staging = Path(str(target) + ".building") + _close_and_remove(staging) + build(staging, log=log) + staging.replace(target) + for suffix in ("-wal", "-shm"): + leftover = Path(str(staging) + suffix) + if leftover.exists(): + leftover.replace(Path(str(target) + suffix)) + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + return target + + +# --------------------------------------------------------------------------- # +# measurement +# --------------------------------------------------------------------------- # +def timed(fn: Callable[[], Any], *, runs: int = 3) -> dict[str, float]: + """Wall-clock seconds for `fn`, run `runs` times: mean/median/max of the individual runs.""" + samples: list[float] = [] + for _ in range(runs): + started = time.perf_counter() + fn() + samples.append(time.perf_counter() - started) + return {"mean": statistics.fmean(samples), "median": statistics.median(samples), + "max": max(samples), "min": min(samples), "runs": len(samples)} + + +def explain(store: SampleStore, sql: str, args: tuple[Any, ...] = ()) -> list[str]: + """`EXPLAIN QUERY PLAN` for one statement, as the one-line shapes the NOTES quote.""" + with store._lock, store._db: + rows = store._db.execute(f"EXPLAIN QUERY PLAN {sql}", args).fetchall() + return [str(row[3]) for row in rows] + + +def _machine() -> dict[str, Any]: + cpu = platform.uname() + model = "unknown" + try: + for line in Path("/proc/cpuinfo").read_text().splitlines(): + if line.lower().startswith("model name"): + model = line.split(":", 1)[1].strip() + break + except OSError: # pragma: no cover - /proc is always there on Linux + pass + usage = shutil.disk_usage(str(cache_dir())) + return { + "cpu": model, + "cores": os.cpu_count(), + "machine": cpu.machine, + "kernel": cpu.release, + "python": platform.python_version(), + "sqlite": sqlite3.sqlite_version, + "store_dir": str(cache_dir()), + "store_dir_free_gb": round(usage.free / 1e9, 1), + "store_dir_on_tmpfs": _is_tmpfs(str(cache_dir())), + } + + +def _is_tmpfs(path: str) -> bool: + """Is the store's directory RAM? A caveat that changes every number in this report.""" + try: + lines = Path("/proc/mounts").read_text() + except OSError: # pragma: no cover + return False + best, best_len, best_type = "", 0, "unknown" + for line in lines.splitlines(): + parts = line.split() + if len(parts) < 3: + continue + mount, kind = parts[1], parts[2] + if path.startswith(mount) and len(mount) > best_len: + best, best_len, best_type = mount, len(mount), kind + del best + return best_type == "tmpfs" + + +def _main_bytes(path: Path) -> int: + """The database file alone — the WAL is reported separately, because *that* is the number + task 03 tunes and folding the two together hides which one moved.""" + return path.stat().st_size if path.exists() else 0 + + +def _size_bytes(path: Path) -> int: + total = 0 + for suffix in ("", "-wal", "-shm"): + candidate = Path(str(path) + suffix) + if candidate.exists(): + total += candidate.stat().st_size + return total + + +def _wal_bytes(path: Path) -> int: + wal = Path(str(path) + "-wal") + return wal.stat().st_size if wal.exists() else 0 + + +@dataclass +class RaceResult: + """One round of "the poller writes while a page reads" — the known-gap-4 race, as numbers.""" + + target: int = 0 # inserts asked for + attempted: int = 0 # inserts actually run (a deadline can stop it early — see MAX_RACE_SECONDS) + inserts: int = 0 # inserts that succeeded + seconds: float = 0.0 + waiters: int = 0 # inserts slower than WAIT_THRESHOLD_MS + max_wait_ms: float = 0.0 + median_wait_ms: float = 0.0 + busy_errors: int = 0 # OperationalError "database is locked" — outlived busy_timeout + notes: list[str] = field(default_factory=list) + + def as_dict(self) -> dict[str, Any]: + return {**asdict(self), "busy_timeout_ms": BUSY_TIMEOUT_MS, + "wait_threshold_ms": WAIT_THRESHOLD_MS} + + +def _record_waits(result: RaceResult, waits: list[float]) -> None: + if not waits: + return + result.waiters = sum(1 for wait in waits if wait > WAIT_THRESHOLD_MS) + result.max_wait_ms = max(waits) + result.median_wait_ms = statistics.median(waits) + + +def race_single_connection(store: SampleStore, *, iterations: int = 100, + max_seconds: float = MAX_RACE_SECONDS) -> RaceResult: + """The one-process race: the poller's `add_many` while a page runs `stats()`. + + Everything in the app shares `SampleStore._lock`, so this cannot produce a SQLite lock + error — it produces the *other* half of the complaint, a poll that waits behind a read (and + a page that waits behind a poll). The number that matters is how long an insert waits for the + mutex, because on today's store that is the read's own full-table cost: one `/api/health` + poll can hold every writer out for as long as it takes to count the table. + + The loop stops at `max_seconds` rather than running all 100 inserts, because a read here + costs seconds and 100 of them would be a five-minute benchmark; the report says how many + were attempted, so the sample size is part of the number. + """ + result = RaceResult(target=iterations) + stop = threading.Event() + reads: list[float] = [] + + def reader() -> None: + while not stop.is_set(): + started = time.perf_counter() + store.stats() + store.series_names("host") + reads.append(time.perf_counter() - started) + + thread = threading.Thread(target=reader, name="perf-reader", daemon=True) + thread.start() + waits: list[float] = [] + started_at = time.perf_counter() + deadline = started_at + max_seconds + try: + while result.attempted < iterations and time.perf_counter() < deadline: + index = result.attempted + result.attempted += 1 + ts = time.time() + samples = [Sample(RACE_SOURCE, RACE_HOST, f"race.metric{column}", float(index), ts) + for column in range(30)] + started = time.perf_counter() + try: + store.add_many(samples) + result.inserts += 1 + except sqlite3.OperationalError as exc: # pragma: no cover - the mutex prevents it + result.busy_errors += 1 + result.notes.append(f"add_many raised {exc}") + waits.append((time.perf_counter() - started) * 1000.0) + finally: + stop.set() + thread.join(timeout=120.0) + store.delete_series(RACE_SOURCE, RACE_HOST) + result.seconds = time.perf_counter() - started_at + _record_waits(result, waits) + if reads: + result.notes.append( + f"{len(reads)} `stats()`+`series_names()` passes in {result.seconds:.1f} s, median " + f"{statistics.median(reads) * 1000:,.0f} ms — the store mutex serialises them, so an " + f"insert above {WAIT_THRESHOLD_MS:.0f} ms waited behind a read, not behind a SQLite " + "lock (one process cannot get one)") + return result + + +def race_two_connections(path: Path, *, iterations: int = 100, busy_timeout_ms: int | None = None, + now: float | None = None, + max_seconds: float = MAX_RACE_SECONDS, + pace_seconds: float = RACE_PACE_SECONDS, + reset_window: bool = False, + maintain_gap_seconds: float = 0.0) -> RaceResult: + """The two-process race: an insert losing to `maintain()`'s write transaction. + + A second `SampleStore` on the same file is the cheapest faithful stand-in for the case + `AGENTS.md` gap 4 proved for this repo — *two servers on one database* — because one + process's mutex hides the SQLite-level contention entirely. `busy_timeout_ms=0` counts raw + `SQLITE_BUSY` replies (what the poller would see with no timeout at all, and the measure of + how much of the window the write lock was held); the production 5 000 ms answers the question + the gap actually asks: does an insert ever outlive the timeout and surface as `store-error`? + + `max_seconds` bounds the pass for the same reason as the other race: with a timeout of five + seconds and a `maintain()` of several, "100 inserts" would be minutes of waiting on purpose. + + `reset_window=True` deletes `last_maintain` before every pass, which puts the maintainer back + into the shape task 01 measured — a whole-retention rollup on every tick — and it is run as the + **control**. A windowed `maintain()` writes almost nothing per minute, so "zero `database is + locked`" would also be what a race that never contended reports; the control is the number that + says this race can still see the failure, and therefore that the zero means the fix. + + `maintain_gap_seconds` is the app's 60 s cadence, compressed: the maintainer idles between + passes instead of restarting the transaction the instant it commits. `measure()` runs both, and + the gap is where the honest pass line lives — see `RACE_MAINTAIN_GAP_SECONDS`. + """ + timeout = BUSY_TIMEOUT_MS if busy_timeout_ms is None else busy_timeout_ms + # prune-safe: the point of this race is the lock, not the retention sweep (see open_store) + writer = open_store(path, extra_retention_hours=2 * MAX_CACHE_AGE_HOURS) + maintainer = open_store(path, extra_retention_hours=2 * MAX_CACHE_AGE_HOURS) + writer._db.execute(f"PRAGMA busy_timeout={timeout}") + result = RaceResult(target=iterations) + stop = threading.Event() + writing = threading.Event() + maintains: list[float] = [] + + def maintainer_loop() -> None: + while not stop.is_set(): + if reset_window: + # outside `writing` on purpose: this is not part of the transaction being raced + maintainer.delete_meta(LAST_MAINTAIN_KEY) + started = time.perf_counter() + writing.set() + try: + # the seed's own clock, so the loop repeats the same write instead of ageing the + # store it is measuring + maintainer.maintain(now=now) + finally: + maintains.append(time.perf_counter() - started) + writing.clear() + if maintain_gap_seconds: + time.sleep(maintain_gap_seconds) + + thread = threading.Thread(target=maintainer_loop, name="perf-maintain", daemon=True) + thread.start() + # Don't start inserting until a write transaction is actually open: un-paced, 100 inserts + # finish in ~50 ms, before the maintainer's first `BEGIN`, and the pass reports "no + # contention" for a race that never happened. `writing` is set just before `maintain()`. + if not writing.wait(timeout=max_seconds): # pragma: no cover - a dead maintainer thread + result.notes.append("the maintainer thread never started a transaction") + time.sleep(0.05) # the flag is set just before `BEGIN`; give the transaction a moment + waits: list[float] = [] + started_at = time.perf_counter() + deadline = started_at + max_seconds + try: + while result.attempted < iterations and time.perf_counter() < deadline: + index = result.attempted + result.attempted += 1 + ts = time.time() + samples = [Sample(RACE_SOURCE, RACE_HOST, f"race.metric{column}", float(index), ts) + for column in range(30)] + started = time.perf_counter() + try: + writer.add_many(samples) + result.inserts += 1 + except sqlite3.OperationalError as exc: + result.busy_errors += 1 + if len(result.notes) < 2: + result.notes.append(f"add_many raised: {exc}") + waits.append((time.perf_counter() - started) * 1000.0) + # pace it: this is the poller's cadence, not a tight loop — 9 sources writing every + # few seconds, so inserts land at every offset inside the write transaction + time.sleep(pace_seconds) + finally: + stop.set() + thread.join(timeout=180.0) + try: + writer.delete_series(RACE_SOURCE, RACE_HOST) + except sqlite3.OperationalError as exc: # pragma: no cover - cleanup only + result.notes.append(f"cleanup skipped: {exc}") + writer.close() + maintainer.close() + result.seconds = time.perf_counter() - started_at + _record_waits(result, waits) + result.notes.insert(0, ( + f"busy_timeout={timeout} ms · {'whole-window (pre-fix control) ' if reset_window else ''}" + f"{'cadence ' + format(maintain_gap_seconds, '.1f') + ' s gap' if maintain_gap_seconds else 'saturated (no gap between passes)'}" + f" · {len(maintains)} `maintain()` write transactions " + f"(median {statistics.median(maintains) * 1000:,.0f} ms) held the WAL write lock across " + f"{result.seconds:.1f} s · {result.busy_errors} of {result.attempted} inserts answered " + f"`database is locked`")) + return result + + +#: The write-side passes, as `(label, seconds after the seed clock, drop `last_maintain` first)`. +#: All three run on the same copy of the same rows, so they differ only in the window: +#: +#: * `steady` is the shape of nearly every real tick — the previous pass recorded the same clock, +#: so each step re-reads its overlap and writes the buckets that filled since; +#: * `catch_up_10m` is the restart-after-a-quiet-interval case, ten minutes of buckets due; +#: * `full_window` is the **pre-fix control**: with no `last_maintain` row the steps scan their whole +#: retention, which is the transaction task 01 measured at 5.4 s. Reproducing it in the same run +#: on the same file is what turns "the race is clean now" into a measurement instead of a hope. +MAINTAIN_PASSES: tuple[tuple[str, float, bool], ...] = ( + ("steady", 0.0, False), + ("catch_up_10m", 600.0, False), + ("full_window", 1_200.0, True), +) + +#: The SELECT half of `store.ROLLUP_INSERT`, derived rather than retyped so the `EXPLAIN QUERY PLAN` +#: the NOTES record is the plan the rollup actually runs (the same reason the count branch is a +#: constant). `SELECT` occurs once in that statement. +ROLLUP_SELECT = "SELECT" + ROLLUP_INSERT.split("SELECT", 1)[1] + + +def measure_maintain(path: Path, *, now: float, log: Callable[[str], None] | None = None) -> dict[str, Any]: + """The write transaction, per window, on a scratch copy — with the WAL around each pass. + + `now` is the seed's own clock. Offsets from it are the only way to ask "what did the next ten + minutes cost?" of a store whose history is anchored to when it was seeded, and the copy keeps + those writes away from the file every later measurement in the phase compares against. + """ + say = log or (lambda _line: None) + out: dict[str, Any] = { + "journal_size_limit_bytes": JOURNAL_SIZE_LIMIT_BYTES, + "seed_now": now, + "passes": {}, + } + with scratch_copy(path) as copy: + out["scratch_bytes"] = _main_bytes(copy) + store = open_store(copy) + try: + for label, offset, reset in MAINTAIN_PASSES: + if reset: + store.delete_meta(LAST_MAINTAIN_KEY) + written: dict[str, int] = {} + + def run_pass(_offset: float = offset, _into: dict[str, int] = written) -> None: + _into.update(store.maintain(now=now + _offset)) + + wal_before = _wal_bytes(copy) + run = timed(run_pass, runs=1) + wal_after = _wal_bytes(copy) + out["passes"][label] = { + "seconds": run["mean"], + "clock": now + offset, + "counts": written, + "rows_written": written["to_minute"] + written["to_hour"], + "wal_before_bytes": wal_before, + "wal_after_bytes": wal_after, + } + say(f" {label}: {run['mean'] * 1000:,.0f} ms, " + f"{written} -> WAL {wal_after / 1e6:,.1f} MB") + # Two more ticks for the WAL criterion: the cap is what the file must stay under, and + # the steady-state growth per tick is what the 140 MB in AGENTS.md gap 4 adds up from. + cycles: list[dict[str, int]] = [] + for _ in range(2): + before = _wal_bytes(copy) + store.maintain(now=now + 1_260.0) + after = _wal_bytes(copy) + cycles.append({"wal_before_bytes": before, "wal_after_bytes": after}) + out["two_steady_cycles"] = cycles + out["wal_within_cap"] = all( + cycle["wal_after_bytes"] <= JOURNAL_SIZE_LIMIT_BYTES for cycle in cycles) + out["store_bytes_after"] = _main_bytes(copy) + # The plans, from the copy, after the first pass put statistics in the file and against + # the windows just timed. The SELECT half of a rollup is what costs and the DELETE half + # is what a planner gets wrong: `SCAN sample` here means the window bought nothing, no + # matter how few rows are inside it. + out["plans"] = { + "rollup_minute_steady": explain(store, ROLLUP_SELECT, ( + RES_MIN, MINUTE, MINUTE, RES_RAW, now - MINUTE - MAINTAIN_SLACK_SECONDS, + now - MINUTE - MAINTAIN_SLACK_SECONDS, MINUTE)), + "rollup_minute_catch_up": explain(store, ROLLUP_SELECT, ( + RES_MIN, MINUTE, MINUTE, RES_RAW, now - MINUTE - MAINTAIN_SLACK_SECONDS, + now - MINUTE - MAINTAIN_SLACK_SECONDS - 600.0, MINUTE)), + "rollup_hour_steady": explain(store, ROLLUP_SELECT, ( + RES_HOUR, HOUR, HOUR, RES_MIN, now - HOUR - MAINTAIN_SLACK_SECONDS, + now - HOUR - MAINTAIN_SLACK_SECONDS, HOUR)), + "prune_raw": explain(store, "DELETE FROM sample WHERE res = ? AND ts < ?", + (RES_RAW, now - RAW_KEEP_HOURS * HOUR)), + } + finally: + store.close() + return out + + +def measure_http(path: Path, *, hosts: tuple[str, ...], + metrics: tuple[str, ...]) -> dict[str, Any]: + """The browser's own numbers, through the real ASGI gate (a session is required). + + `TestClient` is in-process, so this is the handler's cost plus the store's, with no network + — which is the half the operator feels as "the page took forever" once the response is big. + """ + sys_path_fixup() + from fastapi.testclient import TestClient # noqa: PLC0415 - only the HTTP numbers need it + + # tests/ is on sys.path via sys_path_fixup(); these two are the suite's own door in. + from auth_client import ADMIN_PASSWORD, sign_in # noqa: PLC0415 + from auth_contract import FAST_PBKDF2 # noqa: PLC0415 + from dashboards.app import create_app # noqa: PLC0415 + from dashboards.config import Settings # noqa: PLC0415 + + settings = Settings(db_path=str(path), admin_password=ADMIN_PASSWORD, + pbkdf2_iterations=FAST_PBKDF2) + # The lifespan starts the hub's maintenance loop, and that loop calls `maintain()` at the + # wall clock the moment it starts — so this handle gets the prune-safe margin, or booting a + # server to time a read would prune the baseline it is timing. + store = open_store(path, extra_retention_hours=2 * MAX_CACHE_AGE_HOURS) + out: dict[str, Any] = {} + with TestClient(create_app(settings, store)) as client: + sign_in(client) + joined_hosts = ",".join(hosts) + joined_metrics = ",".join(metrics) + page = ",".join(PAGE_HISTORY_METRICS) + out["health"] = timed(lambda: client.get("/api/health"), runs=5) + out["metrics_list"] = timed(lambda: client.get("/api/metrics?source=host"), runs=3) + out["system_history_9x3_1h"] = timed( + lambda: client.get(f"/api/system/history?hosts={joined_hosts}" + f"&metrics={joined_metrics}&range=1h"), runs=5) + out["system_history_9x3_24h"] = timed( + lambda: client.get(f"/api/system/history?hosts={joined_hosts}" + f"&metrics={joined_metrics}&range=24h"), runs=3) + out["system_history_9x3_90d"] = timed( + lambda: client.get(f"/api/system/history?hosts={joined_hosts}" + f"&metrics={joined_metrics}&range=90d"), runs=3) + out["system_history_page_shape_1h"] = timed( + lambda: client.get(f"/api/system/history?hosts={hosts[0]}" + f"&metrics={page}&range=1h"), runs=5) + out["summary"] = timed(lambda: client.get("/api/summary"), runs=3) + store.close() + return out + + +def sys_path_fixup() -> None: + """Make `tests/` importable from a bare script (pytest does it for free via conftest).""" + here = str(Path(__file__).resolve().parent) + if here not in sys.path: + sys.path.insert(0, here) + + +def measure(store: SampleStore, *, race_iterations: int = 100, with_http: bool = True, + race_max_seconds: float = MAX_RACE_SECONDS, + log: Callable[[str], None] | None = None) -> dict[str, Any]: + """Every number task 01 asks for, in one dict. + + Read-only except the two race helpers, which clean up after themselves (the seeded store is + reused by later tasks, so a measurement that leaves 3 000 rows behind would move the next + one's baseline). + """ + say = log or (lambda _line: None) + path = Path(store.path) + now = float(store.get_meta(META_SEED_NOW) or time.time()) + expected = plan(now) + report: dict[str, Any] = { + "generated_at": time.time(), + "seed_version": SEED_VERSION, + "machine": _machine(), + "store": { + "path": str(path), + "bytes": _size_bytes(path), + "wal_bytes": _wal_bytes(path), + "seed_now": now, + "expected": expected.as_dict(), + "counts": counts(store), + }, + "paths": {}, + "plans": {}, + "race": {}, + } + + def cold(fn: Callable[[], Any]) -> Callable[[], Any]: + """Time the *uncached* answer, whatever the short-TTL read caches are holding. + + Task 02 put a cache in front of the two scanned answers, so a report that timed them + naively would compare a cache hit against the baseline's full scan and call it a 1 000x + win. Both numbers belong in the NOTES: the cold one is what the first poll after a process + start pays, the warm one is what the browser's next poll pays. + """ + def run() -> Any: + store.invalidate_read_caches() + return fn() + return run + + say("series_names() — cold (cache invalidated) and cached") + report["paths"]["series_names_all"] = timed(cold(lambda: store.series_names()), runs=3) + report["paths"]["series_names_all_cached"] = timed(lambda: store.series_names(), runs=5) + report["paths"]["series_names_host"] = timed(cold(lambda: store.series_names("host")), runs=3) + report["paths"]["series_names_host_cached"] = timed(lambda: store.series_names("host"), runs=5) + report["paths"]["series_names_host_driveripper"] = timed( + lambda: store.series_names("host", "driveripper"), runs=3) + report["plans"]["series_names_all"] = explain(store, "SELECT DISTINCT metric FROM sample" + " ORDER BY metric") + report["plans"]["series_names_host"] = explain( + store, "SELECT DISTINCT metric FROM sample WHERE source=? ORDER BY metric", ("host",)) + + say("stats() — what the browser-polled /api/health runs, cold and cached") + report["paths"]["stats"] = timed(cold(store.stats), runs=5) + report["paths"]["stats_cached"] = timed(store.stats, runs=5) + report["plans"]["stats_by_res"] = explain( + store, "SELECT res, COUNT(*) FROM sample GROUP BY res") + report["plans"]["stats_events"] = explain(store, "SELECT COUNT(*) FROM event") + + say("pick_res() + history() over 90 d") + host, metric = "driveripper", "cpu.usage_pct" + report["paths"]["pick_res_90d"] = timed( + lambda: [store.pick_res("host", host, metric, span, 600) + for span in (3600.0, 24 * 3600.0, 90 * 24 * 3600.0)], runs=3) + report["paths"]["history_90d"] = timed( + lambda: store.history("host", host, metric, span=90 * 24 * 3600.0, max_points=600), + runs=3) + report["paths"]["history_24h"] = timed( + lambda: store.history("host", host, metric, span=24 * 3600.0, max_points=600), runs=3) + report["paths"]["history_5m"] = timed( + lambda: store.history("host", host, metric, span=300.0, max_points=600), runs=3) + # The statement behind `pick_res` since task 02: three capped counts in one compound, one + # index seek each. EXPLAINed here rather than the old per-`res` COUNT because the NOTES + # compare shapes across the phase, and the shape that runs is the one worth recording. + res_count = " UNION ALL ".join([RES_COUNT_BRANCH] * 3) + res_args: list[object] = [] + for res in (RES_RAW, RES_MIN, RES_HOUR): + # the branch's own parameter order: three label columns, then the seek itself + res_args += [host, metric, res, "host", host, metric, res, now - 90 * 24 * 3600.0, 601] + report["plans"]["pick_res_count"] = explain(store, res_count, tuple(res_args)) + + say("the N x M batch (27 series, the /system page's request) — grouped vs per-series") + pair_hosts = list(HOST_SERIES) + pair_metrics = list(HISTORY_METRICS) + for label, span in (("24h", 24 * 3600.0), ("90d", 90 * 24 * 3600.0)): + report["paths"][f"pick_res_x27_{label}"] = timed( + lambda hs=pair_hosts, ms=pair_metrics, s=span: [ + store.pick_res("host", h, m, s, 600) for h in hs for m in ms], runs=3) + report["paths"][f"pick_res_many_x27_{label}"] = timed( + lambda hs=pair_hosts, ms=pair_metrics, s=span: store.pick_res_many( + "host", hs, ms, s, 600), runs=3) + report["paths"][f"history_x27_{label}"] = timed( + lambda hs=pair_hosts, ms=pair_metrics, s=span: [ + store.history("host", h, m, span=s, max_points=600) + for h in hs for m in ms], runs=3) + report["paths"][f"history_many_x27_{label}"] = timed( + lambda hs=pair_hosts, ms=pair_metrics, s=span: store.history_many( + "host", hs, ms, span=s, max_points=600), runs=3) + # The batched statement is that same branch repeated per `(series, res)` and chunked at + # `COMPOUND_CHUNK`; EXPLAINing three of them is the shape without a 300-line plan in the NOTES. + grouped_args: list[object] = [] + for (h, m), res in zip((("driveripper", "cpu.usage_pct"), ("deskwork", "mem.used_pct"), + ("gateway", "disk.used_pct")), (RES_RAW, RES_MIN, RES_HOUR), + strict=True): + grouped_args += [h, m, res, "host", h, m, res, now - 90 * 24 * 3600.0, 601] + report["plans"]["pick_res_many_branch"] = explain( + store, " UNION ALL ".join([RES_COUNT_BRANCH] * 3), tuple(grouped_args)) + report["plans"]["history_tail"] = explain( + store, "SELECT ts, value FROM (SELECT ts, value FROM sample WHERE source=? AND host=?" + " AND metric=? AND res=? AND ts>=? AND ts<=? ORDER BY ts DESC LIMIT ?)" + " ORDER BY ts ASC", + ("host", host, metric, RES_MIN, now - 90 * 24 * 3600.0, now, 2400)) + + say("maintain() — the write transaction per window, and the WAL around each pass") + # Seeded-clock maintenance on a scratch copy: the store's newest row is at `now`, so timing a + # rollup at the wall clock would measure a slightly different job (and prune the baseline) every + # time the probe runs. The copy is where the passes that *write* belong — a catch-up pass rolls + # ten more minutes of buckets, and the pre-fix whole-window control rewrites 330 000 rows. + report["maintain"] = measure_maintain(path, now=now, log=say) + # The write-side plans (`maintain.plans`) are explained inside `measure_maintain`, on the copy: + # a plan is only worth recording next to the timings that produced it, and only after that + # pass has put query statistics in the file — without statistics the planner reads the table for + # every window, so explaining it beforehand would record a shape production no longer runs. + + say(f"the race: a 30-sample add_many against {race_iterations} concurrent readers/maintains") + single = race_single_connection(store, iterations=race_iterations, + max_seconds=race_max_seconds) + report["race"]["single_connection"] = single.as_dict() + # The two-connection races write (rollups on the maintainer's side, 3 000 raced samples on the + # writer's), so they run against one scratch copy of the same rows: the seed the next task + # measures has to come back exactly as it went in. + with scratch_copy(path, name="race") as arena: + # (1) the saturated harness, unchanged from task 01 so its numbers stay comparable, and + # (2) the same race at the app's cadence, which is the one with a pass line: a poller's + # insert landing inside a rollup waits that rollup out, and windowing is what made the + # wait short enough to never reach `busy_timeout`. + report["race"]["two_connections_prod_timeout"] = race_two_connections( + arena, iterations=race_iterations, now=now, max_seconds=race_max_seconds).as_dict() + report["race"]["two_connections_no_timeout"] = race_two_connections( + arena, iterations=race_iterations, busy_timeout_ms=0, now=now, + max_seconds=race_max_seconds).as_dict() + report["race"]["poller_during_windowed_maintain"] = race_two_connections( + arena, iterations=race_iterations, now=now, max_seconds=race_max_seconds, + maintain_gap_seconds=RACE_MAINTAIN_GAP_SECONDS).as_dict() + report["race"]["poller_during_full_window_maintain"] = race_two_connections( + arena, iterations=race_iterations, now=now, max_seconds=race_max_seconds, + reset_window=True, maintain_gap_seconds=RACE_MAINTAIN_GAP_SECONDS).as_dict() + for key in ("two_connections_prod_timeout", "two_connections_no_timeout"): + report["race"][key]["notes"].append( + "saturated harness (task 01's shape): the maintainer restarts `maintain()` the instant " + "it commits, so the WAL write lock is held ~100% of the wall clock and a writer can be " + "starved however short each pass is. The app maintains once per 60 s — the " + "`poller_during_*` rows are that shape, and they are the ones with a pass line.") + + if with_http: + say("through the app (signed in, TestClient, no network)") + report["http"] = measure_http(path, hosts=expected.hosts, metrics=HISTORY_METRICS) + + # Task 04's budgets, priced against everything above. Computed last on purpose: a budget row + # whose path was never measured is a failing row, so it has to see the finished report. + rows = budget_rows(report) + report["budgets"] = [row.as_dict() for row in rows] + report["budgets_ok"] = all(row.ok for row in rows) + + # `_verify` is cheap and proves the measurements left the seeded shape alone + problem = _verify(path) + report["store_after"] = {"counts": counts(store), "integrity": problem or "unchanged"} + return report + + +# --------------------------------------------------------------------------- # +# the budgets (task 04): the measured "after", promoted to a red test +# --------------------------------------------------------------------------- # +#: Why a table and not `assert elapsed < 0.05` sprinkled through the tests: the same numbers have +#: to appear in three places — the pytest budgets, the probe's report, and +#: `data/logs/NOTES-store-perf.md` — and three copies of a budget drift the moment one is relaxed. +#: One table, one verdict function, and the notes are rendered from it. +#: +#: Every limit is **2-3x the measured "after"** on the seeded store (`AGENTS.md`: a budget asserted +#: on first writing is a number fitted to the machine it was written on), and each row records that +#: measured figure in `basis` so a reader can see the margin instead of trusting it. They are cost +#: *ceilings*, not targets: a path that used to cost a whole-table pass and now costs 1 ms still +#: gets a 10 ms ceiling, because the claim under test is "bounded", not "1 ms". +@dataclass(frozen=True) +class Budget: + """One pinned cost: a report path, the ceiling it may not cross, and where the ceiling came from.""" + + #: which section of a `measure()` report to read: `paths`, `http`, `maintain`, `race` + section: str + #: the key inside that section + key: str + #: the ceiling, in `unit` + limit: float + #: what the limit is compared against: a `timed()` field (`mean`/`median`/`max`) for the + #: `paths`/`http` sections, `seconds` for a `maintain` pass, and the `RaceResult` field name + #: (`max_wait_ms`, `busy_errors`) for a race + statistic: str = "median" + unit: str = "ms" + #: prose: the measured "after" this ceiling was set from, and the margin on it + basis: str = "" + #: an optional *relative* claim — this path must also cost at most `ratio_limit` of the named + #: key in the same section. That is the assertion a cache needs: "0.0 ms" on a machine that + #: got faster for an unrelated reason is uninteresting, "50x cheaper than the scan it replaced" + #: is the scan being gone. + ratio_key: str | None = None + ratio_limit: float | None = None + + def label(self) -> str: + return f"{self.section}.{self.key}[{self.statistic}]" + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +#: The budgets themselves, in the order the NOTES table prints them. Keys are report keys, so +#: `test_every_pinned_budget_names_a_path_the_probe_measures` can fail a budget that points at a +#: measurement nobody takes any more. +BUDGETS: tuple[Budget, ...] = ( + # --- the two reads that used to be a scan per request ------------------------------- + Budget("paths", "series_names_host_cached", 10.0, ratio_key="series_names_host", + ratio_limit=0.02, basis="measured ~0.0 ms cached vs 1,440 ms cold (task 03); the claim " + "is 'the scan is gone', so the ceiling comes with a 50x-relative claim beside it"), + Budget("paths", "stats_cached", 10.0, ratio_key="stats", ratio_limit=0.02, + basis="measured ~0.0 ms cached vs 1,336 ms cold (task 03); `/api/health` runs this " + "every 15 s per tab, which is why both halves are pinned"), + Budget("paths", "history_90d", 25.0, basis="measured 1.9-2.6 ms; task 01's 'do not regress it' " + "line, 10x the after and 6x under the 4.1 ms baseline"), + Budget("paths", "pick_res_many_x27_90d", 50.0, basis="measured 2.7-3.0 ms for 27 series; " + "one grouped statement, 16x headroom"), + Budget("paths", "history_many_x27_90d", 150.0, basis="measured ~30 ms for 27 series at the " + "widest window; 5x headroom on the slowest request the pages can make"), + # --- the same paths over HTTP, gate included ----------------------------------------- + Budget("http", "health", 150.0, basis="measured 0.8-1.4 ms median (task 01: 1,272 ms); " + "task 01's own suggested ceiling, kept because a deployment disk is slower than " + "the dev box"), + Budget("http", "metrics_list", 150.0, basis="measured 1.6-2.0 ms median (task 01: 1,415 ms)"), + Budget("http", "system_history_9x3_1h", 100.0, basis="the raw-resolution case: measured ~6 ms"), + Budget("http", "system_history_9x3_90d", 200.0, basis="measured ~50 ms mean (task 01: 100 ms); " + "4x the after, and under the baseline the phase started from"), + Budget("http", "summary", 150.0, basis="measured ~1 ms; the overview's single call"), + # --- the write side: the transaction that held the WAL write lock -------------------- + Budget("maintain", "steady", 1_000.0, statistic="seconds", basis="measured 2.7 ms steady " + "(task 01: 5,368 ms); task 01's ceiling, 5x under the baseline's *mean* so a " + "catch-up tick cannot cross it either"), + Budget("maintain", "catch_up_10m", 2_000.0, statistic="seconds", + basis="measured 48 ms after ten quiet minutes; the ceiling is the promise that a " + "restart after a quiet spell is still well inside a poller's 5 s busy_timeout"), + Budget("race", "poller_during_windowed_maintain", 1_000.0, statistic="max_wait_ms", + basis="measured 21 ms worst wait (task 01: 5,005 ms median); the wait that matters " + "is the worst one, since that is the one that raises"), + Budget("race", "poller_during_windowed_maintain", 0.0, statistic="busy_errors", unit="count", + basis="measured 0 of 100 (task 01: 5 of 9) — AGENTS.md known gap 4's failure mode, " + "pinned at zero"), +) + +#: Which budgets the *default* test run can price without the seeded store (see +#: `measure_budget_reads`): the cheap-store pass produces these sections and no others. +BUDGET_SECTIONS = ("paths", "http", "maintain", "race") + +#: The keys the two cheap measurement passes produce, pinned by tests against the reports they +#: build. A budget (or the ratio partner of one) has to name a key from the right one of these, +#: because the failure mode of a budget table is a row that quietly stops being priced: rename the +#: measurement, and the ceiling still "passes". +READ_MEASURED_KEYS: tuple[str, ...] = ( + "series_names_host", "series_names_host_cached", "stats", "stats_cached", "history_90d", + "pick_res_many_x27_90d", "history_many_x27_90d", +) +HTTP_MEASURED_KEYS: tuple[str, ...] = ( + "health", "metrics_list", "system_history_9x3_1h", "system_history_9x3_24h", + "system_history_9x3_90d", "system_history_page_shape_1h", "summary", +) +#: The report keys the write-side passes produce, for the same reason. `full_window` is the +#: pre-fix control and deliberately has no budget: a thing is only a control if it is slow. +MAINTAIN_MEASURED_KEYS: tuple[str, ...] = tuple(label for label, _o, _r in MAINTAIN_PASSES) + + +@dataclass(frozen=True) +class BudgetRow: + """One budget, the number it was priced against, and the verdict.""" + + budget: Budget + measured: float | None # in the budget's unit; `None` = the report did not measure it + ok: bool + detail: str = "" + + @property + def margin(self) -> float | None: + """How many times inside the ceiling the measurement is (`None` when unpricables make that + meaningless: a 0.0 ms measurement against a 10 ms ceiling has no finite margin).""" + if self.measured is None or self.measured <= 0.0 or self.budget.limit <= 0.0: + return None + return self.budget.limit / self.measured + + def as_dict(self) -> dict[str, Any]: + return {**self.budget.as_dict(), "measured": self.measured, "ok": self.ok, + "margin": self.margin, "detail": self.detail} + + +def _budget_value(report: dict[str, Any], budget: Budget) -> float | None: + """The measured value for one budget, in its unit, or `None` when the report lacks it.""" + section = report.get(budget.section) or {} + if budget.section == "maintain": + entry = (section.get("passes") or {}).get(budget.key) + return None if entry is None else float(entry["seconds"]) * 1000.0 + entry = section.get(budget.key) + if entry is None: + return None + if budget.section == "race": + # a RaceResult: the fields are already ms / counts, no unit conversion + return None if entry.get(budget.statistic) is None else float(entry[budget.statistic]) + value = entry.get(budget.statistic) + return None if value is None else float(value) * 1000.0 + + +def budget_rows(report: dict[str, Any], + sections: tuple[str, ...] | None = None) -> list[BudgetRow]: + """Price every pinned budget against `report` — the table the NOTES print and the tests assert. + + `sections` narrows the pass to the parts a given report actually contains (the cheap read-only + pass prices `paths` alone, the HTTP pass `http` alone); without it a partial report would report + every other budget as unmeasured, which is true but not what that caller is claiming. + + A budget whose path the report does not measure comes back as a failing row rather than a + missing one: a budget that silently stops being checked is the same failure as a budget that + is crossed, and it is the one a refactor causes. + """ + rows: list[BudgetRow] = [] + for budget in BUDGETS: + if sections is not None and budget.section not in sections: + continue + measured = _budget_value(report, budget) + if measured is None: + rows.append(BudgetRow(budget, None, False, "not measured by this report")) + continue + if measured > budget.limit: + rows.append(BudgetRow(budget, measured, False, + f"over the {fmt_budget(budget)} ceiling")) + continue + if budget.ratio_key is not None and budget.ratio_limit is not None: + other = _budget_value(report, replace(budget, key=budget.ratio_key)) + if other is None: + rows.append(BudgetRow(budget, measured, False, + f"ratio partner {budget.ratio_key} was not measured")) + continue + ratio = (measured / other) if other > 0 else math.inf + if ratio > budget.ratio_limit: + rows.append(BudgetRow(budget, measured, False, f"{ratio:.3f} of the {budget.ratio_key} " + f"cost, must be <= {budget.ratio_limit:.3f}")) + continue + rows.append(BudgetRow(budget, measured, True)) + return rows + + +def budget_violations(report: dict[str, Any], + sections: tuple[str, ...] | None = None) -> list[str]: + """The failing rows as sentences — what a budget test asserts away.""" + return [f"{row.budget.label()} = {fmt_budget_value(row.budget, row.measured)} " + f"(limit {fmt_budget(row.budget)}): {row.detail}" + for row in budget_rows(report, sections) if not row.ok] + + +def missing_budget_keys(report: dict[str, Any], + sections: tuple[str, ...] | None = None) -> list[str]: + """Budgets this report cannot price at all — the stale-pointer check, not the cost check. + + The default suite runs this against a *small* store, where a budget may legitimately be + crossed (a ratio against a 20 us scan is luck) but must never be unmeasurable. + """ + return [row.budget.label() for row in budget_rows(report, sections) + if row.measured is None] + + +def fmt_budget_value(budget: Budget, value: float | None) -> str: + if value is None: + return "n/a" + return _ms(value / 1000.0) if budget.unit == "ms" else f"{value:,.0f}" + + +def fmt_budget(budget: Budget) -> str: + return _ms(budget.limit / 1000.0) if budget.unit == "ms" else f"<= {budget.limit:,.0f}" + + +def render_budgets_markdown(rows: list[BudgetRow]) -> str: + """The budget table: measured, ceiling, margin, verdict — one row per pinned budget.""" + lines = ["| budget | path | measured | ceiling | margin | verdict |", + "| --- | --- | --- | --- | --- | --- |"] + for row in rows: + margin = "—" if row.margin is None else f"{row.margin:,.0f}x" + lines.append(f"| `{row.budget.section}` | `{row.budget.key}` " + f"({row.budget.statistic}) | {fmt_budget_value(row.budget, row.measured)} | " + f"{fmt_budget(row.budget)} | {margin} | " + + ("**inside**" if row.ok else f"**OVER** — {row.detail}") + " |") + return "\n".join(lines) + + +def measure_budget_reads(store: SampleStore, *, runs: int = 5) -> dict[str, Any]: + """Only the paths with a pinned budget, timed — the cheap half of `measure()`. + + Task 04's budget tests must not each pay for the races (minutes) or the write passes, so this + produces the same-shaped `paths` document for the budget keys and nothing else. It is a + *measurement*, not a baseline: the numbers it prints go in the notes next to the full probe + report, and both are on the same seeded file. + """ + host, metric = "driveripper", "cpu.usage_pct" + hosts = list(HOST_SERIES) + metrics = list(HISTORY_METRICS) + paths: dict[str, Any] = {} + paths["series_names_host"] = timed(lambda: (store.invalidate_read_caches(), + store.series_names("host"))[1], runs=runs) + paths["series_names_host_cached"] = timed(lambda: store.series_names("host"), runs=runs) + paths["stats"] = timed(lambda: (store.invalidate_read_caches(), store.stats())[1], runs=runs) + paths["stats_cached"] = timed(store.stats, runs=runs) + paths["history_90d"] = timed( + lambda: store.history("host", host, metric, span=90 * 24 * 3600.0, max_points=600), + runs=runs) + paths["pick_res_many_x27_90d"] = timed( + lambda: store.pick_res_many("host", hosts, metrics, 90 * 24 * 3600.0, 600), runs=runs) + paths["history_many_x27_90d"] = timed( + lambda: store.history_many("host", hosts, metrics, span=90 * 24 * 3600.0, max_points=600), + runs=runs) + return {"budget_runs": runs, "paths": paths} + + +def measure_budget_http(path: str | Path, *, hosts: tuple[str, ...] | None = None, + metrics: tuple[str, ...] | None = None) -> dict[str, Any]: + """The HTTP half of the budgets, through the same signed-in `TestClient` as `measure()`. + + Split out so `tests/test_api.py` can price the endpoint budgets without also paying for the + races and the write passes. `hosts`/`metrics` default to the seeded shape so the request is + the one `/system` makes; a caller passing them explicitly is pricing a *different* request and + should say so in the notes rather than reuse these budgets. + """ + return {"http": measure_http(Path(path), + hosts=hosts or tuple(sorted(HOST_SERIES)), + metrics=metrics or HISTORY_METRICS)} + + +def measure_dev_db(path: str | Path, *, runs: int = 3) -> dict[str, Any]: + """The same three hot reads against a *real* store, opened read-only. + + The answer to "is your synthetic seed representative?" is better as a measurement than as an + argument. `mode=ro` and nothing else: no DDL, no PRAGMA that writes, no `SampleStore` + (constructing one runs the schema and `journal_mode=WAL`, which need write access). Tests + never pass this path — `AGENTS.md`'s testing policy keeps the suite off the operator's + history — but the probe is run by a human, exactly like `data/logs/analyze-watch.py` did in + phase 13, and the numbers it prints belong in the NOTES either way. + """ + uri = f"file:{Path(path)}?mode=ro" + db = sqlite3.connect(uri, uri=True) + out: dict[str, Any] = {"path": str(path), "bytes": Path(path).stat().st_size} + try: + out["counts"] = {row[0]: int(row[1]) for row in db.execute( + "SELECT res, COUNT(*) FROM sample GROUP BY res")} + out["events"] = int(db.execute("SELECT COUNT(*) FROM event").fetchone()[0]) + out["series"] = {row[0]: int(row[1]) for row in db.execute( + "SELECT source, COUNT(*) FROM (SELECT DISTINCT source, host, metric FROM sample)" + " GROUP BY source")} + + def run(sql: str, args: tuple[Any, ...] = ()) -> dict[str, float]: + def call() -> None: + db.execute(sql, args).fetchall() + + return timed(call, runs=runs) + + out["paths"] = { + "series_names_all": run("SELECT DISTINCT metric FROM sample ORDER BY metric"), + "series_names_host": run("SELECT DISTINCT metric FROM sample WHERE source=?" + " ORDER BY metric", ("host",)), + "stats_by_res": run("SELECT res, COUNT(*) FROM sample GROUP BY res"), + "stats_events": run("SELECT COUNT(*) FROM event"), + } + out["plans"] = {key: [str(row[3]) for row in db.execute( + f"EXPLAIN QUERY PLAN {sql}", args).fetchall()] + for key, sql, args in ( + ("series_names_all", "SELECT DISTINCT metric FROM sample ORDER BY metric", ()), + ("stats_by_res", "SELECT res, COUNT(*) FROM sample GROUP BY res", ()), + )} + finally: + db.close() + return out + + +# --------------------------------------------------------------------------- # +# rendering +# --------------------------------------------------------------------------- # +def _ms(seconds: float) -> str: + if seconds >= 1.0: + return f"{seconds * 1000:,.0f} ms" + return f"{seconds * 1000:.1f} ms" + + +def _mb(value: float) -> str: + return f"{value / 1e6:,.1f} MB" + + +def render_markdown(report: dict[str, Any]) -> str: + """The NOTES tables, as markdown, from one report. + + Machine-written so the "before" and the "after" of this phase are the same tables with + different numbers, which is the only form of the comparison that survives a week. + """ + machine = report["machine"] + store = report["store"] + counts_ = store["counts"] + lines: list[str] = [] + push = lines.append + push(f"- seed version: `{report['seed_version']}` · generated " + f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(report['generated_at']))}") + push(f"- cpu: {machine['cpu']} ({machine['cores']} cores, {machine['machine']}, " + f"kernel {machine['kernel']})") + push(f"- python {machine['python']} · sqlite {machine['sqlite']}") + push(f"- store directory: `{machine['store_dir']}` " + f"({'tmpfs — RAM, the faster machine' if machine['store_dir_on_tmpfs'] else 'a real block device'})") + push("") + push("| seeded store | rows | bytes |") + push("| --- | --- | --- |") + push(f"| `{Path(store['path']).name}` | raw {counts_['raw']:,} · 1m {counts_['1m']:,} · " + f"1h {counts_['1h']:,} · events {counts_['events']:,} · " + f"{counts_['series']} series | {_mb(store['bytes'])} (+{_mb(store['wal_bytes'])} WAL) |") + push("") + push("| hot path | mean | median | max |") + push("| --- | --- | --- | --- |") + labels = { + "series_names_all": "`series_names()` — no filter, **cold**", + "series_names_all_cached": "`series_names()` — cached", + "series_names_host": "`series_names('host')` — `/api/metrics?source=host`, **cold**", + "series_names_host_cached": "`series_names('host')` — cached", + "series_names_host_driveripper": "`series_names('host','driveripper')`", + "stats": "`stats()` — `/api/health`, **cold**", + "stats_cached": "`stats()` — cached (what the next `/api/health` poll pays)", + "pick_res_90d": "`pick_res()` x 3 (1 h, 24 h, 90 d) — one series", + "pick_res_x27_24h": "`pick_res()` x 27, one call per series — 24 h", + "pick_res_many_x27_24h": "`pick_res_many()` — the same 27 in one grouped call — 24 h", + "pick_res_x27_90d": "`pick_res()` x 27, one call per series — 90 d", + "pick_res_many_x27_90d": "`pick_res_many()` — the same 27 in one grouped call — 90 d", + "history_x27_24h": "`history()` x 27, one call per series — 24 h", + "history_many_x27_24h": "`history_many()` — the same 27 batched — 24 h", + "history_x27_90d": "`history()` x 27, one call per series — 90 d", + "history_many_x27_90d": "`history_many()` — the same 27 batched — 90 d", + "history_5m": "`history()` — 5 m window", + "history_24h": "`history()` — 24 h window", + "history_90d": "`history()` — 90 d window", + } + for key, label in labels.items(): + entry = report["paths"].get(key) + if entry is None: + continue + push(f"| {label} | {_ms(entry['mean'])} | {_ms(entry['median'])} | {_ms(entry['max'])} |") + maintain = report["maintain"] + for label, entry in maintain["passes"].items(): + push(f"| `maintain()` — {label} | {_ms(entry['seconds'])} | — | — |") + for key, entry in report.get("http", {}).items(): + push(f"| HTTP `{key}` | {_ms(entry['mean'])} | {_ms(entry['median'])} | " + f"{_ms(entry['max'])} |") + push("") + push("`maintain()` passes on a scratch copy of the seeded store — same rows, same device, and " + "the writes kept off the file every later measurement compares against:") + push("") + push("| pass | write transaction | rows written | WAL before → after |") + push("| --- | --- | --- | --- |") + for label, entry in maintain["passes"].items(): + counts_ = entry["counts"] + push(f"| `{label}` | {_ms(entry['seconds'])} | {entry['rows_written']:,} " + f"(1m {counts_['to_minute']:,} · 1h {counts_['to_hour']:,}) | " + f"{_mb(entry['wal_before_bytes'])} → {_mb(entry['wal_after_bytes'])} |") + cycles = maintain["two_steady_cycles"] + push("") + push("Two more steady ticks, WAL after each: " + + " → ".join(_mb(cycle["wal_after_bytes"]) for cycle in cycles) + + f", against the {_mb(maintain['journal_size_limit_bytes'])} `journal_size_limit` cap: " + + ("inside it" if maintain["wal_within_cap"] else "OVER IT") + + f". Scratch file {_mb(maintain['scratch_bytes'])} → " + f"{_mb(maintain['store_bytes_after'])} (the main file does not move; the WAL is the " + "number that grows between checkpoints).") + push("") + push("| race | inserts | waited > " + f"{WAIT_THRESHOLD_MS:.0f} ms | median | max | `database is locked` |") + push("| --- | --- | --- | --- | --- | --- |") + for key, label in (("single_connection", "one connection, `stats()`+`series_names()` looping"), + ("two_connections_prod_timeout", + f"two connections, windowed `maintain()` looping, " + f"busy_timeout={BUSY_TIMEOUT_MS}"), + ("two_connections_no_timeout", + "two connections, windowed `maintain()` looping, busy_timeout=0"), + ("poller_during_windowed_maintain", + f"two connections, windowed `maintain()` at a " + f"{RACE_MAINTAIN_GAP_SECONDS:.1f} s cadence, " + f"busy_timeout={BUSY_TIMEOUT_MS} — **the pass line**"), + ("poller_during_full_window_maintain", + "CONTROL: two connections, whole-window (pre-fix) `maintain()` at the same " + f"cadence, busy_timeout={BUSY_TIMEOUT_MS}")): + entry = report["race"].get(key) + if entry is None: + continue + push(f"| {label} | {entry['inserts']} of {entry['attempted']} attempted " + f"(target {entry['target']}, {entry['seconds']:.1f} s) | {entry['waiters']} | " + f"{entry['median_wait_ms']:,.1f} ms | {entry['max_wait_ms']:,.0f} ms | " + f"{entry['busy_errors']} |") + for note in entry.get("notes", []): + push(f"| ↳ | {note} | | | | |") + push("") + rows = budget_rows(report) + if any(row.measured is not None for row in rows): + push("The task-04 budgets, priced against the run above. Each ceiling is 2-3x the measured " + "\"after\" (and the `basis` column of `store_perf.BUDGETS` says which figure and how " + "much margin), so this table is the regression gate: a path that drifts back toward " + "its baseline turns red here instead of just feeling slow.") + push("") + push(render_budgets_markdown(rows)) + push("") + verdict = "all inside" if all(row.ok for row in rows) else "OVER" + push(f"Budget verdict: **{verdict}** " + f"({sum(1 for row in rows if row.ok)} of {len(rows)} inside).") + push("") + push("EXPLAIN QUERY PLAN shapes (read paths):") + push("") + for key, plan_rows in report["plans"].items(): + push(f"- `{key}`: " + " · ".join(f"`{row}`" for row in plan_rows)) + push("") + push("EXPLAIN QUERY PLAN shapes (`maintain()`, on the copy, after its own statistics landed):") + push("") + for key, plan_rows in maintain.get("plans", {}).items(): + push(f"- `{key}`: " + " · ".join(f"`{row}`" for row in plan_rows)) + push("") + dev = report.get("dev_db") + if dev: + dev_counts = dev["counts"] + dev_total = sum(dev_counts.values()) + dev_series = sum(dev["series"].values()) + dev_by_res = " ".join(f"{key}={value:,}" for key, value in sorted(dev_counts.items())) + push(f"The same reads against the operator's **real** store " + f"(`{dev['path']}`, {_mb(dev['bytes'])}, opened read-only, no test ever does this):") + push("") + push("| read | what is in that store | mean |") + push("| --- | --- | --- |") + push(f"| `series_names()` | {dev_total:,} samples · {dev_series:,} distinct series · " + f"{dev['events']:,} events | {_ms(dev['paths']['series_names_all']['mean'])} |") + host_series = dev["series"].get("host", 0) + push(f"| `series_names('host')` | {host_series:,} host series | " + f"{_ms(dev['paths']['series_names_host']['mean'])} |") + push(f"| `stats()` counts | `{dev_by_res}` | " + f"{_ms(dev['paths']['stats_by_res']['mean'])} + " + f"{_ms(dev['paths']['stats_events']['mean'])} |") + push("") + for key, plan_rows in dev["plans"].items(): + push(f"- `{key}`: " + " · ".join(f"`{row}`" for row in plan_rows)) + push("") + integrity = report.get("store_after", {}).get("integrity", "?") + push(f"Seeded shape after the measurements: `{integrity}` " + f"(raw {report['store_after']['counts']['raw']:,}).") + return "\n".join(lines) + + +def write_report(report: dict[str, Any], out_prefix: Path) -> dict[str, str]: + """Write `.json` and `.md` so the numbers survive the terminal.""" + out_prefix.parent.mkdir(parents=True, exist_ok=True) + json_path = out_prefix.with_suffix(".json") + md_path = out_prefix.with_suffix(".md") + json_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + md_path.write_text(render_markdown(report) + "\n") + return {"json": str(json_path), "md": str(md_path)} diff --git a/tests/test_api.py b/tests/test_api.py index 5e65a4a..6275e3f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -19,6 +19,7 @@ import pytest from fastapi import HTTPException from fastapi.testclient import TestClient +import store_perf from auth_client import sign_in from auth_contract import ADMIN_PASSWORD, FAST_PBKDF2 from dashboards import alerts, auth @@ -34,7 +35,7 @@ from dashboards.sources.llm import ( parse_gufo_metrics, parse_vllm_metrics, ) -from dashboards.store import Event, Sample, SampleStore +from dashboards.store import RES_RAW, Event, Sample, SampleStore from slots_contract import ( CANONICAL_TASK_ID_FIELD, OPTIONAL_NEXT_TOKEN_FIELDS, @@ -3153,6 +3154,84 @@ def test_one_request_per_host_answers_every_per_host_chart( assert gap["series"][0]["points"] == [] +def test_every_history_endpoint_answers_exactly_what_the_per_series_path_did( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Phase 14 task 02: the batched store door moved the cost of an N x M request, not its answer. + + All four history routes now ask `store.history_many` for the whole batch instead of looping + `store.history` per pair. That is only safe if the JSON is the same document the loop built, + so this compares every route against points read straight out of the store with `query()` — + a third path, in neither the loop nor the batch — and pins the series order the pages parse + (host-major, metric-minor) plus the exact key set of one series entry. + """ + with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1", + host_targets=(Target("driveripper", DEAD_URL), + Target("deskwork", DEAD_URL)), + gufo_url=DEAD_URL) as test_client: + store = test_client.app.state.ctx.store # type: ignore[attr-defined] + now = time.time() + writes = [ + ("host", "driveripper", "cpu.usage_pct", [1.0, 2.0, 3.0]), + ("host", "driveripper", "mem.used_pct", [10.0, 20.0]), + ("host", "deskwork", "cpu.usage_pct", [4.0, 5.0, 6.0]), + ("host", "deskwork", "mem.used_pct", [30.0]), + ("disks", "driveripper", "disk.temp_c@nvme0n1", [38.0, 39.0]), + ("llm", "gufo", "llm.decode_tokens_per_second", [40.0, 45.0]), + # a UPS name the simulator does not export, so its 5-second poll cannot add rows to + # the series this assertion is about + ("ups", "notaserial", "ups.charge_pct", [99.0, 100.0]), + ] + for source, host, metric, values in writes: + store.add_many([Sample(source, host, metric, value, now - (len(values) - index) * 60) + for index, value in enumerate(values)]) + + def expected(source: str, host: str, metric: str) -> list[dict[str, float]]: + rows = store.query(source, host, metric, res=RES_RAW, start=now - 3_600.0, tail=True) + return [{"ts": ts, "value": value} for ts, value in rows] + + routes = [ + ("/api/system/history", {"hosts": "driveripper,deskwork", + "metrics": "cpu.usage_pct,mem.used_pct", "range": "1h"}, + [("host", host, metric) for host in ("driveripper", "deskwork") + for metric in ("cpu.usage_pct", "mem.used_pct")]), + ("/api/disks/history", {"hosts": "driveripper", + "metrics": "disk.temp_c@nvme0n1", "range": "1h"}, + [("disks", "driveripper", "disk.temp_c@nvme0n1")]), + ("/api/llm/history", {"engines": "gufo", "metrics": "decode_tokens_per_second", + "range": "1h"}, + [("llm", "gufo", "llm.decode_tokens_per_second")]), + ("/api/ups/history", {"ups": "notaserial", "metrics": "ups.charge_pct", + "range": "1h"}, + [("ups", "notaserial", "ups.charge_pct")]), + ] + for path, params, pairs in routes: + body = test_client.get(path, params=params).json() + assert len(body["series"]) == len(pairs), path + for series, (source, host, metric) in zip(body["series"], pairs, strict=True): + assert set(series) == {"source", "host", "metric", "range_seconds", "points"} + assert (series["source"], series["host"], series["metric"]) == (source, host, metric) + assert series["range_seconds"] == 3_600.0 + assert series["points"] == expected(source, host, metric), f"{path} {metric}" + + # and an empty answer is still an answer: a name with no rows, in a batch with names that + # do, comes back as an empty series rather than a missing entry or a 404 + body = test_client.get("/api/system/history", params={ + "hosts": "driveripper", "metrics": "cpu.usage_pct,gpu.busy_pct@7", + "range": "1h"}).json() + assert [series["points"] for series in body["series"]] == [expected( + "host", "driveripper", "cpu.usage_pct"), []] + + # no metric named is no series asked for: the batch door is never opened with an empty + # list (SQL `IN ()` is a syntax error waiting for that, and 500 is not the answer anyway) + empty = test_client.get("/api/system/history", + params={"hosts": "driveripper", "metrics": ""}).json() + assert empty["hosts"] == ["driveripper"] and empty["series"] == [] + no_hosts = test_client.get("/api/system/history", + params={"hosts": "", "metrics": "cpu.usage_pct"}).json() + assert no_hosts["hosts"] == ["driveripper", "deskwork"], "empty means every configured host" + assert len(no_hosts["series"]) == 2 + + def test_llm_snapshot_keeps_an_engine_that_never_answered_in_the_list( tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A configured-but-dead engine is listed as unreachable, not dropped (invariant 4).""" @@ -4079,3 +4158,93 @@ def test_samples_written_by_the_poller_are_queryable(tmp_path: Path, "range": "15m"}).json()["points"] assert [p["value"] for p in points] == [40.0, 45.0] store.close() + + +# --------------------------------------------------------------------------- # +# phase 14 task 04: the endpoint budgets +# +# `store_perf.BUDGETS` is the one table of ceilings (it is what the probe's report and the notes +# render, so a ceiling relaxed in one place is relaxed everywhere). The ceilings are 2-3x the +# measured "after" on the seeded production-shaped store, which makes them a regression gate +# rather than a stopwatch: the `/system` page's own chart request costs ~2 ms and still gets +# 100 ms, while putting a full-table scan back on a per-request path crosses by three orders of +# magnitude. Wall time lives behind `--store-perf` for that reason; what runs in every `uv run +# pytest` is the key-name pin below, because a budget that silently stops being priced is as +# useless as a budget that is crossed. +# --------------------------------------------------------------------------- # + + +def test_every_http_budget_names_a_route_the_probe_prices(tmp_path: Path) -> None: + """The stale-pointer check for the HTTP half, on a throwaway store. + + `measure_http` is what the priced test runs, and it owns the request shapes; if a key there is + renamed, or a route stops being measured, the priced test would report the gap rather than the + cost — which is why this asserts the key set here, cheaply, on every run. + """ + report = store_perf.measure_budget_http(tmp_path / "budget-keys.sqlite3", + hosts=("driveripper",), metrics=("cpu.usage_pct",)) + assert set(report["http"]) == set(store_perf.HTTP_MEASURED_KEYS), ( + "the HTTP pass and the budget table no longer describe the same requests") + assert store_perf.missing_budget_keys(report, ("http",)) == [] + # and the request the budgets describe is the one the task names: 3 hosts x 9 series + nine = store_perf.HISTORY_METRICS + assert len(nine) == 9, "the 90 d budget is priced against a 3x9 request, not some other shape" + + +def test_metrics_stops_scanning_for_series_names_once_the_hub_is_polling( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The live shape behind one budgeted endpoint, guarded by statements rather than a clock. + + Task 04's watch caught `/api/metrics` answering with its full `SELECT DISTINCT` scan on + practically every call of the running server (535 ms, measured): task 02's cache was invalidated + by every sample write, and the hub writes every 2-300 s, so the cache never lived long enough to + serve. The store-level half of this lives in `tests/test_store.py`; this is the half that says + the *endpoint* is what stops scanning, on the app the operator runs. + """ + db = str(tmp_path / "metrics-busy.sqlite3") + store = SampleStore(db) + store.add_many([Sample("host", "driveripper", "cpu.usage_pct", 1.0, time.time()), + Sample("host", "deskwork", "cpu.usage_pct", 1.0, time.time())]) + app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD, + pbkdf2_iterations=FAST_PBKDF2), store) + try: + with TestClient(app) as http: + sign_in(http) + assert "cpu.usage_pct" in http.get("/api/metrics", params={"source": "host"}).json()["metrics"] + + # a poll of series the store already has, then the same request with the log open + store.add_many([Sample("host", "driveripper", "cpu.usage_pct", 2.0, time.time()), + Sample("host", "deskwork", "cpu.usage_pct", 2.0, time.time())]) + seen: list[str] = [] + # the statement log is the test: `SampleStore._db` is the handle the request runs on + store._db.set_trace_callback(seen.append) + try: + assert "cpu.usage_pct" in http.get("/api/metrics", + params={"source": "host"}).json()["metrics"] + finally: + store._db.set_trace_callback(None) + assert [s for s in seen if "DISTINCT metric" in s] == [], ( + f"`/api/metrics` went back to the table while nothing changed: {seen}") + + # and a source that starts publishing something new is in the very next answer + store.add("host", "driveripper", "gpu.busy_pct@0", 1.0) + assert "gpu.busy_pct@0" in http.get("/api/metrics", + params={"source": "host"}).json()["metrics"] + finally: + store.close() + + +@pytest.mark.store_perf +def test_the_budgeted_http_paths_fit_their_pinned_margins(perf_store_path: Path) -> None: + """`/api/health`, `/api/metrics`, `/api/system/history` (1 h and 90 d), `/api/summary`. + + Priced through the real ASGI gate with a signed-in session, on the seeded 6.4 M-row store — + the same call the operator's browser makes, in-process so no network is measured. This is the + line `TODO.md:7` is about: `/api/health` used to answer in 1.27 s because it counted the whole + `sample` table on every poll, on every tab. + """ + report = store_perf.measure_budget_http(perf_store_path) + print() + print(store_perf.render_budgets_markdown(store_perf.budget_rows(report, ("http",)))) + assert store_perf.budget_violations(report, ("http",)) == [] diff --git a/tests/test_store.py b/tests/test_store.py index c7c16fe..68437b7 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -6,17 +6,33 @@ raw retention was silently empty. These tests fail loudly if that ever happens a """ from __future__ import annotations +import logging +import math +import re +import sqlite3 import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path from typing import Any import pytest +import store_perf from dashboards.store import ( + COMPOUND_CHUNK, HOUR, + JOURNAL_SIZE_LIMIT_BYTES, + LAST_MAINTAIN_KEY, + MAINTAIN_SLACK_SECONDS, MINUTE, RES_HOUR, RES_MIN, RES_RAW, + ROLLUP_INSERT, + SERIES_CACHE_MAX_ENTRIES, + WAL_AUTOCHECKPOINT_PAGES, Event, Sample, SampleStore, @@ -88,16 +104,25 @@ def test_maintain_rolls_raw_into_minutes(db) -> None: def test_maintain_is_idempotent(db) -> None: - """Rollups run on a timer: running twice must not double-count or duplicate rows.""" + """Rollups run on a timer: running twice must not double-count or duplicate rows. + + The second pass is asked for the whole window again (the `last_maintain` row is deleted + first), because since phase 14 task 03 a pass that has recorded its own clock has nothing + left to do — the window is where the work got bounded, and this test is about the rollup being + an `INSERT OR REPLACE` rewrite rather than a duplicate row. `test_a_second_maintain_reads_ + only_the_window_since_the_last_run` is the other half: the pass that does run again at a later + clock re-reads only what became due since. + """ now = time.time() minute = int(now // MINUTE) * MINUTE add_series(db, "host", "h", "m", [10, 20], start=minute - 4 * MINUTE, step=1) first = db.maintain(now=now) rows = db.query("host", "h", "m", res=RES_MIN, start=now - 3600) count = db.count() + assert db.delete_meta("last_maintain") == 1, "the whole-window re-run needs no previous pass" second = db.maintain(now=now) assert rows == db.query("host", "h", "m", res=RES_MIN, start=now - 3600) - assert first["to_minute"] == second["to_minute"] + assert first["to_minute"] == second["to_minute"] > 0 assert db.count() == count, "re-running maintenance duplicated samples" @@ -400,3 +425,1208 @@ def test_stats_reads_last_maintain_through_get_meta(db) -> None: assert db.stats()["last_maintain"] is None db.maintain(now=1_700_000_000.0) assert db.stats()["last_maintain"] == 1_700_000_000.0 + + +# ---- phase 14 task 02: nothing on a per-request path may scan the table ---------------- # +# +# `TODO.md:7` is "pages take forever to load and seem to lock up the database", and the half of +# it a browser feels was two whole-table answers served on every request (`series_names`, +# `stats`) plus three `COUNT`s per series to pick a chart's resolution. The pins below assert the +# two things that make a cache trustworthy rather than fast: it is *invalidated* by the write that +# would make it a lie, and it *expires* so a missed invalidation is a 60-second bug rather than a +# permanent one. +# +# They count SQL statements rather than wall time. `sqlite3`'s trace callback is the query counter +# the task asks for, and it is the right instrument: "the second call did not scan" is a fact +# about statements, and a wall-time assertion here would be a budget that a noisy laptop fails for +# the wrong reason (task 04 owns the budgets). + + +@contextmanager +def query_log(store: SampleStore) -> Iterator[list[str]]: + """Every statement the store executes on this handle while the context is open, in order.""" + seen: list[str] = [] + store._db.set_trace_callback(seen.append) + try: + yield seen + finally: + store._db.set_trace_callback(None) + + +def plan_of(store: SampleStore, sql: str, args: tuple[Any, ...] = ()) -> str: + """`EXPLAIN QUERY PLAN` as one line, so a test can say "seek, not scan" in words.""" + with store._lock: + return " | ".join(row[3] for row in store._db.execute("EXPLAIN QUERY PLAN " + sql, args)) + + +def test_series_names_answers_from_cache_and_a_write_invalidates_it(db) -> None: + """The `/api/metrics` answer is cached — and a write ends the cache, not the clock. + + A cache that outlives a write answers "why is this chart empty" with yesterday's truth, so + the invalidation is the load-bearing half: a new metric name must be visible on the very next + call, and a deleted one must be gone. + """ + now = time.time() + db.add("host", "driveripper", "cpu.usage_pct", 1.0, ts=now) + with query_log(db) as seen: + first = db.series_names("host") + assert [s for s in seen if "DISTINCT metric" in s], "the first call pays the scan once" + + seen.clear() + assert db.series_names("host") == first + assert seen == [], "a cached series_names() must not touch the table at all" + + db.add("host", "driveripper", "gpu.busy_pct@0", 2.0, ts=now + 1) + seen.clear() + assert db.series_names("host") == ["cpu.usage_pct", "gpu.busy_pct@0"], ( + "a metric that appeared after the cached read has to be in the next answer") + assert [s for s in seen if "DISTINCT metric" in s], "the write must have invalidated it" + + +def test_every_write_that_can_change_the_series_list_invalidates_it(db) -> None: + """`add`, `add_many` and `delete_series` — the three writes that move the name set.""" + db.add("host", "a", "cpu.usage_pct", 1.0) + assert db.series_names("host") == ["cpu.usage_pct"] + db.add_many([Sample("host", "a", "mem.used_pct", 2.0, time.time())]) + assert "mem.used_pct" in db.series_names("host") + assert db.delete_series("host", "a", "cpu.usage_pct") == 1 + assert db.series_names("host") == ["mem.used_pct"], "a deleted series must not linger in the list" + + +def test_maintaining_the_store_invalidates_the_series_list(db) -> None: + """Pruning can delete the last row of a name; the list must not promise it afterwards.""" + old = time.time() - (db.raw_keep_seconds + 7200) + add_series(db, "host", "h", "gone.soon", [1, 2], start=old, step=60) + assert db.series_names("host") == ["gone.soon"] + db.maintain(now=time.time()) + assert db.series_names("host") == [] + + +def test_a_write_of_an_already_written_series_leaves_the_series_cache_warm(db) -> None: + """The live defect the task-04 watch caught, pinned so it cannot come back quietly. + + Task 02 invalidated the series cache on *every* sample write. On a store the hub writes to + every 2-300 s that meant the cache never served: `/api/metrics` on the running dev server was + answering with its full `SELECT DISTINCT` scan (535 ms of store mutex) on nearly every call — + the same per-request scan the task existed to remove. A poll of a series the store has already + written cannot change the *set of names*, so it must not cost a scan; only a new name can. + """ + now = time.time() + db.add("host", "driveripper", "cpu.usage_pct", 1.0, ts=now) + assert db.series_names("host") == ["cpu.usage_pct"] + + db.add("host", "deskwork", "cpu.usage_pct", 1.0, ts=now) # the second host of the same name + assert db.series_names("host") == ["cpu.usage_pct"] + + # the next nine polls, of series the store already knows: the answer costs no statement + with query_log(db) as seen: + for poll in range(9): + db.add_many([Sample("host", "driveripper", "cpu.usage_pct", float(poll), now + poll), + Sample("host", "deskwork", "cpu.usage_pct", float(poll), now + poll)]) + # the transaction's own BEGIN/COMMIT ride in the trace; what must not be in there is a + # read, and above all not the `SELECT DISTINCT` this whole task is about + assert not [s for s in seen if s.lstrip().upper().startswith("SELECT")], ( + "a poll should cost its inserts and nothing else") + seen.clear() + assert db.series_names("host") == ["cpu.usage_pct"] + assert seen == [], f"a write of known series must not re-scan the name list: {seen}" + + +def test_a_write_of_a_series_nobody_wrote_yet_still_invalidates_immediately(db) -> None: + """The other half of the rule, in the direction the cache must never be lazy about. + + A new metric name is exactly the event `/api/metrics` exists to report, and a source gaining + one is a deploy — so "eventually" is not good enough and the next call has to see it. + """ + now = time.time() + db.add("host", "driveripper", "cpu.usage_pct", 1.0, ts=now) + assert db.series_names("host") == ["cpu.usage_pct"] + with query_log(db) as seen: + db.add("host", "deskwork", "a.brand.new.metric", 2.0, ts=now + 1) + seen.clear() + assert db.series_names("host") == ["a.brand.new.metric", "cpu.usage_pct"], ( + "a metric nobody has written since the last read must be in the very next answer") + assert "a.brand.new.metric" in db.series_names("host") + # …and a per-device stream on a name the list already carries is a new triple too + db.series_names("host") + db.add_many([Sample("host", "driveripper", "gpu.busy_pct@3", 1.0, now + 2)]) + assert db.series_names("host", "driveripper") == ["cpu.usage_pct", "gpu.busy_pct@3"] + + +def test_a_deleted_series_that_comes_back_is_a_new_series_again(db) -> None: + """`delete_series` empties rows, so the triples it emptied have to be forgotten too. + + Without that, re-creating a deleted series would leave `/api/metrics` answering without it for + a whole TTL window: the write matches a triple the store still believes exists. + """ + db.add("host", "deskwork", "gpu.busy_pct@0", 1.0) + assert db.series_names("host") == ["gpu.busy_pct@0"] + assert db.delete_series("host", "deskwork", "gpu.busy_pct@0") == 1 + assert db.series_names("host") == [] + db.add("host", "deskwork", "gpu.busy_pct@0", 2.0) # the same triple, now a live series again + assert db.series_names("host") == ["gpu.busy_pct@0"] + + +def test_the_series_write_tracking_is_capped(db, monkeypatch: pytest.MonkeyPatch) -> None: + """The existence set is memory, so it has a ceiling and a safe fallback below it. + + Past the cap the store invalidates on every write — task 02's behaviour, correct and slower. + The cap is what keeps a namespace a caller can influence from becoming an unbounded structure. + """ + monkeypatch.setattr("dashboards.store.SERIES_TRACKED_SERIES_MAX", 4) + for index in range(30): + db.add("host", "deskwork", f"metric{index}", 1.0) + assert f"metric{index}" in db.series_names("host"), ( + "giving up the tracking must never cost a name from the list") + assert len(db._known_series) <= 4, f"the tracking grew past its cap: {len(db._known_series)}" + with query_log(db) as seen: + db.add("host", "deskwork", "metric99", 1.0) + db.series_names("host") + assert [s for s in seen if "DISTINCT metric" in s], ( + "past the cap the store must go back to deriving the list from the table") + + +def test_the_series_cache_is_bounded_however_many_names_a_caller_asks_for(db) -> None: + """`/api/metrics?source=&host=` is free text, so the cache is capped, not grown. + + The key space belongs to whoever sends the request; an unbounded dict keyed by it is a slow + memory leak with a query string on the front. + """ + for index in range(SERIES_CACHE_MAX_ENTRIES + 40): + db.series_names("host", f"host{index}") + assert len(db._series_cache) <= SERIES_CACHE_MAX_ENTRIES + + +def test_series_names_expires_on_the_clock(db) -> None: + """The TTL is the ceiling on staleness when nothing wrote.""" + db.add("host", "a", "cpu.usage_pct", 1.0) + db.series_names("host") + db.series_ttl_seconds = 0.0 + with query_log(db) as seen: + db.series_names("host") + assert [s for s in seen if "DISTINCT metric" in s], "an expired entry must be re-derived" + + +def test_invalidate_read_caches_forces_both_answers_back_to_the_table(db) -> None: + """The blunt instrument the cold measurement needs, pinned so it stays blunt. + + `store_perf` asks for a cold `stats()`/`series_names()` against a store whose caches are warm; + if either survived that call, the phase's before/after would be timing a cache hit and calling + it a speed-up. + """ + db.add("host", "a", "cpu.usage_pct", 1.0) + assert db.series_names("host") == ["cpu.usage_pct"] + assert db.stats()["samples_total"] == 1 + with query_log(db) as seen: + db.invalidate_read_caches() + assert seen == [], "invalidating is bookkeeping, not a query" + db.series_names("host") + db.stats() + assert [s for s in seen if "DISTINCT metric" in s], "the name list came back from the scan" + assert [s for s in seen if "COUNT(*) FROM sample" in s], "the counts did too" + + +def test_stats_answers_from_cache_until_the_ttl_expires(db) -> None: + """`/api/health` is polled by every open tab; its two `COUNT` scans may run once per TTL. + + The payload keys are a contract with deployment (`/api/health` is public), so this asserts + the same dict twice and the statements in between: nothing about the answer changes, only how + often the table is asked. + """ + db.add("host", "a", "cpu.usage_pct", 1.0) + db.log_event(Event(time.time(), "system", "a", "recording", "info", "up again")) + with query_log(db) as seen: + first = db.stats() + assert first["samples_total"] == 1 and first["events"] == 1 + assert [s for s in seen if "COUNT(*) FROM sample" in s] + assert [s for s in seen if "COUNT(*) FROM event" in s] + + seen.clear() + second = db.stats() + assert not [s for s in seen if "COUNT(*)" in s], ( + "a cached stats() must not run a COUNT — that 1.3 s is the page load") + assert second == first + + first["samples_by_res"][RES_RAW] = 999 # a caller must not be able to poison the cache + assert db.stats()["samples_by_res"] == second["samples_by_res"] + + db.stats_ttl_seconds = 0.0 + seen.clear() + db.add("host", "a", "cpu.usage_pct", 2.0) + third = db.stats() + assert [s for s in seen if "COUNT(*) FROM sample" in s], "the TTL, not a write, ages counts" + assert third["samples_total"] == 2 + + +def test_stats_caches_the_counts_but_never_the_maintenance_timestamp(db) -> None: + """`last_maintain` is how an operator spots a stalled rollup loop; caching it hides the bug. + + The counts behind `/api/health` may be 30 s old. A `last_maintain` that never moves while the + page says `ok` is exactly the frozen-source failure AGENTS.md invariant 4 calls the one worse + than an outage, so it is read on every call and this test keeps it that way. + """ + db.add("host", "a", "cpu.usage_pct", 1.0) + assert db.stats()["last_maintain"] is None + with query_log(db) as seen: + db.maintain(now=1_700_000_000.0) + seen.clear() + stats = db.stats() + assert stats["last_maintain"] == 1_700_000_000.0 + assert not [s for s in seen if "COUNT(*)" in s], "the counts stay cached; only meta is re-read" + assert [s for s in seen if "FROM meta" in s] + + +def test_pick_res_still_picks_the_finest_resolution_that_fits(db) -> None: + """The capped count must decide exactly what a full `COUNT(*)` decided. + + Cap at `max_points + 1` keeps both predicates — `count == 0` and `count <= max_points` — + identical, so this is a behaviour pin and not a tolerance. + """ + now = time.time() + # raw over budget, minutes under it: the answer is 1m whichever way the rows are counted + add_series(db, "host", "h", "m", range(900), start=now - 900, step=1) + add_series(db, "host", "h", "m", range(30), start=now - 3_600, step=60, res=RES_MIN) + assert db.pick_res("host", "h", "m", span=3_600.0, max_points=600) == RES_MIN + # nothing fits, so the finest resolution that has *any* row wins — an empty one reads as a + # dead sensor, which is the rule the pre-cap implementation had and the cap must not move + assert db.pick_res("host", "h", "m", span=3_600.0, max_points=20) == RES_RAW + assert db.pick_res("host", "h", "absent", span=3_600.0, max_points=600) == RES_HOUR + + +def test_the_resolution_count_is_one_statement_of_index_seeks(db) -> None: + """One statement for all three resolutions, and its plan is a seek, not a scan.""" + now = time.time() + for host in ("a", "b"): + add_series(db, "host", host, "m", range(50), start=now - 300, step=1) + with query_log(db) as seen: + db.pick_res_many("host", ["a", "b"], ["m", "absent"], span=3_600.0, max_points=600) + statements = [s for s in seen if "UNION ALL" in s] + assert len(statements) == 1, ( + f"4 series x 3 resolutions must be one grouped round trip, saw {seen}") + # the trace callback hands back the statement with its bindings expanded, which is the exact + # text that ran — so the plan below is the plan the request paid, not a plan for a lookalike + plan = plan_of(db, statements[0]) + assert "COVERING INDEX" in plan, plan + assert "SCAN sample" not in plan, f"the count fell back to a table scan: {plan}" + + +def test_pick_res_many_agrees_with_pick_res_for_every_pair(db) -> None: + now = time.time() + hosts = ["driveripper", "deskwork", "gateway"] + metrics = ["cpu.usage_pct", "mem.used_pct", "gpu.busy_pct@0", "never.written"] + for host in hosts: + if host == "gateway": + continue # the fleet's GPU-less box: a whole host with no gpu.* rows + for metric in metrics[:3]: + add_series(db, "host", host, metric, range(40), start=now - 400, step=1) + batched = db.pick_res_many("host", hosts, metrics, span=3_600.0, max_points=600) + assert batched == {(host, metric): db.pick_res("host", host, metric, 3_600.0, 600) + for host in hosts for metric in metrics} + assert batched[("gateway", "gpu.busy_pct@0")] == RES_HOUR, "no rows anywhere: the documented default" + assert db.pick_res_many("host", [], metrics, span=3_600.0, max_points=600) == {} + assert db.pick_res_many("host", hosts, [], span=3_600.0, max_points=600) == {} + + +def test_history_many_answers_exactly_what_history_answers_per_series(db) -> None: + """The batched door and the single-series door are the same answer, series for series. + + The shapes a chart depends on are in here too: the raw fallback while rollups lag, the + thinning of an over-budget window, and the newest point surviving both. + """ + now = time.time() + hosts = ["driveripper", "deskwork", "gateway"] + metrics = ["cpu.usage_pct", "mem.used_pct", "never.written"] + for host in hosts: + for index, metric in enumerate(metrics): + add_series(db, "host", host, metric, range(30 + index), start=now - 900, step=30) + # one series with rollups as well, so the resolution decision (not just the fetch) is compared + add_series(db, "host", "driveripper", "cpu.usage_pct", range(20), start=now - 7_200, + step=300, res=RES_MIN) + # one crowded series, for the thinning contract below + add_series(db, "host", "driveripper", "crowded.series", range(100), start=now - 1_000, step=10) + + for span, max_points in ((3_600.0, 600), (3_600.0, 12), (30.0, 600), (90.0, 600)): + batched = db.history_many("host", hosts, [*metrics, "crowded.series"], span=span, + max_points=max_points) + per_series = {(host, metric): db.history("host", host, metric, span=span, + max_points=max_points) + for host in hosts for metric in [*metrics, "crowded.series"]} + assert batched == per_series, f"the batch changed a series (span={span}, points={max_points})" + + # the tail contract survives batching: a thinned series is ascending, inside the budget, and + # still ends on the newest row the window holds — the point a chart cannot afford to lose + thinned = db.history_many("host", ["driveripper"], ["crowded.series"], span=3_600.0, + max_points=12)[("driveripper", "crowded.series")] + full = db.history("host", "driveripper", "crowded.series", span=3_600.0, max_points=600) + # `<= max_points + 1`, not `<= max_points`: the thinning has always been allowed to append the + # newest point back after striding, so a series can land one over budget. That quirk is part + # of the answer the endpoints must keep byte-identically, not something this task fixes. + assert len(full) == 100 and 12 <= len(thinned) <= 13 + assert [ts for ts, _ in thinned] == sorted(ts for ts, _ in thinned) + assert thinned[-1] == full[-1], "the batch must not lose 'now' any more than the loop did" + assert thinned[0] > full[0], "the head is tail-truncated — the documented `tail=True` semantic" + + assert db.history_many("host", [], ["cpu.usage_pct"]) == {} + # a series nobody ever wrote answers as an empty list, per pair — the contract every history + # endpoint shows the browser as an empty plot rather than a 404 + assert db.history_many("host", ["gateway"], ["still.unwritten"]) == { + ("gateway", "still.unwritten"): []} + # duplicate names in a request cost one query each, and still answer once per pair + assert db.history_many("host", ["gateway", "gateway"], ["cpu.usage_pct"]) == db.history_many( + "host", ["gateway"], ["cpu.usage_pct"]) + + +def test_history_many_batches_the_decision_and_not_the_rows(db) -> None: + """One grouped statement for the resolution of the whole batch, whatever its size. + + Recorded deliberately: folding the *rows* into one statement (a window function over the + range, or a compound of the per-series tail selects) measured slower on the seeded store, so + the batch is the decision and the fetch stays a per-series index seek. This pins the half + that did get batched, so a future edit cannot quietly return to three counts per series. + """ + now = time.time() + hosts = [f"host{index}" for index in range(20)] + for host in hosts: + add_series(db, "host", host, "cpu.usage_pct", range(10), start=now - 300, step=30) + with query_log(db) as seen: + db.history_many("host", hosts, ["cpu.usage_pct", "mem.used_pct"], span=3_600.0, + max_points=600) + grouped = [s for s in seen if "UNION ALL" in s] + # 20 hosts x 2 metrics x 3 resolutions = 120 branches = two chunks of COMPOUND_CHUNK + assert len(grouped) == -(-120 // COMPOUND_CHUNK), ( + f"expected one grouped count per {COMPOUND_CHUNK} branches, saw {grouped}") + + +# ---- phase 14 task 03: the rollup window, and the WAL it writes into ------------------- # +# +# `maintain()` used to average every row its source resolution retains, on every tick, inside one +# write transaction: 3.97 M rows of 1-minute history per minute. Task 01 measured that at 5.4 s, +# and at 5 of 9 concurrent poller inserts outliving `busy_timeout=5000` — the +# `store-error: database is locked` line in AGENTS.md known gap 4. The fix is a window per step, +# and the tests below are the two halves of "a window that is only cheaper": the bound is read out +# of the statements that actually ran (never recomputed from the same arithmetic under test), and +# the values come out identical to a whole-table rollup of the same rows. + +#: The `(ceiling, floor)` a rollup statement ran with, pulled out of `sqlite3`'s trace text. +ROLLUP_WINDOW = re.compile( + r"FROM sample WHERE res='(?P\w+)' AND ts < (?P[\d.]+) AND ts >= (?P[\d.]+)") + + +def rollup_windows(statements: list[str]) -> dict[str, tuple[float, float]]: + """`(ceiling, floor)` per rollup step, keyed by the resolution it read from. + + The trace callback hands back the statement with its bindings expanded, so this is the window + the statement ran with — the task's "assert via the EXPLAIN/row count hook, or a sqlite3 + trace", taken literally. Keying on the source resolution is what tells the minute step + (`res='raw'`) from the hour step (`res='1m'`). + """ + found: dict[str, tuple[float, float]] = {} + for statement in statements: + match = ROLLUP_WINDOW.search(statement) + if match: + found[match["res"]] = (float(match["ceiling"]), float(match["floor"])) + return found + + +def bucket_start(ts: float, bucket: float = MINUTE) -> float: + """The aligned bucket a clock falls in — the shape every window floor is snapped to.""" + return math.floor(ts / bucket) * bucket + + +#: Wall clocks for the window-bound tests. `1_800_000_000` is exactly on an hour *and* a minute +#: boundary, so these walk the interesting cases: on it, one second past, half an hour past, and one +#: second short of the next hour. A minute-aligned bound and an hour-aligned one are the same number +#: at offset 0 and 1 980 s apart at offset 1 799, which is exactly the luck this test used to run on +#: (`pytest.approx`'s relative tolerance hid up to 1 800 s of misalignment and not more) — an +#: explicit clock makes every edge a case instead of a coin flip. +WINDOW_CLOCK_OFFSETS = (0, 1, 30, 59, 1_800, 3_539, 3_599) + + +def test_the_window_floor_is_always_aligned_to_its_own_bucket(db) -> None: + """`_rollup_floor` snaps to the step's bucket, at every clock, and never above its bound. + + The alignment is not cosmetic: the ceiling of a rollup step is a *row* bound, so the newest + bucket is a partial one that later passes re-roll, and a floor that lands inside a bucket makes + that re-roll average its own tail rows. Asserting the property over offsets — rather than one + number at one instant — is what keeps a future edit from re-fitting the test to whichever clock + it was written on. + """ + for bucket in (MINUTE, HOUR): + for offset in WINDOW_CLOCK_OFFSETS: + clock = 1_800_000_000.0 + offset + floor = db._rollup_floor(clock, 0.0, bucket) + assert floor % bucket == 0, f"bucket={bucket:g} offset={offset} floor={floor}" + assert floor <= clock - bucket - MAINTAIN_SLACK_SECONDS, "never above the bound" + assert floor > clock - (2 * bucket) - MAINTAIN_SLACK_SECONDS, "never a whole window back" + # …and the two steps really do use different alignments, or the test above proves nothing + clock = 1_800_000_000.0 + 1_799 + assert db._rollup_floor(clock, 0.0, HOUR) != db._rollup_floor(clock, 0.0, MINUTE) + + +@pytest.mark.parametrize("hour_offset", WINDOW_CLOCK_OFFSETS) +def test_the_first_rollup_pass_scans_the_retention_it_has(db, hour_offset: int) -> None: + """No `last_maintain` row means nothing is known, so the window is the whole retention. + + This is the "a first run on a fresh DB scans the full retention window exactly as today" + half of the task, and it is also the restore-from-backup case: a store opened on history it + has never rolled has to be able to reach 47-hour-old raw rows, or its charts above the last + minute stay empty forever. + """ + now = 1_800_000_000.0 + hour_offset + deep = bucket_start(now - 47 * HOUR) + add_series(db, "host", "h", "deep.metric", [7], start=deep, step=1) + with query_log(db) as seen: + db.maintain(now=now) + windows = rollup_windows(seen) + assert windows[RES_RAW] == pytest.approx( + (now - MINUTE - MAINTAIN_SLACK_SECONDS, now - db.raw_keep_seconds)) + assert windows[RES_MIN] == pytest.approx( + (now - HOUR - MAINTAIN_SLACK_SECONDS, now - db.minute_keep_seconds)) + assert db.query("host", "h", "deep.metric", res=RES_MIN, start=deep - 1, + end=deep + MINUTE) == [(deep, 7.0)], ( + "a first pass cannot reach the deep history it retains") + + +@pytest.mark.parametrize("hour_offset", WINDOW_CLOCK_OFFSETS) +def test_a_second_maintain_reads_only_the_window_since_the_last_run(db, hour_offset: int) -> None: + """The cost change, pinned as a bound and as behaviour. + + A bucket that filled after the previous pass is inside the next window and gets rolled. A raw + row that arrives for a bucket ten minutes behind it is not, and that bucket keeps the average + the earlier pass computed — the documented cost of a window, stated here as a passing test + rather than as a surprise. Reaching it in production costs a poller stall measured in minutes + (the poller stamps samples with its own wall clock); the alternative it replaced was + re-averaging 3.97 M rows every minute. + """ + now = 1_800_000_000.0 + hour_offset + bucket_a = bucket_start(now - 10 * MINUTE) + add_series(db, "host", "h", "m", [10, 20], start=bucket_a, step=1) + db.maintain(now=now) + assert db.query("host", "h", "m", res=RES_MIN, start=bucket_a - 1, + end=bucket_a + MINUTE) == [(bucket_a, 15.0)] + + second = now + 60 + # Two full minutes behind the ceiling, so both rows of the pair are always inside the next + # window: with the bucket only one step behind, a clock that lands within a second of the + # boundary puts the pair's second row above the ceiling and the bucket rolls 40.0 instead. + bucket_b = bucket_start(second - 3 * MINUTE) + add_series(db, "host", "h", "m", [40, 50], start=bucket_b, step=1) + add_series(db, "host", "h", "m", [1000], start=bucket_a + 30, step=1) + with query_log(db) as seen: + counts = db.maintain(now=second) + + windows = rollup_windows(seen) + # The bound: one minute (one hour) of rows plus the overlap, not 48 h of raw (not 30 d of 1m). + # Each floor is snapped to *its own step's* bucket — the minute step to a minute, the hour step + # to an hour — which is what `hour_offset` walks, and both are far inside retention. + assert windows[RES_RAW] == pytest.approx( + (second - MINUTE - MAINTAIN_SLACK_SECONDS, + bucket_start(now - MINUTE - MAINTAIN_SLACK_SECONDS, MINUTE))) + assert windows[RES_MIN] == pytest.approx( + (second - HOUR - MAINTAIN_SLACK_SECONDS, + bucket_start(now - HOUR - MAINTAIN_SLACK_SECONDS, HOUR))) + assert windows[RES_RAW][1] > second - db.raw_keep_seconds + assert windows[RES_MIN][1] > second - db.minute_keep_seconds + # …and the behaviour that bound implies, in both directions + assert db.query("host", "h", "m", res=RES_MIN, start=bucket_b - 1, + end=bucket_b + MINUTE) == [(bucket_b, 45.0)], "the due bucket must be rolled" + assert db.query("host", "h", "m", res=RES_MIN, start=bucket_a - 1, + end=bucket_a + MINUTE) == [(bucket_a, 15.0)], ( + "the row behind the window must not rewrite an older bucket") + assert counts["to_minute"] == 1, f"only the due bucket was written: {counts}" + assert db.get_meta(LAST_MAINTAIN_KEY) == str(second) + + +@pytest.mark.parametrize("hour_offset", WINDOW_CLOCK_OFFSETS) +def test_a_missed_interval_is_a_catch_up_scan_not_a_gap(db, hour_offset: int) -> None: + """Twelve minutes with no pass, then one: every bucket in between is rolled. + + The window reaches one bucket plus slack *behind* the previous pass's ceiling rather than + starting at it, and that overlap is the whole difference between "the app was stopped for + twelve minutes" and "twelve minutes of charts are silently empty". One row is placed + deliberately inside the overlap: above the first pass's ceiling, below the clock the first pass + recorded, so a bound of exactly `last_maintain` would skip it for ever. + """ + # An explicit clock, for the same reason as the test above: the overlap row's bucket and the + # window's floor are both snapped, and the count that proves the catch-up happened depends on + # which side of a boundary they land on. + first_at = 1_800_000_000.0 + hour_offset - 20 * MINUTE + db.maintain(now=first_at) # an otherwise empty store: this only records the clock + aligned = bucket_start(first_at) + for index in range(10): + add_series(db, "host", "h", "m", [float(index)], start=aligned + index * MINUTE + 5, step=1) + inside_the_overlap = first_at - 100 + add_series(db, "host", "h", "m", [99], start=inside_the_overlap, step=1) + + second = first_at + 12 * MINUTE + counts = db.maintain(now=second) + # `end` is explicit because `query` defaults it to the wall clock, and this test's clock is its + # own: a default would silently answer "no rows" for buckets that are in the store. + minutes = db.query("host", "h", "m", res=RES_MIN, start=0, end=second) + assert counts["to_minute"] == 11, f"ten buckets plus the overlap row: {counts}" + assert [value for _ts, value in minutes] == [99.0, *[float(step) for step in range(10)]], ( + "a missed interval must be caught up, not skipped") + assert minutes[0][0] <= inside_the_overlap, "the overlap row has its bucket" + + +@pytest.mark.parametrize("sub_second", [0.0, 0.5, 30.0, 59.5]) +@pytest.mark.parametrize("hour_offset", [0, 600, 1800, 3_540]) +def test_windowed_rollups_answer_exactly_what_one_full_window_rollup_answers( + tmp_path, hour_offset: int, sub_second: float) -> None: + """The equality the task asks for: the cost moves, not one averaged value. + + Two stores, the same rows. One is maintained twice with an hour between the passes, so its + second pass reads a window; the other gets every row up front and is maintained once over the + whole retention at the later clock — the shape task 01 measured. Every row of every resolution + must match, `ts` and `value` exactly. `INSERT OR REPLACE` over a complete bucket is what makes + that safe, and the bucket-aligned floor (`_rollup_floor`) is what makes it true: a floor that + cut into a bucket would instead leave the *tail* rows of a split minute as its "average". + """ + # `first_at` is parametrised across an hour on purpose: the windows land on bucket boundaries + # or in the middle of them depending on where the clock fell, and the boundary cases are the + # ones a partial bucket can hide in. + first_at = bucket_start(time.time() - 6 * HOUR, HOUR) + hour_offset + sub_second + second_at = first_at + HOUR + 90.0 # an hour and a half: straddles an hour bucket boundary + + def seed(store: SampleStore, *, late: bool) -> None: + for series, base in (("cpu.usage_pct", 0.0), ("mem.used_pct", 100.0)): + # three hours of raw at the fleet's 10 s cadence, all of it complete by `first_at` + add_series(store, "host", "h", series, + [base + step for step in range(1_080)], + start=first_at - 4 * HOUR, step=10) + if late: + # the buckets that fill in the hour between the two passes, unaligned on purpose + for series, base in (("cpu.usage_pct", 0.0), ("mem.used_pct", 100.0)): + add_series(store, "host", "h", series, + [base + step for step in range(180)], + start=first_at + 95.0, step=20) + + def dump(store: SampleStore) -> list[tuple[str, str, str, str, float, float]]: + with store._lock, store._db: + return store._db.execute( + "SELECT source, host, metric, res, ts, value FROM sample" + " ORDER BY source, host, metric, res, ts").fetchall() + + windowed = SampleStore(tmp_path / "windowed.sqlite3") + reference = SampleStore(tmp_path / "reference.sqlite3") + try: + seed(windowed, late=False) + windowed.maintain(now=first_at) + seed(windowed, late=True) + windowed.maintain(now=second_at) + windowed.maintain(now=second_at + 60) # a third tick, the steady state the app lives in + + seed(reference, late=True) + assert reference.get_meta(LAST_MAINTAIN_KEY) is None, "a fresh store is a full window" + reference.maintain(now=second_at + 60) + + windowed_rows, reference_rows = dump(windowed), dump(reference) + assert {row[3] for row in windowed_rows} == {RES_RAW, RES_MIN, RES_HOUR}, ( + f"the comparison needs rolled rows on both steps: {sorted({r[3] for r in windowed_rows})}") + assert len([row for row in windowed_rows if row[3] == RES_MIN]) > 200, ( + "the equality has to cover a lot of buckets, not a token few") + assert windowed_rows == reference_rows, ( + "the window changed a rollup value (or a bucket a full scan would have written)") + finally: + windowed.close() + reference.close() + + +def test_a_re_rolled_bucket_only_ever_gains_rows(db) -> None: + """The progressive correction the whole-table scan used to provide still happens. + + A pass whose ceiling lands mid-bucket writes a partial average — as it always did — and the + next pass must re-read that bucket **from its own start** and publish the full one. A window + floor that cut into the bucket instead would re-read its tail and overwrite 2.5 with 4.5, which + is an invented number wearing a real chart's clothes (AGENTS.md rule 2). + """ + minute = bucket_start(time.time() - 10 * MINUTE) + add_series(db, "host", "h", "m", [0, 1, 2, 3, 4, 5], start=minute, step=10) + first = minute + 155.0 # ceiling = minute + 35: four of the six rows are in the window + db.maintain(now=first) + partial = db.query("host", "h", "m", res=RES_MIN, start=minute - 1, end=minute + MINUTE) + assert partial == [(minute, 1.5)], "a boundary bucket is rolled from its start, partially" + + db.maintain(now=first + 60) + rolled = db.query("host", "h", "m", res=RES_MIN, start=minute - 1, end=minute + MINUTE) + assert rolled == [(minute, 2.5)], ( + f"the re-roll must add the remaining rows, not replace them: {rolled}") + db.maintain(now=first + 120) + assert db.query("host", "h", "m", res=RES_MIN, start=minute - 1, + end=minute + MINUTE) == rolled, "a settled bucket keeps its value" + + +def test_a_corrupt_last_maintain_row_falls_back_to_the_full_window(db, caplog) -> None: + """A bound the store cannot read is the expensive answer, said once, never a crash. + + `maintain()` is the only writer of that row, so a non-number in it means something outside the + app edited the database. Rolling the whole window again rewrites identical rows; guessing a + window from a value that is not a timestamp is how a rollup starts skipping buckets in + silence, and the maintenance loop swallowing an exception is worse still (invariant 4). + """ + now = time.time() + db.set_meta(LAST_MAINTAIN_KEY, "not-a-timestamp") + with caplog.at_level(logging.WARNING, logger="dashboards.store"), query_log(db) as seen: + db.maintain(now=now) + assert rollup_windows(seen)[RES_RAW] == pytest.approx( + (now - MINUTE - MAINTAIN_SLACK_SECONDS, now - db.raw_keep_seconds)) + assert [rec for rec in caplog.records if "last_maintain" in rec.message], ( + "drift in the row that bounds every rollup is worth one line") + assert "not-a-timestamp" not in caplog.text, "the log quotes the key, not the stranger's value" + assert db.get_meta(LAST_MAINTAIN_KEY) == str(now), "the pass repairs the row it reads" + + +def test_a_failing_optimize_is_a_log_line_and_not_a_failed_tick(db, monkeypatch, caplog) -> None: + """Statistics are bookkeeping: `maintain()` may not turn them into an outage. + + `_optimize` runs outside the rollup transaction, so the only way it can hurt is by raising — and + a `store-error` event for a planner hint is worse than a slightly slower query (AGENTS.md + invariant 4: a source that stops answering is the failure, and the maintenance loop's health is + reported the same way). The stub quotes a message of its own; the log line names the pragma and + the consequence, nothing else. + """ + class _Boom: + def execute(self, _sql: str) -> None: + raise sqlite3.Error("database is locked") + + monkeypatch.setattr(db, "_db", _Boom()) + with caplog.at_level(logging.WARNING, logger="dashboards.store"): + db._optimize() # must not raise + assert "PRAGMA optimize failed" in caplog.text + assert "database is locked" not in caplog.text, "the tick's own words, not the driver's" + + +def test_the_perf_probe_explains_the_statement_that_runs(db) -> None: + """The probe's rollup `EXPLAIN` is derived from `ROLLUP_INSERT`, so its bindings must match. + + A hand-copied lookalike is how a NOTES table ends up describing a statement nobody runs — and + when the rollup grew its seventh binding (the target resolution), the copy in the probe was the + thing that broke. Preparing the derived statement with the probe's own arity is the pin: retyping + it, or letting the two drift, raises here instead of in the middle of a measurement run. + """ + whole_select = "SELECT" + ROLLUP_INSERT.split("SELECT", 1)[1] + assert whole_select == store_perf.ROLLUP_SELECT + assert ROLLUP_INSERT.count("?") == store_perf.ROLLUP_SELECT.count("?") == 7 + with db._lock: + db._db.execute("EXPLAIN QUERY PLAN " + store_perf.ROLLUP_SELECT, + (RES_MIN, MINUTE, MINUTE, RES_RAW, 1.0, 0.0, MINUTE)).fetchall() + + +def test_the_wal_pragmas_are_set_on_every_connection(db) -> None: + """The WAL half of task 03, pinned as the settings that ship. + + `journal_size_limit` is the lever: a WAL that grows past it is truncated to it at the next + checkpoint instead of staying at a 140 MB high-water mark (AGENTS.md known gap 4). + `wal_autocheckpoint` is SQLite's own default and is set so the knob is visible beside it, and + `busy_timeout` stays at 5 000 ms because the fix is a shorter write, not a longer wait. + All the pragmas are per connection, so this asserts them on a handle built by `SampleStore` and + not on whatever SQLite happens to have guessed. + """ + with db._lock: + assert db._db.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal" + assert db._db.execute("PRAGMA synchronous").fetchone()[0] == 1 + assert db._db.execute("PRAGMA busy_timeout").fetchone()[0] == 5000 + assert db._db.execute("PRAGMA wal_autocheckpoint").fetchone()[0] == WAL_AUTOCHECKPOINT_PAGES + assert db._db.execute("PRAGMA journal_size_limit").fetchone()[0] == JOURNAL_SIZE_LIMIT_BYTES + assert JOURNAL_SIZE_LIMIT_BYTES == 64 * 1024 * 1024 == 67108864 + # `automatic_analyze` (off, so no poller's commit pays for a statistics refresh) is set-only in + # this build — reading it back returns no row — so its effect is pinned where it is observable, + # in `test_maintain_leaves_query_statistics`. + + +def test_maintain_leaves_query_statistics(db) -> None: + """`maintain()` is the app's only statistics refresh, and the windowed rollup needs them. + + Without `sqlite_stat1` the planner cannot price a range and reads the table for every window: + on the seeded store the windowed rollup cost 328 ms of `SCAN sample` matching no rows at all, + and 1 ms with `SEARCH sample USING INDEX sqlite_autoindex_sample_1` after one analyse. So the + statistics are half of "cheaper for the same rows", and they are refreshed here rather than by + SQLite's own automatic ANALYZE — which would run inside whichever poller's commit tripped its + change counter, the exact path this task is shortening. On a database this size a scan is free, + so the assertion is that the statistics exist, not what plan they produce; the plan is pinned at + production scale in the `store_perf` report. + """ + def stat_rows() -> int: + with db._lock: + if db._db.execute("SELECT 1 FROM sqlite_master WHERE name='sqlite_stat1'").fetchone() is None: + return 0 + return int(db._db.execute("SELECT COUNT(*) FROM sqlite_stat1").fetchone()[0]) + + assert stat_rows() == 0, "a store that has never been maintained has no statistics" + db.add("host", "gateway", "cpu.usage_pct", 1.0) + db.maintain(now=time.time()) + assert stat_rows() > 0, "maintenance is the one place the planner's statistics get refreshed" + + +# ---- phase 14: the production-shaped store the performance work measures ---- # +# +# `TODO.md:7` is "pages take forever to load and seem to lock up the database", and a claim +# about speed needs a measured "before". The fixture below is that before: an on-disk store with +# the deployed retention semantics (`raw_keep_hours=48`, `minute_keep_days=30`, +# `hour_keep_days=400` — the `SampleStore` defaults, i.e. what a 90 d chart really reads is 1m +# and 1h, never raw beyond 48 h), the fleet's real metric names, the real 10 s cadence, ~2 000 +# events and the `meta` rows the app writes. ~6.4 M rows in ~700 MB, which is the dev DB's order +# of magnitude (`AGENTS.md` gap 4 records 636 MB / 3.4 M rows for a store only days old). +# +# Why the fixture is not the dev DB: the testing policy keeps `uv run pytest` hermetic, and a +# fixture that read the operator's live history would change behaviour when that history does. +# Why it is not an in-memory store: `:memory:` hides the WAL and the lock behaviour, which are +# half of what this phase measures. Why it is not `tmp_path`: pytest's tmp dir on this box is +# `tmpfs`, i.e. RAM, and a database timed on RAM is a flattering number nobody deploys — +# `store_perf.cache_dir()` defaults to a real block device and says so in its report. +# +# The default-run tests here assert **shape and determinism**; the wall-time budgets task 04 pins +# are behind `-m store_perf --store-perf` (and in `scripts/store_perf_probe.py`, which writes the +# NOTES tables) because a budget asserted on first writing is a number fitted to the machine it was +# written on. The budgets themselves live in `store_perf.BUDGETS` — one table, priced by the probe, +# the report, and the two budget tests below, because three copies of a budget drift the moment one +# is relaxed. The `perf_store`/`perf_store_path` fixtures are in `tests/conftest.py`: the HTTP +# budgets in `tests/test_api.py` measure the same file. + + + + +def test_the_seed_seeds_the_retention_the_app_actually_keeps() -> None: + """The fixture's retention must be `SampleStore`'s own defaults, not a story about them. + + A budget measured on a store seeded to a *different* retention is a budget for a database + that does not exist: `pick_res`/`history` costs depend on how deep each resolution goes. + """ + fresh = SampleStore(":memory:") + try: + assert fresh.raw_keep_seconds == pytest.approx(store_perf.RAW_KEEP_HOURS * HOUR) + assert fresh.minute_keep_seconds == pytest.approx(store_perf.MINUTE_KEEP_DAYS * 24 * HOUR) + assert fresh.hour_keep_seconds == pytest.approx(store_perf.HOUR_KEEP_DAYS * 24 * HOUR) + finally: + fresh.close() + assert store_perf.CADENCE_SECONDS == 10.0, "the raw window's cadence is HOST_INTERVAL_SECONDS" + + +def test_the_seeded_store_is_the_shape_the_budget_tests_measure(perf_store: SampleStore) -> None: + """Row counts, series, retention windows, events, meta — asserted, never timed.""" + now = store_perf.seed_now(perf_store) + expected = store_perf.plan(now) + + with perf_store._lock, perf_store._db: # one pass over the table, not three + rows = {row[0]: row for row in perf_store._db.execute( + "SELECT res, COUNT(*), MIN(ts), MAX(ts) FROM sample WHERE source='host' GROUP BY res")} + + assert expected.raw_rows == expected.series_total * expected.raw_steps + assert expected.minute_rows_seeded == expected.series_total * expected.minute_steps + assert expected.hour_rows_seeded == expected.series_total * expected.hour_steps + assert sum(len(store_perf.HOST_SERIES[name]) for name in expected.hosts) == ( + expected.series_total) + assert rows[RES_RAW][1] == expected.raw_rows, ( + "the raw window is inserted, not rolled: its count is the one number the seed owns " + "outright, so a mismatch here means the shape drifted from the plan") + assert rows[RES_MIN][1] >= expected.minute_rows_seeded + assert rows[RES_HOUR][1] >= expected.hour_rows_seeded + # …and the rollups `maintain()` added inside the windows are there too, or the fixture + # would be a store whose charts draw nothing above the raw window + assert rows[RES_MIN][1] > expected.minute_rows_seeded + assert rows[RES_HOUR][1] > expected.hour_rows_seeded + + # Retention: nothing lives past its resolution's window, and each window is deep enough to + # be the one the pages read (a 90 d chart reads 1m/1h; a 5 m chart reads raw). + assert rows[RES_RAW][2] >= now - store_perf.RAW_KEEP_HOURS * HOUR - 1.0 + assert rows[RES_RAW][3] <= now + 1.0 + assert rows[RES_MIN][2] >= now - store_perf.MINUTE_KEEP_DAYS * 24 * HOUR - 1.0 + assert rows[RES_HOUR][2] >= now - store_perf.HOUR_KEEP_DAYS * 24 * HOUR - 1.0 + + assert perf_store.hosts("host") == list(expected.hosts) + assert len(perf_store.series_names("host")) == len(store_perf.BASE_SERIES) + len( + store_perf.GPU_SERIES), "the union: gateway has no gpu.* series, the other two do" + assert len(perf_store.series_names("host", "gateway")) == len(store_perf.BASE_SERIES) + assert len(perf_store.series_names("host", "driveripper")) == ( + len(store_perf.BASE_SERIES) + len(store_perf.GPU_SERIES)) + + assert perf_store.count() == rows[RES_RAW][1] + rows[RES_MIN][1] + rows[RES_HOUR][1] + + events = perf_store.recent_events(store_perf.EVENT_ROWS) + assert len(events) == store_perf.EVENT_ROWS + assert {event.source for event in events} == {"system"} + + # The `meta` rows the app writes: `last_maintain` from the build's own `maintain()` call, + # plus this fixture's own provenance. Phase 11's auth rows are not seeded — the app writes + # them the first time a test boots a server on this store. + assert perf_store.get_meta("last_maintain") is not None + assert perf_store.get_meta(store_perf.META_SEED_VERSION) == store_perf.SEED_VERSION + + # Magnitude, not exact bytes: a fixture two orders of magnitude smaller would not catch a + # full-table scan, and the number that matters is in the NOTES (dev DB 636 MB, this ~700 MB). + size = Path(perf_store.path).stat().st_size + assert size > 100 * 1024 * 1024, f"seeded store is {size / 1e6:.0f} MB — too small to measure" + + +def test_the_rollups_in_the_seed_came_from_the_raw_rows_and_run_twice_the_same( + perf_store: SampleStore, +) -> None: + """Determinism, in both directions the phase depends on. + + 1. the seed is a pure function of `(shape, seed clock)`, so a rebuild is byte-identical and + a "before/after" comparison is comparing the same data — asserted by recomputing stored + raw values from `store_perf._value`; + 2. the rollup rows are the real `maintain()` arithmetic over those raw values (the mean of + the raw samples in the bucket), and running maintenance again changes nothing. + """ + now = store_perf.seed_now(perf_store) + host, metric = "driveripper", "cpu.usage_pct" + + first_rows = perf_store.query("host", host, metric, res=RES_RAW, + start=now - store_perf.RAW_KEEP_HOURS * HOUR, + limit=60, tail=False) + assert first_rows, "the raw window is empty" + offset = round((first_rows[0][0] - (now - store_perf.RAW_KEEP_HOURS * HOUR)) + / store_perf.CADENCE_SECONDS) + assert [value for _ts, value in first_rows] == [ + store_perf.seed_value(metric, offset + index) for index in range(len(first_rows))], ( + "stored raw values are not the seed function's output — the seed is no longer a pure " + "function of the shape, so before/after numbers would not be comparable") + + # one complete bucket inside the raw window, recomputed by hand + bucket_ts = first_rows[0][0] + 30 * store_perf.CADENCE_SECONDS + bucket_ts = int(bucket_ts // MINUTE) * MINUTE + raw_in_bucket = perf_store.query("host", host, metric, res=RES_RAW, start=bucket_ts, + end=bucket_ts + MINUTE - 0.001) + assert len(raw_in_bucket) == 6, raw_in_bucket + rolled = perf_store.query("host", host, metric, res=RES_MIN, start=bucket_ts, + end=bucket_ts + MINUTE - 0.001) + assert rolled == [(float(bucket_ts), pytest.approx( + sum(value for _ts, value in raw_in_bucket) / len(raw_in_bucket)))] + + def shape() -> dict[str, tuple[int, float, float]]: + """Rows, and the time window each resolution holds — the exact half of the checksum. + + The total of `value` is deliberately *not* in here. `maintain()` writes rollups with + `INSERT OR REPLACE`, which deletes and re-inserts the row, so a pass that changes no value + still moves every rowid it touched, and a `SUM(value)` scanned in rowid order accumulates + those doubles in a different sequence — a correctly idempotent pass then prints a + different total (2.8e5 out of 2.5e15 was how this test failed once the seeded file had + been maintained a few times). Counts and windows are exact; the sum is compared to a + tolerance below, and the *values* are compared exactly, in primary-key order, by + `values_slice()`. + """ + with perf_store._lock, perf_store._db: + return {row[0]: (row[1], row[2], row[3]) for row in perf_store._db.execute( + "SELECT res, COUNT(*), COALESCE(MIN(ts),0), COALESCE(MAX(ts),0)" + " FROM sample GROUP BY res")} + + def totals() -> dict[str, float]: + with perf_store._lock, perf_store._db: + return {row[0]: row[1] for row in perf_store._db.execute( + "SELECT res, COALESCE(SUM(value),0) FROM sample GROUP BY res")} + + def values_slice(res: str, limit: int = 4000) -> list[tuple[str, str, str, float, float]]: + """A deterministic slice of the table, in primary-key order, compared exactly. + + `ORDER BY source, host, metric, ts` walks the primary-key index rather than the rowids, + so this is the same list of rows before and after regardless of where maintenance moved + them — which is the assertion the aggregate cannot make: not one stored value rewritten. + """ + with perf_store._lock, perf_store._db: + return perf_store._db.execute( + "SELECT source, host, metric, ts, value FROM sample WHERE res=?" + " ORDER BY source, host, metric, ts LIMIT ?", (res, limit)).fetchall() + + before, before_totals = shape(), totals() + slices = {res: values_slice(res) for res in (RES_RAW, RES_MIN, RES_HOUR)} + again = perf_store.maintain(now=now) + after, after_totals = shape(), totals() + + assert after == before, ( + "maintenance is on a timer: a second pass over the same data may not add or drop a row, " + "or move a resolution's time window") + for res, total in after_totals.items(): + assert total == pytest.approx(before_totals[res], rel=1e-9), ( + f"{res}: the stored total moved more than float accumulation can explain — a rollup " + f"re-averaged a bucket it had already rolled") + for res, rows in slices.items(): + assert values_slice(res) == rows, ( + f"{res}: a row's value changed on the second pass — the rollup is not idempotent") + assert again["pruned"] == 0, "nothing in this store is past its retention" + assert perf_store.count() == sum(row[0] for row in before.values()) + + +def test_the_race_measurements_leave_the_seeded_shape_alone(perf_store: SampleStore) -> None: + """A measurement that dirtied the baseline would move the next one — so this is a test. + + The race inserts 30-sample polls (`store_perf.RACE_SOURCE`, never a seeded series) and the + task-04 comparison depends on the seeded counts being what they were. + """ + before = store_perf.counts(perf_store) + result = store_perf.race_single_connection(perf_store, iterations=5, max_seconds=15.0) + after = store_perf.counts(perf_store) + + assert result.attempted >= 1 and result.inserts == result.attempted + assert after == before, f"the race left rows behind: {before} → {after}" + with perf_store._lock, perf_store._db: + leftovers = int(perf_store._db.execute( + "SELECT COUNT(*) FROM sample WHERE source=?", + (store_perf.RACE_SOURCE,)).fetchone()[0]) + assert leftovers == 0 + + +def test_the_batched_answers_match_the_per_series_answers_on_the_real_shaped_store( + perf_store: SampleStore) -> None: + """The equality that lets the endpoints switch doors: same series, same order, same cost class. + + On the store this phase measures its budgets on, at the two windows where the resolution + decision actually differs (24 h reads 1m, 90 d reads 1h), for the 27-series request + `/api/system/history` answers for a whole page. A capped count that decided a resolution + differently from a full `COUNT(*)` would move a chart onto the wrong resolution — the same + points averaged together, which is the kind of wrong that still renders. + """ + hosts = list(store_perf.HOST_SERIES) + metrics = list(store_perf.HISTORY_METRICS) + for span in (24 * 3_600.0, 90 * 24 * 3_600.0): + res_map = perf_store.pick_res_many("host", hosts, metrics, span, 600) + assert res_map == {(host, metric): perf_store.pick_res("host", host, metric, span, 600) + for host in hosts for metric in metrics} + batched = perf_store.history_many("host", hosts, metrics, span=span, max_points=600) + assert batched == {(host, metric): perf_store.history("host", host, metric, + span=span, max_points=600) + for host in hosts for metric in metrics}, ( + f"the batched path changed a series at span={span:.0f}") + # and the series themselves are still the shape the charts parse: ascending, budgeted, + # ending at the newest row the window holds. `HISTORY_METRICS` is the *request* the page + # makes and the seed holds a subset of it (`cpu.temp_c` is not one of the seeded names), + # so the absent ones are asserted empty rather than skipped — an empty series is an answer. + held = {(host, metric) for host, names in store_perf.HOST_SERIES.items() for metric in names} + for (host, metric), points in batched.items(): + if (host, metric) not in held: + assert points == [], f"{host}/{metric} is not in the seed: it must stay empty" + continue + assert points, f"{host}/{metric} at {span} came back empty on a store full of it" + assert 0 < len(points) <= 601, "the budget plus the point the thinning appends back" + assert [ts for ts, _value in points] == sorted(ts for ts, _value in points) + + +def test_two_maintain_cycles_leave_the_seeded_wal_under_the_cap(perf_store: SampleStore) -> None: + """Criterion 3, on the store this phase measures its numbers on. + + `journal_size_limit` is the ceiling; the windowed rollup is the reason the file has hardly + anything to grow with. The assertion is on the cap rather than a byte count because the cap is + the promise — a WAL that stays under 64 MB across ticks is what AGENTS.md known gap 4 needed, + and a run that left a bigger file behind would have to be explained by something other than + `maintain()`. + """ + path = Path(perf_store.path) + now = store_perf.seed_now(perf_store) + with perf_store._lock, perf_store._db: # start from a checkpointed file, so this measures the cycles + perf_store._db.execute("PRAGMA wal_checkpoint(TRUNCATE)") + wal_before = store_perf.wal_bytes(path) + for _ in range(2): + perf_store.maintain(now=now) + wal_after = store_perf.wal_bytes(path) + # 8 MB of headroom is generous for a tick that writes one bucket per series: it is there so the + # test says something about the cycles rather than about the filesystem's rounding. + assert wal_after <= JOURNAL_SIZE_LIMIT_BYTES, ( + f"WAL {wal_after / 1e6:,.1f} MB after two cycles, cap " + f"{JOURNAL_SIZE_LIMIT_BYTES / 1e6:,.0f} MB") + assert wal_after <= wal_before + 8 * 1024 * 1024, ( + f"two steady ticks grew the WAL {wal_before:,} → {wal_after:,} bytes") + + +# --------------------------------------------------------------------------- # +# task 04: the budgets +# +# Two kinds of test, because the phase's own rule is that a cost claim is pinned as a shape *and* +# priced, and the two need different stores. The shape half runs on the little `db` fixture in +# every `uv run pytest`: the budget table has to be internally sound and every budget has to name +# a measurement that still exists, so a budget cannot rot into a comment by pointing at a report +# key somebody renamed. The price half runs on the seeded store behind `--store-perf`, where the +# numbers mean something; a budget priced on a 40-row store would be a budget for a database +# nobody has, which is the failure AGENTS.md warns about. +# --------------------------------------------------------------------------- # + + +def test_the_budget_table_is_internally_coherent() -> None: + """Every pinned budget is a checkable claim about a real report path. + + Cheap, and it is the test that keeps the rest honest: a budget with a limit of 0 is always + crossed (or never, if the statistic is a count), a ratio pointing at a key in another section + silently measures nothing, and two rows on the same measurement with different ceilings is the + moment a table stops meaning one thing. + """ + assert store_perf.BUDGETS, "there are no budgets" + seen: set[tuple[str, str, str]] = set() + for budget in store_perf.BUDGETS: + assert budget.section in store_perf.BUDGET_SECTIONS, f"unknown section {budget.section}" + assert budget.key, "a budget must name the report key it prices" + assert budget.limit >= 0.0, f"{budget.label()} has a negative ceiling" + assert budget.unit in {"ms", "count"}, f"{budget.label()} has an unknown unit {budget.unit}" + assert budget.basis, f"{budget.label()} records no reasoning for its ceiling" + identity = (budget.section, budget.key, budget.statistic) + assert identity not in seen, f"{identity} is pinned twice, with different ceilings possible" + seen.add(identity) + # every budget's key has to be a key the pass that prices it actually produces + measured = (store_perf.READ_MEASURED_KEYS if budget.section == "paths" + else store_perf.HTTP_MEASURED_KEYS if budget.section == "http" + else store_perf.MAINTAIN_MEASURED_KEYS if budget.section == "maintain" else None) + if measured is not None: + assert budget.key in measured, (f"{budget.label()} names {budget.key}, which " + f"{budget.section} does not measure") + if budget.ratio_key is not None: + # `_budget_value` swaps only the key, so the partner has to live in the same section + # and be priced the same way, or the ratio compares two different things. The partner + # is usually *not* itself a budget — nobody promises the cold scan a ceiling; it is the + # denominator that says the cached answer is not a scan in disguise. + assert measured is not None and budget.ratio_key in measured, ( + f"{budget.label()} is relative to {budget.ratio_key}, which {budget.section} " + "does not measure") + assert budget.ratio_limit is not None and 0.0 < budget.ratio_limit < 1.0, ( + f"{budget.label()} must claim a *fraction* of its partner") + # the cached reads are the ones the phase claims are no longer scans: that claim is the ratio, + # so both cached paths carry one rather than a bare 10 ms that a fast laptop always beats + ratios = {budget.key for budget in store_perf.BUDGETS if budget.ratio_key is not None} + assert {"series_names_host_cached", "stats_cached"} <= ratios + + +def test_every_read_budget_names_a_measurement_that_still_exists(db) -> None: + """The stale-pointer check, on the little store, in the default run. + + `measure_budget_reads` is what the priced test calls; if a key there stops matching a key in + `BUDGETS` the priced test would quietly price nothing and pass. Crossings are not asserted + here — a 40-row store makes every ceiling trivially true and every ratio a coin flip. + """ + report = store_perf.measure_budget_reads(db, runs=2) + assert set(report["paths"]) == set(store_perf.READ_MEASURED_KEYS), ( + "the read pass and the budget table no longer describe the same measurements") + assert store_perf.missing_budget_keys(report, ("paths",)) == [] + rows = store_perf.budget_rows(report, ("paths",)) + assert len(rows) == sum(1 for budget in store_perf.BUDGETS + if budget.section == "paths") + assert all(row.measured is not None and row.measured >= 0.0 for row in rows) + # the ratio partner is looked up by swapping the key, so it has to resolve on the same report + for row in rows: + if row.budget.ratio_key is not None: + partner = store_perf._budget_value( + report, replace(row.budget, key=row.budget.ratio_key)) + assert partner is not None, f"{row.budget.label()} has an unpriceable partner" + + +@pytest.mark.store_perf +def test_the_seeded_store_reads_fit_their_pinned_budgets(perf_store: SampleStore) -> None: + """The read budgets, priced on the store this phase's numbers were taken from. + + Ceiling 2-3x the measured "after" (see each row's `basis`), so a slower deployment disk does + not turn red for the wrong reason — but the *scan* paths are what these catch: put the + whole-table `COUNT(*)` back in `stats()` and the cached row crosses on the ratio alone, which + is the half a generous absolute ceiling cannot see. + """ + report = store_perf.measure_budget_reads(perf_store) + print() + print(store_perf.render_budgets_markdown(store_perf.budget_rows(report, ("paths",)))) + assert store_perf.budget_violations(report, ("paths",)) == [] + + +@pytest.mark.store_perf +def test_the_write_side_fits_its_pinned_budgets(perf_store: SampleStore) -> None: + """`maintain()`'s write transaction and the poller racing it — the gap-4 budgets. + + Priced on a scratch copy (`measure_maintain`'s rule: a rollup really writes, and the cached + baseline has to come back unchanged), at the seed's own clock. The race needs two connections, + so it runs on its own copy too; the whole-window control is *not* budgeted here — it is what + `test_the_baseline_report_measures_every_hot_path` uses to prove the race can still see a + failure, and it is supposed to be slow. + """ + path = Path(perf_store.path) + now = store_perf.seed_now(perf_store) + maintain = store_perf.measure_maintain(path, now=now) + report = {"maintain": maintain} + print() + print(store_perf.render_budgets_markdown(store_perf.budget_rows(report, ("maintain",)))) + assert store_perf.budget_violations(report, ("maintain",)) == [] + + with store_perf.scratch_copy(path, name="budget-race") as arena: + race = store_perf.race_two_connections(arena, iterations=100, now=now, + max_seconds=60.0, + maintain_gap_seconds=store_perf + .RACE_MAINTAIN_GAP_SECONDS) + report = {"race": {"poller_during_windowed_maintain": race.as_dict()}} + print() + print(store_perf.render_budgets_markdown(store_perf.budget_rows(report, ("race",)))) + assert store_perf.budget_violations(report, ("race",)) == [] + + +@pytest.mark.store_perf +def test_the_baseline_report_measures_every_hot_path(perf_store: SampleStore) -> None: + """The measurement itself, inside pytest: every path task 01 names, with its EXPLAIN shape. + + Opt-in (`-m store_perf --store-perf`) because it takes minutes, and skipped by default so + the shape tests above still run in every `uv run pytest`. The numbers that go in + `data/logs/NOTES-store-perf.md` come from `scripts/store_perf_probe.py` with 100 race + iterations; this runs the same code with a short race so the *report's shape* is guarded. + """ + report = store_perf.measure(perf_store, race_iterations=10, race_max_seconds=10.0, + with_http=True, log=lambda line: print(line, flush=True)) + + assert {"machine", "store", "paths", "plans", "race", "http", "maintain"} <= report.keys() + for key in ("series_names_all", "series_names_host", "stats", "pick_res_90d", + "history_5m", "history_24h", "history_90d", + # task 02: both halves of a cached answer, so the before/after compares cold to + # cold instead of calling a cache hit a 1 000x speed-up of a full scan + "series_names_all_cached", "series_names_host_cached", "stats_cached", + # task 02: the batched door against the loop it replaced, at both widths + "pick_res_x27_24h", "pick_res_many_x27_24h", "history_x27_24h", + "history_many_x27_24h", "pick_res_x27_90d", "pick_res_many_x27_90d", + "history_x27_90d", "history_many_x27_90d"): + assert report["paths"][key]["runs"] >= 1 and report["paths"][key]["max"] > 0.0 + for key in store_perf.HTTP_MEASURED_KEYS: + assert key in report["http"], f"{key} is a hot path the operator's browser pays" + assert set(report["http"]) == set(store_perf.HTTP_MEASURED_KEYS) + # task 04: the report is the budget table's other customer. A budget crossed here is the phase + # saying "the regression is back", and a budget this report cannot price is a budget nobody + # checks any more — which is why `budget_rows` reports the missing ones as failures. + assert store_perf.missing_budget_keys(report) == [] + assert report["budgets_ok"], "a pinned budget is crossed:\n" + "\n".join( + store_perf.budget_violations(report)) + for key, plan_rows in report["plans"].items(): + assert plan_rows, f"{key}: no EXPLAIN QUERY PLAN row — the statement was not explained" + + # one connection: the store lock serialises, so an insert never sees SQLITE_BUSY here + assert report["race"]["single_connection"]["busy_errors"] == 0 + # two connections: the gap-4 failure mode is a *raised* error, and a short-race pass here still + # has to have attempted inserts for that to mean anything. The saturated rows (task 01's shape) + # are reported for comparability only; the pass line is the cadence-shaped one, and the + # whole-window control beside it is what proves the race can still see the failure. + for key in ("two_connections_prod_timeout", "two_connections_no_timeout", + "poller_during_windowed_maintain", "poller_during_full_window_maintain"): + entry = report["race"][key] + assert entry["attempted"] >= 1 and entry["attempted"] <= 10 + assert entry["inserts"] + entry["busy_errors"] == entry["attempted"] + assert entry["notes"], "every race says what it actually ran" + windowed = report["race"]["poller_during_windowed_maintain"] + assert windowed["busy_errors"] == 0, ( + f"a poller insert lost to a windowed rollup: {windowed}") + assert report["race"]["poller_during_full_window_maintain"]["waiters"] >= 1, ( + "the control pass must contend, or the windowed zero proves nothing") + # the write transaction, per window, on a scratch copy — windowing has to show up as a shorter + # write for the same rows, which is the whole point of task 03 + passes = report["maintain"]["passes"] + assert set(passes) == {label for label, _offset, _reset in store_perf.MAINTAIN_PASSES} + assert all(entry["seconds"] > 0.0 for entry in passes.values()) + assert passes["full_window"]["seconds"] > passes["steady"]["seconds"], ( + "the control measured no slower than the windowed pass — the copy or the window is wrong") + assert passes["full_window"]["rows_written"] > passes["steady"]["rows_written"] + # criterion 1's cost bound. Ten minutes of missed buckets is ten times the rollup work of one + # minute and costs under twice the wall clock at this size because the fixed costs dominate; the + # slack is the prune, which is deliberately **not** windowed and therefore deletes as many rows + # as aged out during the jump whatever the rollup window was. + assert passes["catch_up_10m"]["seconds"] <= 2 * passes["steady"]["seconds"] + 0.10, ( + f"catch-up {passes['catch_up_10m']['seconds']:.3f} s vs steady " + f"{passes['steady']['seconds']:.3f} s") + # criterion 2's shape, at production scale: a window the planner can seek. Without statistics it + # prices a range as a fraction of the table and reads the table for it (`SCAN sample`) — which is + # how a "windowed" rollup came to cost 328 ms while matching nothing. + for key, plan_rows in report["maintain"]["plans"].items(): + assert plan_rows, f"{key}: nothing was explained" + assert not any(row.startswith("SCAN") for row in plan_rows), ( + f"{key} still reads the whole table: {plan_rows}") + assert report["maintain"]["wal_within_cap"], ( + f"the WAL escaped journal_size_limit across two ticks: {report['maintain']['two_steady_cycles']}") + assert report["store_after"]["integrity"] == "unchanged", ( + f"the measurements changed the baseline: {report['store_after']['integrity']}") + + print() + print(store_perf.render_markdown(report))