"""History store: rollups, retention, and the honesty of an empty chart. The rollup queries are plain SQL with positional bindings, and a mismatch there used to raise on every maintenance tick while the poller swallowed it — so every window past the raw retention was silently empty. These tests fail loudly if that ever happens again. """ from __future__ import annotations import time from typing import Any import pytest from dashboards.store import ( HOUR, MINUTE, RES_HOUR, RES_MIN, RES_RAW, Event, Sample, SampleStore, samples_from_flat, select_window, ) def add_series(db, source: str, host: str, metric: str, values, *, start: float, step: float, res: str = RES_RAW) -> None: for i, value in enumerate(values): # the private handle is the point: writing at an explicit resolution is what # the resolution-choice tests need db._db.execute( "INSERT INTO sample (source, host, metric, res, ts, value) VALUES (?,?,?,?,?,?)", (source, host, metric, res, start + i * step, float(value)), ) db._db.commit() def test_store_starts_in_wal_mode(db) -> None: """Readers poll while the writer commits; without WAL the API would hit locked errors.""" assert db._db.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal" def test_add_and_query_roundtrip(db) -> None: now = time.time() db.add("host", "driveripper", "cpu.usage_pct", 42.5, ts=now) points = db.query("host", "driveripper", "cpu.usage_pct", start=now - 60) assert points == [(now, 42.5)] def test_query_is_bounded_and_ascending(db) -> None: base = 1_700_000_000.0 add_series(db, "host", "h", "m", [3, 1, 2], start=base, step=10) assert db.query("host", "h", "m", start=base - 1) == [(base, 3.0), (base + 10, 1.0), (base + 20, 2.0)] assert len(db.query("host", "h", "m", start=base - 1, limit=2)) == 2 def test_query_tail_keeps_the_newest_rows(db) -> None: base = 1_700_000_000.0 add_series(db, "host", "h", "m", range(100), start=base, step=10) head = db.query("host", "h", "m", start=base - 1, limit=3) tail = db.query("host", "h", "m", start=base - 1, limit=3, tail=True) assert [v for _, v in head] == [0.0, 1.0, 2.0] assert [v for _, v in tail] == [97.0, 98.0, 99.0] assert tail == sorted(tail), "tail must still come back oldest-first" def test_samples_from_flat_drops_unknown_and_none() -> None: payload: dict[str, Any] = {"a": 1.0, "b": None, "c": "nope"} samples = samples_from_flat("host", "gw", payload, ts=5.0) assert [(s.metric, s.value) for s in samples] == [("a", 1.0)] def test_maintain_rolls_raw_into_minutes(db) -> None: """Regression: the rollup statement had 7 placeholders and 6 bindings.""" now = time.time() # two full, aligned minute buckets — see the hour test for why alignment matters minute = int(now // MINUTE) * MINUTE add_series(db, "host", "h", "cpu.usage_pct", [10, 20, 30, 40], start=minute - 4 * MINUTE, step=1) add_series(db, "host", "h", "cpu.usage_pct", [1, 1, 1, 1], start=minute - 3 * MINUTE, step=1) counts = db.maintain(now=now) assert counts["to_minute"] > 0 minutes = db.query("host", "h", "cpu.usage_pct", res=RES_MIN, start=now - 3600) assert minutes, "no 1m rollup was written" assert [round(value) for _ts, value in minutes] == [25, 1] def test_maintain_is_idempotent(db) -> None: """Rollups run on a timer: running twice must not double-count or duplicate rows.""" now = time.time() minute = int(now // MINUTE) * MINUTE add_series(db, "host", "h", "m", [10, 20], start=minute - 4 * MINUTE, step=1) first = db.maintain(now=now) rows = db.query("host", "h", "m", res=RES_MIN, start=now - 3600) count = db.count() second = db.maintain(now=now) assert rows == db.query("host", "h", "m", res=RES_MIN, start=now - 3600) assert first["to_minute"] == second["to_minute"] assert db.count() == count, "re-running maintenance duplicated samples" def test_maintain_rolls_minutes_into_hours(db) -> None: now = time.time() # buckets are aligned to the epoch (`ts/3600*3600`), not to "hours from now", so the # fixtures start on a boundary or the two hours mix into a third, averaged bucket hour = int(now // HOUR) * HOUR # two *complete* hours of minute buckets: the current and previous hour are still # filling in, and maintenance must not touch them (see the partial-bucket test) add_series(db, "host", "h", "m", [1] * 60, start=hour - 5 * HOUR, step=60, res=RES_MIN) add_series(db, "host", "h", "m", [3] * 60, start=hour - 4 * HOUR, step=60, res=RES_MIN) counts = db.maintain(now=now) assert counts["to_hour"] > 0 hours = db.query("host", "h", "m", res=RES_HOUR, start=now - 86_400) assert sorted(round(v) for _, v in hours) == [1.0, 3.0] def test_maintain_prunes_expired_raw(db) -> None: now = time.time() old = now - (db.raw_keep_seconds + 7200) add_series(db, "host", "h", "m", [1, 2], start=old, step=60) add_series(db, "host", "h", "fresh", [1], start=now - 60, step=1) counts = db.maintain(now=now) assert counts["pruned"] >= 2 assert db.query("host", "h", "m", start=0) == [] def test_maintain_records_timestamp_in_stats(db) -> None: db.maintain() stats = db.stats() assert stats["last_maintain"] is not None assert "samples_by_res" in stats and "path" in stats def test_pick_res_prefers_the_finest_resolution_that_fits(db) -> None: now = time.time() add_series(db, "host", "h", "m", list(range(500)), start=now - 3600, step=1) assert db.pick_res("host", "h", "m", 3600, 600) == RES_RAW add_series(db, "host", "h", "m", list(range(4000)), start=now - 7200, step=1) assert db.pick_res("host", "h", "m", 7200, 600) == RES_RAW, "raw still fits" add_series(db, "host", "h", "m", list(range(120)), start=now - 7200, step=60, res=RES_MIN) # 4500 raw points blow the 600 budget, so the minute rollup takes over assert db.pick_res("host", "h", "m", 7200, 600) == RES_MIN def test_pick_res_never_picks_an_empty_resolution(db) -> None: """A fresh install has raw only: choosing 1h because it 'fits' would chart nothing.""" now = time.time() add_series(db, "host", "h", "m", [1] * 2000, start=now - 7200, step=1) assert db.pick_res("host", "h", "m", 86_400, 600) == RES_RAW def test_history_falls_back_to_raw_when_rollups_lag(db) -> None: now = time.time() add_series(db, "host", "h", "m", list(range(1500)), start=now - 1500, step=1) points = db.history("host", "h", "m", span=3600, max_points=600) assert len(points) > 0, "empty chart while raw data exists" assert len(points) <= 600 def test_history_always_keeps_the_newest_point(db) -> None: now = time.time() values = list(range(2000)) add_series(db, "host", "h", "m", values, start=now - 2000, step=1) points = db.history("host", "h", "m", span=3600, max_points=50) assert points[-1][1] == pytest.approx(values[-1]) def test_series_names_and_hosts(db) -> None: db.add("host", "a", "cpu.usage_pct", 1.0) db.add("host", "a", "mem.used_pct", 2.0) db.add("host", "b", "cpu.usage_pct", 3.0) db.add("llm", "gufo", "llm.decode_tokens_per_second", 4.0) assert db.series_names("host", "a") == ["cpu.usage_pct", "mem.used_pct"] assert db.hosts("host") == ["a", "b"] assert db.series_names("llm") == ["llm.decode_tokens_per_second"] def test_delete_series_cleans_up(db) -> None: db.add("host", "a", "gone.metric", 1.0) db.add("host", "a", "keep.metric", 2.0) assert db.delete_series("host", "a", "gone.metric") == 1 assert db.series_names("host", "a") == ["keep.metric"] def test_events_roundtrip_and_filter(db) -> None: now = time.time() db.log_event(Event(ts=now, source="host", host="a", kind="down", severity="error", message="a is not answering", data={"fails": 3})) db.log_event(Event(ts=now + 1, source="nut", host="cyberpower", kind="state", severity="warn", message="OL → OB")) everything = db.recent_events(10) assert [e.host for e in everything] == ["cyberpower", "a"] assert everything[-1].data == {"fails": 3} assert [e.source for e in db.recent_events(10, source="nut")] == ["nut"] assert len(db.recent_events(1)) == 1 # the API hands out plain JSON, so the boundary has to be serialisable assert everything[0].as_json()["message"] == "OL → OB" def test_event_as_json_is_serialisable(db) -> None: event = Event(ts=1.0, source="host", host="a", kind="up", severity="info", message="ok") payload = event.as_json() assert payload["severity"] == "info" and payload["data"] is None def test_count_filters_by_resolution(db) -> None: now = time.time() db.add("host", "h", "m", 1.0, ts=now) add_series(db, "host", "h", "m", [2.0], start=now, step=1, res=RES_MIN) assert db.count() == 2 assert db.count(res=RES_RAW) == 1 assert db.count(res=RES_HOUR) == 0 def test_add_many_reports_inserted_rows(db) -> None: now = time.time() rows = [Sample("host", "h", f"m{i}", float(i), now) for i in range(5)] assert db.add_many(rows) == 5 assert db.count() == 5 def test_store_reopens_without_losing_history(tmp_path) -> None: path = tmp_path / "reopen.sqlite3" first = SampleStore(path) now = time.time() first.add("host", "h", "m", 7.0, ts=now) first.close() second = SampleStore(path) try: assert second.query("host", "h", "m", start=now - 5) == [(now, 7.0)] finally: second.close() def test_select_window_maps_range_onto_presets() -> None: assert select_window(60, [300, 3600, 86_400]) == 300 assert select_window(3600, [300, 3600, 86_400]) == 3600 assert select_window(86_401, [300, 3600, 86_400]) == 86_400 def test_rollup_does_not_average_a_partial_minute(db) -> None: """A bucket is only rolled once its whole window is in the past, or it gets rewritten.""" now = time.time() add_series(db, "host", "h", "m", [1, 1, 1], start=now - 5, step=1) # current minute db.maintain(now=now) assert db.query("host", "h", "m", res=RES_MIN, start=now - 600) == [] def test_hour_constant_is_used_in_retention(db) -> None: assert db.raw_keep_seconds == pytest.approx(48 * HOUR) # ---- a 300-second source (phase 09: `disks` samples SMART every 5 minutes) -- # # # The rollup SQL groups by minute and hour buckets, so a source slower than a minute # puts one raw row in every fifth bucket and nothing in the other four. Two things had to # be true and are asserted rather than assumed: an empty bucket produces *no* rollup row # (inventing a value for a minute nothing was sampled is rule 2), and the hour bucket # still averages the samples it did get. SMART_STEP = 300.0 def test_a_300s_source_rolls_into_minutes_without_inventing_the_empty_ones(db) -> None: now = time.time() # five SMART polls at exactly the phase's cadence (-30, -25, -20, -15, -10 minutes), # so every bucket is fully in the past (+60 s slack) and maintenance may roll all five start = int(now // MINUTE) * MINUTE - 30 * MINUTE add_series(db, "disks", "driveripper", "disk.temp_c@nvme0n1", [38, 39, 38, 40, 37], start=start, step=SMART_STEP) counts = db.maintain(now=now) minutes = db.query("disks", "driveripper", "disk.temp_c@nvme0n1", res=RES_MIN, start=0) assert counts["to_minute"] == 5 assert [value for _ts, value in minutes] == [38.0, 39.0, 38.0, 40.0, 37.0], ( "a bucket holding one sample rolls up that sample, unchanged") assert [int(ts) for ts, _value in minutes] == [ int(start + index * SMART_STEP // MINUTE * MINUTE) for index in range(5)], ( "every fifth minute bucket has a row; the empty ones must have none") assert db.count(res=RES_MIN) == 5, "empty buckets became fabricated rows" def test_a_300s_source_averages_into_the_hour_bucket_it_lands_in(db) -> None: now = time.time() # aligned to the epoch, and old enough that the hour bucket is complete: 12 samples # five minutes apart inside one hour average to their mean, not to a sample count hour = int(now // HOUR) * HOUR - 3 * HOUR add_series(db, "disks", "driveripper", "disk.power_on_hours@nvme0n1", [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 124], start=hour, step=SMART_STEP) db.maintain(now=now) db.maintain(now=now) # the 1h pass reads 1m rows, so the first pass only makes them hours = db.query("disks", "driveripper", "disk.power_on_hours@nvme0n1", res=RES_HOUR, start=0) assert len(hours) == 1 assert hours[0][0] == hour assert hours[0][1] == pytest.approx((11 * 100 + 124) / 12) def test_a_300s_series_is_queryable_at_every_resolution(db) -> None: """A 5-minute series over a 1h window is 12 points; `pick_res` must find them raw.""" now = time.time() add_series(db, "disks", "driveripper", "disk.reallocated@nvme0n1", [8] * 12, start=now - 3600 + 60, step=SMART_STEP) assert db.pick_res("disks", "driveripper", "disk.reallocated@nvme0n1", 3600, 600) == RES_RAW points = db.history("disks", "driveripper", "disk.reallocated@nvme0n1", span=3600) assert len(points) == 12, "the chart is empty while raw history exists" db.maintain(now=now) db.maintain(now=now) # with rollups present, a long window reads the rolled-up series, not 288 raw rows assert db.pick_res("disks", "driveripper", "disk.reallocated@nvme0n1", 24 * 3600, 600) in (RES_MIN, RES_RAW) def test_a_300s_source_is_pruned_by_the_existing_retention(db) -> None: """288 rows a day per series is fine for 48h and not fine forever — rule 9.""" now = time.time() expired = now - db.raw_keep_seconds - 6 * SMART_STEP add_series(db, "disks", "h", "disk.temp_c@sda", [30] * 4, start=expired, step=SMART_STEP) add_series(db, "disks", "h", "disk.temp_c@sdb", [30], start=now - SMART_STEP, step=SMART_STEP) counts = db.maintain(now=now) assert counts["pruned"] >= 4 assert db.query("disks", "h", "disk.temp_c@sda", start=0) == [] assert len(db.query("disks", "h", "disk.temp_c@sdb", start=0)) == 1 # ---- meta: the durable key/value state (phase 11) --------------------------- # def test_get_meta_answers_none_for_a_key_nobody_wrote(db) -> None: """`None` and `""` are different sentences here. `auth.verifier` missing means "this box has never been seeded" (fail closed at startup), while a value means "the credential lives here". A store that returned `""` for an absent row would read as a corrupt one instead. """ assert db.get_meta("no.such.key") is None db.set_meta("empty.on.purpose", "") assert db.get_meta("empty.on.purpose") == "" def test_meta_round_trips_through_a_reopen(db) -> None: db.set_meta("auth.session_secret", "ab" * 32) assert db.get_meta("auth.session_secret") == "ab" * 32 db.close() reopened = SampleStore(db.path) try: assert reopened.get_meta("auth.session_secret") == "ab" * 32 finally: reopened.close() def test_set_meta_replaces_instead_of_adding(db) -> None: """One row per key (`INSERT OR REPLACE`), because `key` is the primary key. An `INSERT` alone would raise on the second write, which is the change-password path (`auth.update_verifier`) failing after the user pressed Save. """ db.set_meta("auth.verifier", "first") db.set_meta("auth.verifier", "second") assert db.get_meta("auth.verifier") == "second" (rows,) = db._db.execute( # the private handle is the point: exactly one row, not two "SELECT COUNT(*) FROM meta WHERE key='auth.verifier'").fetchone() assert rows == 1 def test_delete_meta_reports_whether_a_row_was_there(db) -> None: """The rowcount is the whole point: the password-reset procedure has to say what it did.""" db.set_meta("auth.verifier", "x") assert db.delete_meta("auth.verifier") == 1 assert db.delete_meta("auth.verifier") == 0 assert db.get_meta("auth.verifier") is None def test_delete_meta_touches_only_the_key_it_names(db) -> None: """A reset that deletes one row too many deletes the operator's only credential.""" db.set_meta("auth.verifier", "the-verifier") db.set_meta("auth.session_secret", "the-secret") db.set_meta("last_maintain", "1700000000.0") assert db.delete_meta("auth.session_secret") == 1 assert db.get_meta("auth.session_secret") is None assert db.get_meta("auth.verifier") == "the-verifier" assert db.get_meta("last_maintain") == "1700000000.0" def test_stats_reads_last_maintain_through_get_meta(db) -> None: """One path to the table. `stats()` reported `None` before any maintenance ran and the timestamp afterwards; that behaviour is pinned here while the read goes through the same helper everything else uses, and it is why the helper is called outside `stats()`'s own lock (`threading.Lock` is not reentrant — call it inside and the dashboard deadlocks on the first `/api/health`). """ assert db.stats()["last_maintain"] is None db.maintain(now=1_700_000_000.0) assert db.stats()["last_maintain"] == 1_700_000_000.0