diff --git a/.agents/phases/todo/24_dev_mtls_rollout/00_phase.md b/.agents/phases/complete/24_dev_mtls_rollout/00_phase.md similarity index 100% rename from .agents/phases/todo/24_dev_mtls_rollout/00_phase.md rename to .agents/phases/complete/24_dev_mtls_rollout/00_phase.md diff --git a/.agents/phases/todo/24_dev_mtls_rollout/01_dev_bundle_export.md b/.agents/phases/complete/24_dev_mtls_rollout/01_dev_bundle_export.md similarity index 100% rename from .agents/phases/todo/24_dev_mtls_rollout/01_dev_bundle_export.md rename to .agents/phases/complete/24_dev_mtls_rollout/01_dev_bundle_export.md diff --git a/.agents/phases/todo/24_dev_mtls_rollout/02_installer_fetch_bundle.md b/.agents/phases/complete/24_dev_mtls_rollout/02_installer_fetch_bundle.md similarity index 100% rename from .agents/phases/todo/24_dev_mtls_rollout/02_installer_fetch_bundle.md rename to .agents/phases/complete/24_dev_mtls_rollout/02_installer_fetch_bundle.md diff --git a/.agents/phases/todo/24_dev_mtls_rollout/03_fleet_rollout.md b/.agents/phases/complete/24_dev_mtls_rollout/03_fleet_rollout.md similarity index 100% rename from .agents/phases/todo/24_dev_mtls_rollout/03_fleet_rollout.md rename to .agents/phases/complete/24_dev_mtls_rollout/03_fleet_rollout.md diff --git a/.agents/phases/todo/24_dev_mtls_rollout/04_verification.md b/.agents/phases/complete/24_dev_mtls_rollout/04_verification.md similarity index 100% rename from .agents/phases/todo/24_dev_mtls_rollout/04_verification.md rename to .agents/phases/complete/24_dev_mtls_rollout/04_verification.md diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.err b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.md b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.md new file mode 100644 index 0000000..c462c8a --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.md @@ -0,0 +1,11 @@ +**Phase 24 final verification — all green** + +- **Verified (read-only re-run, 62/62 PASS)** via `data/logs/verify-24-mtls.py`: nine sources `ok`; `/api/certs` all three `active` with `deployed_bundle_rev == bundle_rev == 2`; `agent.tls: true` + `agent.version 1.1.0` on every host; driveripper 16/16 drives live; gateway absent from `/disks` but `disks:gateway ok`; all served target URLs `https://`, no userinfo; pre-flip backup present; no transport/cert alert. +- **Criterion 1 (export):** `data/certs/{driveripper,deskwork,gateway}.bundle` re-checked byte-identical (sha256) to `GET /api/certs//bundle`, mode `0600`, under gitignored `data/`; live arms re-probed (issued→file, unknown/`issued:false`→one-line note + no file, wrong password→exit 2, nothing written). +- **Criterion 2 (`--fetch-bundle`):** 54 sandbox pins in `tests/test_deploy_scripts.py` pass — byte-identical `0640` install + TLS unit lines, refusals (unissued / 401 / non-bundle body / dead dashboard) before `need_root` with no write, token absent from token file, unit, stdout and sudoers hint. +- **Criterion 3–4:** `./scripts/dev-server.sh status` nine `ok` + bundles block; screenshots `data/logs/24-task04-{certs,disks,system}.png` exist. +- **Criterion 7:** no behavior change — `agent/` untouched, `src/` diff is a `certs.ensure_certs` docstring only; token-mode, revocation-before-dial and bundle-route tests green. +- **Tests/lint:** `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov --cov-fail-under=94` → **1542 passed, 15 skipped, coverage 98.94%**; `ruff check src tests agent` → All checks passed; `pyright` → 0 errors; `node --check` clean; `uv run pytest -m live --live` → **11 passed**. +- **Docs:** README.md (all 1144 lines) + AGENTS.md reviewed against the live project — every config.py env var is in README and both templates, route set / severity thresholds / test counts / coverage figure match, all repo paths, cited test names and `Dash.*`/`DashCharts.*` helpers resolve, phase-24 doc pins green. **Verified consistent — no changes needed.** +- **Notable:** left `fastapi 0.153.0 → 0.158.0` (validate reports it as advisory; no dependency was touched, rule 8 conditions the check on deps moving). Killed two abandoned `/tmp/fbtest` loopback stub processes left by task 02's manual probe; dev server unaffected. +- **Next pending phase:** `.agents/phases/todo/25_masthead_control_placement/`. diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.md.status b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.validate b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.validate new file mode 100644 index 0000000..eb5485a --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__00_phase.a1.validate @@ -0,0 +1,66 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 13%] +........................................................................ [ 18%] +........................................................................ [ 23%] +.........................................................s.............. [ 27%] +........................................................................ [ 32%] +........................................................................ [ 36%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 55%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 69%] +sssssssssss............................................................. [ 73%] +........................................................................ [ 78%] +........................................................................ [ 83%] +........................................................................ [ 87%] +........................................................................ [ 92%] +...........................................................sss.......... [ 97%] +............................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1542 passed, 15 skipped, 1 warning in 165.53s (0:02:45) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.158.1 wheel diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.err b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.md b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.md new file mode 100644 index 0000000..453c777 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.md @@ -0,0 +1,15 @@ +**Implemented** +- `scripts/dev-certs-export.py` (new, stdlib `urllib`): `--url/--password(+$DASH_ADMIN_PASSWORD)/--names/--out-dir`; signs in via public `POST /api/auth/login`, reads the token out of `Set-Cookie`, `GET /api/certs//bundle` per name; writes `0600` via `os.open` (birth mode + re-chmod) into gitignored `data/certs/`. Arms: issued→file, `issued:false`/`configured:false`→one-line note + no file, other statuses→`!` line; exit 0 all-resolved / 1 a refused host (others still attempted) / 2 no sign-in. Reads the route only — never `ensure_certs`, so it cannot mint. +- `scripts/dev-server.sh`: `export_bundles` after the readiness probe (names split from `HOST_TARGETS`, failure = `|| echo` warning, never fails `start`); `bundles` block (names/sizes/modes) printed by `status` and as its own subcommand. +- 19 new tests in `tests/test_deploy_scripts.py`: real `create_app` served by uvicorn on `127.0.0.1:0`, byte-identity vs the route, `0600` (+ re-chmod of a loosened file), unissued/unconfigured notes, wrong-password & no-server exit 2 writing nothing, idempotence, "never mints" (serials/revs/root unchanged), no PEM/token/password printed, name-escape refusal, and the dev-server composition pins (order, warning-only, `status` block, `CERTDIR` under `data/`). + +**Verification** — `bash .agents/validate.sh`: `uv run pytest --cov --cov-fail-under=94` → **1513 passed, 15 skipped**, coverage **98.94%** (floor 94%); `ruff check src tests agent` clean (also clean on `scripts/`); `pyright` 0 errors; `node --check` ran. `uv run pytest -m live --live` → **11 passed**. Real boot: `./scripts/dev-server.sh restart` exported `data/certs/deskwork.bundle` (1430 B, `mode 0600`, byte-identical to the route; verified over HTTP) and printed the unissued note for `driveripper`/`gateway`; `status` still reads nine sources `ok`. + +**Docs** — README (Quickstart `bundles` subcommand + export paragraph, new *Managing the certificates* dev paragraph, rollout step 1 note, Testing paragraph) and AGENTS.md (invariant 6 export rules, `certs.py` bullet, Commands block + count 1494→1513, new testing-policy bullet, gap 11 dev-side half closed). + +**Decisions / deviations** +- Kept the logic in the standalone script as the task specifies; `scripts/` is outside `[tool.coverage.run] source` (like the other dev scripts), so the gate number is unchanged — the new code is instead exercised end-to-end as a subprocess on every arm plus in-process helper tests. +- Cookie name is imported from `dashboards.auth` (with an `ImportError` fallback) so it cannot drift; `--password` also falls back to the env var. +- No live-infra changes: only the local dev-server restart and a read-only export over loopback. No fleet change, no bundle installed, no `HOST_TARGETS` edited — tasks 02-04 are untouched. Pre-existing `uv pip list --outdated` advisory (fastapi 0.153.0→0.156.8) left alone: dependencies were not touched here. + +**Next pending task:** `.agents/phases/todo/24_dev_mtls_rollout/02_installer_fetch_bundle.md` diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.md.status b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.validate b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.validate new file mode 100644 index 0000000..9069742 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__01_dev_bundle_export.a1.validate @@ -0,0 +1,66 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 14%] +........................................................................ [ 18%] +........................................................................ [ 23%] +.........................................................s.............. [ 28%] +........................................................................ [ 32%] +........................................................................ [ 37%] +........................................................................ [ 42%] +........................................................................ [ 47%] +........................................................................ [ 51%] +........................................................................ [ 56%] +........................................................................ [ 61%] +........................................................................ [ 65%] +...........................................sssssssssss.................. [ 70%] +........................................................................ [ 75%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 89%] +........................................................................ [ 94%] +..............................sss....................................... [ 98%] +................ [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1513 passed, 15 skipped, 1 warning in 156.26s (0:02:36) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.156.8 wheel diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.err b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.md b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.md new file mode 100644 index 0000000..3005d2d --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.md @@ -0,0 +1,19 @@ +All green. Final state: + +**Implemented** +- `deploy/install-agent.sh`: `--fetch-bundle --cert-name --session ` — `python3` `urllib` GETs `/api/certs//bundle`, writes it to a `mktemp` file that becomes `$BUNDLE_SRC`, so the existing shape check → `0640 root:agent` install → unit `--cert-bundle`+`RuntimeDirectory` → `--status` probe all run on fetched bytes unchanged. +- Order: flag checks → fetch → `validate_bundle` → `need_root`; refusals (unissued `issued:false`, `401`/`403`/`5xx`/`404`, no dashboard, non-bundle 200 body) print one line and write nothing; `trap cleanup_fetch EXIT` sweeps the temp key on every path. +- Refusals added before `need_root`: partial flag set (exit 1), `--fetch-bundle` + local `--cert-bundle` (two sources), fetch with `--status`/`--uninstall`. +- Token hygiene: travels to python via env (not argv), only the URL is logged, `unset` before any write, never in the token file/unit/bundle/log nor in the sudoers hint's re-quoted command; `validate_bundle` gained a flag-name label so a fetched refusal says `--fetch-bundle` and the file path still says `--cert-bundle`. +- `--help` documents the three flags + copy-pasteable recipe. + +**Tests/lint/coverage** — `uv run pytest` → **1535 passed, 15 skipped**; `bash .agents/validate.sh` → **exit 0** (coverage 98.94%, floor 94%; ruff clean; pyright 0 errors; `node --check` ok; hygiene clean). 12 new pins in `tests/test_deploy_scripts.py` run the real `create_app` on a socket (byte-identical `0640` install + TLS unit lines, unissued/`configured:false`/401 refusals with no stub call logged, non-bundle 200 bodies, dead server, token grep of every file + both streams + sudoers hint, all-or-nothing/two-source/mode refusals, fetch-before-shape-before-root ordering). Four deliberate script mutations were used to confirm the pins bite. + +**Docs** — README: rollout step 1 is now the `--fetch-bundle` registration run (manual signed-in `curl` demoted to the fallback), plus the flag list and the dev-export paragraph; AGENTS.md: invariant 6 (session token as fetch credential), installer deployment paragraph, gap 11 (task 02 landed), testing-policy pin, pytest count. + +**Decisions / deviations** +- Pins drive the real `served_dashboard` socket (not a look-alike `http.server`); a canned `http.server` covers only the shapes the app cannot answer (200-with-HTML, empty 200). +- Added two refusals beyond the task list (fetch with `--status`/`--uninstall`; per-flag value check follows existing style) — silent ignoring of a credential flag was the alternative. +- Live infra: **no live changes made**. Verified read-only against the running dev dashboard: fetched `deskwork` (then correctly stopped at the unprivileged root boundary, token absent from output, temp dir empty) and got the one-line unissued refusal for `gateway`. The per-host rollout is task 03. + +Next pending task: `.agents/phases/todo/24_dev_mtls_rollout/03_fleet_rollout.md` (agent 1.1.0 + bundle on deskwork/driveripper/gateway, `HOST_TARGETS` flipped per host — live, operator-gated root on two hosts). diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.md.status b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.validate b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.validate new file mode 100644 index 0000000..08ed78c --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__02_installer_fetch_bundle.a1.validate @@ -0,0 +1,66 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 13%] +........................................................................ [ 18%] +........................................................................ [ 23%] +.........................................................s.............. [ 27%] +........................................................................ [ 32%] +........................................................................ [ 37%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 51%] +........................................................................ [ 55%] +........................................................................ [ 60%] +........................................................................ [ 65%] +.................................................................sssssss [ 69%] +ssss.................................................................... [ 74%] +........................................................................ [ 78%] +........................................................................ [ 83%] +........................................................................ [ 88%] +........................................................................ [ 92%] +....................................................sss................. [ 97%] +...................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1535 passed, 15 skipped, 1 warning in 162.74s (0:02:42) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.156.8 wheel diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.err b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.md b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.md new file mode 100644 index 0000000..79b5468 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.md @@ -0,0 +1,18 @@ +Final state verified. Summary of this task's work: + +**Fleet rollout (live, all three hosts now mTLS-only)** +| host | how | proof | +|---|---|---| +| `gateway` | staged installer + 1.1.0, dry-run diff → `--force`, restart, CA-only `--status` probe, then URL flip | `answering over TLS (bundle rev 2)` | +| `deskwork` | same; its `zz-devices.conf` drop-in (`DeviceAllow=/dev/nvme0n1`, SYS_ADMIN caps) survived the rewrite | same, SMART still reads | +| `driveripper` | gap-10 line `--as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN --force`; diff moved comments + TLS lines only | same, **16/16 drives with temps before and after** | + +`.env.dev` `HOST_TARGETS` is now `driveripper/deskwork/gateway=https://…:9120` with no `http://`, no userinfo (pre-flip file backed up in `data/env-backup/`); `/certs` reads `bundle_rev == deployed_bundle_rev == 2` on all three; `dev-server.sh status` = nine sources `ok`. Every installer ran twice, once without `--force` to read the guard's diff. A canary revoke on `gateway` proved the bite (`agent certificate revoked (serial …)` on `host:gateway` + `disks:gateway`, page-side too), then restored. + +**Defects found and fixed** +- `data/logs/NOTES-24-rollout.md` § *the defect that stopped the first flip*: the stored root predated the SKI fix → strict dials failed `Missing Subject Key Identifier`. Repaired with `POST /api/certs/root/rotate` (not the `DELETE FROM meta` reset), pinned by `tests/test_certs.py::test_a_stored_root_without_an_identifier_still_boots_and_root_rotate_repairs_it` and a new `certs.ensure_certs` docstring paragraph. +- The live suite could not dial the fleet (3 failures, `CERTIFICATE_VERIFY_FAILED: self-signed certificate in chain`): `tests/test_live_fleet.py` gained an `agent_tls` fixture (read-only `sqlite3` authority whose `set_meta`/`delete_meta` raise, `certs.write_material` + `collectors.agent_tls_map`), and the four host tests now go through `fetch_snapshot`/`fetch_host_snapshot`/`fetch_disk_snapshot` with `tls=` → **11 passed, 0 failed**. + +**Docs**: `README.md` (rollout order, reverse-tunnel route, `--status --bind` caveat, pre-SKI CA rule, rollback rule, live-suite note) and `AGENTS.md` (fleet record now mTLS + corrected sudo/staging facts, gap 10 trap closed, gap 11 resolved into rotation hygiene, gap 12, testing policy, `--status` address rule). + +**Gates**: `bash .agents/validate.sh` → **EXIT=0** (1536 passed / 15 skipped, ruff clean, pyright clean, `node --check` clean); `pytest -m live --live` → EXIT=0; evidence in gitignored `data/logs/` (`NOTES-24-rollout.md`, `24-*-install.log`, `24-fleet-{before,after,end-state}.txt`, `24-certs-end-state.txt`, `24-units-after-rollout.txt`, `24-mtls-{system,certs}.png`). Only the advisory `fastapi 0.153.0 → 0.157.0` was left alone (no dependency was touched here). Nothing staged, committed, or moved in `.agents/`. diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.md.status b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.validate b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.validate new file mode 100644 index 0000000..e9faf63 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__03_fleet_rollout.a1.validate @@ -0,0 +1,66 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 13%] +........................................................................ [ 18%] +........................................................................ [ 23%] +.........................................................s.............. [ 27%] +........................................................................ [ 32%] +........................................................................ [ 37%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 51%] +........................................................................ [ 55%] +........................................................................ [ 60%] +........................................................................ [ 64%] +..................................................................ssssss [ 69%] +sssss................................................................... [ 74%] +........................................................................ [ 78%] +........................................................................ [ 83%] +........................................................................ [ 88%] +........................................................................ [ 92%] +.....................................................sss................ [ 97%] +....................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1536 passed, 15 skipped, 1 warning in 167.02s (0:02:47) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.157.0 wheel diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.err b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.md b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.md new file mode 100644 index 0000000..43b6caa --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.md @@ -0,0 +1,19 @@ +Everything is green. Final state of the work: + +**Implemented (task 04 — read-only verification of the mTLS end state)** +- `data/logs/verify-24-mtls.py` + `24-task04-endstate.txt`: 57 signed-in GET assertions — nine sources `ok`; `/api/certs` all three `active` with `deployed_bundle_rev == bundle_rev == 2`; `agent.tls: true` + `agent.bundle_rev` + `agent.version 1.1.0` on every host; driveripper 16/16 drives with temp+hours (same single `nvme5n1` alarm chip, no `smart: null`), deskwork `nvme0n1`; gateway absent from `/api/disks/snapshot` while `disks:gateway` still `ok`; `/api/summary` fleet block identical, **no transport/certificate alert**; `/api/config` + `.env.dev` all `https://`, no userinfo; pre-flip backup present; bundles in gitignored `data/certs/` at `0600`. +- `data/logs/24-task04-verify.cjs` → `24-task04-{certs,disks,system}.png` (+ `.log`/`.out`): the same claims as rendered — `#2 → #2` with no `stale` chip on all three rows, deskwork's GPU card with four live values, four masthead dots all green. +- Negative control per the task: named phase 15's `verify-15-tls-e2e.py`/`.out` as the standing proof and ran the two suite pins (`tests/test_collectors.py`, `tests/test_api.py`) → 3 passed, logged in `24-task04-revocation-pins.log`. **Nothing revoked, posted or restarted on the fleet.** +- Doc-consistency pass (this phase's final pass): README dev quick-start and `HOST_TARGETS` config row now lead with bundle registration; `AGENTS.md` transport bullet, `Commands` block, gap 11 (verification record + pins named) and gap 12 (phase closed) updated, plus a pre-existing duplicated sentence in gap 11 de-duplicated; `.env.example`/`.env.dev.example` now document `https://`-no-userinfo as the shape a new host gets, token mode as the compatibility path. +- Added 6 regression pins to `tests/test_deploy_scripts.py` so the docs can't slide back: setup docs must name `--fetch-bundle`/`--cert-name`/`--session`, and every template `HOST_TARGETS` example URL must be `https://` with no userinfo. + +**Gates (exact)** +- `uv run pytest` → **1542 passed, 15 skipped**, exit 0; `uv run pytest -m live --live` → **11 passed**, exit 0. +- `bash .agents/validate.sh` → **EXIT=0**: coverage **98.94%** (floor 94; `certs.py`/`api.py`/`app.py`/`auth.py` 100%), ruff `All checks passed!`, pyright `0 errors, 0 warnings, 0 informations`, `node --check` clean, repo-hygiene clean. +- `./scripts/dev-server.sh status` → `status: ok`, nine sources ok, three bundles exported. + +**Docs**: `README.md`, `AGENTS.md`, `.env.example`, `.env.dev.example` (see above); transcript `data/logs/NOTES-24-verification.md`. + +**Decisions**: no live revoke/restart (read-only by rule); left the advisory `fastapi 0.153.0 → 0.157.0` alone — no dependency was touched. + +**Next pending task**: none in `todo/24_dev_mtls_rollout` (this completes the phase); `25_masthead_control_placement` is next. diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.md.status b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.validate b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.validate new file mode 100644 index 0000000..d506987 --- /dev/null +++ b/.agents/reports/24_dev_mtls_rollout/24_dev_mtls_rollout__04_verification.a1.validate @@ -0,0 +1,66 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 13%] +........................................................................ [ 18%] +........................................................................ [ 23%] +.........................................................s.............. [ 27%] +........................................................................ [ 32%] +........................................................................ [ 36%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 55%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 69%] +sssssssssss............................................................. [ 73%] +........................................................................ [ 78%] +........................................................................ [ 83%] +........................................................................ [ 87%] +........................................................................ [ 92%] +...........................................................sss.......... [ 97%] +............................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1542 passed, 15 skipped, 1 warning in 165.02s (0:02:45) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.158.0 wheel diff --git a/.env.dev.example b/.env.dev.example index 583705f..d1b465f 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -1,29 +1,35 @@ -# Local development overrides. Copy to .env.dev and edit the addresses and tokens to yours. -# This file is gitignored and holds the fleet's bearer tokens: NEVER commit it. +# Local development overrides. Copy to .env.dev and edit the addresses to yours. +# This file is gitignored and holds the fleet's credentials: NEVER commit it. # cp .env.dev.example .env.dev # chmod 0600 .env.dev # # Dev dials the fleet exactly the way the deployed dashboard does — no tunnels, no -# forwarded ports, nothing listening on localhost. Per monitored host, once: +# forwarded ports, nothing listening on localhost. Per monitored host, once: register it +# with the bundle this dashboard signs (issue one on /certs, or POST +# /api/certs//rotate, if the name has none yet): # -# sudo deploy/install-agent.sh --bind
--port 9120 +# sudo deploy/install-agent.sh --bind
--port 9120 \ +# --fetch-bundle --cert-name --session # -# It installs agent/host_agent.py as the dashboards-agent systemd service, bound to the -# address you dial, and prints its bearer token exactly once. No root shell on the host? -# deploy/agent-sudoers is the one-line grant that lets the fleet's `ops` account run the -# installer (and running the installer without root prints that exact line). Put each -# printed token into HOST_TARGETS below as URL userinfo; config.parse_targets moves it -# into an Authorization header, so Target.url, /api/config and every error message stay -# credential-free. +# That installs agent/host_agent.py as the dashboards-agent systemd service, bound to the +# address you dial, with this dashboard's own root CA and the host's client certificate in +# /etc/dashboards-agent.certs — so the URL below is https:// with NO userinfo: the +# handshake is the auth, the agent never reads a token on that path, and a host revoked on +# /certs is refused before a socket opens (certs.py, collectors.AgentTls). A monitored host +# that cannot route back to a dev box registers over a reverse tunnel instead: +# `ssh -R 18099:127.0.0.1:8090 ` with `--fetch-bundle http://127.0.0.1:18099`. +# No root shell on the host? deploy/agent-sudoers is the one-line grant that lets the +# fleet's `ops` account run the installer (and running the installer without root prints +# that exact line). # -# A host whose agent runs in TLS mode (--cert-bundle) is an https:// url with NO userinfo: -# the handshake is the auth there, the agent never reads a token on that path, and the -# dashboard verifies the agent against the root CA it minted itself (certs.py). Mixing the -# two (`https://TOKEN@host`) works — the token is simply unused — and is how a rollout leaves -# one host on tokens while another is on certificates. +# The compatibility path is a token-mode agent: install with no --cert-bundle / --fetch- +# bundle and it prints its bearer token exactly once, which goes in the URL as userinfo +# (`http://TOKEN@host:9120`); config.parse_targets moves it into an Authorization header, +# so Target.url, /api/config and every error message stay credential-free. Nothing in this +# fleet runs that way any more. # # The host names below are placeholders (RFC 2606) — use your own. -export HOST_TARGETS="driveripper=http://TOKEN@driveripper.example.net:9120 deskwork=http://TOKEN@deskwork.example.net:9120 gateway=http://TOKEN@gateway.example.net:9120" +export HOST_TARGETS="driveripper=https://driveripper.example.net:9120 deskwork=https://deskwork.example.net:9120 gateway=https://gateway.example.net:9120" # Physical-disk health (phase 09): scripts/capture-fixtures.sh records one # `smartctl -i -A -H` per whole disk per host into tests/fixtures/smartctl__.txt. diff --git a/.env.example b/.env.example index cd0ce9d..1d392b2 100644 --- a/.env.example +++ b/.env.example @@ -48,13 +48,16 @@ DASH_PBKDF2_ITERATIONS=600000 # --------------------------------------------------------------------------- # # System dashboard: hosts running agent/host_agent.py -# Space or comma separated name=url. A bearer token goes in the URL userinfo -# (http://TOKEN@host:9120) and is never sent to the browser. An https:// url instead -# dials a certificate-authenticated agent: the fetch verifies it against this app's own root CA -# and presents this app's own client certificate (certs.py), and a host revoked there is refused -# before any connection. Per host, so one host can be on https while its neighbours are on http. +# Space or comma separated name=url. The shape a new host gets is https:// with NO userinfo: +# register it with `install-agent.sh --fetch-bundle --cert-name +# --session ` and the fetch verifies the agent against this app's own root CA and +# presents this app's own client certificate (certs.py); a host revoked on /certs is refused +# before any connection. The compatibility shape is a token-mode agent's URL userinfo +# (http://TOKEN@host:9120), which config.parse_targets moves into an Authorization header and +# never sends to the browser. The choice is per host, so a rollout can leave one host on +# tokens while its neighbour is on certificates. # --------------------------------------------------------------------------- # -# HOST_TARGETS=driveripper=http://SECRET@driveripper.example.net:9120 deskwork=http://SECRET@deskwork.example.net:9120 +# HOST_TARGETS=driveripper=https://driveripper.example.net:9120 deskwork=https://deskwork.example.net:9120 HOST_TARGETS= HOST_INTERVAL_SECONDS=10 HOST_TIMEOUT_SECONDS=4 diff --git a/AGENTS.md b/AGENTS.md index d6271a2..1c4b471 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,6 +103,19 @@ page still renders. no line of the stored key), the root key is never written to the temp dir the client material lives in, and a bundle carries the root *certificate* — never the key — because an agent only has to verify its clients and a stolen agent install must not be a stolen CA. + **A session token used to fetch a bundle is the same case** (phase 24): the installer's + `--fetch-bundle --cert-name --session ` sends that token on exactly one + request, hands it to its `python3` child through the **environment** rather than argv, prints + only the URL in its log line, `unset`s it before anything is written, and never repeats it in + the sudoers hint's re-quoted command (which names `--bind`/`--port` and nothing else). + `tests/test_deploy_scripts.py` signs in for a real token and then greps every file the run + could have touched plus both output streams. + **A bundle exported for the dev loop is the same case** (phase 24): + `scripts/dev-certs-export.py` writes only into the gitignored `data/certs/` tree, creates each + file with `os.open` so `0600` is its *birth* mode (and re-chmods a file that already existed + looser), dials `GET /api/certs//bundle` over HTTP instead of importing `ensure_certs` — so + it cannot mint even by accident — and prints one line per host with no PEM, no token and no + password in it (`tests/test_deploy_scripts.py` greps the output for all three). 7. **Verify against the deployed upstream; never guess a metric name.** Field maps in `sources/llm.py` and `sources/nut.py` were captured from the running engines and NUT 2.8.5, and are pinned by fixtures in `tests/fixtures/`. Re-capture with @@ -463,7 +476,14 @@ Responsibilities, one line each: a write, so a page refresh cannot bootstrap an authority the operator deleted. The material is read with (task 03): `collectors.agent_tls_map` turns `AppContext.certs` into the per-host dial pair, and because the revocation answer is `agent_status()` called per poll, a revoke in this - module is what stops the dial — no restart, no cache, no second list to keep in sync. + module is what stops the dial — no restart, no cache, no second list to keep in sync. There is a + **third door and it is deliberately outside this module** (phase 24): + `scripts/dev-certs-export.py` is the dev loop's exporter — stdlib `urllib`, signs in through + `POST /api/auth/login`, reads the token out of the `Set-Cookie`, and downloads each + `HOST_TARGETS` name's bundle into gitignored `data/certs/` at `0600`. It cannot mint because it + imports neither read; it dials the route the `/certs` page downloads from, which is what makes the + exported file byte-identical to the row. `scripts/dev-server.sh` calls it after the readiness + probe and treats a failure as a warning line — the server being up is the thing the probe proved. - `severity.py` — the only place thresholds live: `THRESHOLDS`/`INVERTED` per metric, `classify()` → `ok`/`warn`/`fail`/`None`, plus `attach()`/`with_severity()` to put the verdict beside a value. No measurement, no verdict: `None` in → `None` out, never a green pass for @@ -917,8 +937,10 @@ That scheme is the history API's key — treat it as a public interface. - **Transport is direct dial: dev is the prod topology, repo-wide (phase 10 task 05).** The designed shape is one `dashboards-agent` **systemd service per host**, installed by `ops` through the `deploy/agent-sudoers` grant with `deploy/install-agent.sh`, bound to - the address the dashboard dials, its bearer token carried in the `HOST_TARGETS` URL - userinfo; driveripper's upsd serves the LAN on 3051 with the host and laptop ranges allowed + the address the dashboard dials — since phase 24 registered **with a certificate bundle** + (`--fetch-bundle`, so its `HOST_TARGETS` URL is `https://` and carries no userinfo); what was + once the token in the URL is the compatibility path for a host nobody has bundled. driveripper's + upsd serves the LAN on 3051 with the host and laptop ranges allowed by the **firewall**, never by a `upsd.conf` line. The old ssh-tunnel script is deleted and neither `.env.dev` nor `.env.dev.example` declares a forwarded port — dev dials the same addresses the deployed dashboard does, and `scripts/capture-fixtures.sh` reads @@ -953,11 +975,28 @@ That scheme is the history API's key — treat it as a public interface. `/api/health` today reads **all nine sources `ok`**: `host:driveripper`, `host:deskwork`, `host:gateway`, `disks:driveripper`, `disks:deskwork`, `disks:gateway`, `nut`, `llm:gufo`, `llm:vllm`. - **Since phase 15 that bearer-token transport is the compatibility path, not the designed one:** - the agent can serve `https://` from an installed bundle and the dashboard dials it with its own - client certificate, but no bundle is installed on any host here, so `HOST_TARGETS` stays - `http://` + userinfo until somebody runs the rollout in README's *Protecting an agent, host by - host* (gap 11 says what that needs and why flipping the URL first turns a host red). + **The fleet is mTLS since 2026-10-21 (phase 24 task 03), so the bearer-token transport is now + history rather than the designed shape.** All three hosts run `agent/host_agent.py` **1.1.0** + from an installed bundle, every unit carries `--cert-bundle /etc/dashboards-agent.certs` (plus + `RuntimeDirectory=dashboards-agent`, and still its `--token-file` line), and `HOST_TARGETS` is + `"driveripper=https://10.1.0.126:9120 deskwork=https://10.1.0.51:9120 + gateway=https://10.1.1.27:9120"` — **no `http://` entry and no userinfo anywhere**: the + certificate is the credential (the pre-flip `.env.dev` is backed up in `data/env-backup/`). `agent.tls: true` + `agent.bundle_rev: 2` in + every payload, `deployed_bundle_rev == bundle_rev` for all three on `/certs`, and + `./scripts/dev-server.sh status` reads the nine sources `ok`. The per-host order was the safety: + install agent 1.1.0 + bundle → `systemctl restart` → the CA-only `--status` probe answers → + *then* that host's URL flips, so no source was red longer than its own host's restart. + `data/logs/NOTES-24-rollout.md` is the transcript (sha256 of what was staged where, the unit + diffs the re-run guard printed, the before/after SMART tables, the screenshots). Two facts in it + are load-bearing for the next host: the fetch needed an **SSH reverse tunnel** (no monitored host + can route back to a dev box, so `ssh -R 18099:127.0.0.1:8090 ` + + `--fetch-bundle http://127.0.0.1:18099` is how registration reaches the dashboard), and + `--status` takes its address from `--bind`, so run bare it dials `127.0.0.1` and reports "no + answer" about a healthy host. **Rolling a host back is not just the URL**: a TLS-mode agent + serves https only, so the `--cert-bundle` line and `/etc/dashboards-agent.certs` both have to go + with it. And the fleet now trusts **this** dashboard's CA — a second deployment with its own + `certs.*` rows cannot dial these agents until it carries the same authority or re-issues and + re-installs every bundle. Phase 10's proof that the old drop was not this repo's firewall stands as history in `data/logs/NOTES-verify.md` — do not re-add a tunnel on the strength of it, and re-run the probe rather than trusting either note. @@ -1015,16 +1054,25 @@ That scheme is the history API's key — treat it as a public interface. `upsc`, `udevadm`, the `/etc/ups` reads, and the `tee /etc/ups/upsd.users|upsd.conf` + `systemctl restart|reload nut-server` pair documented in `deploy/nut-sudoers`); `deskwork` grants an unrelated set (`btrfs scrub`, `snapper`, `setfacl /srv/*`, `osbuild-auto.sh`). - `deskwork` grants `ops` neither the installer nor `visudo`, so its install went through root by - the operator. **`driveripper` does grant the installer** — `(root) NOPASSWD: - /home/ops/agent-install/deploy/install-agent.sh`, an Oct-18 snapshot of the script staged under - the operator's home (`sudo -n -l`, re-checked 2026-10-20) — and that grant is still the trap, - not the shortcut: it points at that Oct-18 **snapshot**, which carries neither this phase's - flags, nor the filter removal, nor the re-run guard, so running the granted path replaces the - `User=root` + `CapabilityBoundingSet` unit that gap 10 resolved with the snapshot's own - `User=dashboards-agent`, capability-less unit and re-breaks all 16 SMART reads — **silently**, - because the guard that refuses a differing re-run (task 03) lives only in this repo's script, - which therefore has to be re-staged before any of it means anything there. The template now writes the live unit's shape: no device filter at all + Neither `deskwork` nor `driveripper` grants `visudo`, so a sudoers change is the operator's + action. **`deskwork` and `driveripper` both grant the installer** — + `(root) NOPASSWD: /home/ops/agent-install/deploy/install-agent.sh` — and both also grant + `(root) NOPASSWD: /usr/bin/systemctl restart *`. The earlier record + here said deskwork's `ops` had no installer grant and that its install went through the + operator's root; that is out of date — `sudo -n -l` on 2026-10-21 lists the grant, and phase 24 + task 03 installed deskwork's bundle as `ops`. The grant names a **path**, and until phase 24 + that path held an Oct-18 snapshot of the script which carried neither the later flags, nor the + filter removal, nor the re-run guard — running it would have replaced driveripper's `User=root` + + `CapabilityBoundingSet` unit with the snapshot's capability-less one, **silently**. That trap + is now closed on all three hosts: `/home/ops/agent-install/` is `ops`-owned, so the repo's + `deploy/install-agent.sh` (sha256 `ffbcd91f…`) and `agent/host_agent.py` (**1.1.0**) were staged + over it without root, and each snapshot was left beside the grant as + `install-agent.sh.oct18-backup`, which sudo will not run. **Re-staging is therefore the first + step of any install, not an optional one** — the grant is only as current as the file it points + at. The `systemctl restart *` grant on those two hosts is the other half of an unattended + install: a bundle changes the unit, and the unit only takes effect on a restart that + `systemctl enable --now` will not perform. + The template now writes the live unit's shape: no device filter at all (phase 17 task 01 — systemd's default `auto`, with `PrivateDevices=false`) and `--as-root` plus `--caps CAP_SYS_RAWIO,CAP_SYS_ADMIN` for the `User=root` and `CapabilityBoundingSet=CAP_SYS_RAWIO CAP_SYS_ADMIN` lines (task 02). The old allow-list's @@ -1121,15 +1169,24 @@ uv sync --python 3.14 # deps + editable install cp .env.example .env # production-shaped config (DASH_ADMIN_PASSWORD seeds # the login on the first boot of a database — with no # seed and no stored verifier the server exits non-zero) -cp .env.dev.example .env.dev # dev-only: the real hosts, tokens in URL userinfo +cp .env.dev.example .env.dev # dev-only: the real hosts (this fleet's URLs are + # `https://` with no userinfo — the bundle is the cred) set -a; . ./.env.dev; set +a # dev dials the fleet the prod way, no tunnels ./scripts/dev-server.sh restart # keep http://127.0.0.1:8090 up; sign in with the - # DASH_ADMIN_PASSWORD from .env.dev (`testpass`) + # DASH_ADMIN_PASSWORD from .env.dev (`testpass`), then + # export every issued HOST_TARGETS bundle to data/certs/ ./scripts/dev-server.sh status # per-source ok/DOWN — unauthenticated: /api/health +./scripts/dev-server.sh bundles # what that export left: names, sizes, modes ./scripts/dev-server.sh logs -uv run pytest # hermetic: 1494 passed + 15 skipped (11 live, +uv run python scripts/dev-certs-export.py --url http://127.0.0.1:8090 \ + --password "$DASH_ADMIN_PASSWORD" --names + # the export on its own (phase 24): reads + # GET /api/certs//bundle and writes + # data/certs/.bundle at 0600 — mints nothing + +uv run pytest # hermetic: 1542 passed + 15 skipped (11 live, # 4 store_perf), no network uv run pytest --cov # coverage gate (98.94%; floor 94%) uv run pytest -m live --live # opt-in: real fleet / engines / upsd @@ -1177,12 +1234,13 @@ bash .agents/validate.sh # the phase gate: the four checks abov Deployment: `Containerfile` (two-stage, `uv sync --frozen`, runs as uid 10001 on `0.0.0.0:8090`, SQLite under `/var/lib/dashboards` — put a volume there) and `deploy/install-agent.sh` per monitored host (systemd unit, own user, `--bind`/`--port`/`--token`/`--rotate-token`/ -`--cert-bundle`/`--as-root`/`--caps`/`--force`/`--status`/`--uninstall`; it stores the token +`--cert-bundle`/`--fetch-bundle`/`--cert-name`/`--session`/`--as-root`/`--caps`/`--force`/ +`--status`/`--uninstall`; it stores the token `0640 root:dashboards-agent` — the unit hands the agent the *path* and the agent opens it as its own user, so that user must be able to read it and no other account may, and a `--cert-bundle` file is installed to `/etc/dashboards-agent.certs` at that same mode after a shape check that runs before anything is -touched and never dials the dashboard (the agent re-reads the bundle live for the denylist, so +touched (with a local file this script dials nothing; the fetched path is the one dial, below), so 0600 would fail the unit exactly like it fails the token file; the task file's `0600` is the intent — *root and the agent's own user only* — and 0640 is the mode that achieves it) — never puts a token on argv, and runs under `ProtectSystem=strict` with **no device @@ -1202,7 +1260,19 @@ PEM blocks there and the agent deletes the key once the SSL context holds it), k bundle's CA with **no** client certificate via a `python3` `ssl` one-liner — task 02's `CERT_OPTIONAL` + `/health` exemption is what makes that probeable, and there is no external TLS binary. A re-run without the flag leaves an installed bundle in place, and `--uninstall` removes -it with the token — and neither is guarded, because both are deliberate. What is guarded is the +it with the token — and neither is guarded, because both are deliberate. **Registration can fetch +that bundle** (phase 24 task 02): `--fetch-bundle --cert-name +--session ` dials `GET /api/certs//bundle` with `python3` `urllib` (no +`curl` assumption on a monitored host) in the install run and hands the bytes to that same +`--cert-bundle` path, so the shape check, the `0640` install, the unit's TLS lines and the +`--status` probe behave identically for a fetched and a downloaded bundle. The three flags are +all-or-nothing, are refused together with a local `--cert-bundle` and with `--status`/`--uninstall`, +and the fetch plus the shape check run **before** `need_root`: a 401, an unissued name, a proxy's +200-with-HTML or a dashboard that never answers is one line naming the status and no partial +install. The session is a **fetch credential only** — it travels to python in the environment (argv +is world-readable through `ps`, which is why the agent's token is a file), is `unset` as soon as +the bytes are in hand, appears in no written artefact and not even in the sudoers hint's re-quoted +command, and the fetched temp file is removed by an EXIT trap on every path. What is guarded is the unit itself (phase 17 task 03): it is rendered to a temp file first, and if an installed `$UNIT` differs the run prints `refusing to overwrite …` plus the `diff -u` on stdout and exits 1 before the unit write, the `daemon-reload` and the `enable`, so a redeploy cannot undo a hand-fix and an @@ -1212,7 +1282,9 @@ which is what the TLS rollout's `--cert-bundle` step needs on a token-mode host, so a run that rewrote `ExecStart` leaves the already-running agent on its old settings until `systemctl restart dashboards-agent` — and `--status` chooses http from https by reading the unit file, so probing before that restart reports "no answer" for a change that landed correctly (the -rollout in README restarts for exactly this reason). No root shell? The one sudoers +rollout in README restarts for exactly this reason). It takes its **address** from the flags rather +than the unit, so a bare `--status` probes `127.0.0.1:9120` and answers "no answer" about an agent +bound to a LAN address that is healthy — always pass the same `--bind`/`--port` the install did. No root shell? The one sudoers line that lets `ops` run the installer is `deploy/agent-sudoers`; run the script without root and it prints that line with its own resolved path.) The quadlet unit + env template + nginx/SNI live in the deployment repo, like every other @@ -1293,6 +1365,16 @@ branch `main`); there is no CI on it, so `bash .agents/validate.sh` is the only fleet, engines, and upsd. Skipped unless `--live`; each test skips when its source is unset, so partial fleet availability is normal. **Live tests must not write** — a battery test on hardware something is plugged into is not a test. + **An `https://` agent is dialled with the stored authority, through `fetch_snapshot` and never a + hand-rolled `client.get`** (phase 24 task 03). The module-scoped `agent_tls` fixture reads the + `certs.*` rows out of the same SQLite file the app uses — through a read-only `sqlite3` URI whose + `set_meta`/`delete_meta` **raise**, because "live tests do not write" includes not minting or + repairing an authority from a test — and hands `collectors.agent_tls_map` the material + `certs.write_material` laid down, exactly as `app.build_hub` does. The failure this removes is the + misleading kind: a raw `get` against a healthy mTLS agent dies with `CERTIFICATE_VERIFY_FAILED: + self-signed certificate in certificate chain`, which reads like a dead host and is really a test + that forgot to present a certificate. No store file, or no authority in it, skips — the same + partial-environment rule as an unreachable host. - The agent's real-`/proc` test (`tests/test_agent.py`) touches only the box running pytest, so it stays in the default run and is *not* marked `live`. - `tests/auth_contract.py` is why the HTTP suite can start a server at all: every app in a test @@ -1346,6 +1428,29 @@ branch `main`); there is no CI on it, so `bash .agents/validate.sh` is the only faked, which is how a `0640` mode, a byte-identical bundle and the https `--status` probe are proven without a root shell or a host. **A `0600` in a task file is not a fact about this repo**: the agent opens its own bundle and re-reads it for the denylist, so the group bit is load-bearing. +- **A bundle source is validated once, whichever source it is** (phase 24 task 02). The + installer's `--fetch-bundle` is tested through the same `served_dashboard` socket as the export, + because the claim is the same claim: the file a host installs is the route's bytes. The pins that + go with it are the ones a second install path would break — byte-identical `0640` install with + the unit's TLS lines; `issued: false`, a 401 and a 200-with-HTML each refused with their own + sentence, no stub call logged and no file written (the stub log is the proof the refusal came + *before* the root boundary); a temp `TMPDIR` left empty on the success path and on every refusal + path, including the one where the shape check is what stopped the run; the three flags + all-or-nothing and refused beside a local `--cert-bundle`, beside `--status`/`--uninstall`; and + one label-aware validator, pinned by demanding that the local path still reads `--cert-bundle …` + and never names the flag that was not used. +- **A script that is an HTTP client is tested over a real socket** (phase 24). The dev bundle + exporter is run as a subprocess against a real `create_app(...)` served by uvicorn on + `127.0.0.1:0` in a thread, not through `TestClient`: what has to be proven is that it signs in + through the public route, reads the session token back out of a `Set-Cookie` header, and sends it + as `Authorization: Bearer` — none of which a transport-level test client has to serialise. The + pins that go with it are the ones that keep the artifact honest: the file byte-identical to the + route's bytes, `0600` (including the re-`chmod` of a file somebody loosened), no file and one + note for `issued: false` and for `configured: false`, exit 2 with nothing written for a refused + sign-in, **no mint** (the serials, revisions and root are the same after the run), and no PEM, + token or password in the output. The dev-server composition is pinned on the served text too — + export after the readiness probe, failure as `|| echo` and no `return 1` — because "it runs + later, and never fails `start`" is the part a refactor loses. - **A live measurement is re-derivable, so a verdict can be re-checked without re-watching** (phase 23). `scripts/watch-llm-window.py` writes the series it judged (value, the `Δ` the payload carried, the engine's own busy count, the raw token counter) beside its verdict, and @@ -2086,19 +2191,73 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl `/snapshot`. So the operator's `--force` run lands the new rationale prose and changes nothing systemd acts on (the three captures and the exact probe list are in `data/logs/NOTES-17-installer-repro.md`; no live change was made while pinning this phase). Two facts that came out of reading the live host, both load-bearing for the - next install: the sudo-granted `/home/ops/agent-install` snapshot is the Oct-18 file (no + next install: the sudo-granted `/home/ops/agent-install` path held the Oct-18 file (no `--as-root`, no `--caps`, no `--force`, no `--cert-bundle`, still nine `DeviceAllow` lines) and - it **refuses** the recipe with `unknown argument: --as-root` and exit 2 before `need_root`, so - that path cannot apply this recipe at all until the repo's script is re-staged — while a - re-run of it *without* the flags is still the silent clobber the guard exists for. (The other + it **refused** the recipe with `unknown argument: --as-root` and exit 2 before `need_root`, so + that path could not apply this recipe at all — while a re-run of it *without* the flags was + still the silent clobber the guard exists for. **Resolved on 2026-10-21:** phase 24 task 03 + staged the repo's script and agent 1.1.0 into `/home/ops/agent-install/` on all three hosts + (the directory is `ops`-owned, so no root needed) before running the granted path, and left the + snapshot beside it as a file sudo will not execute. The rule that came out of it is in the + fleet record above: re-stage first, because a path grant is only as current as the file. (The other fact from that read — an applied change reaches the file but not the running process until something restarts it — is in the Deployment paragraph, and README's rollout now restarts.) `tests/fixtures/smartctl_driveripper_*.txt` (captured via `sudo`) remain the ground truth for the drives' shape. -11. **The CA is durable, the collector dials with it, and no bundle is installed with one** - (phase 15 tasks 01-05, landed 2026-10-20). `certs.py` mints the CA and the dashboard's client - certificate into `meta` on the first boot of a database — verified against the running dev - store: the five `certs.*` rows exist +11. **The fleet runs on installed bundles; what is left is rotation hygiene** + (phase 15 tasks 01-05 landed 2026-10-20; phase 24 task 03 rolled the fleet out 2026-10-21). + The rollout is done and verified: gateway, deskwork and driveripper each run `agent/host_agent.py` + **1.1.0** with `/etc/dashboards-agent.certs` installed `0640 root:dashboards-agent`, their units + carry `--cert-bundle` + `RuntimeDirectory=dashboards-agent` (and keep `--token-file`), + `HOST_TARGETS` is `https://` for all three with no userinfo, `agent.bundle_rev` reads `2` on + every host and equals what `/api/certs` issued, and the nine sources are `ok`. The transcript is + `data/logs/NOTES-24-rollout.md`, and task 04 re-read the whole end state **read-only** afterwards + (`data/logs/verify-24-mtls.py` → `24-task04-endstate.txt`, plus the rendered proof + `data/logs/24-task04-verify.cjs` → `24-task04-{certs,disks,system}.png`): nine sources `ok`, + `status: active` with `deployed_bundle_rev == bundle_rev` on all three rows, `agent.tls` + + `agent.bundle_rev` + `agent.version 1.1.0` in every host payload, driveripper's 16 drives and + deskwork's `nvme0n1` reporting temperature, hours and wear, deskwork's GPU card drawn on + `/system`, gateway still absent from `/disks` with `disks:gateway` still `ok`, no `http://` and + no userinfo in any URL the dashboard serves, and **no transport or certificate row in + `/api/summary`'s alerts**. It revoked nothing: the standing negative control for "a revoked + serial is refused" remains phase 15's `data/logs/verify-15-tls-e2e.py` (transcript + `verify-15-tls-e2e.out`) plus the two suite pins — `tests/test_collectors.py`'s + `test_a_revoked_agent_is_refused_before_anything_is_dialled` and `tests/test_api.py`'s + `test_a_revoked_agent_fails_both_of_its_sources_without_a_dial`. + + The **docs** are part of this end state, because a fleet comes back to `http://` through a + setup file somebody copies: `tests/test_deploy_scripts.py` now pins that `README.md`, + `AGENTS.md` and both tracked env templates name the registration trio (`--fetch-bundle`, + `--cert-name`, `--session`), and that every `HOST_TARGETS` example line in the templates is + `https://` with no userinfo (an empty `HOST_TARGETS=` stays allowed — unconfigured is not + misconfigured). The token-mode dial itself is unchanged and still documented as the + compatibility path; what may not come back is it being the shape a template seeds. + + Two things that gap learned belong with this record, because both bite before anybody reads a + dashboard: + + * **A `certs.root.cert` row older than the SKI/AKI fix signs chains nobody can verify.** "A + stored row always wins" is the invariant that keeps a CA from rotating with the process, and + its cost is that a root minted before the extension keeps being used forever: a leaf issued + afterwards carries a correct AKI pointing at an identifier its issuer does not have, and + `ssl.create_default_context()` — strict mode, the context `AgentTls.ssl_context()` builds — + answers `CERTIFICATE_VERIFY_FAILED: Missing Subject Key Identifier`. The agent is healthy, + `--status` answers over TLS, and the dashboard calls the host unreachable. The repair is + `POST /api/certs/root/rotate`, a dashboard restart (the CA and client pair are startup + material) and a fresh bundle per host — never `DELETE FROM meta`, which throws away the + authority instead of replacing it. Pinned by + `tests/test_certs.py::test_a_stored_root_without_an_identifier_still_boots_and_root_rotate_repairs_it`, + which also pins the other half: reading such a table must **not** fail closed, because + `/certs` is the page that fixes it. + * **These bundles came from the dev-box authority.** The fleet trusts the `certs.*` rows in the + database `./scripts/dev-server.sh` runs against, and nothing else dials these agents (a + `ss` sweep on each host saw only the dev box on 9120). A second deployment with its own + authority cannot read this fleet until it carries the same root or re-issues and reinstalls; + that is a deployment-repo decision, and the honest version is one authority per fleet. + + Everything below is the phase-15/24 history that led here. + `certs.py` mints the CA and the dashboard's client certificate into `meta` on the first boot of + a database — verified against the running dev store: the five `certs.*` rows exist and did not change across two restarts — signs per-host server certificates into `certs.agents`, and does reissue, revocation, and root rotation. Task 02 gave `agent/host_agent.py` its TLS mode (see the agent bullet above: one installed bundle, the @@ -2109,9 +2268,22 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl socket opens** — spy-pinned in `tests/test_collectors.py` and again through the real `build_hub` wiring in `tests/test_api.py`, where the revocation is written into the same `meta` table the running app reads. So both halves are proven against each other with no network and - no hardware. What is still missing is an **installed** bundle: `HOST_TARGETS` is `http://` for - every host here and no agent runs `--cert-bundle`, so deleting `certs.*` costs nothing today - and everything once bundles are placed (which is what the reset message warns about). Proving + no hardware. What phase 15 left missing — an **installed** bundle, which is what makes + `DELETE FROM meta` the expensive command the reset message warns about — is what task 03 + supplied: deleting `certs.*` now costs the fleet's identity, and every host has to be + re-bundled. Phase 24 + task 01 has since closed the **dev-side** half of that sentence: `./scripts/dev-server.sh + restart` exports every issued bundle into gitignored `data/certs/.bundle` at `0600`, so + putting one on a host is an `scp` plus `install-agent.sh --cert-bundle` rather than a hand-run + signed `curl` (`scripts/dev-certs-export.py`, invariant 6's export rules, pinned by + `tests/test_deploy_scripts.py` against a real uvicorn-served `create_app`). Task 02 has since + removed even that hand-run from the rollout: `install-agent.sh --fetch-bundle --cert-name + --session ` fetches `GET /api/certs//bundle` during the install and feeds + the bytes to the unchanged validator + `0640` install + TLS-unit path, refusing a 401, an + unissued name or a non-bundle body before it writes anything, with the session token absent from + every artefact the run leaves (pinned the same way, over the same socket) — which is the path + all three hosts were installed through, over an SSH reverse tunnel because no monitored host + can route back to a laptop. Proving the dial also found one defect in task 01's output: the leaves carried no `AuthorityKeyIdentifier`, and OpenSSL's strict mode — which is what Python's `ssl.create_default_context()` turns on, and that is the context this dial builds — refuses @@ -2155,23 +2327,25 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl followed by `agent certificate revoked` on both jobs with no dial, the dashboard's own client identity untouched by a *host* revoke (the list is serials, not hosts), and a revoked **client** serial refused 403 by the running agent on `/health` too the moment the re-sealed bundle is in - place (`data/logs/verify-15-tls-e2e.py`, transcript `verify-15-tls-e2e.out`). What is left of - the phase is the live rollout (07): the fleet's agents still run token mode, so `HOST_TARGETS` - stays `http://` here until somebody installs a bundle on a host — which needs root on that host - (`ops` has no installer grant on `deskwork`, and `driveripper`'s unit is reproducible only with - phase 17's `--as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN --force` line, gap 10), and flipping - the URL before the bundle is installed turns that host red by construction. - Re-checked from the dev box the same day, and worth knowing before that rollout: all three - agents answer `/health` with `version` **1.0.0** (`agent_version` is the `/snapshot` spelling), - so none of them carries task 02's TLS code either — the rollout ships the repo's - `agent/host_agent.py` (1.1.0) *together with* the - bundle, never the bundle alone. `driveripper`'s installed copy differs from the repo's by - exactly that TLS block; its `_smart_pass` and the phase-16 budget scale-up are already there. - Copying 1.1.0 into place is not an `install-agent.sh` re-run on that host (gap 10) — even - though `ops` there is sudo-granted a staged copy of the installer (see the fleet record above), - which is the one route that must not be taken until that snapshot is re-staged from this repo: - the Oct-18 copy predates phase 17's template *and* its re-run guard, so it would rewrite that - unit silently — and no live change was made while pinning phase 16. + place (`data/logs/verify-15-tls-e2e.py`, transcript `verify-15-tls-e2e.out`). + What was left was the live rollout, and phase 24 task 03 ran it on 2026-10-21, host by host, in + the order that keeps every number honest: install **1.1.0 together with the bundle** (never the + bundle alone — the installed agents were 1.0.0 and do not know `--cert-bundle`), restart the + unit, let the CA-only `--status` probe answer, and only *then* flip that host's URL, because a + URL flipped before its bundle is installed turns the host red by construction. All three: the + trivial `gateway`, the drop-in-carrying `deskwork`, and `driveripper` with the gap-10 line + (`--as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN --force`), whose dry-run diff moved comments and + the TLS lines only — `User=root` and `CapabilityBoundingSet=CAP_SYS_RAWIO CAP_SYS_ADMIN` came + through byte-identical, and all 16 drives read live on both sides of the rewrite (same single + `nvme5n1` alarm chip). Every host ran the installer **twice**, once without `--force` to read + what task 17's guard would refuse before letting it apply. Staging the repo's script and agent + into the sudo-granted `/home/ops/agent-install/` path was step one on all three (gap 10), and + fetching was reached through an SSH reverse tunnel because no monitored host can route back to + the dev box. `data/logs/NOTES-24-rollout.md` is the transcript: commands, unit diffs, the + before/after SMART tables, the screenshots, and the two operational traps named at the head of + this gap. What the fleet now *never* does is answer an agent over plain `http` or accept a + bearer token on the certificate path — the TODO's "the client should never allow non-mtls + communication" is the shipped state, and a new host has to be registered the same way. 12. **The nav collapses on a phone; the page content below it does not** (phase 18's verification pass, 2026-10-20). At 360px `/`'s fleet table measures 450-510 px inside a 320 px `section.card`, and `.card { overflow: hidden }` clips the tail columns (`WORST DISK`, @@ -2191,8 +2365,11 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl left-justified rows at every width, it is no longer sticky, the phone menu slides, and both `⚙ Account` and `⚷ Certs` render on all seven signed-in pages including the two they name (task 04, reversing phase 15's "no self-link" rule at the operator's request). The other, - `24_dev_mtls_rollout` (never accepting a non-mTLS agent — fleet gap 11's rollout, half of it), - is still in `todo/`. + `24_dev_mtls_rollout` (never accepting a non-mTLS agent) is closed: its fleet half landed on + 2026-10-21 and task 04 re-read the end state read-only the same day — gap 11. No `http://` + agent URL and no token-mode agent is left on this fleet, and the setup templates no longer + seed one (`tests/test_deploy_scripts.py` keeps it that way). What remains of phase 24 is + ordinary rotation hygiene, not a missing step. ## Locked decisions (and why) diff --git a/README.md b/README.md index 68bb805..c92a8c3 100644 --- a/README.md +++ b/README.md @@ -44,25 +44,46 @@ Developing against the real fleet from a laptop — dev dials exactly the addres dashboard dials, so there is nothing to tunnel and no port to forward: ```bash -cp .env.dev.example .env.dev # your host addresses + agent tokens (chmod 0600) -sudo deploy/install-agent.sh --bind --port 9120 +cp .env.dev.example .env.dev # your host addresses (chmod 0600); this fleet's + # entries are `https://` and carry no userinfo +sudo deploy/install-agent.sh --bind --port 9120 --fetch-bundle \ + --cert-name --session # once per monitored host: the agent as a systemd - # service, printing its bearer token exactly once + # service, registered with the bundle this dashboard + # signs for it (the token-mode install — no + # `--cert-bundle` — still works and prints its bearer + # token exactly once; see *Protecting an agent*) set -a; . ./.env.dev; set +a uv run dashboards # foreground ./scripts/dev-server.sh restart # or backgrounded, with a pidfile + logs +./scripts/dev-server.sh bundles # the bundles it exported, with sizes and modes ``` Local dev signs in with the `DASH_ADMIN_PASSWORD` in that file — the word the template's comment names (`testpass`, what the hermetic suite uses) — and without it a fresh database exits non-zero -for the same reason production does. `.env.dev` is gitignored and `0600`: it holds the agent -tokens and the upsd password too. +for the same reason production does. `.env.dev` is gitignored and `0600`: it holds the upsd +password, and it held the agent bearer tokens until every host of this fleet moved onto an +installed bundle — a TLS `HOST_TARGETS` entry carries no userinfo, so the credential that used to +sit in the URL is now a certificate on the host and a `meta` row here. -Each `HOST_TARGETS` entry carries that token as URL userinfo +`dev-server.sh restart` also **exports the agent bundles** while it is up: one line per +`HOST_TARGETS` name, and each host's stored bundle written to `data/certs/.bundle` at `0600` +(`data/` is gitignored, so no key this writes can be committed). It is a read of +`GET /api/certs//bundle` and nothing else — the authority stays the one in the database — so +the file is byte-identical to the row and ready to `scp` into `install-agent.sh --cert-bundle` — +though the normal path now needs neither: `install-agent.sh --fetch-bundle --cert-name +--session ` downloads that same route on the host during its own install (see +*Protecting an agent, host by host*). A host with nothing issued writes no file and prints the route +that issues one. See *Managing the certificates*. + +Each **token-mode** `HOST_TARGETS` entry carries its token as URL userinfo (`driveripper=http://@:9120`); `parse_targets` moves it into an `Authorization` -header, so `Target.url`, `/api/config` and every error message stay credential-free. No root -shell on a host? `deploy/agent-sudoers` is the one-line grant that lets `ops` run the installer, -and running the installer without root prints that exact line. upsd is dialled at +header, so `Target.url`, `/api/config` and every error message stay credential-free. A host that +runs from an installed bundle is `https://:9120` with **no userinfo at all** — the client +certificate is the credential there, and this fleet is all `https://` today (see +*Protecting an agent, host by host*). No root shell on a host? `deploy/agent-sudoers` is the +one-line grant that lets `ops` run the installer, and running the installer without root prints +that exact line. upsd is dialled at `NUT_HOST`/`NUT_PORT` on the port it really answers on (3051 on driveripper, not the stock 3493) — see the NUT section below for who may connect. @@ -81,6 +102,9 @@ python3 host_agent.py --host 0.0.0.0 --port 9120 --token-file /etc/dashboards-ag python3 host_agent.py --host 0.0.0.0 --port 9120 --cert-bundle /etc/dashboards-agent.certs sudo deploy/install-agent.sh # systemd unit + user + token sudo deploy/install-agent.sh --cert-bundle agent.bundle # ...or the TLS install (bundle from /certs) +sudo deploy/install-agent.sh --fetch-bundle https://dashboards.example.com \ + --cert-name deskwork --session --bind --port 9120 + # ...the same TLS install, bundle fetched in the run sudo deploy/install-agent.sh --as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN # ...or a kernel-gated host (SMART passthrough, below) sudo deploy/install-agent.sh --force # ...only when it must overwrite a unit it would change @@ -123,6 +147,19 @@ one installed before phase 09 has no SMART pass to run. Opt out silently with `- The drive truth this is all developed against lives in `tests/fixtures/smartctl_driveripper_*.txt`: 16 real drives, recorded with serials scrubbed, each one parsed by a test. +`deploy/install-agent.sh --fetch-bundle --cert-name --session ` is the same +TLS install with the download done inside the registration run: it asks +`GET /api/certs//bundle`, writes the answer to a temp file, and hands it to the +identical validator + `0640` install + unit path. The three flags are all-or-nothing, they are +refused together with a local `--cert-bundle` (one install, one bundle source) and with +`--status`/`--uninstall`, and the fetch and the shape check both run **before** the root +boundary — a dashboard that answers 401, or that has issued nothing for that name, stops the run +with its own one-line sentence and no partial install. The session token is a *fetch credential +only*: it is sent on that one request, never printed (the sudoers hint re-quotes the command +without it), never written to the token file, the unit, the bundle or the log, and unset the +moment the bytes are in hand. The temp copy is removed on every exit path, so the installed +`/etc/dashboards-agent.certs` is the only place the fetched key survives. + **TLS mode is `--cert-bundle` (env `AGENT_CERT_BUNDLE`)**: one installed PEM the dashboard signed — this host's server certificate and key, the dashboard's root CA, a `# BUNDLE-REV ` line and the `# REVOKED ` denylist. The agent then serves `https://` and answers every @@ -219,42 +256,101 @@ that the *next poll* refuses to dial (see `collectors.AgentTls` above): | `POST /api/certs//revoke` | marks the identity revoked, and **re-seals every other bundle** because the `# REVOKED` denylist lives inside each of them — so the fleet has new bundles to collect, while their `bundle_rev` does not move (their identity did not change). There is no unrevoke: the fix is a rotate | | `POST /api/certs/root/rotate` | a new root, a new client certificate, every agent re-issued, and `{"ok": true, "restart_required": true}` — the CA and the client pair are startup material, so the table is new at once while the running process keeps dialling with what it loaded. Saying so beats pretending a hot-swap that does not exist | +**During development there is no second authority.** The CA the first boot minted into the +`meta` table is the one everything reads, and `scripts/dev-certs-export.py` is the dev loop's half +of that: `./scripts/dev-server.sh restart` runs it after the readiness probe, it signs in to the +local server, downloads every `HOST_TARGETS` name's bundle through `GET /api/certs//bundle`, +and writes each one to `data/certs/.bundle` at `0600` (`os.open`, so the mode is the file's +birth mode) in gitignored `data/`. Per host it prints exactly one line — the file it wrote with its +size, or `no bundle issued yet — issue one on /certs` — and nothing at all is written for a host +with nothing issued. It is a **read** of the same route the `/certs` download uses, never the +startup path, so a restart cannot rotate an identity: the bytes on disk are the bytes in the row. +A sign-in that fails is exit 2 and a `!` line (a `429` says so with its wait), and +`dev-server.sh` prints that as a warning rather than failing `start`, because the server is +provably up and the authority is untouched. `status` — or `dev-server.sh bundles` — lists what is +there with sizes and modes. + ### Protecting an agent, host by host -Three steps, and they are order-dependent because the dashboard picks its dial from the URL: +Four steps, and they are order-dependent because the dashboard picks its dial from the URL. **A URL +flipped before its bundle is installed turns that host red by construction**, so the URL flips +last, after the probe answers. ```bash -# 1. one bundle per host, from the dashboard that owns the authority (a signed-in GET; it is -# byte-identical to the stored row and it carries that host's private key, so `curl -o` it -# somewhere private rather than `cat`ting it into a shell history) -curl -s -H "Authorization: Bearer $SESSION" -o agent-deskwork.bundle \ - http://127.0.0.1:8090/api/certs/deskwork/bundle +# 0. issue the host a bundle if it has none: POST /api/certs//rotate (the Rotate button on +# /certs) is also how a host gets its *first* one, and `GET …/bundle` then answers the bytes. +# An unissued name is what makes step 1 say "no bundle issued for ". -# 2. install it beside the token — as root on the host, 0640 root:dashboards-agent. The same run -# copies the repo's `agent/host_agent.py`, which is how a pre-1.1.0 agent gains TLS mode at all -# (an older file does not know the `--cert-bundle` flag). The unit gains `--cert-bundle` and its -# RuntimeDirectory — and then restart the service as root, because the installer's -# `systemctl enable --now` only *starts* a unit, so an already-running agent keeps serving its -# old mode until something restarts it. -scp agent-deskwork.bundle -ops:/tmp/ -ssh -ops sudo /tmp/install-agent.sh --cert-bundle /tmp/agent-deskwork.bundle --force +# 1. register the host: one run installs the agent *and* fetches its bundle from the dashboard +# that owns the authority. The name is the HOST_TARGETS name; the token is a signed-in +# session and it buys this one GET — it is not stored on the host. As root, 0640 +# root:dashboards-agent, after a shape check on the fetched bytes that runs before anything +# is written, so a 401 or an unissued name leaves this host exactly as it was. +# Stage the repo's script and agent first (`../agent/host_agent.py` is where the installer +# reads the agent from, and the fleet's sudo grant names a path, not a checkout): +scp deploy/install-agent.sh -ops:/home/ops/agent-install/deploy/ +scp agent/host_agent.py -ops:/home/ops/agent-install/agent/ +ssh -ops sudo /home/ops/agent-install/deploy/install-agent.sh --bind --port 9120 \ + --fetch-bundle https://dashboards.example.com --cert-name deskwork --session "$SESSION" \ + --force + +# the fallback, for a host that cannot reach the dashboard: the signed-in GET by hand (its +# bytes are identical to the stored row and it carries that host's private key, so `curl -o` +# it somewhere private rather than `cat`ting it into a shell history — and against a local dev +# server this file already exists, written by `./scripts/dev-server.sh restart`), then +# --cert-bundle instead of --fetch-bundle. +# curl -s -H "Authorization: Bearer $SESSION" -o agent-deskwork.bundle \ +# http://127.0.0.1:8090/api/certs/deskwork/bundle +# scp agent-deskwork.bundle -ops:/tmp/ +# A *dev* dashboard on a laptop is usually unreachable from a monitored host in the other +# direction too (the tunnel route is one-way). An SSH reverse tunnel keeps the registration +# path rather than the scp: `ssh -R 18099:127.0.0.1:8090 -ops` and then +# `--fetch-bundle http://127.0.0.1:18099`. + +# 2. restart it as root: the same run copies the repo's `agent/host_agent.py`, which is how a +# pre-1.1.0 agent gains TLS mode at all (an older file does not know the `--cert-bundle` +# flag), and the unit gains `--cert-bundle` and its RuntimeDirectory — but `systemctl enable +# --now` only *starts* a unit, so an already-running agent keeps serving its old mode until +# something restarts it. ssh -ops sudo systemctl restart dashboards-agent -ssh -ops sudo /tmp/install-agent.sh --status # https, with the CA and no client cert +# the CA-only https probe — pass the SAME --bind the unit uses, because the probe takes its +# address from the flags and would otherwise dial 127.0.0.1 and report "no answer" for an +# agent bound to a LAN address that is in fact healthy. +ssh -ops sudo /home/ops/agent-install/deploy/install-agent.sh --bind --port 9120 --status # 3. point that host's HOST_TARGETS entry at https with NO userinfo token — the certificate is # the auth now — and restart the dashboard. `/certs` should then read issued == deployed rev. export HOST_TARGETS="deskwork=https://:9120 …" ``` -`--force` in step 2 is not a shortcut: installing a bundle *is* a change to that host's unit (the +`--force` in step 1 is not a shortcut: installing a bundle *is* a change to that host's unit (the `--cert-bundle` flag and its `RuntimeDirectory`), so without it the run prints the diff and stops -by design — read it, then re-run with the flag. The same rule covers every change the installer -applies, which is why the `--status` line above comes *after* the restart: the probe reads the -unit file to decide between http and https, so pointing it at a process that has not been -restarted yet reports "no answer" for a bundle that installed perfectly. -`install-agent.sh --bind … --port 9120` prints the `https://:9120` line for a bundle install, -so step 3 is a copy rather than a guess. A host left on `http://` keeps working over its bearer -token — the rollout is per host, and one revoked or expired certificate never blinks the others. +by design — read it, then re-run with the flag. Reading it first is cheap and worth doing on a host +with a hand-fixed unit (run the same command once without the flag, study the diff, run it again +with it). The same rule covers every change the installer applies, which is why the `--status` line +above comes *after* the restart: the probe reads the unit file to decide between http and https, so +pointing it at a process that has not been restarted yet reports "no answer" for a bundle that +installed perfectly. `install-agent.sh --bind … --port 9120` prints the `https://:9120` line +for a bundle install, so step 3 is a copy rather than a guess. A host left on `http://` keeps +working over its bearer token — the rollout is per host, and one revoked or expired certificate +never blinks the others. + +Two things the first full rollout learned, both cheap to hit and expensive to guess at: + +* **A CA row older than the identifier fix must be rotated before any bundle is installed.** A + root minted without a `SubjectKeyIdentifier` is still readable, still starts, and still signs — + but `ssl.create_default_context()` (the strict mode every dial here builds) rejects every chain + it produces, with `CERTIFICATE_VERIFY_FAILED: Missing Subject Key Identifier`, so the host comes + up healthy on `https` and reads `unreachable (ConnectError)` from the dashboard. `POST + /api/certs/root/rotate` is the repair — a new root with its identifiers, a new client identity, + every agent re-issued, then a dashboard restart and a fresh bundle per host. Never a + `DELETE FROM meta` for this: the rotate button exists precisely so the operator keeps the + authority in one place. +* **Rolling back is not just the URL.** A TLS-mode agent serves `https` only, and the unit keeps + its `--token-file` line (so the token survives on the box, ready for this). Restoring an old + `http://@…` entry does not bring the host back on its own: the `--cert-bundle` line has to + come out of the unit *and* `/etc/dashboards-agent.certs` has to go, because the installer reuses + an installed bundle on a run that was not given the flag. A name that is not in `HOST_TARGETS` answers `404 {"configured": false}` **before** any signing, any dial, and any `meta` write: a typo that minted a key would leave a certificate nobody can install @@ -476,7 +572,7 @@ documentation and is never loaded, and `scripts/dev-server.sh` exports `.env.dev | Variable | Meaning | | ------------------------------------- | -------------------------------------------------------------- | -| `HOST_TARGETS` | `name=url` per host running the agent; a bearer token may ride in the URL userinfo (`http://TOKEN@host:9120`). An `https://` URL is dialled as a *protected* agent instead: verified against the dashboard's own root CA and presented with its client certificate, with a revoked host refused before the dial (`certs.py`, `collectors.AgentTls`) | +| `HOST_TARGETS` | `name=url` per host running the agent. The shape a new host gets is `https://host:9120` with **no userinfo** — a certificate-authenticated agent, verified against this app's own root CA and presented with its client certificate, with a revoked host refused before the dial (`certs.py`, `collectors.AgentTls`). The compatibility shape is a token-mode agent's URL userinfo (`http://TOKEN@host:9120`), moved into an `Authorization` header by `parse_targets` so it never reaches `/api/config` or an error message | | `GUFO_BASE_URL` / `VLLM_BASE_URL` | LLM engines; unset means *not deployed here*, and its half of the page disappears | | `RATE_WINDOW_SECONDS` | the **host's** Δ window (default 10) — bytes/sec and jiffies→CPU %, measured over what a 10 s poll can fill; each value carries the window it actually got, and `/system`'s footnote prints that, not this default. A window the poll loop cannot honour — the ~0.2 s catch-up after a fetch slower than the interval, or a base older than 2× the window — answers one `n/a` instead of a spike, and a rate 8× above the last one is held until a window measured from a **different** sample confirms it (phase 13). The LLM rates do **not** use it: they are measured over `LLM_INTERVAL_SECONDS` (phase 23) | | `LLM_INTERVAL_SECONDS` | the `/llm` poll cadence (default 2 s) **and** the Δ window behind every token rate — one estimator per measuring cadence, so a slot that starts reads steady within two polls instead of ramping over five. Slowing the poll slows the window with it; there is no separate LLM window knob, and a window narrower than the interval would answer `n/a` every poll | @@ -806,7 +902,10 @@ certs.py ─ root CA + client cert, minted once at startup (unreadable row → n uv run pytest # unit + API tests, fully mocked, no network uv run pytest --cov # coverage gate set -a; . ./.env.dev; set +a # the live suite reads the same env as the app -uv run pytest -m live --live # optional: real fleet / engines / upsd +uv run pytest -m live --live # optional: real fleet / engines / upsd — an + # `https://` HOST_TARGETS entry is dialled with the + # authority in DASH_DB (read-only), so a box with no + # store skips the host tests instead of failing them uv run pytest -m store_perf --store-perf -s # optional: the store's hot-path timings (phase 14) uv run python scripts/store_perf_probe.py # the same measurements as a report in data/logs/ DASH_WATCH_SECONDS=1500 uv run python scripts/watch-store-perf.py @@ -885,7 +984,12 @@ completes a real client-certificate handshake with the material a startup would `tests/test_agent.py` runs the matching TLS matrix against the real agent handler, and `tests/test_deploy_scripts.py` runs the installer in a sandbox where `install`/`chown` are faked, so the `0640` mode, the byte-identical bundle and the `--status` probe (against a live TLS agent) are -proven without a root shell. +proven without a root shell. The same file runs `scripts/dev-certs-export.py` against a real +`create_app(...)` served by uvicorn on a loopback port (phase 24) — a thread and a socket rather +than `TestClient`, because the unit under test is an HTTP client that reads its session out of a +`Set-Cookie` header — and pins the export against the route: `0600`, byte-identical, one line per +host, no file for an unissued one, no mint and no PEM in the output, and a wrong password exiting +non-zero having written nothing. `tests/store_perf.py` is the second opt-in marker's machinery (`live` is the other). It seeds a **production-shaped diff --git a/deploy/install-agent.sh b/deploy/install-agent.sh index caec073..e258bd0 100755 --- a/deploy/install-agent.sh +++ b/deploy/install-agent.sh @@ -4,6 +4,9 @@ # sudo ./deploy/install-agent.sh # loopback only (read it locally) # sudo ./deploy/install-agent.sh --bind 10.0.0.5 # serve the LAN, token required # sudo ./deploy/install-agent.sh --cert-bundle agent.bundle # serve TLS (phase 15) +# sudo ./deploy/install-agent.sh --fetch-bundle https://dashboards.example.com \ +# --cert-name deskwork --session --bind 10.1.0.51 --port 9120 +# # registration fetches its own bundle # sudo ./deploy/install-agent.sh --as-root --caps CAP_SYS_RAWIO,CAP_SYS_ADMIN # # a kernel-gated host (see below) # ./deploy/install-agent.sh --status # unit state + /health, no root @@ -23,7 +26,7 @@ # The certificate bundle (--cert-bundle , fetched by the operator from the dashboard's # GET /api/certs//bundle or the /certs page) is the TLS install: one PEM holding this # host's server certificate and key, the dashboard's root CA, a `# BUNDLE-REV ` line and -# the revocation denylist. This script never dials the dashboard for it. The bundle is +# the revocation denylist. With a local file this script dials nothing at all. The bundle is # validated before anything is touched — a bundle the agent could not load must stop the # install with its reason, not leave a unit that exits at startup — and then installed to # /etc/dashboards-agent.certs, the token-file idiom: the agent opens the bundle *itself* at @@ -35,6 +38,31 @@ # the agent exempts for probes. Re-running without the flag leaves an installed bundle in # place, the token-reuse idiom. # +# --fetch-bundle is that same install with the download done for you, because registration is +# the moment a client should get its identity (TODO.md:3 — "distribute them to clients during +# registration so we never need non-mtls servers"). It pulls +# GET /api/certs//bundle and hands the bytes to the path above, so there is no +# curl-a-key-across-the-network step for an operator to forget; the shape check, the 0640 +# install, the unit's TLS lines and the --status probe all then run on the fetched bytes +# unchanged. +# +# --fetch-bundle URL the dashboard's own base URL — this is the one dial this script makes +# --cert-name NAME the HOST_TARGETS name the certificate is issued for +# --session TOKEN the signed-in session token, a **fetch credential only** +# +# The three are all-or-nothing, and they are refused together with a local --cert-bundle: one +# install, one bundle source. The token is used for that one request and nothing else — never +# printed, never written to the token file, the unit, the bundle or a log, and unset as soon as +# the bundle is in hand (it is a bearer credential for a signed-in surface, so `ps` exposure +# for the seconds of the fetch is the accepted cost on a single-admin box; the file it buys is +# the artifact, not the credential). The fetch and the shape check both run before the root +# boundary, so a dashboard that answers 401 — or says nothing is issued for that name — stops +# the run with its own sentence and no partial install: +# +# sudo ./deploy/install-agent.sh --bind --port 9120 \ +# --fetch-bundle https://dashboards.example.com --cert-name \ +# --session +# # The agent is read-only by construction (it answers GET and writes nothing), runs as its own # system user (unless --as-root says otherwise, below), and gets the usual systemd lockdown. It # still needs to read /proc, /sys and statvfs(), which the hardening below leaves alone. @@ -111,6 +139,17 @@ PORT="9120" TOKEN="${AGENT_TOKEN:-}" ROTATE=0 BUNDLE_SRC="" +# Which flag the bytes in $BUNDLE_SRC came from, so a shape refusal names the flag the operator +# typed. Empty means `--cert-bundle` (the default inside `validate_bundle`). +BUNDLE_LABEL="" +# --fetch-bundle: the three parts of "fetch my bundle in this install run". All three or none +# (checked below, before any write), and never together with a local --cert-bundle. +FETCH_URL="" +CERT_NAME="" +SESSION_TOKEN="" +# The temp file the fetched bytes land in, and the trap that guarantees it is gone whatever the +# rest of the run does. Its name is never printed. +FETCH_TMP="" # --as-root / --caps: the kernel-gate variant of the unit (phase 17 task 02). Both off means # the default install — dedicated user, empty CapabilityBoundingSet — byte-for-byte what this # script wrote before the flags existed. @@ -134,6 +173,9 @@ while [ $# -gt 0 ]; do --token) TOKEN="$2"; shift 2 ;; --rotate-token) ROTATE=1; shift ;; --cert-bundle) BUNDLE_SRC="$2"; shift 2 ;; + --fetch-bundle) FETCH_URL="$2"; shift 2 ;; + --cert-name) CERT_NAME="$2"; shift 2 ;; + --session) SESSION_TOKEN="$2"; shift 2 ;; --as-root) AS_ROOT=1; shift ;; --caps) [ $# -ge 2 ] || { @@ -167,6 +209,43 @@ if [ -n "$CAPS" ] && [ "$AS_ROOT" -ne 1 ]; then exit 1 fi +# The same reasoning for the fetch: a flag combination is a mistake about what to install, and +# it gets its own sentence before anything else happens. Three parts because a URL without a +# name is a route nobody can guess and a name without a token is a 401 the operator would have +# to diagnose — so all three or none. And `--fetch-bundle` beside `--cert-bundle` is two +# answers to "where is the bundle?", which is a contradiction rather than a preference. +FETCH_GIVEN=0 +for _flag in "$FETCH_URL" "$CERT_NAME" "$SESSION_TOKEN"; do + # an `if`, not `[ -n … ] && FETCH_GIVEN=…`: under `set -e` a failing test as the last command + # of the loop body would end the run on the first unset flag instead of counting them. + if [ -n "$_flag" ]; then + FETCH_GIVEN=$((FETCH_GIVEN + 1)) + fi +done +if [ "$FETCH_GIVEN" -gt 0 ] && [ "$FETCH_GIVEN" -lt 3 ]; then + echo "--fetch-bundle, --cert-name and --session go together (see --help)" >&2 + exit 1 +fi +if [ "$FETCH_GIVEN" -eq 3 ] && [ -n "$BUNDLE_SRC" ]; then + echo "--fetch-bundle and --cert-bundle are two bundle sources; pick one" >&2 + exit 1 +fi +# The fetch belongs to an install. `--status` probes a port and `--uninstall` deletes files; a +# run of either would fetch a private key for nothing and leave it in a temp file. +if [ "$FETCH_GIVEN" -eq 3 ] && [ "$MODE" != "install" ]; then + echo "--fetch-bundle needs an install (it means nothing to --status or --uninstall)" >&2 + exit 1 +fi + +cleanup_fetch() { + # The fetched bytes are somebody's private key. Registered as an EXIT trap when the fetch + # starts, so no path — a refused shape, a refused unit re-run, a failing systemctl — leaves + # them behind in /tmp. Runs before the trap is armed too, and does nothing then. + if [ -n "$FETCH_TMP" ] && [ -e "$FETCH_TMP" ]; then + rm -f "$FETCH_TMP" + fi +} + need_root() { if [ "$(id -u)" -ne 0 ]; then # The dev workflow deploys over ssh as the unprivileged `ops` account, so "run as root" @@ -269,23 +348,28 @@ validate_bundle() { # stdlib; a monitored host has no cryptography) and the dashboard is the only signer, so # a second copy of this shape check would only ever disagree about nothing. The three PEM # blocks are positional: agent certificate, agent key, root CA certificate. + # + # $2 names the flag the bytes came from, so a refusal says what the operator typed rather than + # a temp file they did not. One validator, two bundle sources: a local --cert-bundle file and + # the fetched bytes of --fetch-bundle have to satisfy exactly the same shape, because the agent + # cannot tell them apart either. if ! command -v python3 >/dev/null 2>&1; then - echo "--cert-bundle needs python3 (3.10 or newer) to be validated" >&2 + echo "${2:---cert-bundle} needs python3 (3.10 or newer) to validate the bundle" >&2 exit 2 fi - if python3 - "$1" <<'PY' 1>&2 + if python3 - "$1" "${2:---cert-bundle}" <<'PY' 1>&2 import re import sys -path = sys.argv[1] +path, flag = sys.argv[1], sys.argv[2] try: with open(path, encoding="utf-8", errors="replace") as handle: text = handle.read() except OSError as exc: - print(f"--cert-bundle {path} cannot be read: {exc.strerror or exc}") + print(f"{flag} {path} cannot be read: {exc.strerror or exc}") raise SystemExit(1) if not text.strip(): - print(f"--cert-bundle {path} is empty") + print(f"{flag} {path} is empty") raise SystemExit(1) revision = None for line in text.splitlines(): @@ -299,7 +383,7 @@ for line in text.splitlines(): revision = None break if revision is None: - print(f"--cert-bundle {path} has no `# BUNDLE-REV ` line — that line is how the" + print(f"{flag} {path} has no `# BUNDLE-REV ` line — that line is how the" " dashboard tells the bundle it issued from a stale one, so a bundle without it" " is not one this app signed") raise SystemExit(1) @@ -307,7 +391,7 @@ blocks = re.findall(r"-----BEGIN [A-Z0-9 ]+-----.*?-----END [A-Z0-9 ]+-----", te certificates = [b for b in blocks if b.startswith("-----BEGIN CERTIFICATE-----")] keys = [b for b in blocks if not b.startswith("-----BEGIN CERTIFICATE-----")] if len(certificates) != 2 or len(keys) != 1: - print(f"--cert-bundle {path} must hold exactly the agent certificate, the agent key and" + print(f"{flag} {path} must hold exactly the agent certificate, the agent key and" f" the root CA certificate — found {len(certificates)} certificates and" f" {len(keys)} keys. Re-download it from GET /api/certs//bundle") raise SystemExit(1) @@ -318,15 +402,118 @@ PY exit 2 } +fetch_bundle() { + # Registration is where a client gets its identity, so the install run pulls this host's + # bundle from the dashboard instead of an operator carrying a file across the network by hand + # (phase 24: "distribute them to clients during registration so we never need non-mtls + # servers"). Two properties are load-bearing, and both are the validator's reasoning. + # + # * **Before `need_root`.** A dashboard that answers 401, or says nothing is issued for that + # name, or does not answer at all, stops the run here with its own sentence — no user, no + # files, no unit. A refusal is not a sudoers problem, and a half-install is worse than none. + # * **Then the existing shape check.** The bytes land in a temp file and become `$BUNDLE_SRC`, + # so the PEM check, the `0640` install, the unit's TLS lines and the `--status` probe see a + # fetched bundle exactly as they see a downloaded one. One install path, two sources. + # + # python3 is already required (the shape check and the TLS probe both use it), so this adds no + # dependency and no assumption that a monitored host has `curl`. The token reaches python in + # the **environment**, never on its argv: argv is world-readable through `ps`, which is the + # same reason the agent's own token is a file rather than a flag — a second copy of a bearer + # credential on a command line would only widen what `ps` shows. + if ! command -v python3 >/dev/null 2>&1; then + echo "--fetch-bundle needs python3 (3.10 or newer) to fetch the bundle" >&2 + exit 2 + fi + FETCH_TMP="$(mktemp "${TMPDIR:-/tmp}/dashboards-agent-bundle.XXXXXX")" + trap cleanup_fetch EXIT + if ! FETCH_SESSION="$SESSION_TOKEN" python3 - "$FETCH_URL" "$CERT_NAME" "$FETCH_TMP" <<'PY' 1>&2 +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + +base, name, destination = sys.argv[1].rstrip("/"), sys.argv[2], sys.argv[3] +token = os.environ.get("FETCH_SESSION", "") +route = f"/api/certs/{urllib.parse.quote(name, safe='')}/bundle" +request = urllib.request.Request(f"{base}{route}", + headers={"authorization": f"Bearer {token}"}) +try: + with urllib.request.urlopen(request, timeout=20) as response: + status, body = response.status, response.read() +except urllib.error.HTTPError as refused: + # A refusal is the answer, and it carries the reason: this app's 404 body says whether the + # name is configured and whether anything was issued, which are three different operator + # actions (fix the URL, issue a certificate, re-sign in). + status, body = refused.code, refused.read() +except (urllib.error.URLError, TimeoutError, OSError) as unreachable: + reason = getattr(unreachable, "reason", None) or unreachable + print(f"--fetch-bundle: no dashboard answered at {base}: {reason}") + raise SystemExit(1) + +try: + parsed = json.loads(body.decode("utf-8", "replace")) +except ValueError: + parsed = None +payload = parsed if isinstance(parsed, dict) else None + + +def detail() -> str: + """The dashboard's own sentence, or the first bytes of whatever it sent instead. + + Every refusal this prints is text the app writes for an operator, and no response body of + a *successful* fetch ever reaches the output — only the bytes land in the file. + """ + if isinstance(payload, dict) and isinstance(payload.get("detail"), str) and payload["detail"]: + return str(payload["detail"]) + text = body.decode("utf-8", "replace").strip().replace("\n", " ") + return text[:160] or "no body" + + +if status == 200: + if not body.strip(): + print(f"--fetch-bundle: GET {route} answered 200 with an empty body — nothing to install") + raise SystemExit(1) + with open(destination, "wb") as handle: + handle.write(body) + raise SystemExit(0) +if (status == 404 and isinstance(payload, dict) + and payload.get("configured") is True and payload.get("issued") is False): + # The invariant-3 shape: a configured host with no certificate yet. Its own sentence, because + # the fix is a click on /certs and not a re-typed URL or a new token. + print(f"--fetch-bundle: no bundle issued for {name} — issue it from /certs first" + f" (POST /api/certs/{name}/rotate mints one; GET {route} answered 404)") + raise SystemExit(1) +print(f"--fetch-bundle: GET {route} answered {status}: {detail()}") +raise SystemExit(1) +PY + then + exit 2 + fi + # A fetch credential only: from here the install needs the bytes, not the session. The env + # assignment above was per-command, so the only copy left is the shell variable. + unset SESSION_TOKEN + BUNDLE_SRC="$FETCH_TMP" + BUNDLE_LABEL="--fetch-bundle" + echo " fetched the certificate bundle for $CERT_NAME from $FETCH_URL" +} + # NOT named `install`: bash resolves a function before an external command of the same name, # so an `install()` here turns the `install -d` / `install -m` calls below into infinite # recursion instead of creating $AGENT_DIR. Phase 10 task 02 found exactly that on a live # host: the run looped on the SMART block forever and installed nothing. do_install() { + # Registration fetches its own bundle, before the root boundary for the same reason the shape + # check is there: a dashboard that refuses is answered with its own sentence, and a run that + # cannot get a bundle writes nothing at all. On success this is just another $BUNDLE_SRC. + if [ -n "$FETCH_URL" ]; then + fetch_bundle + fi # Validate before need_root: a bad bundle is refused with its own reason whether or not # this shell is root, so a wrong file says "wrong file" and not "add a sudoers line". if [ -n "$BUNDLE_SRC" ]; then - validate_bundle "$BUNDLE_SRC" + validate_bundle "$BUNDLE_SRC" "$BUNDLE_LABEL" fi need_root --bind "$BIND" --port "$PORT" [ -f "$AGENT_SOURCE" ] || { echo "no agent next to this script: $AGENT_SOURCE" >&2; exit 1; } @@ -534,9 +721,11 @@ none: HOST_TARGETS="hostname=https://$BIND:$PORT" -After a rotate or a root rotation on the dashboard, re-download the bundle and re-run this -script with --cert-bundle; the agent picks a changed denylist up without a restart, but a -new certificate needs one. +After a rotate or a root rotation on the dashboard, fetch the new bundle and re-run this +script: --fetch-bundle --cert-name --session does the download and the +install in one go, and --cert-bundle still works from a file already on this box. The +agent picks a changed denylist up without a restart, but a new certificate needs one, and the +run above only *starts* the unit — restart the service to put the new bundle into effect. NEXT else cat </bundle`), which is + the request-time read (`certs.read_state`) and never the startup one (`ensure_certs`). So + "generate certs during dev" means *export what the first boot minted*: running this against + a dashboard changes no certificate, and a bundle it writes is byte-identical to the row. +* **Not a place a key is committed.** Everything it writes lands under the repo's gitignored + `data/` tree (the default), and a bundle carries that host's **private key**, so the file is + created by `os.open` with `0600` as its birth mode and re-chmodded in case it already + existed looser. Keep `--out-dir` inside `data/`. + +Stdlib only (`urllib`), the way `install-agent.sh` reaches for `python3`: this runs on the dev +box against a loopback URL, and a helper that needs the app's own dependency set to sign in +would be a helper that cannot run before `uv sync`. The session token is read out of the +`Set-Cookie` of the public `POST /api/auth/login` and sent back as `Authorization: Bearer` — +the bearer carrier is the one the gate exempts from CSRF, and it never needs a cookie jar. + +Exit codes: `0` every name resolved to a file or to the "nothing issued" note, `1` a host the +route refused for some other reason, `2` no sign-in (wrong password, no password, no server). +A sign-in failure is never silent, because the operator's next move is `install-agent.sh`. +""" + +from __future__ import annotations + +import argparse +import email.message +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +#: The non-committed tree `TODO.md:3` asks for. `data/` is in `.gitignore`, so no key this +#: writes can be committed by a scoped `git add` of a phase's work. +DEFAULT_OUT_DIR = ROOT / "data" / "certs" +TIMEOUT_SECONDS = 10.0 + +try: # the package owns the cookie name; importing it is what stops that drifting + from dashboards.auth import SESSION_COOKIE +except ImportError: # a bare interpreter outside the project venv can still run this + SESSION_COOKIE = "dash_session" + +#: `--names` accepts the shape `HOST_TARGETS` has (space or comma separated) so the caller can +#: hand over what it already parsed instead of this script re-parsing a config it must not +#: disagree with. +_SPLIT_RE = re.compile(r"[,\s]+") + + +class ExportError(Exception): + """One sentence for the operator: what failed, and where to look next.""" + + +def split_names(values: list[str] | None) -> list[str]: + """The host names to export, from one space/comma-separated string or several.""" + return [part for part in _SPLIT_RE.split(" ".join(values or []).strip()) if part] + + +def _base(url: str) -> str: + """A base URL with no trailing slash, so joining a route path cannot double one.""" + return url.strip().rstrip("/") + + +def _json(body: bytes) -> dict[str, object] | None: + try: + parsed = json.loads(body.decode("utf-8", "replace")) + except ValueError: + return None + return parsed if isinstance(parsed, dict) else None + + +def _detail(body: bytes) -> str: + """The server's own sentence, or the first bytes of whatever it sent instead. + + Every refusal this script can print is one the app writes for an operator + (`wrong password`, `too many failed sign-ins — try again in 120s`) and carries no + credential, which is the only reason echoing it is allowed at all (AGENTS.md invariant 6). + The response body of a *successful* sign-in never reaches the output. + """ + parsed = _json(body) + if parsed is not None and isinstance(parsed.get("detail"), str): + return str(parsed["detail"]) + text = body.decode("utf-8", "replace").strip().replace("\n", " ") + return text[:160] or "no body" + + +def _request( + url: str, + *, + method: str = "GET", + body: bytes | None = None, + headers: dict[str, str] | None = None, +) -> tuple[int, bytes, email.message.Message | None]: + """One HTTP call answered as `(status, body, headers)`. + + urllib raises for every 4xx/5xx, and here a 404 **is** the answer: `{"configured": false}` + and `{"configured": true, "issued": false}` are two different facts about a host. So the + status comes back instead of an exception, and only a call that never reached the dashboard + raises. + """ + request = urllib.request.Request(url, data=body, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS) as response: + return response.status, response.read(), response.headers + except urllib.error.HTTPError as exc: # a refusal from the app: read it, do not raise + return exc.code, exc.read(), exc.headers + except (urllib.error.URLError, TimeoutError, OSError) as exc: + reason = getattr(exc, "reason", None) or exc + raise ExportError(f"no dashboard answered at {url}: {reason}") from exc + + +def _cookie(headers: email.message.Message | None, name: str) -> str: + """The value of one `Set-Cookie` from a response, with no cookie jar.""" + if headers is None: + return "" + for line in headers.get_all("set-cookie") or []: + for chunk in line.split(";"): + key, sep, value = chunk.strip().partition("=") + if sep and key == name: + return value + return "" + + +def sign_in(base: str, password: str) -> str: + """The session token for this password, or an :class:`ExportError`. + + `/api/auth/login` is the one route an anonymous caller may POST (AGENTS.md invariant 12), + and the token it mints travels **only** in its `Set-Cookie` — never in a body field a page + could log — so reading it back out of the header is the whole job. A `429` says so with its + `Retry-After` in the sentence: a dev loop that trips its own throttle should wait, not + retype. + """ + status, body, headers = _request( + f"{base}/api/auth/login", + method="POST", + body=json.dumps({"password": password}).encode("utf-8"), + headers={"content-type": "application/json"}, + ) + if status != 200: + raise ExportError(f"sign-in at {base} refused ({status}): {_detail(body)}") + token = _cookie(headers, SESSION_COOKIE) + if not token: + raise ExportError(f"sign-in at {base} answered 200 with no {SESSION_COOKIE} cookie") + return token + + +def fetch_bundle(base: str, name: str, token: str) -> bytes | str: + """The bundle **bytes** for an issued host, or the one-line **note** for one with nothing. + + The two 404 shapes are the invariant-3 pair and both mean "write no file": a host with no + certificate yet, and a name this dashboard does not dial at all. The second cannot happen + when the caller passes `HOST_TARGETS` names, so it is handled as a line rather than a + crash — a config typo must not stop the other hosts' bundles. + """ + route = f"/api/certs/{urllib.parse.quote(name, safe='')}/bundle" + status, body, _headers = _request(f"{base}{route}", + headers={"authorization": f"Bearer {token}"}) + if status == 200 and body: + return body + payload = _json(body) + if status == 404 and payload is not None: + if payload.get("configured") is True: + return (f"{name}: no bundle issued yet — issue one on /certs " + f"(POST /api/certs/{name}/rotate mints it)") + return f"{name}: not in this dashboard's HOST_TARGETS — nothing to export" + raise ExportError(f"{name}: GET {route} answered {status}: {_detail(body)}") + + +def export_one(base: str, name: str, token: str, out_dir: Path) -> str: + """One host's output line: where its bundle went, or why nothing was written. + + Raises :class:`ExportError` for a host the route refused for a reason that is not "nothing + issued" — the caller counts those and exits non-zero, because a bundle nobody downloaded is + a host that stays on `http://`. + """ + result = fetch_bundle(base, name, token) + if isinstance(result, str): + return f" - {result}" + path, unchanged = write_bundle(out_dir, name, result) + return (f" ✓ {name}: {path} ({len(result)} bytes, mode 0600" + f"{', unchanged' if unchanged else ''})") + + +def write_bundle(out_dir: Path, name: str, data: bytes) -> tuple[Path, bool]: + """Write one bundle at `0600`; returns (path, whether the bytes were already there). + + `os.open` rather than `open`, so `0600` is the file's **birth** mode: there is no window in + which a private key exists on disk at the umask default. The `chmod` after covers the + already-existing file, for which `O_CREAT`'s mode argument is ignored. + """ + if not name or Path(name).name != name: + # Names come from HOST_TARGETS, which the route has already validated, but this is the + # one function that decides where a key lands on disk. Say it plainly and write nothing. + raise ExportError(f"{name!r} is not a usable host name — nothing written") + out_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + path = out_dir / f"{name}.bundle" + previous = path.read_bytes() if path.exists() else None + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(data) + os.chmod(path, 0o600) + return path, previous == data + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Export the dev dashboard's stored agent bundles to a non-committed " + "directory (one file per configured host that has an issued bundle).", + ) + parser.add_argument("--url", required=True, + help="the dashboard's base URL, e.g. http://127.0.0.1:8090") + parser.add_argument("--password", default=os.environ.get("DASH_ADMIN_PASSWORD", ""), + help="the sign-in password (defaults to $DASH_ADMIN_PASSWORD, which " + "dev-server.sh already loads from .env.dev)") + parser.add_argument("--names", nargs="*", default=None, metavar="HOST", + help="the configured host names, space or comma separated (from " + "HOST_TARGETS — parsed by the caller, never re-parsed here)") + parser.add_argument("--out-dir", default=str(DEFAULT_OUT_DIR), + help=f"where to write .bundle (default: {DEFAULT_OUT_DIR}; keep " + "it inside the gitignored data/ tree)") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + names = split_names(args.names) + if not names: + print(" - no host names given (HOST_TARGETS unset?) — nothing to export") + return 0 + if not args.password: + print(" ! no sign-in password: pass --password or set DASH_ADMIN_PASSWORD " + "(dev-server.sh loads it from .env.dev)", file=sys.stderr) + return 2 + + base = _base(args.url) + try: + token = sign_in(base, args.password) + except ExportError as exc: + print(f" ! {exc}", file=sys.stderr) + return 2 + + out_dir = Path(args.out_dir) + failed = 0 + for name in names: + try: + print(export_one(base, name, token, out_dir)) + except ExportError as exc: + print(f" ! {exc}", file=sys.stderr) + failed += 1 + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/dev-server.sh b/scripts/dev-server.sh index 863c1ea..dda4c32 100755 --- a/scripts/dev-server.sh +++ b/scripts/dev-server.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Local development server: start|stop|status|restart|logs +# Local development server: start|stop|status|restart|logs|bundles # # ./scripts/dev-server.sh restart # reads .env.dev if present, runs on DASH_PORT # @@ -11,10 +11,54 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd)" cd "$ROOT" PIDFILE="$ROOT/data/dev-server.pid" LOG="$ROOT/data/logs/dev.log" +# Phase 24: where the stored agent bundles are materialised for `install-agent.sh` to hand to a +# host. Never committed — a bundle carries that host's private key — because `data/` is +# gitignored, and the CA itself stays in the database (`certs.py`), never in this directory. +CERTDIR="$ROOT/data/certs" mkdir -p "$ROOT/data/logs" running() { [ -f "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; } +# The CA is durable in the database, so "generate certs during dev" means *export what the first +# boot minted*: one .bundle per configured host in gitignored data/certs/, ready to `scp` +# to the host it belongs to. The names come out of HOST_TARGETS exactly the way `parse_targets` +# reads it (space or comma separated `name=url`), so this script invents no host of its own and +# re-parses nothing inside the exporter. +# +# A failure here is a warning line, never a start failure: the readiness probe proved the server +# is up, and a bundle export that cannot sign in costs a dev convenience — not the dashboard, and +# not the authority (the exporter only ever GETs the bundle route; it cannot mint). +export_bundles() { + local port="$1" names=() entry targets="${HOST_TARGETS:-}" + for entry in ${targets//,/ }; do # space or comma separated, like config.parse_targets + names+=("${entry%%=*}") + done + if [ ${#names[@]} -eq 0 ]; then + echo " - no bundles exported (HOST_TARGETS unset)" + return 0 + fi + .venv/bin/python scripts/dev-certs-export.py \ + --url "http://127.0.0.1:${port}" \ + --password "${DASH_ADMIN_PASSWORD:-}" \ + --names "${names[@]}" \ + --out-dir "$CERTDIR" \ + || echo " ! bundle export failed — check DASH_ADMIN_PASSWORD in .env.dev (the server is up and the CA is untouched)" +} + +# The exported-bundles block: names, sizes and modes, straight off the disk. `status` prints it, +# because "is this host's bundle there yet?" is the question `status` gets asked; `bundles` is +# the same block on its own. +bundles() { + echo "bundles exported ($CERTDIR):" + local f found=0 + for f in "$CERTDIR"/*.bundle; do + [ -e "$f" ] || continue + found=1 + printf ' %s %s bytes mode 0%s\n' "${f##*/}" "$(wc -c <"$f")" "$(stat -c %a "$f")" + done + [ "$found" -eq 1 ] || echo " (none — restart the server, or run scripts/dev-certs-export.py)" +} + start() { if running; then echo "already running (pid $(cat "$PIDFILE")) — http://127.0.0.1:${DASH_PORT:-8090}" @@ -33,6 +77,7 @@ start() { if running && curl -sf -o /dev/null "http://127.0.0.1:${PORT}/api/health"; then echo "started pid $(cat "$PIDFILE") — http://127.0.0.1:${PORT}" echo " sign in with DASH_ADMIN_PASSWORD from .env.dev" + export_bundles "$PORT" else echo "FAILED to start; last lines of $LOG:" tail -20 "$LOG" @@ -80,7 +125,9 @@ case "${1:-start}" in | .venv/bin/python -c "import json,sys; d=json.load(sys.stdin); print('status:', d['status']); [print(' ', s['key'], 'ok' if s['ok'] else 'DOWN', (s['error'] or '')[:90]) for s in d['sources']]" else echo "not running" - fi ;; + fi + bundles ;; + bundles) bundles ;; logs) tail -n 40 "$LOG" ;; - *) echo "usage: $0 start|stop|restart|status|logs"; exit 2 ;; + *) echo "usage: $0 start|stop|restart|status|logs|bundles"; exit 2 ;; esac diff --git a/src/dashboards/certs.py b/src/dashboards/certs.py index 65f8494..4f7c155 100644 --- a/src/dashboards/certs.py +++ b/src/dashboards/certs.py @@ -567,6 +567,19 @@ def ensure_certs(store: MetaStore, *, now: float | None = None) -> CertsState: into a fleet-wide outage with no operator action available, so this function generates only what is absent and never overwrites a value it read. + One consequence of that rule is worth reading before filing the next "every host is red" + report: **a stored row also wins when it is older than this module's own shape.** A root minted + before `SubjectKeyIdentifier` was added keeps being used forever — it still parses, its key + still matches, its signatures are valid, so nothing here rejects it — and yet every leaf it + signs is refused by `ssl.create_default_context()` (OpenSSL's strict mode, the context the + collector's dial builds) with `Missing Subject Key Identifier`. That is a *shape* problem in the + authority, not a corruption, so the answer is :func:`rotate_root` — a new root with its + identifiers, a new client identity, every agent re-issued — rather than the `DELETE FROM meta` + reset, which would throw the authority away along with the flaw. Phase 24 task 03 found this on + a live store on the way to installing the fleet's first bundles, and + `tests/test_certs.py::test_a_stored_root_without_an_identifier_still_boots_and_root_rotate_repairs_it` + pins both halves: readable enough that `/certs` can still be reached, rotatable enough to fix. + Half-populated rows generate the missing half, which is what a crash between two `set_meta` calls leaves behind — and a *newly generated* root takes its client certificate with it, because a leaf signed by the previous root is a credential no agent in the fleet can verify. An empty diff --git a/tests/test_certs.py b/tests/test_certs.py index eb55c36..d21b553 100644 --- a/tests/test_certs.py +++ b/tests/test_certs.py @@ -25,13 +25,15 @@ import socket import ssl import tempfile import threading +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import pytest from cryptography import x509 from cryptography.exceptions import InvalidSignature -from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec, rsa from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey from cryptography.hazmat.primitives.asymmetric.types import PublicKeyTypes from cryptography.hazmat.primitives.serialization import ( @@ -371,6 +373,146 @@ def test_every_certificate_carries_the_identifiers_a_strict_verifier_requires() assert context.verify_flags & ssl.VERIFY_X509_STRICT, "this is the mode being satisfied" +def _legacy_authority() -> FakeMeta: + """The meta rows a pre-identifier `certs.py` left in a database: a root **with no SKI**. + + This is the shape phase 24 task 03 found on a live store. The CA row was minted before + `SubjectKeyIdentifier` landed, and "a stored row always wins" means no later start re-mints + it, so every leaf signed afterwards carries an AKI naming an identifier its issuer does not + have. Written by hand here rather than by an old revision of the module, because the point is + to read what a database can hold, not what this module would choose to write. + + The real clock, for the same reason the handshake test below uses it: `ssl` judges the + validity window itself, and the fixed `NOW` would read as an expired certificate. + """ + def pem_key(key: Any) -> str: + return key.private_bytes(Encoding.PEM, PrivateFormat.PKCS8, + NoEncryption()).decode("ascii") + + def pem_cert(cert: x509.Certificate) -> str: + return cert.public_bytes(Encoding.PEM).decode("ascii") + + def subject(name: str) -> x509.Name: + return x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, name)]) + + moment = datetime.now(tz=UTC) + root_key = ec.generate_private_key(ec.SECP256R1()) + root = (x509.CertificateBuilder().subject_name(subject(certs.ROOT_CN)) + .issuer_name(subject(certs.ROOT_CN)) + .public_key(root_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(moment - timedelta(days=1)) + .not_valid_after(moment + certs.ROOT_LIFETIME) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(root_key, hashes.SHA256())) + client_key = ec.generate_private_key(ec.SECP256R1()) + client = (x509.CertificateBuilder().subject_name(subject(certs.CLIENT_CN)) + .issuer_name(subject(certs.ROOT_CN)) + .public_key(client_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(moment - timedelta(days=1)) + .not_valid_after(moment + certs.LEAF_LIFETIME) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .sign(root_key, hashes.SHA256())) + return FakeMeta({certs.META_ROOT_KEY: pem_key(root_key), + certs.META_ROOT_CERT: pem_cert(root), + certs.META_CLIENT_KEY: pem_key(client_key), + certs.META_CLIENT_CERT: pem_cert(client), + certs.META_AGENTS: "{}"}) + + +def _strict_dial_error(root_pem: str, leaf_pem: str, leaf_key_pem: str, + tmp_path: Path) -> str | None: + """Serve `leaf_pem` and connect with the context the collector builds. + + Returns `None` when the handshake completes, or the verification error's own sentence. + `create_default_context` is the whole point: it is what `collectors.AgentTls.ssl_context` + calls, so this is the dial the fleet actually makes and not a laxer stand-in. + """ + here = tmp_path / "strict" + here.mkdir(parents=True, exist_ok=True) + (here / "leaf.pem").write_text(leaf_pem) + (here / "leaf.key").write_text(leaf_key_pem) + (here / "ca.pem").write_text(root_pem) + + server = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server.load_cert_chain(str(here / "leaf.pem"), str(here / "leaf.key")) + client = ssl.create_default_context(cafile=str(here / "ca.pem")) + + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(15) + port = listener.getsockname()[1] + + def serve() -> None: + try: + raw, _peer = listener.accept() + except OSError: + return + try: + with server.wrap_socket(raw, server_side=True) as tls: + tls.recv(1) + except OSError: + pass # the server's half of a handshake the client refused + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + try: + with socket.create_connection(("127.0.0.1", port), timeout=15) as raw, \ + client.wrap_socket(raw, server_hostname="localhost") as tls: + tls.sendall(b"x") + return None + except ssl.SSLCertVerificationError as exc: + return str(exc) + finally: + listener.close() + thread.join(timeout=15) + + +def test_a_stored_root_without_an_identifier_still_boots_and_root_rotate_repairs_it( + tmp_path: Path) -> None: + """A CA older than the identifier fix is a rotatable CA, not a bricked dashboard. + + Three claims, all of them learned on 2026-10-21 while moving the fleet onto installed bundles: + + * `ssl.create_default_context()` — strict mode — refuses a chain whose **CA** carries no + Subject Key Identifier, so every leaf such a root signs is rejected by the dashboard's own + dial. The live store hit exactly this: its `certs.root.cert` row predated the extension, and + a leaf minted afterwards (with its own SKI and a correct AKI) still failed the handshake + with `CERTIFICATE_VERIFY_FAILED: Missing Subject Key Identifier`. + * Nothing may fail closed over it. `read_state` reads those rows because `/certs` is the page + that fixes the problem — a startup that refused this table would leave the operator no + **Rotate root** button, and `DELETE FROM meta` is the last resort rather than the first. + * `rotate_root` is therefore the repair, and it has to actually repair the chain rather than + only bump a revision: a new root with its identifiers, a new client identity, every agent + re-issued under it, and the strict dial accepting the result. + """ + store = _legacy_authority() + state = certs.read_state(store) + with pytest.raises(x509.ExtensionNotFound): + root_of(state).extensions.get_extension_for_class(x509.SubjectKeyIdentifier) + + # A leaf issued *now*, with every identifier current `certs.py` adds, is still unverifiable: + # the gap is in the issuer, and no care at signing time reaches it. + state = issue(store, state, "localhost", ["localhost", "127.0.0.1"]) + before = bundle_pems(entry_of(state, "localhost").bundle) + refused = _strict_dial_error(state.root_cert, before["cert"], before["key"], tmp_path) + assert refused is not None, ( + "a strict client accepted a chain from a CA with no Subject Key Identifier, so this " + "OpenSSL is not the strict mode the fleet's dial runs in and the pin below means nothing") + assert "certificate verify failed" in refused.lower() + + rotated = certs.rotate_root(store, state) + entry = entry_of(rotated, "localhost") + assert entry.bundle_rev == 2, "every host is re-issued under the new root, one bump each" + assert cert_of(rotated.root_cert).extensions.get_extension_for_class( + x509.SubjectKeyIdentifier), "the rotated root carries the identifier its leaves name" + after = bundle_pems(entry.bundle) + accepted = _strict_dial_error(rotated.root_cert, after["cert"], after["key"], tmp_path) + assert accepted is None, f"rotating the root must make the chain dialable: {accepted}" + + def test_the_client_certificate_chains_to_the_root_and_is_a_client_identity() -> None: _store, state = boot() client = cert_of(state.client_cert) diff --git a/tests/test_deploy_scripts.py b/tests/test_deploy_scripts.py index 6b5e33b..87c6378 100644 --- a/tests/test_deploy_scripts.py +++ b/tests/test_deploy_scripts.py @@ -1,4 +1,4 @@ -"""Contract tests for the deployment scripts in `deploy/`. +"""Contract tests for the scripts an operator runs: `deploy/` and `scripts/dev-*.sh`/`.py`. There is no shell test framework here — and adding one would buy nothing a subprocess does not — but phase 10 rests on a sentence that lives in **two** files: the sudoers grant in @@ -8,25 +8,41 @@ silently, and the failure is an install that cannot happen on a host nobody is s So these tests read the file, run the script the way the operator does (unprivileged, which exits before the script can touch anything), and require the two lines to be the same line apart from the path the script was invoked through. + +The same reasoning covers `scripts/dev-certs-export.py` and the `scripts/dev-server.sh` call that +runs it (phase 24): the exporter is what turns the durable CA into a file an operator can `scp`, +and the composition — *after* the readiness probe, *as* a warning line — is as much of the +contract as the bytes it writes. So both files are run here and read here. """ from __future__ import annotations import difflib +import importlib.util import json import os import re import shutil import stat import subprocess +import sys import threading -from http.server import ThreadingHTTPServer +import time +from collections.abc import Iterator +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any +import httpx import pytest +import uvicorn -from dashboards import certs +from auth_contract import ADMIN_PASSWORD, FAST_PBKDF2 +from dashboards import auth, certs +from dashboards.app import create_app +from dashboards.config import Settings, Target +from dashboards.store import SampleStore ROOT = Path(__file__).resolve().parent.parent DEPLOY = ROOT / "deploy" @@ -227,6 +243,56 @@ def test_no_operational_file_mentions_the_tunnel_machinery() -> None: assert not offenders, "tunnel-era names/ports are back:\n" + "\n".join(offenders) +# --- the mTLS registration is the documented default (phase 24) --------------------------- + +#: The three flags that make a host's **install** its registration moment: the run that installs +#: the agent also pulls that host's bundle from the dashboard that owns the authority. The +#: token-mode install is still supported (a host nobody has bundled keeps its bearer token), but +#: it is the compatibility path now, and a setup file that leads with it is a recipe for a fleet +#: of `http://` agents — which is the exact state `TODO.md:3` asked to be rid of. +REGISTRATION_FLAGS = ("--fetch-bundle", "--cert-name", "--session") + +#: The files that tell an operator what to type, per host. +REGISTRATION_DOCS = ("README.md", "AGENTS.md", ".env.example", ".env.dev.example") + + +def _host_target_lines(path: Path) -> list[str]: + """Every `HOST_TARGETS` assignment line in a file, comments included. + + The commented example in `.env.example` counts, because it is the line a reader copies. + """ + return [line for line in path.read_text().splitlines() + if re.search(r"^\s*#?\s*(export\s+)?HOST_TARGETS\s*=", line)] + + +@pytest.mark.parametrize("name", REGISTRATION_DOCS) +def test_the_setup_docs_teach_registration_with_a_bundle(name: str) -> None: + """The per-host recipe a reader copies is the bundle fetch, not the bearer token.""" + text = (ROOT / name).read_text() + for flag in REGISTRATION_FLAGS: + assert flag in text, f"{name} stopped naming {flag} — the fetch is no longer documented" + + +@pytest.mark.parametrize("name", (".env.example", ".env.dev.example")) +def test_the_template_target_urls_are_https_and_credential_free(name: str) -> None: + """A `HOST_TARGETS` line an operator copies is `https://` with no userinfo. + + Pinning the *example* is what keeps this true: prose can and should still document the + token-mode shape (it is a supported dial, `collectors.fetch_snapshot` keeps its path byte-for- + byte), but the line that gets pasted into a real config must not seed a non-mTLS fleet. The + empty `HOST_TARGETS=` is allowed — unconfigured is a different fact from misconfigured. + """ + lines = _host_target_lines(ROOT / name) + assert lines, f"{name} lost its HOST_TARGETS documentation entirely" + for line in lines: + value = line.split("=", 1)[1] + urls = re.findall(r"\w+://[^\s\"']+", value) + if not urls: + continue # the empty assignment + bad = [u for u in urls if not u.startswith("https://") or "@" in u] + assert not bad, f"{name} seeds a non-mTLS or userinfo-bearing target: {bad}" + + # --- shell syntax (the only gate these scripts get) ------------------------------------- @@ -1544,3 +1610,727 @@ def test_a_token_mode_status_probe_stays_plain_http(tmp_path: Path) -> None: assert _installer_status(result.stdout).startswith("(no answer"), ( "token mode must probe plain http — nothing listens here, and the refusal is the point" ) + + +# --- scripts/dev-certs-export.py (phase 24 task 01) --------------------------------------- + +#: The dev exporter, and the dev server that calls it. Both are **run** below and **read** +#: below: the bytes a host installs are the contract of the first, and the composition of the +#: second (export *after* the readiness probe, failure *as* a warning) is the contract of the +#: dev loop that has to stop producing `http://` agents. +EXPORTER = ROOT / "scripts" / "dev-certs-export.py" +DEV_SERVER = ROOT / "scripts" / "dev-server.sh" + +#: The invariant-3 pair, as two configured hosts: one with a bundle in `meta`, one without. +ISSUED_HOST = "issuedhost" +UNISSUED_HOST = "unissuedhost" + + +def _load_exporter() -> Any: + """The exporter as a module — it is a script, not a package member, so import it by path.""" + spec = importlib.util.spec_from_file_location("dev_certs_export", EXPORTER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@contextmanager +def served_dashboard(tmp_path: Path) -> Iterator[tuple[str, SampleStore]]: + """A real `create_app(...)` on a real socket, with one host's bundle already issued. + + uvicorn in a thread rather than `TestClient`, because the unit under test is an **HTTP + client**: the exporter signs in over the public route and reads the session token back out + of a `Set-Cookie` header, and a transport-level test client can pass that without ever + serialising a header. Port 0, so nothing here can collide with a port another test bound. + + The store is injected and closed here, and the two `HOST_TARGETS` entries point at a port + nothing listens on: the app under test must never dial, and a dead target is what keeps + this hermetic. Every other source is switched off explicitly, so an operator running the + suite with `.env.dev` sourced still gets a test of two hosts and no fleet. + """ + db = tmp_path / "export.sqlite3" + store = SampleStore(db) + state = certs.ensure_certs(store) + certs.issue_agent_cert(store, state, ISSUED_HOST, san=[ISSUED_HOST]) + settings = Settings( + db_path=str(db), + admin_password=ADMIN_PASSWORD, + pbkdf2_iterations=FAST_PBKDF2, + host_targets=(Target(name=ISSUED_HOST, url="http://127.0.0.1:1"), + Target(name=UNISSUED_HOST, url="http://127.0.0.1:1")), + host_interval=3600, + host_timeout=0.2, + gufo_url="", + vllm_url="", + nut_host="", + ) + server = uvicorn.Server(uvicorn.Config(create_app(settings, store), host="127.0.0.1", + port=0, log_level="error", lifespan="on")) + thread = threading.Thread(target=server.run, daemon=True, name="dev-certs-export-test") + thread.start() + try: + for _ in range(2_000): + if server.started: + break + time.sleep(0.01) + assert server.started, "the test dashboard never came up (see the captured stderr)" + port = int(server.servers[0].sockets[0].getsockname()[1]) + yield f"http://127.0.0.1:{port}", store + finally: + server.should_exit = True + thread.join(timeout=60) + store.close() + + +def _export(base: str, out_dir: Path, names: list[str], *, + password: str | None = ADMIN_PASSWORD) -> subprocess.CompletedProcess[str]: + """Run the exporter exactly the way `dev-server.sh` does, as a subprocess.""" + args = [sys.executable, str(EXPORTER), "--url", base, "--out-dir", str(out_dir)] + if password is not None: + args += ["--password", password] + if names: + args += ["--names", *names] + return _run(args) + + +def _served_bundle(base: str, name: str) -> bytes: + """What `GET /api/certs//bundle` answers, fetched the way the exporter fetches it.""" + with httpx.Client(base_url=base, timeout=10.0) as client: + login = client.post("/api/auth/login", json={"password": ADMIN_PASSWORD}) + login.raise_for_status() + token = login.cookies[auth.SESSION_COOKIE] + answer = client.get(f"/api/certs/{name}/bundle", + headers={"authorization": f"Bearer {token}"}) + answer.raise_for_status() + return answer.content + + +def test_the_export_writes_the_issued_bundle_byte_identical_at_0600(tmp_path: Path) -> None: + """The task's headline: the file is the stored bundle, at a mode that keeps it private. + + Byte-identity is compared against the **route** rather than the `meta` row because the + question this answers is "is the file I am about to `scp` the file the dashboard issued?", + and a bundle that re-signed between the two would make that question unanswerable. + """ + with served_dashboard(tmp_path) as (base, _store): + out = tmp_path / "certs" + result = _export(base, out, [ISSUED_HOST]) + assert result.returncode == 0, result.stderr + path = out / f"{ISSUED_HOST}.bundle" + assert path.is_file(), f"no bundle exported; stdout was:\n{result.stdout}" + assert stat.S_IMODE(path.stat().st_mode) == 0o600, ( + "a bundle carries a private key; 0600 is its birth mode and stays its mode" + ) + assert path.read_bytes() == _served_bundle(base, ISSUED_HOST) + assert ISSUED_HOST in result.stdout and "0600" in result.stdout + + +def test_an_unissued_host_writes_no_file_and_names_the_certs_route(tmp_path: Path) -> None: + """`issued: false` is a fact, not a failure: one line, no file, exit 0 (invariant 3).""" + with served_dashboard(tmp_path) as (base, _store): + out = tmp_path / "certs" + result = _export(base, out, [ISSUED_HOST, UNISSUED_HOST]) + assert result.returncode == 0, result.stderr + assert not (out / f"{UNISSUED_HOST}.bundle").exists() + note = next(line for line in result.stdout.splitlines() if UNISSUED_HOST in line) + assert "no bundle issued" in note and "/certs" in note, note + assert "!" not in note, f"an unissued host must not read as a failure: {note}" + + +def test_a_name_outside_host_targets_is_a_note_and_not_a_crash(tmp_path: Path) -> None: + """The `{"configured": false}` arm: impossible from `HOST_TARGETS`, still never a crash.""" + with served_dashboard(tmp_path) as (base, _store): + out = tmp_path / "certs" + result = _export(base, out, ["nosuchhost"]) + assert result.returncode == 0, result.stderr + assert not out.exists(), "nothing was issued, so nothing — not even a directory" + assert "HOST_TARGETS" in result.stdout + + +def test_a_wrong_password_exits_non_zero_and_writes_nothing(tmp_path: Path) -> None: + """A dev server that cannot be signed in to says so loudly (and the CA stays untouched).""" + with served_dashboard(tmp_path) as (base, _store): + out = tmp_path / "certs" + result = _export(base, out, [ISSUED_HOST], password="definitely-not-the-password") + assert result.returncode == 2, f"a sign-in failure must not look like success: {result}" + assert not out.exists() + assert "sign-in" in result.stderr.lower() + + +def test_no_password_at_all_is_a_refusal_and_not_a_hang(tmp_path: Path) -> None: + """`--password` is optional only because `$DASH_ADMIN_PASSWORD` is the dev source of it.""" + env = {key: value for key, value in os.environ.items() if key != "DASH_ADMIN_PASSWORD"} + result = subprocess.run( + [sys.executable, str(EXPORTER), "--url", "http://127.0.0.1:1", + "--out-dir", str(tmp_path / "certs"), "--names", ISSUED_HOST], + capture_output=True, text=True, check=False, env=env, timeout=60, + ) + assert result.returncode == 2 + assert "DASH_ADMIN_PASSWORD" in result.stderr + assert not (tmp_path / "certs").exists() + + +def test_no_names_at_all_is_a_quiet_zero(tmp_path: Path) -> None: + """A dashboard with no `HOST_TARGETS` has nothing to export and is not broken.""" + result = _export("http://127.0.0.1:1", tmp_path / "certs", []) + assert result.returncode == 0, result.stderr + assert "nothing to export" in result.stdout + assert not (tmp_path / "certs").exists() + + +def test_a_second_export_is_idempotent_and_says_unchanged(tmp_path: Path) -> None: + """`restart` runs this every time; a dev loop that rewrote keys on every boot would be noise.""" + with served_dashboard(tmp_path) as (base, _store): + out = tmp_path / "certs" + first = _export(base, out, [ISSUED_HOST]) + assert first.returncode == 0, first.stderr + path = out / f"{ISSUED_HOST}.bundle" + before = path.read_bytes() + path.chmod(0o644) # somebody loosened it; the export is what puts the mode back + second = _export(base, out, [ISSUED_HOST]) + assert second.returncode == 0, second.stderr + assert path.read_bytes() == before, "the stored bundle must not move between exports" + assert stat.S_IMODE(path.stat().st_mode) == 0o600, "an existing file is re-chmodded" + assert "unchanged" in second.stdout + + +def test_the_export_reads_the_authority_and_never_mints_anything(tmp_path: Path) -> None: + """The difference between `read_state` and `ensure_certs`, pinned as an observable. + + "The dev server should generate certs" is the TODO's wording; the safe reading is *export + what the first boot minted*. If this script ever issued, a stray `restart` would rotate a + fleet's identities — so the serials, the revisions and the root all have to be the same + bytes after the run as before it. + """ + with served_dashboard(tmp_path) as (base, store): + def fingerprint() -> tuple[str, ...]: + listed = certs.read_state(store) + rows = tuple(f"{e.name}:{e.serial}:{e.bundle_rev}:{e.status}" + for e in certs.read_agents(store)) + return (f"root:{listed.root_serial}", *rows) + + before = fingerprint() + assert _export(base, tmp_path / "certs", + [ISSUED_HOST, UNISSUED_HOST, "nosuchhost"]).returncode == 0 + assert fingerprint() == before, "the exporter changed the certificate authority" + + +def test_the_export_prints_no_key_material_and_no_password(tmp_path: Path) -> None: + """Invariant 6, on the one path that holds both a session token and a private key.""" + with served_dashboard(tmp_path) as (base, _store): + result = _export(base, tmp_path / "certs", [ISSUED_HOST, UNISSUED_HOST]) + printed = result.stdout + result.stderr + assert "BEGIN" not in printed, "a PEM block reached the terminal (and the dev log)" + assert ADMIN_PASSWORD not in printed + assert auth.SESSION_COOKIE not in printed, "the token itself must never be printed" + + +def test_the_exporter_splits_names_the_way_host_targets_looks() -> None: + """One argument with spaces in it, three arguments, commas: all the same list.""" + module = _load_exporter() + assert module.split_names(["alpha beta", "gamma,delta"]) == ["alpha", "beta", "gamma", "delta"] + assert module.split_names([" alpha,,beta "]) == ["alpha", "beta"] + assert module.split_names([]) == [] + assert module.split_names(None) == [] + + +def test_the_exporter_reads_the_cookie_the_app_sets() -> None: + """`auth.SESSION_COOKIE` is imported, not restated — a rename must not silently break dev.""" + module = _load_exporter() + assert module.SESSION_COOKIE == auth.SESSION_COOKIE + + +def test_write_bundle_refuses_a_name_that_would_leave_the_directory(tmp_path: Path) -> None: + """The route gates names long before this, but this is the function that picks a path.""" + module = _load_exporter() + for bad in ("../escape", "nested/x", "/absolute", "", "."): + with pytest.raises(module.ExportError): + module.write_bundle(tmp_path, bad, b"secret") + assert list(tmp_path.iterdir()) == [] + + +def test_the_bundles_only_ever_land_under_the_gitignored_data_tree() -> None: + """No key, bundle or PEM outside `data/`: the exporter's default and the caller's flag.""" + module = _load_exporter() + assert module.DEFAULT_OUT_DIR == ROOT / "data" / "certs" + gitignore = (ROOT / ".gitignore").read_text() + assert re.search(r"^data/$", gitignore, re.MULTILINE), "data/ is the gitignored tree" + dev_server = DEV_SERVER.read_text() + assignments = [line for line in dev_server.splitlines() if line.startswith("CERTDIR=")] + assert assignments == ['CERTDIR="$ROOT/data/certs"'], ( + f"the dev server must name exactly one export dir, under data/: {assignments}" + ) + assert '--out-dir "$CERTDIR"' in dev_server, "the export must be told where it writes" + + +def _shell_function(text: str, name: str) -> str: + """One function's body, so a pin can talk about *that* block and not the whole file.""" + match = re.search(rf"^{re.escape(name)}\(\) \{{$", text, re.MULTILINE) + assert match is not None, f"{name}() is no longer a plain function in dev-server.sh" + body = text[match.end():] + end = body.index("\n}") + return body[:end] + + +def test_dev_server_exports_the_bundles_after_the_readiness_probe() -> None: + """The order is the whole safety of the dev loop: no bundle export against a dead server.""" + text = DEV_SERVER.read_text() + probe = text.index("curl -sf -o /dev/null") + call = text.index('export_bundles "$PORT"') + assert probe < call, "the export must run only once /api/health has answered" + assert "dev-certs-export.py" in _shell_function(text, "export_bundles") + body = _shell_function(text, "export_bundles") + assert "http://127.0.0.1:${port}" in body, "the exporter dials the dev server and nothing else" + # The names are HOST_TARGETS' names, split by the caller: the exporter re-parses no config. + assert "${entry%%=*}" in body + + +def test_a_failed_bundle_export_never_fails_the_dev_server_start() -> None: + """A dev-convenience failure is a warning line: the readiness probe already proved the app.""" + body = _shell_function(DEV_SERVER.read_text(), "export_bundles") + assert "|| echo" in body, "a failed export must print a warning rather than propagate" + for escape in ("return 1", "exit 1", "set -e"): + assert escape not in body, f"the export can now abort `start` ({escape})" + + +def test_dev_server_status_prints_the_exported_bundles_block() -> None: + """`status` answers "is this host's bundle there yet?" without a signed-in curl.""" + text = DEV_SERVER.read_text() + body = _shell_function(text, "bundles") + assert "bundles exported" in body and ".bundle" in body + assert "mode" in body, "the block shows the 0600, which is the thing worth checking" + assert re.search(r"^ bundles ;;$", text, re.MULTILINE), "`status` must print the block" + assert "bundles)" in text, "`bundles` is a subcommand of its own" + + +def test_a_dashboard_that_never_answers_is_named_rather_than_guessed(tmp_path: Path) -> None: + """Nothing listened: the line says which URL, the exit is non-zero, nothing is written.""" + result = _export("http://127.0.0.1:1", tmp_path / "certs", [ISSUED_HOST]) + assert result.returncode == 2 + assert "no dashboard answered" in result.stderr and "127.0.0.1:1" in result.stderr + assert not (tmp_path / "certs").exists() + + +def test_an_unexpected_answer_on_the_bundle_route_is_not_read_as_unissued(tmp_path: Path) -> None: + """Only the two 404 shapes are notes. A 401 means the caller is not authorised, which is news.""" + with served_dashboard(tmp_path) as (base, _store): + module = _load_exporter() + with pytest.raises(module.ExportError) as raised: + module.fetch_bundle(base, ISSUED_HOST, "not-a-session-token") + assert "401" in str(raised.value) + + +def test_one_host_the_route_refuses_never_stops_the_others(tmp_path: Any, + monkeypatch: pytest.MonkeyPatch) -> None: + """The aggregation `main` owns: every name is attempted, and a failure moves the exit code.""" + module = _load_exporter() + attempted: list[str] = [] + + def fake_fetch(base: str, name: str, token: str) -> bytes | str: + attempted.append(name) + if name == "bad": + raise module.ExportError("bad: GET /api/certs/bad/bundle answered 500: boom") + return b"bundle-bytes" + + monkeypatch.setattr(module, "sign_in", lambda base, password: "a-token") + monkeypatch.setattr(module, "fetch_bundle", fake_fetch) + code = module.main(["--url", "http://127.0.0.1:1", "--password", "p", + "--names", "good bad later", "--out-dir", str(tmp_path / "certs")]) + assert attempted == ["good", "bad", "later"], "one refused host must not end the run" + assert code == 1, "a bundle nobody exported is a host that stays on http:// — say so" + assert (tmp_path / "certs" / "good.bundle").is_file() + assert not (tmp_path / "certs" / "bad.bundle").exists() + + +# --- fetching the bundle during registration (phase 24 task 02) --------------------------- + +#: `install-agent.sh --fetch-bundle --cert-name --session `: the install +#: run pulls its own bundle from the dashboard, so a host is never registered onto a token URL +#: because an operator forgot a `curl`. The contract has three parts — the bytes that land are +#: the bytes the route serves, a refusal writes *nothing*, and the session token is a fetch +#: credential that appears in no artefact the run leaves behind. +FETCH_UNISSUED = "no bundle issued for" +FETCH_ALL_OR_NOTHING = "--fetch-bundle, --cert-name and --session go together" +FETCH_TWO_SOURCES = "two bundle sources" +FETCH_MODE_REFUSAL = "--fetch-bundle needs an install" + + +def _sign_in(base: str) -> str: + """A real session token for the served dashboard — the credential `--session` carries.""" + with httpx.Client(base_url=base, timeout=10.0) as client: + login = client.post("/api/auth/login", json={"password": ADMIN_PASSWORD}) + login.raise_for_status() + return login.cookies[auth.SESSION_COOKIE] + + +def _fetch_run(tmp_path: Path, *flags: str, + ) -> tuple[subprocess.CompletedProcess[str], Path, Path]: + """One sandboxed install with a sandbox `TMPDIR`, so the fetched temp file is observable. + + The fetch writes to `${TMPDIR:-/tmp}`, so pointing TMPDIR into the sandbox is what turns + "the fetched private key never stays behind" into an assertion instead of a hope — and it + catches the temp file on the refusal paths too, where nothing else was written to look at. + """ + script, call_log, env = _sandbox_installer(tmp_path) + tmpdir = tmp_path / "tmp" + tmpdir.mkdir(parents=True, exist_ok=True) + result = _install(script, {**env, "TMPDIR": str(tmpdir)}, *flags) + return result, tmpdir, call_log + + +def _fetch(base: str, name: str, token: str) -> list[str]: + """The three fetch flags, spelled once.""" + return ["--fetch-bundle", base, "--cert-name", name, "--session", token] + + +@contextmanager +def served_body(*, status: int = 200, body: bytes = b"wrong", + media_type: str = "text/html") -> Iterator[str]: + """A loopback port that answers every GET with one canned body. + + Only for the shapes the real route cannot produce: a proxy's 200 error page, a 200 with no + bytes at all. The issued host, the unissued host and a refused session are all real answers + from `served_dashboard`, and those are the legs that matter most. + """ + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(status) + self.send_header("content-type", media_type) + self.send_header("content-length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + + def log_message(self, *_args: Any, **_kwargs: Any) -> None: + pass # the installer's answer is the assertion, not this server's access log + + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + try: + yield f"http://127.0.0.1:{httpd.server_address[1]}" + finally: + httpd.shutdown() + httpd.server_close() + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_a_fetched_bundle_installs_byte_identical_at_0640(tmp_path: Path) -> None: + """The registration leg: fetch, validate, install, TLS unit — one run, no hand-carried file. + + Byte-identity is against the same route's answer as task 01's export, because the claim the + installer makes is the same one: the file on the host *is* the row in `meta`. Everything + after the fetch is the existing `--cert-bundle` path, so the mode, the unit's TLS lines and + the https hint are asserted here as unchanged rather than as new. + """ + with served_dashboard(tmp_path) as (base, _store): + token = _sign_in(base) + result, tmpdir, call_log = _fetch_run( + tmp_path, "--bind", "10.0.0.5", "--port", "9120", *_fetch(base, ISSUED_HOST, token)) + if result.returncode != 0: + pytest.fail(f"the fetch install failed:\n{result.stdout}\n{result.stderr}") + + cert_path = tmp_path / "target" / "certs" + assert cert_path.read_bytes() == _served_bundle(base, ISSUED_HOST) + mode = stat.S_IMODE(cert_path.stat().st_mode) + assert mode == 0o640, f"bundle mode is {oct(mode)}, expected 0o640 (group = agent user)" + + unit = (tmp_path / "target" / "dashboards-agent.service").read_text() + assert f"--cert-bundle {cert_path}" in unit + assert "RuntimeDirectory=dashboards-agent" in unit + assert 'HOST_TARGETS="hostname=https://10.0.0.5:9120"' in result.stdout + assert f"fetched the certificate bundle for {ISSUED_HOST} from {base}" in result.stdout + assert "chown root:dashboards-agent" in call_log.read_text() + assert list(tmpdir.iterdir()) == [], "the fetched private key outlived the install" + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_an_unissued_bundle_is_refused_before_anything_is_written(tmp_path: Path) -> None: + """`issued: false` is one sentence naming the fix, and the host stays exactly as it was. + + This is the invariant-3 answer on the install path: the route says "configured, nothing + issued", and the installer must not read that as a bundle, as a sudoers problem, or as a + reason to create a service user. The stub log is the proof nothing privileged ran: `id -u` + is the first privileged call a real install makes, and it is absent. + """ + with served_dashboard(tmp_path) as (base, _store): + token = _sign_in(base) + result, tmpdir, call_log = _fetch_run( + tmp_path, "--bind", "10.0.0.5", *_fetch(base, UNISSUED_HOST, token)) + assert result.returncode == 2, f"a refused fetch keeps exit 2: {result.returncode}" + assert result.stdout == "", "a refusal is a failure: nothing on stdout" + assert FETCH_UNISSUED in result.stderr and UNISSUED_HOST in result.stderr + assert "/certs" in result.stderr, "the sentence names where the certificate is issued" + assert GRANT_PREFIX not in result.stderr, "a refusal is not a sudoers problem" + assert not call_log.exists(), "refused before the root boundary: no privileged command ran" + assert not (tmp_path / "target").exists(), "a refused fetch writes no files at all" + assert list(tmpdir.iterdir()) == [], "the temp file is gone even though it was written" + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_a_name_the_dashboard_does_not_dial_is_refused_with_its_own_404(tmp_path: Path) -> None: + """`{"configured": false}` is a third fact, and it is not read as "nothing issued". + + The route refuses a name outside `HOST_TARGETS` before any signing, so a typo in + `--cert-name` must say *that* — an operator who is told "no bundle issued" would go looking + for a certificate that was never supposed to exist for a host this dashboard does not dial. + """ + with served_dashboard(tmp_path) as (base, _store): + token = _sign_in(base) + result, tmpdir, call_log = _fetch_run( + tmp_path, "--bind", "10.0.0.5", *_fetch(base, "nosuchhost", token)) + assert result.returncode == 2 + assert "404" in result.stderr, result.stderr + assert FETCH_UNISSUED not in result.stderr, "not configured is not the same as unissued" + assert "HOST_TARGETS" in result.stderr, "the dashboard's own sentence is echoed" + assert not call_log.exists() + assert list(tmpdir.iterdir()) == [] + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_a_refused_session_names_the_http_status(tmp_path: Path) -> None: + """A stale or wrong session is a 401 on the wire, and the run says 401. + + Not "no bundle" and not a sudoers hint: the operator's move is a fresh sign-in, and the + only way they learn that is the status the dashboard actually answered. + """ + with served_dashboard(tmp_path) as (base, _store): + result, tmpdir, call_log = _fetch_run( + tmp_path, "--bind", "10.0.0.5", *_fetch(base, ISSUED_HOST, "not-a-session-token")) + assert result.returncode == 2 + assert "401" in result.stderr, result.stderr + assert "sign in required" in result.stderr, "the dashboard's own sentence is echoed" + assert not call_log.exists() + assert not (tmp_path / "target").exists() + assert list(tmpdir.iterdir()) == [] + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +@pytest.mark.parametrize( + ("status", "body", "reason"), + [ + (200, b"502 from the proxy", "# BUNDLE-REV"), + (200, b"", "empty body"), + (500, b"upstream exploded", "500"), + ], + ids=["proxy-error-page", "empty-200", "server-error"], +) +def test_a_body_that_is_not_a_bundle_never_becomes_an_install( + tmp_path: Path, status: int, body: bytes, reason: str +) -> None: + """The shapes the dashboard cannot answer with — and the check that catches each one. + + A 200 with a proxy's HTML page is the dangerous one: it is a *success* to any client that + only looks at the status, and installing it produces a unit that exits at startup. It falls + through to the existing shape check, which is the reason one validator serves both bundle + sources. The refusal names `--fetch-bundle` (the flag that was used), not the temp file. + """ + with served_body(status=status, body=body) as base: + result, tmpdir, call_log = _fetch_run( + tmp_path, "--bind", "10.0.0.5", *_fetch(base, ISSUED_HOST, "a-token")) + assert result.returncode == 2, f"{status}/{len(body)} bytes must be refused" + assert reason in result.stderr, result.stderr + assert "--fetch-bundle" in result.stderr + assert not call_log.exists() + assert not (tmp_path / "target").exists() + assert list(tmpdir.iterdir()) == [] + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_a_dashboard_that_never_answers_stops_the_fetch(tmp_path: Path) -> None: + """Nothing listening on the dashboard URL: the line names it, and no install happens.""" + result, tmpdir, call_log = _fetch_run( + tmp_path, "--bind", "10.0.0.5", + *_fetch("http://127.0.0.1:1", ISSUED_HOST, "a-token")) + assert result.returncode == 2 + assert "no dashboard answered" in result.stderr and "127.0.0.1:1" in result.stderr + assert not call_log.exists() + assert list(tmpdir.iterdir()) == [] + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_the_session_token_appears_in_no_written_artefact(tmp_path: Path) -> None: + """Completion criterion: the token is a fetch credential, not a stored one. + + The test signs in for a real token and then searches **every** file the run could have + touched plus both output streams. The agent's own bearer token is expected in the token + file and on stdout — that is a different secret, deliberately printed once — so the pin is + specifically that this session value is nowhere. + """ + with served_dashboard(tmp_path) as (base, _store): + token = _sign_in(base) + assert token and token != ADMIN_PASSWORD + result, _tmpdir, call_log = _fetch_run( + tmp_path, "--bind", "10.0.0.5", "--port", "9120", + *_fetch(base, ISSUED_HOST, token)) + assert result.returncode == 0, f"install failed:\n{result.stderr}" + + where = tmp_path / "target" + assert token not in (where / "token").read_text(), "the token file is the agent's, not ours" + assert token not in (where / "certs").read_text(), "a bundle is not a place for a session" + assert token not in (where / "dashboards-agent.service").read_text() + assert token not in call_log.read_text(), "a privileged call must not carry it either" + assert token not in result.stdout + result.stderr + + hits = [str(path.relative_to(tmp_path)) for path in tmp_path.rglob("*") + if path.is_file() and token in path.read_text(encoding="utf-8", errors="replace")] + assert hits == [], f"the session token was written to {hits}" + + +@pytest.mark.parametrize( + "flags", + [ + ["--fetch-bundle", "http://127.0.0.1:8090"], + ["--cert-name", "deskwork"], + ["--session", "a-token"], + ["--fetch-bundle", "http://127.0.0.1:8090", "--cert-name", "deskwork"], + ["--cert-name", "deskwork", "--session", "a-token"], + ], + ids=["url", "name", "token", "url+name", "name+token"], +) +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_the_fetch_flags_are_all_or_nothing(tmp_path: Path, flags: list[str]) -> None: + """Half a fetch is a refusal, not a guess: no URL, no name, or no token means nothing dials. + + A URL without a name has no route to fetch, and a name without a token is a 401 the + operator would have to diagnose from a stack of unrelated output. Both are answered with + one sentence before the root boundary and before any write — the `--caps` reasoning again. + """ + script, call_log, env = _sandbox_installer(tmp_path) + result = _install(script, env, "--bind", "127.0.0.1", "--port", "9199", *flags) + assert result.returncode == 1, "a flag mistake keeps exit 1, not the fetch's 2" + assert result.stdout == "" + assert FETCH_ALL_OR_NOTHING in result.stderr + assert GRANT_PREFIX not in result.stderr, "a flag mistake is not a sudoers problem" + assert not call_log.exists(), "refused before a single privileged command" + assert not (tmp_path / "target").exists() + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_two_bundle_sources_are_refused_rather_than_ordered(tmp_path: Path) -> None: + """`--fetch-bundle` beside `--cert-bundle` is a contradiction: one install, one answer. + + "Which one wins" has no good answer here — a stale file and a fresh bundle are two + different identities, and silently preferring either would install a certificate the + operator did not mean to install. So the run stops. + """ + bundle = tmp_path / "agent.bundle" + bundle.write_text(_fake_bundle_text(), encoding="ascii") + script, call_log, env = _sandbox_installer(tmp_path) + result = _install(script, env, "--bind", "127.0.0.1", "--port", "9199", + *_fetch("http://127.0.0.1:8090", "deskwork", "a-token"), + "--cert-bundle", str(bundle)) + assert result.returncode == 1 + assert FETCH_TWO_SOURCES in result.stderr + assert GRANT_PREFIX not in result.stderr + assert not call_log.exists() + assert not (tmp_path / "target").exists() + + +@pytest.mark.parametrize("mode", ["--status", "--uninstall"], ids=["status", "uninstall"]) +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_fetching_a_bundle_is_refused_outside_an_install(tmp_path: Path, mode: str) -> None: + """The fetch belongs to the run that installs, or a key is fetched to be thrown away. + + `--status` answers about a port and `--uninstall` removes files; neither has a use for the + bytes, and a silent "flags ignored" on the uninstall path would leave an operator believing + a bundle had been refreshed. + """ + script, call_log, env = _sandbox_installer(tmp_path) + result = _install(script, env, mode, + *_fetch("http://127.0.0.1:8090", "deskwork", "a-token")) + assert result.returncode == 1 + assert FETCH_MODE_REFUSAL in result.stderr + assert not call_log.exists(), "and refused before the mode's own privileged work" + + +def test_help_teaches_the_registration_fetch() -> None: + """The three flags and the recipe are learnable from `--help`, like every other flag here.""" + stdout = _run([BASH, str(INSTALLER), "--help"]).stdout + assert "--fetch-bundle URL" in stdout + assert "--cert-name NAME" in stdout + assert "--session TOKEN" in stdout + assert "GET /api/certs//bundle" in stdout + assert "fetch credential only" in stdout, "the token's scope has to be learnable, not folklore" + recipe = "--fetch-bundle https://dashboards.example.com --cert-name " + assert recipe in stdout.replace("\\\n", " ").replace(" ", " "), ( + "the recipe line must be copy-pasteable from --help" + ) + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_the_sudoers_hint_never_repeats_the_session_token(tmp_path: Path) -> None: + """The hint quotes a command, so it quotes only the flags that are safe to re-type. + + `need_root` ends with `sudo …` so the operator can copy it. Handing that + string the real argv would print a bearer credential into the terminal, the ssh log and + whoever's scrollback — and the fetch runs *before* the root boundary, so this is the one + path where a good fetch and a refusal share a run. The fetch succeeding here is also the + proof of the order: an unprivileged run can only reach the hint after fetching. + """ + with served_dashboard(tmp_path) as (base, _store): + token = _sign_in(base) + result = _installer_without_root("--bind", "10.0.0.5", "--port", "9120", + *_fetch(base, ISSUED_HOST, token)) + assert result.returncode == 1 + assert _grant_lines(result.stderr), "the fetch succeeded, so what remains is the grant" + assert f"fetched the certificate bundle for {ISSUED_HOST}" in result.stdout + assert token not in result.stdout + result.stderr + assert "--session" not in result.stderr, "the echoed command must not name the flag either" + + +def test_the_session_token_is_confined_to_the_fetch() -> None: + """Structural pin: the install body — everything that writes — cannot see the token. + + `unset SESSION_TOKEN` is the promise; this is the proof that the promise is enforceable + rather than aspirational. Every file the installer writes is written inside `do_install` + (the token file, the bundle, the unit), so if the name does not appear anywhere in that + function or in the unit template, no artefact can carry it — the same style of pin + `tests/test_auth.py` and `tests/test_certs.py` hold over the two modules that hold secrets. + """ + text = INSTALLER.read_text() + assert 'unset SESSION_TOKEN' in text, "the fetch must forget the credential when it is done" + assert 'FETCH_SESSION="$SESSION_TOKEN" python3' in text, ( + "the token reaches python in the environment: argv is world-readable through `ps`, so a" + " second copy on a command line would widen the exposure instead of limiting it" + ) + body = text.split("do_install() {", 1)[1] + assert "SESSION_TOKEN" not in body, "nothing that writes may read the fetch credential" + assert text.count("SESSION_TOKEN") <= 8, ( + "the name belongs to the parser, the all-or-nothing check and the fetch; every further" + " mention is another place a bearer token could be printed or written" + ) + + +def test_the_fetch_and_its_shape_check_both_run_before_the_root_boundary() -> None: + """Text pin for the order the no-partial-install promise depends on. + + Behaviourally, a refusal as `ops` proves the refusal comes first; this proves the *shape* + check is not quietly moved after `need_root` (where a bad fetched bundle would produce a + sudoers hint instead of a reason, on the host where the operator can least afford it). + """ + body = INSTALLER.read_text().split("do_install() {", 1)[1] + fetch = body.index("fetch_bundle\n") + shape = body.index('validate_bundle "$BUNDLE_SRC"') + root = body.index("need_root --bind") + assert fetch < shape < root, "fetch, then shape, then root — in that order" + assert 'trap cleanup_fetch EXIT' in INSTALLER.read_text(), ( + "the fetched key is cleaned up on every exit path, not only the happy one" + ) + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_a_local_bundle_still_validates_with_its_own_flag_name(tmp_path: Path) -> None: + """The validator grew a label, and the pre-existing path must read exactly as before. + + `--fetch-bundle` needed the refusal to name the flag that was used; the `--cert-bundle` + path must not start naming a flag nobody passed. That is the whole change, so this is the + whole pin — the file-local behaviour around it is covered by the phase-15 tests. + """ + path = tmp_path / "revless.bundle" + path.write_text(_fake_bundle_text().replace("# BUNDLE-REV 3\n", ""), encoding="ascii") + result = _installer_without_root("--cert-bundle", str(path)) + assert result.returncode == 2 + assert result.stderr.startswith("--cert-bundle"), result.stderr + assert "--fetch-bundle" not in result.stderr diff --git a/tests/test_live_fleet.py b/tests/test_live_fleet.py index e2a2b30..776e019 100644 --- a/tests/test_live_fleet.py +++ b/tests/test_live_fleet.py @@ -22,12 +22,14 @@ from __future__ import annotations import asyncio import os +import sqlite3 +from collections.abc import Iterator import httpx import pytest from card_contract import assert_card_contract -from dashboards import collectors +from dashboards import certs, collectors from dashboards.config import Settings, load from dashboards.rates import RateEstimator from dashboards.severity import classify @@ -46,25 +48,88 @@ def settings() -> Settings: return load() +class _StoredAuthority: + """`certs.MetaStore` over an existing database, opened **read-only**. + + The live suite has to dial the fleet the way the app does, and since the phase-24 rollout that + means presenting this dashboard's own client certificate. The authority lives in the `meta` + table of the store the dev server owns, so this is the smallest honest reader of it: `mode=ro` + on the URI, `get_meta` the only method that does anything, and the two write methods a + *refusal* rather than a no-op — this file's docstring says nothing in here may write, and + minting or repairing an authority from a test would be the worst write it could make. + """ + + def __init__(self, path: str) -> None: + self._conn = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True) + + def close(self) -> None: + self._conn.close() + + def get_meta(self, key: str) -> str | None: + row = self._conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone() + return None if row is None else str(row[0]) + + def set_meta(self, key: str, value: str) -> None: + raise AssertionError(f"live tests are read-only: refused to write {key!r}") + + def delete_meta(self, key: str) -> int: + raise AssertionError(f"live tests are read-only: refused to delete {key!r}") + + +@pytest.fixture(scope="module") +def agent_tls(settings: Settings) -> Iterator[dict[str, collectors.AgentTls]]: + """One :class:`collectors.AgentTls` per `https://` host, from the authority in the store. + + Empty when no `HOST_TARGETS` entry is https — every host is then dialled on the pre-phase-15 + token path, which is what an `http://` fleet still is — and skipped when the store or its + `certs.*` rows are absent, because a box running the live suite without the dev database is a + normal partial environment rather than a failure (the same rule that lets half a fleet be + reachable). + + `certs.read_state` is the request-time read and never `ensure_certs`: a live test may not mint + a root CA, and the material this writes is removed on teardown. + """ + targets = settings.host_targets + if not any(target.url.startswith("https://") for target in targets): + yield {} + return + if not os.path.exists(settings.db_path): + pytest.skip(f"no store at {settings.db_path} to read the CA from") + authority = _StoredAuthority(settings.db_path) + try: + state = certs.read_state(authority) + except (RuntimeError, sqlite3.Error) as exc: + authority.close() + pytest.skip(f"no readable authority in {settings.db_path}: {type(exc).__name__}") + material = certs.write_material(state) + try: + yield collectors.agent_tls_map(material, authority, targets) + finally: + certs.remove_material(material) + authority.close() + + @pytest.fixture def host_collector() -> HostCollector: return HostCollector(rates=RateEstimator(span=10.0)) async def test_every_host_agent_answers_and_parses(settings: Settings, - host_collector: HostCollector) -> None: + host_collector: HostCollector, + agent_tls: dict[str, collectors.AgentTls]) -> None: """Each HOST_TARGETS entry must answer /snapshot and yield the metrics the page renders.""" if not settings.host_targets: pytest.skip("HOST_TARGETS is unset") required = {"cpu.usage_pct", "mem.used_pct", "disk.used_pct"} async with httpx.AsyncClient(timeout=settings.host_timeout) as client: for target in settings.host_targets: + tls = agent_tls.get(target.name) # Poll twice: usage_pct is a Δ/jiffies rate, and a rate has no second point yet # on the first scrape. That is the design (the agent never remembers who polled # it), so a live test that scraped once would be asserting a false failure. - await collectors.fetch_host_snapshot(client, target, host_collector) + await collectors.fetch_host_snapshot(client, target, host_collector, tls=tls) await asyncio.sleep(1.2) - result = await collectors.fetch_host_snapshot(client, target, host_collector) + result = await collectors.fetch_host_snapshot(client, target, host_collector, tls=tls) metrics = {sample.metric for sample in result.samples} assert result.payload["info"]["hostname"], f"{target.name}: no hostname" missing = required - metrics @@ -76,7 +141,8 @@ async def test_every_host_agent_answers_and_parses(settings: Settings, async def test_every_host_serves_the_hero_card_contract(settings: Settings, - host_collector: HostCollector) -> None: + host_collector: HostCollector, + agent_tls: dict[str, collectors.AgentTls]) -> None: """The phase-03 `/system` card paths exist on the **real** agents, not just the captures. Values may honestly be `n/a` on real hardware (gateway reports no CPU sensor and no GPU @@ -87,16 +153,18 @@ async def test_every_host_serves_the_hero_card_contract(settings: Settings, pytest.skip("HOST_TARGETS is unset") async with httpx.AsyncClient(timeout=settings.host_timeout) as client: for target in settings.host_targets: - await collectors.fetch_host_snapshot(client, target, host_collector) + tls = agent_tls.get(target.name) + await collectors.fetch_host_snapshot(client, target, host_collector, tls=tls) await asyncio.sleep(1.2) - result = await collectors.fetch_host_snapshot(client, target, host_collector) + result = await collectors.fetch_host_snapshot(client, target, host_collector, tls=tls) assert_card_contract(result.payload) assert result.payload["cpu"]["temp_c"] == result.payload["temperatures"]["cpu_c"], ( f"{target.name}: the card and /api/summary would report different CPU temps" ) -async def test_agent_disk_rows_carry_the_smart_shape(settings: Settings) -> None: +async def test_agent_disk_rows_carry_the_smart_shape( + settings: Settings, agent_tls: dict[str, collectors.AgentTls]) -> None: """Phase-09: a redeployed agent answers `smart` per physical disk of `/snapshot`. Agents are redeployed by hand, so an agent whose rows carry no `smart` key at all @@ -111,10 +179,16 @@ async def test_agent_disk_rows_carry_the_smart_shape(settings: Settings) -> None seen_smart = 0 async with httpx.AsyncClient(timeout=settings.host_timeout) as client: for target in settings.host_targets: - response = await client.get(f"{target.url}/snapshot", headers=target.headers) - if response.status_code >= 400: + try: + # through `fetch_snapshot`, not a hand-rolled `get`, because that is the function + # that owns the certificate dial: an `https://` agent needs this app's own CA and + # client leaf, and a test dialling it with the platform trust store would fail on + # a healthy fleet (`CERTIFICATE_VERIFY_FAILED: self-signed certificate in chain`). + response = await collectors.fetch_snapshot(client, target, + tls=agent_tls.get(target.name)) + except collectors.UpstreamError: continue # reachability has its own live test - devices = (response.json().get("disk_io") or {}).get("devices") or [] + devices = (response.get("disk_io") or {}).get("devices") or [] if devices and not all("smart" in device for device in devices): continue # agent predates the smart pass — redeploy it to test this for device in devices: @@ -138,7 +212,7 @@ async def test_agent_disk_rows_carry_the_smart_shape(settings: Settings) -> None async def test_disks_source_charts_the_physical_drives_the_agent_reports( - settings: Settings) -> None: + settings: Settings, agent_tls: dict[str, collectors.AgentTls]) -> None: """Phase 09 task 02: the `disks` source over a real agent snapshot. This is the production code path — the fetch the poller runs, not a hand-built dict — @@ -153,8 +227,9 @@ async def test_disks_source_charts_the_physical_drives_the_agent_reports( async with httpx.AsyncClient(timeout=settings.disks_timeout) as client: for target in settings.host_targets: try: - result = await collectors.fetch_disk_snapshot(client, target, collector, - timeout=settings.disks_timeout) + result = await collectors.fetch_disk_snapshot( + client, target, collector, timeout=settings.disks_timeout, + tls=agent_tls.get(target.name)) except collectors.UpstreamError: continue # reachability has its own live test checked += 1