phase: 15_agent_mtls

All green. Report: **Verified / implemented (final pass of `15_agent_mtls`)** - Wrote `data/logs/verify-15-tls-e2e.py`: a real `host_agent.py --cert-bundle` + a real dashboard process over a scratch DB (`HOST_TARGETS=loop=https://…`) — **27/27 PASS**, transcript `verify-15-tls-e2e.out`, notes `data/logs/NOTES-tls-rollout.md`. - Read-only checks against the running dev store: 5 `certs.*` rows byte-identical across a restart; served bundle == the `meta` row (1430 B); 0 events containing `-----BEGIN`; unsigned `/certs`→`303 ?next=`, unsigned `/api/certs`→`401`, signed-in `200`; screenshots `data/logs/15-final-{certs,system}.png`. - Docs (step 6, full review): README — added *Protecting an agent, host by host* (the rollout task-07 left out), a `tests/test_certs.py`/TLS-matrix/sandbox-installer paragraph, and fixed two stale “the one write endpoint in the app” claims (`NUT_ALLOW_ACTIONS`, Battery-test buttons). AGENTS.md — test count 1424→1427, bundle-fetch command in *Commands*, a Testing-policy bullet on structurally pinning a key-holding module, fleet transport bullet (token mode is now the compatibility path), gap 11 final state. **Tests / lint (full suite 100% green)** - `uv run pytest` → **1427 passed, 15 skipped**, exit 0 · `bash .agents/validate.sh` → **exit 0** (coverage **98.94%** vs 94% floor, `certs.py` 100%; ruff “All checks passed”; pyright 0 errors; `node --check` clean; hygiene clean) - `uv run pytest -m live --live` → **11 passed / 0 failed** · `./scripts/dev-server.sh status` → all 9 sources `ok` - `uv pip list --outdated` → empty after bumping `fastapi 0.152.0 → 0.153.0` (`uv.lock` only); full suite re-run green. **Completion criteria** - Fresh-DB root CA, idempotent: ✅ (tests + live store hashes) · agent TLS matrix incl. wrong-CA/certless 403/`/health` CA-only/revoked 403 + live denylist re-read: ✅ - `https` `host:`/`disks:` `ok` + revoked says “agent certificate revoked” with no dial: ✅ on loopback (real agent + real dashboard) and spy-pinned hermetically; **not** on the fleet (below) - Signed-in `/certs` + rotate/revoke/root-rotate + byte-identical bundle download: ✅ · installer bundle/unit/`--status` over TLS: ✅ (0640, see deviation) - `pytest`/`validate.sh`/`--outdated`: ✅ · token-mode byte-identical, engine/UPS untouched, public surface unchanged (route walk): ✅ - **Live fleet TLS (deskwork+gateway flip): NOT done** — no live changes. It needs root on hosts without the installer grant (`deskwork`), has no both-ways-working window (`http://` dashboard vs https-only agent), and would deploy the agent from this not-yet-committed tree. Exact procedure recorded in README + `data/logs/NOTES-tls-rollout.md`. **Deviations**: bundle mode `0640 root:dashboards-agent` instead of the task file’s `0600` (the agent re-reads its own bundle for the denylist; 0600 breaks the unit — already recorded in AGENTS.md); the loopback dashboard substitutes for the fleet flip. Incidental: a stray `pkill` of mine killed the dev server mid-pass; restarted and verified healthy. **Next pending phase**: `.agents/phases/todo/16_agent_smart_pass_budget` (then `17_installer_device_filter`).
2026-10-20 10:02:16 -04:00
parent 569ca25fdb
commit cffbac5771
72 changed files with 7168 additions and 216 deletions
@@ -12,15 +12,21 @@ apart from the path the script was invoked through.

 from __future__ import annotations

+import json
 import os
 import re
 import shutil
 import stat
 import subprocess
+import threading
+from http.server import ThreadingHTTPServer
 from pathlib import Path
+from typing import Any

 import pytest

+from dashboards import certs
+
 ROOT = Path(__file__).resolve().parent.parent
 DEPLOY = ROOT / "deploy"
 INSTALLER = DEPLOY / "install-agent.sh"
@@ -291,6 +297,7 @@ ROOT_BOUNDARY_STUBS = ("id", "getent", "useradd", "usermod", "chown", "systemctl
 SANDBOX_REWRITES = {
    'AGENT_DIR="/usr/local/lib/dashboards-agent"': 'AGENT_DIR="{root}/lib"',
    'TOKEN_FILE="/etc/dashboards-agent.token"': 'TOKEN_FILE="{root}/token"',
+    'CERT_FILE="/etc/dashboards-agent.certs"': 'CERT_FILE="{root}/certs"',
    'UNIT="/etc/systemd/system/$SERVICE.service"': 'UNIT="{root}/dashboards-agent.service"',
 }

@@ -361,6 +368,10 @@ def test_install_path_installs_into_its_own_paths_and_prints_the_token_once(tmp_
    assert "User=dashboards-agent" in unit
    assert f"--token-file {token_path}" in unit, "the unit must use the file, never --token"
    assert "--host 10.0.0.5 --port 9120" in unit
+    # A token-only install runs no TLS, so it gets neither the bundle flag nor the directory
+    # the bundle unpacks into (the agent falls back to a private mkdtemp without one).
+    assert "--cert-bundle" not in unit, "no bundle installed means no TLS flag on a token install"
+    assert "RuntimeDirectory=dashboards-agent" not in unit
    # DevicePolicy=strict denies every node not listed, and systemd opens /dev/null for stdin
    # before it execs python: without this line the unit dies at spawn (status 208/STDIN).
    assert "DevicePolicy=strict" in unit
@@ -396,3 +407,370 @@ def test_token_file_is_readable_by_the_service_user_and_nobody_else(tmp_path: Pa
    mode = stat.S_IMODE((tmp_path / "target" / "token").stat().st_mode)
    assert mode == 0o640, f"token file mode is {oct(mode)}, expected 0o640 (group = the agent user)"
    assert not mode & 0o007, "the token file must never be world-readable"
+
+
+# --- the certificate bundle (phase 15 task 06) --------------------------------------------
+
+#: Where the installed bundle lives — one constant in the script, pinned here so a rename
+#: fails a test instead of stranding every already-written unit file.
+CERT_FILE_PATH = "/etc/dashboards-agent.certs"
+
+
+def _fake_bundle_text(revision: str = "3") -> str:
+    """A syntactically real bundle: agent cert, agent key, root CA, and the rev line.
+
+    Shape only. The installer's validator greps PEM blocks by text exactly like
+    `resolve_bundle` does (a monitored host has no `cryptography`), so a well-formed *fake*
+    is all an unprivileged validation run needs; the functional probe tests below use a
+    real bundle minted by `dashboards.certs`.
+    """
+
+    def block(kind: str, body: str) -> str:
+        return f"-----BEGIN {kind}-----\n{body}\n-----END {kind}-----\n"
+
+    return (
+        f"# BUNDLE-REV {revision}\n"
+        + block("CERTIFICATE", "AGENTCERT")
+        + block("PRIVATE KEY", "AGENTKEY")
+        + block("CERTIFICATE", "ROOTCA")
+    )
+
+
+def _installer_status(stdout: str) -> str:
+    """The line `status()` prints as `health: …`, parsed as JSON when it is JSON."""
+    for line in stdout.splitlines():
+        if line.startswith("health: "):
+            body = line.removeprefix("health: ").strip()
+            try:
+                return str(json.loads(body)["ok"])
+            except (json.JSONDecodeError, KeyError):
+                return body
+    pytest.fail(f"the probe printed no health line:\n{stdout}")
+
+
+@pytest.mark.parametrize(
+    ("shape", "reason"),
+    [
+        ("missing", "cannot be read"),
+        ("empty", "is empty"),
+        ("no-revision", "BUNDLE-REV"),
+        ("two-keys", "the agent certificate, the agent key and the root CA"),
+    ],
+)
+def test_a_bad_cert_bundle_is_refused_with_its_reason_before_anything_else(
+    tmp_path: Path, shape: str, reason: str
+) -> None:
+    """Completion criterion: `--cert-bundle /nonexistent` refuses with the reason, touches nothing.
+
+    The validation runs *before* `need_root`, so the failure names the file rather than
+    sending the operator to add a sudoers line for a typo in the flag's argument — and
+    nothing after the validation (user, files, unit) can have run.
+    """
+    if shape == "missing":
+        path = tmp_path / "absent.bundle"
+    elif shape == "empty":
+        path = tmp_path / "empty.bundle"
+        path.write_text("\n", encoding="ascii")
+    elif shape == "no-revision":
+        path = tmp_path / "revless.bundle"
+        path.write_text(_fake_bundle_text().replace("# BUNDLE-REV 3\n", ""), encoding="ascii")
+    else:  # a bundle with the root *key* in it is exactly what the shape check refuses
+        path = tmp_path / "leaky.bundle"
+        extra = "-----BEGIN PRIVATE KEY-----\nROOTKEY\n-----END PRIVATE KEY-----\n"
+        path.write_text(_fake_bundle_text() + extra, encoding="ascii")
+    result = _installer_without_root("--cert-bundle", str(path))
+    assert result.returncode == 2, "the bundle refusal keeps its own exit code (2)"
+    assert result.stdout == "", "a refusal goes to stderr, not stdout"
+    assert reason in result.stderr
+    assert str(path) in result.stderr, "the message names the file the operator passed"
+    assert GRANT_PREFIX not in result.stderr, "a bad bundle is not a sudoers problem"
+
+
+def test_a_well_formed_bundle_passes_validation_and_reaches_the_root_check(
+    tmp_path: Path,
+) -> None:
+    """Validation is shape-only and never dials: the next refusal is the usual sudoers one."""
+    path = tmp_path / "agent.bundle"
+    path.write_text(_fake_bundle_text(), encoding="ascii")
+    result = _installer_without_root("--bind", "10.0.0.5", "--cert-bundle", str(path))
+    assert result.returncode == 1
+    assert _grant_lines(result.stderr), "the bundle validated; what remains is the root boundary"
+
+
+def test_installer_help_documents_the_cert_bundle_and_its_path() -> None:
+    result = _run([BASH, str(INSTALLER), "--help"])
+    assert result.returncode == 0
+    assert "--cert-bundle" in result.stdout
+    assert CERT_FILE_PATH in result.stdout
+
+
+def test_the_bundle_lands_at_the_token_file_idiom_in_the_script() -> None:
+    """`0640 root:$AGENT_USER` + chown: the agent opens the bundle itself and re-reads it."""
+    text = INSTALLER.read_text()
+    assert f'CERT_FILE="{CERT_FILE_PATH}"' in text
+    assert 'install -m 0640 "$BUNDLE_SRC" "$CERT_FILE"' in text
+    assert 'chown root:"$AGENT_USER" "$CERT_FILE"' in text
+    # 0600 would fail the unit the way it would fail the token file: the service user opens
+    # the bundle at startup and re-reads the denylist live. Pin the reasoning, not just the mode.
+    assert "0600 root-owned file would fail the unit" in text
+    assert "re-reads it when its stamp changes" in text
+
+
+def test_the_tls_unit_lines_come_from_one_bundle_branch() -> None:
+    """Flag + RuntimeDirectory are emitted together, or the unit is a token-mode unit."""
+    text = INSTALLER.read_text()
+    match = re.search(r'\n  if \[ -f "\$CERT_FILE" \]; then(.*?)\n  fi\n', text, re.DOTALL)
+    assert match, "the TLS unit lines must come from one `if [ -f \"$CERT_FILE\" ]` branch"
+    block = match.group(1)
+    assert 'TLS_EXEC_ARGS=" --cert-bundle $CERT_FILE"' in block
+    assert "RuntimeDirectory=dashboards-agent" in block
+    assert "${TLS_EXEC_ARGS}" in text, "ExecStart must carry the flag from that branch"
+
+
+def test_status_probes_tls_with_the_bundle_ca_and_no_client_cert() -> None:
+    """The status branch: unit says --cert-bundle → python3 ssl probe, never openssl, no cert."""
+    status_body = INSTALLER.read_text().split("status() {", 1)[1].split("\n}", 1)[0]
+    assert "grep -q -- '--cert-bundle'" in status_body
+    assert "PROTOCOL_TLS_CLIENT" in status_body
+    assert "load_verify_locations" in status_body
+    assert "https://" in status_body
+    assert "load_cert_chain" not in status_body, "the installer holds no client certificate"
+    assert 'http://$BIND:$PORT/health' in status_body, "token mode keeps the plain probe"
+    assert "openssl" not in INSTALLER.read_text(), "no new binary dependency on a monitored host"
+
+
+@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root")
+def test_a_bundle_install_lands_the_file_the_unit_flag_and_the_runtime_directory(
+    tmp_path: Path,
+) -> None:
+    """The install body with a bundle: byte-identical file at 0640, TLS unit, https hint."""
+    bundle = tmp_path / "agent.bundle"
+    bundle.write_text(_fake_bundle_text(), encoding="ascii")
+    script, call_log, env = _sandbox_installer(tmp_path)
+    result = subprocess.run(
+        [BASH, str(script), "--bind", "10.0.0.5", "--port", "9120", "--cert-bundle", str(bundle)],
+        capture_output=True,
+        text=True,
+        env=env,
+        timeout=60,
+        check=False,
+    )
+    assert result.returncode == 0, f"install failed:\n{result.stdout}\n{result.stderr}"
+
+    cert_path = tmp_path / "target" / "certs"
+    assert cert_path.is_file(), "install -m never landed the bundle"
+    assert cert_path.read_text(encoding="ascii") == _fake_bundle_text(), (
+        "the installed bundle must be byte-identical to the downloaded one"
+    )
+    mode = stat.S_IMODE(cert_path.stat().st_mode)
+    assert mode == 0o640, f"bundle mode is {oct(mode)}, expected 0o640 (group = the agent user)"
+    assert not mode & 0o007, "the bundle holds the agent's key: never world-readable"
+
+    unit = (tmp_path / "target" / "dashboards-agent.service").read_text()
+    assert f"--cert-bundle {cert_path}" in unit
+    assert "RuntimeDirectory=dashboards-agent" in unit, (
+        "ProtectSystem=strict leaves the agent no writable file but this, and the bundle "
+        "unpacks its PEM blocks there"
+    )
+    assert "--token-file" in unit, "the dual-mode line stays — rollout is host by host"
+    assert 'HOST_TARGETS="hostname=https://10.0.0.5:9120"' in result.stdout, (
+        "a TLS install points the dashboard at https, with no token in the URL"
+    )
+
+    calls = call_log.read_text()
+    assert calls.count("chown root:dashboards-agent") == 2, "token and bundle both chowned"
+
+
+@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root")
+def test_an_unrelated_rerun_reuses_the_installed_bundle(tmp_path: Path) -> None:
+    """The token-reuse idiom: a re-run without the flag neither rewrites nor drops the bundle."""
+    script, _call_log, env = _sandbox_installer(tmp_path)
+    target = tmp_path / "target"
+    target.mkdir(parents=True)
+    cert_path = target / "certs"
+    cert_path.write_text(_fake_bundle_text(revision="7"), encoding="ascii")
+    result = subprocess.run(
+        [BASH, str(script), "--bind", "10.0.0.5"],
+        capture_output=True,
+        text=True,
+        env=env,
+        timeout=60,
+        check=False,
+    )
+    assert result.returncode == 0, f"re-run failed:\n{result.stdout}\n{result.stderr}"
+    assert cert_path.read_text(encoding="ascii") == _fake_bundle_text(revision="7")
+    assert "reusing the installed certificate bundle" in result.stdout
+    unit = (target / "dashboards-agent.service").read_text()
+    assert f"--cert-bundle {cert_path}" in unit, "an installed bundle keeps the unit in TLS mode"
+
+
+@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root")
+def test_uninstall_removes_the_bundle_with_the_token(tmp_path: Path) -> None:
+    """An uninstalled agent leaves no private key behind."""
+    script, call_log, env = _sandbox_installer(tmp_path)
+    target = tmp_path / "target"
+    (target / "lib").mkdir(parents=True)
+    leftovers = {
+        target / "token": "tok\n",
+        target / "certs": _fake_bundle_text(),
+        target / "lib" / "host_agent.py": "# stub\n",
+        target / "dashboards-agent.service": "[Unit]\n",
+    }
+    for path, body in leftovers.items():
+        path.write_text(body, encoding="ascii")
+    result = subprocess.run(
+        [BASH, str(script), "--uninstall"],
+        capture_output=True,
+        text=True,
+        env=env,
+        timeout=60,
+        check=False,
+    )
+    assert result.returncode == 0, f"uninstall failed:\n{result.stdout}\n{result.stderr}"
+    for path in leftovers:
+        assert not path.exists(), f"uninstall left {path} on disk"
+    assert "systemctl disable --now dashboards-agent" in call_log.read_text()
+
+
+# --- the --status probe, end to end against the real TLS agent -----------------------------
+
+
+class DictMeta:
+    """The three-method `certs.MetaStore` slice, backed by a dict (the `test_auth.py` idiom)."""
+
+    def __init__(self) -> None:
+        self.rows: dict[str, str] = {}
+
+    def get_meta(self, key: str) -> str | None:
+        return self.rows.get(key)
+
+    def set_meta(self, key: str, value: str) -> None:
+        self.rows[key] = value
+
+    def delete_meta(self, key: str) -> int:
+        return 1 if self.rows.pop(key, None) is not None else 0
+
+
+def _mint_bundle() -> str:
+    """A real bundle from the dashboard's own CA — the artifact the operator downloads."""
+    store = DictMeta()
+    state = certs.ensure_certs(store)
+    state = certs.issue_agent_cert(store, state, "localhost", ["127.0.0.1", "localhost"])
+    entry = state.agent("localhost")
+    assert entry is not None
+    return entry.bundle
+
+
+def _serve_tls_agent(host_agent: Any, bundle_text: str, where: Path) -> tuple[Any, int]:
+    """The real agent handler on a TLS-wrapped loopback port, exactly as the unit runs it."""
+    path = where / "installed.certs"
+    path.write_text(bundle_text, encoding="ascii")
+    bundle, error = host_agent.TlsBundle.load(str(path))
+    assert error is None and bundle is not None
+    host_agent.Handler.collector = host_agent.Collector("/", smart=False)
+    host_agent.Handler.token = None
+    host_agent.Handler.bundle = bundle
+    httpd = ThreadingHTTPServer(("127.0.0.1", 0), host_agent.Handler)
+    httpd.socket = bundle.context.wrap_socket(httpd.socket, server_side=True)
+    threading.Thread(target=httpd.serve_forever, daemon=True).start()
+    return httpd, httpd.server_address[1]
+
+
+def _run_status(script: Path, port: int, env: dict[str, str]) -> str:
+    result = subprocess.run(
+        [BASH, str(script), "--status", "--bind", "127.0.0.1", "--port", str(port)],
+        capture_output=True,
+        text=True,
+        env=env,
+        timeout=60,
+        check=False,
+    )
+    assert result.returncode == 0, f"--status exited {result.returncode}:\n{result.stderr}"
+    return result.stdout
+
+
+@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root")
+def test_the_tls_status_probe_reads_health_with_the_ca_and_no_client_cert(
+    tmp_path: Path, host_agent_module: Any
+) -> None:
+    """The task-02 ASSUMPTION, proven through the installer: CERT_OPTIONAL + the /health
+    exemption let a probe that holds only the bundle's CA read /health on a TLS agent."""
+    bundle = _mint_bundle()
+    httpd, port = _serve_tls_agent(host_agent_module, bundle, tmp_path)
+    try:
+        script, _call_log, env = _sandbox_installer(tmp_path / "installer")
+        target = tmp_path / "installer" / "target"
+        target.mkdir(parents=True)
+        (target / "dashboards-agent.service").write_text(
+            f"ExecStart=python3 host_agent.py --cert-bundle {target / 'certs'}\n", encoding="ascii"
+        )
+        (target / "certs").write_text(bundle, encoding="ascii")
+        assert _installer_status(_run_status(script, port, env)) == "True"
+    finally:
+        httpd.shutdown()
+        httpd.server_close()
+        host_agent_module.Handler.bundle = None
+
+
+@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root")
+def test_the_tls_status_probe_verifies_the_chain_and_says_no_to_another_authority(
+    tmp_path: Path, host_agent_module: Any
+) -> None:
+    """The probe loads the CA for a reason: a server signed by a different root is not health."""
+    httpd, port = _serve_tls_agent(host_agent_module, _mint_bundle(), tmp_path)
+    try:
+        script, _call_log, env = _sandbox_installer(tmp_path / "installer")
+        target = tmp_path / "installer" / "target"
+        target.mkdir(parents=True)
+        (target / "dashboards-agent.service").write_text(
+            f"ExecStart=python3 host_agent.py --cert-bundle {target / 'certs'}\n", encoding="ascii"
+        )
+        (target / "certs").write_text(_mint_bundle(), encoding="ascii")  # a *different* CA
+        stdout = _run_status(script, port, env)
+    finally:
+        httpd.shutdown()
+        httpd.server_close()
+        host_agent_module.Handler.bundle = None
+    line = _installer_status(stdout)
+    assert line != "True" and line.startswith("(no answer"), f"the probe must refuse: {line}"
+
+
+@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root")
+def test_the_tls_status_probe_names_the_bundle_when_it_cannot_read_it(tmp_path: Path) -> None:
+    """The bundle is 0640 root:agent — an unprivileged --status cannot read it, and says so."""
+    script, _call_log, env = _sandbox_installer(tmp_path)
+    target = tmp_path / "target"
+    target.mkdir(parents=True)
+    (target / "dashboards-agent.service").write_text(
+        f"ExecStart=python3 host_agent.py --cert-bundle {target / 'certs'}\n", encoding="ascii"
+    )
+    cert = target / "certs"
+    cert.write_text(_fake_bundle_text(), encoding="ascii")
+    cert.chmod(0o000)
+    line = _installer_status(_run_status(script, 1, env))
+    assert "cannot read" in line and "run --status as root" in line, line
+
+
+@pytest.mark.skipif(shutil.which("curl") is None, reason="curl is not installed on this box")
+@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root")
+def test_a_token_mode_status_probe_stays_plain_http(tmp_path: Path) -> None:
+    """The other branch: no bundle in the unit → the unchanged plain probe, not https."""
+    script, _call_log, env = _sandbox_installer(tmp_path)
+    target = tmp_path / "target"
+    target.mkdir(parents=True)
+    (target / "dashboards-agent.service").write_text(
+        "ExecStart=python3 host_agent.py --token-file /etc/dashboards-agent.token\n",
+        encoding="ascii",
+    )
+    result = subprocess.run(
+        [BASH, str(script), "--status", "--bind", "127.0.0.1", "--port", "1"],
+        capture_output=True,
+        text=True,
+        env=env,
+        timeout=60,
+        check=False,
+    )
+    assert result.returncode == 0
+    assert _installer_status(result.stdout).startswith("(no answer"), (
+        "token mode must probe plain http — nothing listens here, and the refusal is the point"
+    )