phase: 24_dev_mtls_rollout

**Phase 24 final verification — all green**

- **Verified (read-only re-run, 62/62 PASS)** via `data/logs/verify-24-mtls.py`: nine sources `ok`; `/api/certs` all three `active` with `deployed_bundle_rev == bundle_rev == 2`; `agent.tls: true` + `agent.version 1.1.0` on every host; driveripper 16/16 drives live; gateway absent from `/disks` but `disks:gateway ok`; all served target URLs `https://`, no userinfo; pre-flip backup present; no transport/cert alert.
- **Criterion 1 (export):** `data/certs/{driveripper,deskwork,gateway}.bundle` re-checked byte-identical (sha256) to `GET /api/certs/<name>/bundle`, mode `0600`, under gitignored `data/`; live arms re-probed (issued→file, unknown/`issued:false`→one-line note + no file, wrong password→exit 2, nothing written).
- **Criterion 2 (`--fetch-bundle`):** 54 sandbox pins in `tests/test_deploy_scripts.py` pass — byte-identical `0640` install + TLS unit lines, refusals (unissued / 401 / non-bundle body / dead dashboard) before `need_root` with no write, token absent from token file, unit, stdout and sudoers hint.
- **Criterion 3–4:** `./scripts/dev-server.sh status` nine `ok` + bundles block; screenshots `data/logs/24-task04-{certs,disks,system}.png` exist.
- **Criterion 7:** no behavior change — `agent/` untouched, `src/` diff is a `certs.ensure_certs` docstring only; token-mode, revocation-before-dial and bundle-route tests green.
- **Tests/lint:** `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov --cov-fail-under=94` → **1542 passed, 15 skipped, coverage 98.94%**; `ruff check src tests agent` → All checks passed; `pyright` → 0 errors; `node --check` clean; `uv run pytest -m live --live` → **11 passed**.
- **Docs:** README.md (all 1144 lines) + AGENTS.md reviewed against the live project — every config.py env var is in README and both templates, route set / severity thresholds / test counts / coverage figure match, all repo paths, cited test names and `Dash.*`/`DashCharts.*` helpers resolve, phase-24 doc pins green. **Verified consistent — no changes needed.**
- **Notable:** left `fastapi 0.153.0 → 0.158.0` (validate reports it as advisory; no dependency was touched, rule 8 conditions the check on deps moving). Killed two abandoned `/tmp/fbtest` loopback stub processes left by task 02's manual probe; dev server unaffected.
- **Next pending phase:** `.agents/phases/todo/25_masthead_control_placement/`.

tests/test_certs.py

@@ -25,13 +25,15 @@ import socket
 import ssl
 import tempfile
 import threading
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
 
 import pytest
 from cryptography import x509
 from cryptography.exceptions import InvalidSignature
-from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import ec, rsa
 from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey
 from cryptography.hazmat.primitives.asymmetric.types import PublicKeyTypes
 from cryptography.hazmat.primitives.serialization import (
@@ -371,6 +373,146 @@ def test_every_certificate_carries_the_identifiers_a_strict_verifier_requires()
     assert context.verify_flags & ssl.VERIFY_X509_STRICT, "this is the mode being satisfied"
 
 
+def _legacy_authority() -> FakeMeta:
+    """The meta rows a pre-identifier `certs.py` left in a database: a root **with no SKI**.
+
+    This is the shape phase 24 task 03 found on a live store. The CA row was minted before
+    `SubjectKeyIdentifier` landed, and "a stored row always wins" means no later start re-mints
+    it, so every leaf signed afterwards carries an AKI naming an identifier its issuer does not
+    have. Written by hand here rather than by an old revision of the module, because the point is
+    to read what a database can hold, not what this module would choose to write.
+
+    The real clock, for the same reason the handshake test below uses it: `ssl` judges the
+    validity window itself, and the fixed `NOW` would read as an expired certificate.
+    """
+    def pem_key(key: Any) -> str:
+        return key.private_bytes(Encoding.PEM, PrivateFormat.PKCS8,
+                                 NoEncryption()).decode("ascii")
+
+    def pem_cert(cert: x509.Certificate) -> str:
+        return cert.public_bytes(Encoding.PEM).decode("ascii")
+
+    def subject(name: str) -> x509.Name:
+        return x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, name)])
+
+    moment = datetime.now(tz=UTC)
+    root_key = ec.generate_private_key(ec.SECP256R1())
+    root = (x509.CertificateBuilder().subject_name(subject(certs.ROOT_CN))
+            .issuer_name(subject(certs.ROOT_CN))
+            .public_key(root_key.public_key())
+            .serial_number(x509.random_serial_number())
+            .not_valid_before(moment - timedelta(days=1))
+            .not_valid_after(moment + certs.ROOT_LIFETIME)
+            .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
+            .sign(root_key, hashes.SHA256()))
+    client_key = ec.generate_private_key(ec.SECP256R1())
+    client = (x509.CertificateBuilder().subject_name(subject(certs.CLIENT_CN))
+              .issuer_name(subject(certs.ROOT_CN))
+              .public_key(client_key.public_key())
+              .serial_number(x509.random_serial_number())
+              .not_valid_before(moment - timedelta(days=1))
+              .not_valid_after(moment + certs.LEAF_LIFETIME)
+              .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
+              .sign(root_key, hashes.SHA256()))
+    return FakeMeta({certs.META_ROOT_KEY: pem_key(root_key),
+                     certs.META_ROOT_CERT: pem_cert(root),
+                     certs.META_CLIENT_KEY: pem_key(client_key),
+                     certs.META_CLIENT_CERT: pem_cert(client),
+                     certs.META_AGENTS: "{}"})
+
+
+def _strict_dial_error(root_pem: str, leaf_pem: str, leaf_key_pem: str,
+                       tmp_path: Path) -> str | None:
+    """Serve `leaf_pem` and connect with the context the collector builds.
+
+    Returns `None` when the handshake completes, or the verification error's own sentence.
+    `create_default_context` is the whole point: it is what `collectors.AgentTls.ssl_context`
+    calls, so this is the dial the fleet actually makes and not a laxer stand-in.
+    """
+    here = tmp_path / "strict"
+    here.mkdir(parents=True, exist_ok=True)
+    (here / "leaf.pem").write_text(leaf_pem)
+    (here / "leaf.key").write_text(leaf_key_pem)
+    (here / "ca.pem").write_text(root_pem)
+
+    server = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    server.load_cert_chain(str(here / "leaf.pem"), str(here / "leaf.key"))
+    client = ssl.create_default_context(cafile=str(here / "ca.pem"))
+
+    listener = socket.socket()
+    listener.bind(("127.0.0.1", 0))
+    listener.listen(1)
+    listener.settimeout(15)
+    port = listener.getsockname()[1]
+
+    def serve() -> None:
+        try:
+            raw, _peer = listener.accept()
+        except OSError:
+            return
+        try:
+            with server.wrap_socket(raw, server_side=True) as tls:
+                tls.recv(1)
+        except OSError:
+            pass  # the server's half of a handshake the client refused
+
+    thread = threading.Thread(target=serve, daemon=True)
+    thread.start()
+    try:
+        with socket.create_connection(("127.0.0.1", port), timeout=15) as raw, \
+                client.wrap_socket(raw, server_hostname="localhost") as tls:
+            tls.sendall(b"x")
+        return None
+    except ssl.SSLCertVerificationError as exc:
+        return str(exc)
+    finally:
+        listener.close()
+        thread.join(timeout=15)
+
+
+def test_a_stored_root_without_an_identifier_still_boots_and_root_rotate_repairs_it(
+        tmp_path: Path) -> None:
+    """A CA older than the identifier fix is a rotatable CA, not a bricked dashboard.
+
+    Three claims, all of them learned on 2026-10-21 while moving the fleet onto installed bundles:
+
+    * `ssl.create_default_context()` — strict mode — refuses a chain whose **CA** carries no
+      Subject Key Identifier, so every leaf such a root signs is rejected by the dashboard's own
+      dial. The live store hit exactly this: its `certs.root.cert` row predated the extension, and
+      a leaf minted afterwards (with its own SKI and a correct AKI) still failed the handshake
+      with `CERTIFICATE_VERIFY_FAILED: Missing Subject Key Identifier`.
+    * Nothing may fail closed over it. `read_state` reads those rows because `/certs` is the page
+      that fixes the problem — a startup that refused this table would leave the operator no
+      **Rotate root** button, and `DELETE FROM meta` is the last resort rather than the first.
+    * `rotate_root` is therefore the repair, and it has to actually repair the chain rather than
+      only bump a revision: a new root with its identifiers, a new client identity, every agent
+      re-issued under it, and the strict dial accepting the result.
+    """
+    store = _legacy_authority()
+    state = certs.read_state(store)
+    with pytest.raises(x509.ExtensionNotFound):
+        root_of(state).extensions.get_extension_for_class(x509.SubjectKeyIdentifier)
+
+    # A leaf issued *now*, with every identifier current `certs.py` adds, is still unverifiable:
+    # the gap is in the issuer, and no care at signing time reaches it.
+    state = issue(store, state, "localhost", ["localhost", "127.0.0.1"])
+    before = bundle_pems(entry_of(state, "localhost").bundle)
+    refused = _strict_dial_error(state.root_cert, before["cert"], before["key"], tmp_path)
+    assert refused is not None, (
+        "a strict client accepted a chain from a CA with no Subject Key Identifier, so this "
+        "OpenSSL is not the strict mode the fleet's dial runs in and the pin below means nothing")
+    assert "certificate verify failed" in refused.lower()
+
+    rotated = certs.rotate_root(store, state)
+    entry = entry_of(rotated, "localhost")
+    assert entry.bundle_rev == 2, "every host is re-issued under the new root, one bump each"
+    assert cert_of(rotated.root_cert).extensions.get_extension_for_class(
+        x509.SubjectKeyIdentifier), "the rotated root carries the identifier its leaves name"
+    after = bundle_pems(entry.bundle)
+    accepted = _strict_dial_error(rotated.root_cert, after["cert"], after["key"], tmp_path)
+    assert accepted is None, f"rotating the root must make the chain dialable: {accepted}"
+
+
 def test_the_client_certificate_chains_to_the_root_and_is_a_client_identity() -> None:
     _store, state = boot()
     client = cert_of(state.client_cert)