From 44d61d3ab9df548aae2eb7586095200bc758933a Mon Sep 17 00:00:00 2001 From: ducoterra Date: Thu, 22 Oct 2026 20:24:01 -0400 Subject: [PATCH] phase: 28_gateway_agent_proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Phase 28 final pass — complete, 100 % green.** All five tasks were already `complete/`; this pass verified the criteria, re-ran the live half, and did the docs pass. - Gate `bash .agents/validate.sh` **rc=0**: `1650 passed, 15 skipped`, coverage **98.97 %** (floor 94), `ruff` clean, `pyright` **0 errors**, `node --check` clean, hygiene clean (`/tmp/validate-done.log`). - Live: `pytest -m live --live` **11 passed**; `./scripts/dev-server.sh status` nine sources `ok`; mTLS re-read of `gateway` → agent **1.2.0**, all three sections `available: true`, 6/6 frontends routed, 11 nginx servers + `$targetBackend`, **19** listeners, no core errors (`data/logs/28-finalpass-live.out`); installed agent file byte-identical to this tree (`b0e9ebf1b718…`). - Criteria: 1 ✅ pytest/coverage · 2 ✅ three sections over a captured tree, `false`/`null` + one `errors` line each elsewhere with `cpu`/`mem`/`disk_io`/`system` intact · 3 ✅ `AGENT_VERSION` 1.2.0 + SCHEMA names all three · 4 ✅ six fixtures verbatim, capture refuses a bad read, 116 socket address columns hold no globally routable endpoint · 5 ✅ gateway runs 1.2.0 and answers over mTLS (transcript `data/logs/28-task05-rollout.md`, `28-finalpass-live.out`) · 6 ✅ validate · 7 ✅ additive: HEAD's 1.1.0 and this 1.2.0 `collect()`ed over one `--root` are key-for-key equal plus the three new sections, `--no-proxy` equal exactly (`data/logs/verify-28-final.py` → 41 PASS, `28-final-verify.out`). - Docs fixed (README/AGENTS/Containerfile, all re-green): README's opening said "Four dashboards" above a five-row table and said the new sections are "stored in the snapshot" (they are in the **agent's** document — no `/api` payload, no `sample` row, nothing charts them until phase 29); `Containerfile`'s header still said "three dashboards"; AGENTS's coverage figure (98.94 → **98.97 %**), invariant 6 gained the captured-socket-table identifier rule, the agent bullet gained "phase 28 stops at the wire", and the fleet record names the stash-and-compare control plus the volatile listener count. Full record: `data/logs/NOTES-28-final.md`. - Deviations: none functional; no git index/history touched, phase files untouched (`PLAN.md` and `complete/` unchanged). `uv pip list --outdated` still lists 4 packages (fastapi 0.153→0.161 etc.) — advisory only, dependencies untouched by this phase, left for a deliberate pass. - Next pending phase: `.agents/phases/todo/29_gateway_dashboard_page/` (the page that consumes these three sections). --- .../28_gateway_agent_proxy/00_phase.md | 0 .../01_haproxy_section.md | 0 .../02_nginx_section.md | 0 .../03_socket_section.md | 0 .../04_version_and_capture.md | 0 .../05_rollout_gateway.md | 0 .../28_gateway_agent_proxy__00_phase.a1.err | 0 .../28_gateway_agent_proxy__00_phase.a1.md | 8 + ...gateway_agent_proxy__00_phase.a1.md.status | 1 + ..._gateway_agent_proxy__00_phase.a1.validate | 71 + ...way_agent_proxy__01_haproxy_section.a1.err | 0 ...eway_agent_proxy__01_haproxy_section.a1.md | 11 + ...ent_proxy__01_haproxy_section.a1.md.status | 1 + ...gent_proxy__01_haproxy_section.a1.validate | 66 + ...teway_agent_proxy__02_nginx_section.a1.err | 0 ...ateway_agent_proxy__02_nginx_section.a1.md | 17 + ...agent_proxy__02_nginx_section.a1.md.status | 1 + ..._agent_proxy__02_nginx_section.a1.validate | 67 + ...eway_agent_proxy__03_socket_section.a1.err | 0 ...teway_agent_proxy__03_socket_section.a1.md | 15 + ...gent_proxy__03_socket_section.a1.md.status | 1 + ...agent_proxy__03_socket_section.a1.validate | 67 + ...agent_proxy__04_version_and_capture.a1.err | 0 ..._agent_proxy__04_version_and_capture.a1.md | 13 + ...proxy__04_version_and_capture.a1.md.status | 1 + ..._proxy__04_version_and_capture.a1.validate | 68 + ...way_agent_proxy__05_rollout_gateway.a1.err | 0 ...eway_agent_proxy__05_rollout_gateway.a1.md | 16 + ...ent_proxy__05_rollout_gateway.a1.md.status | 1 + ...gent_proxy__05_rollout_gateway.a1.validate | 68 + AGENTS.md | 182 +- Containerfile | 2 +- README.md | 100 +- agent/host_agent.py | 1076 ++++++++++- scripts/capture-gateway-config.sh | 352 ++++ src/dashboards/collectors.py | 29 +- tests/fixtures/gateway_haproxy.cfg | 57 + tests/fixtures/gateway_nginx_conf_d_http.conf | 214 +++ tests/fixtures/gateway_nginx_main.conf | 63 + .../gateway_nginx_stream_d_stream.conf | 40 + tests/fixtures/gateway_proc_net_tcp.txt | 58 + tests/fixtures/gateway_proc_net_tcp6.txt | 26 + tests/test_agent.py | 1673 ++++++++++++++++- tests/test_collectors.py | 49 + tests/test_deploy_scripts.py | 387 +++- 45 files changed, 4759 insertions(+), 42 deletions(-) rename .agents/phases/{todo => complete}/28_gateway_agent_proxy/00_phase.md (100%) rename .agents/phases/{todo => complete}/28_gateway_agent_proxy/01_haproxy_section.md (100%) rename .agents/phases/{todo => complete}/28_gateway_agent_proxy/02_nginx_section.md (100%) rename .agents/phases/{todo => complete}/28_gateway_agent_proxy/03_socket_section.md (100%) rename .agents/phases/{todo => complete}/28_gateway_agent_proxy/04_version_and_capture.md (100%) rename .agents/phases/{todo => complete}/28_gateway_agent_proxy/05_rollout_gateway.md (100%) create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.err create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.md create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.md.status create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.validate create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.err create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.md create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.md.status create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.validate create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.err create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.md create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.md.status create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.validate create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.err create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.md create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.md.status create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.validate create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.err create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.md create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.md.status create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.validate create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.err create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.md create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.md.status create mode 100644 .agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.validate create mode 100755 scripts/capture-gateway-config.sh create mode 100644 tests/fixtures/gateway_haproxy.cfg create mode 100644 tests/fixtures/gateway_nginx_conf_d_http.conf create mode 100644 tests/fixtures/gateway_nginx_main.conf create mode 100644 tests/fixtures/gateway_nginx_stream_d_stream.conf create mode 100644 tests/fixtures/gateway_proc_net_tcp.txt create mode 100644 tests/fixtures/gateway_proc_net_tcp6.txt diff --git a/.agents/phases/todo/28_gateway_agent_proxy/00_phase.md b/.agents/phases/complete/28_gateway_agent_proxy/00_phase.md similarity index 100% rename from .agents/phases/todo/28_gateway_agent_proxy/00_phase.md rename to .agents/phases/complete/28_gateway_agent_proxy/00_phase.md diff --git a/.agents/phases/todo/28_gateway_agent_proxy/01_haproxy_section.md b/.agents/phases/complete/28_gateway_agent_proxy/01_haproxy_section.md similarity index 100% rename from .agents/phases/todo/28_gateway_agent_proxy/01_haproxy_section.md rename to .agents/phases/complete/28_gateway_agent_proxy/01_haproxy_section.md diff --git a/.agents/phases/todo/28_gateway_agent_proxy/02_nginx_section.md b/.agents/phases/complete/28_gateway_agent_proxy/02_nginx_section.md similarity index 100% rename from .agents/phases/todo/28_gateway_agent_proxy/02_nginx_section.md rename to .agents/phases/complete/28_gateway_agent_proxy/02_nginx_section.md diff --git a/.agents/phases/todo/28_gateway_agent_proxy/03_socket_section.md b/.agents/phases/complete/28_gateway_agent_proxy/03_socket_section.md similarity index 100% rename from .agents/phases/todo/28_gateway_agent_proxy/03_socket_section.md rename to .agents/phases/complete/28_gateway_agent_proxy/03_socket_section.md diff --git a/.agents/phases/todo/28_gateway_agent_proxy/04_version_and_capture.md b/.agents/phases/complete/28_gateway_agent_proxy/04_version_and_capture.md similarity index 100% rename from .agents/phases/todo/28_gateway_agent_proxy/04_version_and_capture.md rename to .agents/phases/complete/28_gateway_agent_proxy/04_version_and_capture.md diff --git a/.agents/phases/todo/28_gateway_agent_proxy/05_rollout_gateway.md b/.agents/phases/complete/28_gateway_agent_proxy/05_rollout_gateway.md similarity index 100% rename from .agents/phases/todo/28_gateway_agent_proxy/05_rollout_gateway.md rename to .agents/phases/complete/28_gateway_agent_proxy/05_rollout_gateway.md diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.err b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.md b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.md new file mode 100644 index 0000000..9ef9536 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.md @@ -0,0 +1,8 @@ +**Phase 28 final pass — complete, 100 % green.** All five tasks were already `complete/`; this pass verified the criteria, re-ran the live half, and did the docs pass. + +- Gate `bash .agents/validate.sh` **rc=0**: `1650 passed, 15 skipped`, coverage **98.97 %** (floor 94), `ruff` clean, `pyright` **0 errors**, `node --check` clean, hygiene clean (`/tmp/validate-done.log`). +- Live: `pytest -m live --live` **11 passed**; `./scripts/dev-server.sh status` nine sources `ok`; mTLS re-read of `gateway` → agent **1.2.0**, all three sections `available: true`, 6/6 frontends routed, 11 nginx servers + `$targetBackend`, **19** listeners, no core errors (`data/logs/28-finalpass-live.out`); installed agent file byte-identical to this tree (`b0e9ebf1b718…`). +- Criteria: 1 ✅ pytest/coverage · 2 ✅ three sections over a captured tree, `false`/`null` + one `errors` line each elsewhere with `cpu`/`mem`/`disk_io`/`system` intact · 3 ✅ `AGENT_VERSION` 1.2.0 + SCHEMA names all three · 4 ✅ six fixtures verbatim, capture refuses a bad read, 116 socket address columns hold no globally routable endpoint · 5 ✅ gateway runs 1.2.0 and answers over mTLS (transcript `data/logs/28-task05-rollout.md`, `28-finalpass-live.out`) · 6 ✅ validate · 7 ✅ additive: HEAD's 1.1.0 and this 1.2.0 `collect()`ed over one `--root` are key-for-key equal plus the three new sections, `--no-proxy` equal exactly (`data/logs/verify-28-final.py` → 41 PASS, `28-final-verify.out`). +- Docs fixed (README/AGENTS/Containerfile, all re-green): README's opening said "Four dashboards" above a five-row table and said the new sections are "stored in the snapshot" (they are in the **agent's** document — no `/api` payload, no `sample` row, nothing charts them until phase 29); `Containerfile`'s header still said "three dashboards"; AGENTS's coverage figure (98.94 → **98.97 %**), invariant 6 gained the captured-socket-table identifier rule, the agent bullet gained "phase 28 stops at the wire", and the fleet record names the stash-and-compare control plus the volatile listener count. Full record: `data/logs/NOTES-28-final.md`. +- Deviations: none functional; no git index/history touched, phase files untouched (`PLAN.md` and `complete/` unchanged). `uv pip list --outdated` still lists 4 packages (fastapi 0.153→0.161 etc.) — advisory only, dependencies untouched by this phase, left for a deliberate pass. +- Next pending phase: `.agents/phases/todo/29_gateway_dashboard_page/` (the page that consumes these three sections). diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.md.status b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.validate b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.validate new file mode 100644 index 0000000..3be66bd --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__00_phase.a1.validate @@ -0,0 +1,71 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 12%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 30%] +.......s................................................................ [ 34%] +........................................................................ [ 38%] +........................................................................ [ 43%] +........................................................................ [ 47%] +........................................................................ [ 51%] +........................................................................ [ 56%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 69%] +....................................sssssssssss......................... [ 73%] +........................................................................ [ 77%] +........................................................................ [ 82%] +........................................................................ [ 86%] +........................................................................ [ 90%] +........................................................................ [ 95%] +.......................sss.............................................. [ 99%] +......... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 206 4 98% 304, 308-309, 326 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3496 36 99% +Required test coverage of 94% reached. Total coverage: 98.97% +1650 passed, 15 skipped, 1 warning in 157.81s (0:02:37) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + coverage 7.16.2 7.17.0 wheel +fastapi 0.153.0 0.161.1 wheel +pycparser 3.11 3.12 wheel +ruff 0.17.1 0.17.2 wheel diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.err b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.md b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.md new file mode 100644 index 0000000..6c8b633 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.md @@ -0,0 +1,11 @@ +**Task 28/01 complete — green.** + +- `agent/host_agent.py`: new `haproxy` snapshot section — `haproxy_config` argument (+`--haproxy-config`, `--no-proxy`), module-level `parse_haproxy_config()` / `_scan_haproxy_config()`, `Collector.haproxy()`, wired into `snapshot()` before `errors`. Shape matches the task's assumptions; `:::22`→`addr "::"`, port `22`; a portless target keeps `port: null`; `use_backend %[…]` contributes no name; `listen` becomes the pair haproxy defines; design-rules + SCHEMA docstring updated (`AGENT_VERSION` left at 1.1.0 — task 04 owns the bump). +- Three-way `available` pinned: `true` / `false` (missing file, one `haproxy: … not found …` line) / `null` (mode-000, a directory, oversized, non-haproxy text) — each exactly **one** line, snapshot and `/health` intact. `--no-proxy` removes the key and the line, Collector-level and through `main()`. +- `src/dashboards/collectors.py`: `ADDON_ERROR_PREFIXES = ("smart", "haproxy")` — defect found while implementing. The task mandates the absence line, but `degraded` (that list) is `alerts.py`'s input, so shipping it unfiltered would have put a red banner + overview WARN on every host with no haproxy after the upgrade. Filtered there, still verbatim in `agent.errors` for phase 29. +- Tests: 19 new (17 `tests/test_agent.py`, 2 `tests/test_collectors.py`); three existing `errors == []` assertions now filter add-on lines via `core_errors()`. +- `tests/fixtures/gateway_haproxy.cfg`: **read-only** ssh capture, body byte-identical to `gateway:/etc/haproxy/haproxy.cfg` (mode 0644 — so the agent user can read it; no credential in it). Parsed end-to-end: 6 frontends / 6 backends, all `default_backend`s resolve. +- Docs: README agent section, AGENTS.md agent + `collectors.py` bullets, gateway's fleet row, pytest tally. +- Gate: `bash .agents/validate.sh` **rc=0** — 1568 passed / 15 skipped, coverage 98.94 % (floor 94), ruff clean, pyright 0 errors, `node --check` clean. Only the pre-existing advisory (fastapi 0.153.0→0.159.1; no dependency touched). +- Dev server restarted: 9 sources `ok`; `/api/system/snapshot` unchanged on the live fleet (agents are 1.1.0, so no `haproxy` key at all). Evidence: `data/logs/NOTES-28-haproxy.md`. +- No live change made to any host; no git index/history touched; phase files unmodified. Cosmetic note for task 05/29: after a fleet upgrade, a non-haproxy host's collapsed "More detail" banner will print the absence line — same as a smart line today; alerts are unaffected. diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.md.status b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.validate b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.validate new file mode 100644 index 0000000..f786bc9 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__01_haproxy_section.a1.validate @@ -0,0 +1,66 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 13%] +........................................................................ [ 18%] +........................................................................ [ 22%] +........................................................................ [ 27%] +.........s.............................................................. [ 31%] +........................................................................ [ 36%] +........................................................................ [ 40%] +........................................................................ [ 45%] +........................................................................ [ 50%] +........................................................................ [ 54%] +........................................................................ [ 59%] +........................................................................ [ 63%] +........................................................................ [ 68%] +..........................sssssssssss................................... [ 72%] +........................................................................ [ 77%] +........................................................................ [ 81%] +........................................................................ [ 86%] +........................................................................ [ 90%] +........................................................................ [ 95%] +.............sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 206 5 98% 296-297, 301-302, 319 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3496 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1568 passed, 15 skipped, 1 warning in 165.97s (0:02:45) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.159.1 wheel diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.err b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.md b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.md new file mode 100644 index 0000000..c3d21f2 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.md @@ -0,0 +1,17 @@ +**Implemented (phase 28 task 02 — agent `nginx` section)** +- `agent/host_agent.py`: additive `nginx` section — `Collector.nginx(errors)` + `nginx_include_files()` + pure `nginx_include_paths()` / `parse_nginx_config()`; brace-block scanner (`_nginx_tokens`/`_nginx_statements`/`_collect_nginx`) so quoted `;`/`{`, comments, nested `server`/`location`/`upstream`/`map` all parse; `--nginx-config` flag; shared `_config_path`/`read_config`/`_config_label` now used by both proxy sections; SCHEMA docstring + design-rules bullet updated. +- Three-way `available` with exactly one `nginx:` line per arm ("not found" pinned); includes followed **one level**, bounded 8 files / 1 MB (excess dropped with one line naming the count, section stays `available: true`); `files` = main + merged docs in the config's own spelling; `proxy_pass $var` served with `variable` + `target: null` (never merged); `--no-proxy` now removes `haproxy` **and** `nginx`. +- `collectors.ADDON_ERROR_PREFIXES` grew `nginx` in the same commit (AGENTS.md rule) + new pin that the agent-side and dashboard-side lists are equal (that pin was claimed but missing — defect fixed). +- Verified the parser read-only against the **real** gateway tree (`cat` over ssh): 11 servers, both maps, 17 `$targetBackend` entries, the `127.0.0.1:8081` literal, stream `proxy_pass $targetBackend` unmerged. No live change made (install is task 05). + +**Tests / lint** (`uv run …`) +- `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov --cov-fail-under=94` → **1603 passed, 15 skipped, 98.94%** (was 1568; 34 new nginx cases, one 6-way parametrise over malformed files). +- `uv run ruff check src tests agent` → All checks passed; `uv run pyright` → 0 errors. +- `uv run pytest -m live --live` → 11 passed; `./scripts/dev-server.sh status` → 9 sources `ok`. +- Agent-file coverage via `coverage run --branch --source=agent`: 94%, new code fully covered except the unreachable defensive `except` in `_config_label` (>90% criterion met). + +**Docs**: `README.md` (nginx add-on paragraph), `AGENTS.md` (agent bullet for the `nginx` section, `ADDON_ERROR_PREFIXES` list, gateway fleet row, test tally). Transcript: `data/logs/NOTES-28-nginx.md` (gitignored). + +**Deviations**: map entry keys carry `variable`/`key` with the `$` dropped on both sides of the proxy_pass↔map join (one canonical spelling instead of the task's mixed `$targetBackend`/`targetBackend`); map values that are not addresses report `target: null` rather than a host-shaped guess. `uv pip list --outdated` still reports a pre-existing `fastapi 0.153.0 → 0.159.1` (validate.sh prints it as an advisory only; deps untouched by this task). + +**Next pending task**: `28_gateway_agent_proxy/03_socket_section.md`. diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.md.status b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.validate b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.validate new file mode 100644 index 0000000..ffd1829 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__02_nginx_section.a1.validate @@ -0,0 +1,67 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 13%] +........................................................................ [ 17%] +........................................................................ [ 22%] +........................................................................ [ 26%] +............................................s........................... [ 31%] +........................................................................ [ 35%] +........................................................................ [ 40%] +........................................................................ [ 44%] +........................................................................ [ 48%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 66%] +.............................................................sssssssssss [ 71%] +........................................................................ [ 75%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 93%] +................................................sss..................... [ 97%] +.................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 206 5 98% 302-303, 307-308, 325 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3496 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1603 passed, 15 skipped, 1 warning in 162.84s (0:02:42) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.159.1 wheel diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.err b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.md b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.md new file mode 100644 index 0000000..09104c4 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.md @@ -0,0 +1,15 @@ +Task 03 (`sockets` section) is complete. + +**Implemented** — `agent/host_agent.py`: +- `parse_proc_net_tcp(text, family)` keeps only `0A` LISTEN → `{family, addr, port}` and `01` ESTABLISHED → `{family, local, remote}`; every other state is skipped silently. `_proc_net_addr` unrolls `/proc`'s per-32-bit-word little-endian spelling through `socket.inet_ntop`, so wildcards stay literal `0.0.0.0` / `::` and a bad word yields `addr: null` (never `0`); a bad port is `null` too. +- `Collector.read_proc_table()` bounds each table to `PROC_NET_MAX_LINES` (4096, `/proc`'s header among them) and reports the excess as **one** line naming the path; `Collector.sockets(errors)` reads v4/v6 independently — either answering gives `available: true`, a missing `tcp6` is silent (kernel without IPv6), a denied table always costs one line, neither answering gives `available: null` plus one combined line. There is deliberately **no `false` arm** (`false` beside empty lists would read "no listeners"). +- `document["sockets"]` is attached inside `Collector.snapshot()` only when `proxy_enabled`, wrapped in `_safe`, so an unexpected exception costs one `sockets:` line and nothing else — `/snapshot` and `/health` never fail. `--no-proxy` removes the key with no error line. +- `src/dashboards/collectors.py`: `ADDON_ERROR_PREFIXES = ("smart", "haproxy", "nginx", "sockets")` so absence/errors do not turn `/system` red. + +**Tests** — `tests/test_agent.py`: v4+v6 parsing incl. wildcard `0.0.0.0`/`::`, every non-`0A`/`01` state skipped, malformed endpoint words, header line ignored, `--root` fake-tree reads, additivity (only `ts`/`duration_ms`/`sockets`/`errors` move), both-missing → one line naming both paths, missing `tcp6` silent, denied table costs one line, truncation reported once, `--no-proxy` (payload, `--once`, `/schema`), `_safe` fallback, `inet_ntop` raising, and a real-`/proc` shape test. `tests/test_collectors.py`: `sockets:` lines excluded from `degraded`, prefix tuple pinned. + +**Verification** — `bash .agents/validate.sh`: **1630 passed, 15 skipped**; coverage **98.94%** (floor 94% met); `ruff` clean; `pyright` 0 errors. Real read-only `--once` on this box: 14 listeners / 8 established, matching `ss -ltnH` exactly, no error lines. No live-infrastructure changes; no git index/history commands. + +**Docs** — README agent section + AGENTS.md `sockets` rules and the add-on-prefix sentence. + +**Next pending task, not started**: `04_version_and_capture.md`. diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.md.status b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.validate b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.validate new file mode 100644 index 0000000..917c233 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__03_socket_section.a1.validate @@ -0,0 +1,67 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 13%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 26%] +.......................................................................s [ 30%] +........................................................................ [ 35%] +........................................................................ [ 39%] +........................................................................ [ 43%] +........................................................................ [ 48%] +........................................................................ [ 52%] +........................................................................ [ 56%] +........................................................................ [ 61%] +........................................................................ [ 65%] +........................................................................ [ 70%] +................sssssssssss............................................. [ 74%] +........................................................................ [ 78%] +........................................................................ [ 83%] +........................................................................ [ 87%] +........................................................................ [ 91%] +........................................................................ [ 96%] +...sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 206 5 98% 303-304, 308-309, 326 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3496 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1630 passed, 15 skipped, 1 warning in 449.51s (0:07:29) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.159.1 wheel diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.err b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.md b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.md new file mode 100644 index 0000000..8fc4c3a --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.md @@ -0,0 +1,13 @@ +**Task 04 complete — `AGENT_VERSION` 1.2.0, the SCHEMA, and the gateway captures.** + +**Agent** — `agent/host_agent.py`: `AGENT_VERSION` `1.1.0` → `1.2.0`; the `GET /schema` block already names `haproxy`/`nginx`/`sockets` (each with its `available`), `--no-proxy`, `--haproxy-config`, `--nginx-config`. `tests/test_agent.py` pins the constant plus all of that; the phase-15 TLS pin keeps its schema assertions and no longer competes over the number. + +**Capture** — new `scripts/capture-gateway-config.sh` (`GATEWAY_SSH`, default `gateway-ops`): one `ssh … cat` per document (`scp` would copy `/proc/net/tcp` as 0 bytes), no sudo, haproxy.cfg + `nginx.conf` **plus every document its own glob `include` names** + `/proc/net/tcp{,6}`, provenance header naming the remote path and script, and a refusal (`exit 1`, nothing written, temp removed) for a failed/empty read, an unnameable include dir or a missing glob. Ran it: 7 fixtures now in `tests/fixtures/` (haproxy, nginx main + `conf_d_http` + `stream_d_stream`, both socket tables). + +**Deviation from the task's "scrubs nothing" assumption** — configs stay verbatim (domains, `proxy_pass` targets, key *paths*); the two socket tables now replace **only globally routable endpoints** with documentation space (RFC 5737/3849), idempotent and column-width-preserving, because a global IPv6 here is EUI-64 (MAC-derived) and an inbound visitor's address is not ours to publish — the smartctl serial rule (invariant 6). Documented in README + AGENTS.md and pinned in both test files. + +**Tests** — `test_agent.py`: 7 tests over the captures (header/lineage, the merged include tree, SNI map + `proxy_pass $var` unmerged, every table row decodes, config↔socket join on 22/443/4443/8443/9120, no globally routable endpoint, one collector answering all three sections). `test_deploy_scripts.py`: 10 pins (alias dialed and nothing else, no `sudo`/IP in code, full tree recorded, domains verbatim, dead gateway + empty read leave fixtures alone, the two refusal messages, substitution behaviour + idempotence). + +**Live probe (read-only)** — `data/logs/verify-28-task04.py` → `28-task04-probe.out`: gateway `active`, installed **1.1.0**, CA-only `/health` 200, `/snapshot` 403 `client certificate required`, mTLS snapshot has every pre-1.2.0 section with live numbers and **none** of the three new ones → **PASS: no behaviour change**. + +**Gate** — `bash .agents/validate.sh`: **1650 passed, 15 skipped**, coverage **98.94%**, ruff clean, pyright 0 errors, `node --check` clean, hygiene clean (only the pre-existing `fastapi` advisory); `pytest -m live --live` with `.env.dev`: **11 passed**. Docs updated in README + AGENTS.md (capture script, the three capture rules, the fleet read). Nothing staged or committed; new script/fixtures are untracked for task 05. diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.md.status b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.validate b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.validate new file mode 100644 index 0000000..28cd25f --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__04_version_and_capture.a1.validate @@ -0,0 +1,68 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 12%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 30%] +.......s................................................................ [ 34%] +........................................................................ [ 38%] +........................................................................ [ 43%] +........................................................................ [ 47%] +........................................................................ [ 51%] +........................................................................ [ 56%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 69%] +....................................sssssssssss......................... [ 73%] +........................................................................ [ 77%] +........................................................................ [ 82%] +........................................................................ [ 86%] +........................................................................ [ 90%] +........................................................................ [ 95%] +.......................sss.............................................. [ 99%] +......... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 206 5 98% 303-304, 308-309, 326 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3496 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1650 passed, 15 skipped, 1 warning in 457.28s (0:07:37) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.159.1 wheel diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.err b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.md b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.md new file mode 100644 index 0000000..d6b7ad5 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.md @@ -0,0 +1,16 @@ +**Task 05 complete — gateway host now runs agent 1.2.0, verified over mTLS.** + +- Re-staged `/home/ops/agent-install/` on `gateway-ops` (gap-10 rule): `host_agent.py` 1.2.0 (`b0e9ebf1…`, md5 `9e062e4f…`) + installer (`ffbcd91f…`), old files snapshotted as `*-20261022-backup` (names the sudo grant doesn't name). +- Installed via the granted path with the unit's own args (`--bind 10.1.1.27 --port 9120`); **re-run guard printed nothing** — `systemctl cat` byte-identical before/after (`26bd812…`); bundle + token reused, not rewritten. +- `sudo systemctl restart dashboards-agent` → `active`, pid 20708, `https … client cert required, rev 2`. +- Verified: CA-only `--status` answers `{"ok":true,…,"version":"1.2.0"}` and still 403s `/snapshot`; mTLS probe (`data/logs/verify-28-task05.py` → `28-task05-verify.out`, exit 0) asserts version 1.2.0, all three new keys, `haproxy.available=true` (6 routed frontends/6 backends), `nginx.available=true` (11 servers, `$targetBackend` map, 19 entries), `sockets.available=true` (17 listens incl. **443**), and every pre-1.2.0 section intact with live numbers, no core error. +- Dashboard side untouched: `./scripts/dev-server.sh status` → **nine sources `ok`**; `/api/system/snapshot` gateway row `agent.version 1.2.0`, no `degraded`; `/api/disks/snapshot` still 2 hosts. +- Transcript: `data/logs/28-task05-rollout.md` (+ `28-task05-raw.log`) — hashes, ports and counts only, no config body, no address, no credential. + +**Gates:** `bash .agents/validate.sh` → **exit 0**: `uv run pytest --cov --cov-fail-under=94` **1650 passed / 15 skipped**, coverage **98.94 %**; `ruff check src tests agent` clean; `pyright` clean; `node --check` clean; hygiene clean. Re-run after doc edits `uv run pytest -q` → exit 0; doc pins `tests/test_auth_docs.py tests/test_deploy_scripts.py` → **141 passed**; `uv run pytest -m live --live` → **11 passed** (read-only). + +**Docs:** `AGENTS.md` fleet record (gateway on 1.2.0, other two still 1.1.0 and why that costs nothing; the rollout procedure, byte-identical unit, verification pointers) + gap 11's end-state line; `README.md` new "Upgrading an already-registered host is one file, not a rollout" paragraph (bundle-on-disk vs `--cert-bundle` flag, silent re-run guard, restart, both probes). + +**Deviations:** no code change (as the task states); `uv pip list --outdated` shows pre-existing `fastapi 0.153.0 → 0.159.1` drift left alone — this task moved no dependencies and a bump belongs in its own change. + +**Next pending:** phase 28 has no further tasks; `29_gateway_dashboard_page/01_…` is next in `todo/`. diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.md.status b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.validate b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.validate new file mode 100644 index 0000000..a53b8e3 --- /dev/null +++ b/.agents/reports/28_gateway_agent_proxy/28_gateway_agent_proxy__05_rollout_gateway.a1.validate @@ -0,0 +1,68 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 8%] +........................................................................ [ 12%] +........................................................................ [ 17%] +........................................................................ [ 21%] +........................................................................ [ 25%] +........................................................................ [ 30%] +.......s................................................................ [ 34%] +........................................................................ [ 38%] +........................................................................ [ 43%] +........................................................................ [ 47%] +........................................................................ [ 51%] +........................................................................ [ 56%] +........................................................................ [ 60%] +........................................................................ [ 64%] +........................................................................ [ 69%] +....................................sssssssssss......................... [ 73%] +........................................................................ [ 77%] +........................................................................ [ 82%] +........................................................................ [ 86%] +........................................................................ [ 90%] +........................................................................ [ 95%] +.......................sss.............................................. [ 99%] +......... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 206 5 98% 303-304, 308-309, 326 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3496 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1650 passed, 15 skipped, 1 warning in 455.76s (0:07:35) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.159.1 wheel diff --git a/AGENTS.md b/AGENTS.md index 7dd3bc9..7d59dab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,11 +116,20 @@ page still renders. looser), dials `GET /api/certs//bundle` over HTTP instead of importing `ensure_certs` — so it cannot mint even by accident — and prints one line per host with no PEM, no token and no password in it (`tests/test_deploy_scripts.py` greps the output for all three). + **A captured socket table is the same case** (phase 28): `scripts/capture-gateway-config.sh` + keeps every config byte (a `server_name` and its `proxy_pass` are the ingress map the page draws, + and a key *path* is a filename, not a key) and rewrites **only** the two address columns of + `/proc/net/tcp{,6}`, and only when `ipaddress` calls the address globally routable — a Linux + global IPv6 is EUI-64, so it encodes this box's NIC, and an inbound visitor's address was never + ours to publish. Documentation space (RFC 5737 / RFC 3849), first-seen order, same width, same + `::ffff:` spelling, idempotent; ports, states and every other column stay verbatim. + `tests/test_agent.py` fails the day a fixture grows a routable endpoint back. 7. **Verify against the deployed upstream; never guess a metric name.** Field maps in `sources/llm.py` and `sources/nut.py` were captured from the running engines and NUT 2.8.5, and are pinned by fixtures in `tests/fixtures/`. Re-capture with `scripts/capture-fixtures.sh` (and `scripts/capture-node-exporter.sh` when comparing - exporters) rather than trusting a README or a memory of an upstream. + exporters, `scripts/capture-gateway-config.sh` for the agent's `haproxy`/`nginx`/`sockets` + ground truth) rather than trusting a README or a memory of an upstream. 8. **Keep dependencies current on purpose.** LLM-written version floors are usually stale. Check after touching dependencies: `uv pip list --outdated` must come back empty. Python is 3.14 (`requires-python >=3.14`, ruff `py314`, pyright `3.14`, Containerfile @@ -194,7 +203,8 @@ store.py ── SQLite (WAL): sample(source, host, metric, res, ts) + event ─ Responsibilities, one line each: - `agent/host_agent.py` — stdlib-only (3.10+), reads `/proc`, `/sys/block`, `/sys/class/drm`, - `statvfs`; serves `GET /snapshot` + `GET /info` + `GET /schema` + `GET /health`, plus an + `statvfs` — plus, for the proxy add-on, the haproxy and nginx configs and `/proc/net/tcp{,6}` + on the host that has them; serves `GET /snapshot` + `GET /info` + `GET /schema` + `GET /health`, plus an HTML status page at `GET /`. Refuses to bind off-loopback without a token **or** a certificate bundle (exit 2). Counters stay counters: the dashboard computes rates so the agent never remembers who polled it last. @@ -238,6 +248,105 @@ Responsibilities, one line each: *system* source red every four minutes over an add-on that page never draws. The refresher is single-flight and keeps the old cache on failure, so a bad pass retries on the next expiry instead of hammering the disk. + **The proxy-config add-on (phase 28 task 01)** is the `haproxy` section: `Collector.haproxy()` + reads `--haproxy-config` (default `/etc/haproxy/haproxy.cfg`, resolved under `--root` so the + fake-tree tests read the fake tree, one size-bounded read per snapshot) and + `parse_haproxy_config()` turns its text into `{frontends: [{name, binds: [{addr, port}], + default_backend, backends}], backends: [{name, servers: [{name, target, port}]}]}`. Three rules + keep it honest. (a) `available` is `smartctl_available`'s three-way answer — `true` read and + parsed, `false` no file at that path, `null` a file that could not be read or holds no haproxy + section — and each non-`true` arm costs exactly **one** `haproxy:` line for the pass, the missing + one saying **not found** (pinned wording, the same cross-module contract as smartctl's, because + the dashboard reads that word to answer "is haproxy installed?"). (b) The section reports the + config's own text and infers nothing about a request: a bind keeps the address as written + (`:::22` splits at the **last** colon into `"::"` and `22`, so an IPv6 bind survives, and + `bind :8080` keeps `addr: null` because `*` is a spelling the config did not use), a `server` + with no port keeps `port: null` and never `0`, a `use_backend` whose target is a fetch + expression (`%[req.hdr(host),map_dom(...)]`) contributes **no name** rather than an expression + where a backend belongs, and everything outside `bind`/`default_backend`/`use_backend`/`server` + is ignored without a word. The frontend→backend routing is therefore the names, which the + dashboard joins. (c) A `listen` section is reported as the frontend+backend pair haproxy itself + defines it as, so a config written in that shorthand cannot read as "this gateway routes + nothing". The section is absent from the document with `--no-proxy` (silence, the `--no-smart` + precedent — "not collected" stays distinguishable from "collected and empty"), it is additive in + the payload, and nothing in it can fail `/snapshot` or `/health`. + `tests/fixtures/gateway_haproxy.cfg` is the verbatim capture the parser is pinned to. + **The same add-on's `nginx` section (phase 28 task 02)** reads `--nginx-config` (default + `/etc/nginx/nginx.conf`) **plus the files its own `include` lines name, one level deep** — the + gateway keeps its vhosts in `conf.d/*.conf` and its SNI routing in `stream.d/*.conf`, so the + main config alone is not the picture. `parse_nginx_config(main_text, included)` is the pure + half over text, and it needs a real scanner rather than haproxy's indent loop because nginx + nests by `{}`: `_nginx_tokens` keeps a `;` inside a quoted value as data (the real config writes + `add_header … "max-age=31536000; includeSubDomains" always;`), `_nginx_statements` builds the + block tree and raises only for a file nginx itself would refuse to start on, and the walk fills + `server` blocks (`listen`, `server_name`, `proxy_pass`), `upstream` blocks and `map` blocks. + Same three rules as `haproxy`: `available` is three-way with exactly one `nginx:` line per arm + (the missing one saying **not found**, the pinned wording the dashboard reads), `--no-proxy` + removes the key in silence, and nothing in the section can fail `/snapshot` or `/health` — + including a config that will not tokenize, which is `available: null` plus its one line with + every other section intact. What it refuses to do is interpret: `listen` keeps the address as + written (`[::]:8443` → `"::"`, a bare `80` → `addr: null`) and ignores the `ssl`/`proxy_protocol` + flags, `server_name _` stays the literal `_`, a map's value becomes a `target` only when it is + an address (the `$connection_upgrade` map's `upgrade`/`close` are header words, and calling one + a host is inventing a fact to fill a field), and a `proxy_pass $var` travels with `variable` + (the `$` dropped once here, so both sides of the join spell it identically) and a **null + `target`** — the map decides per connection, so a merged target would be rule 2 with a page's + clothes on. Include-following is one level and bounded (8 files / 1 MB of included text, the + smart pass-budget shape): the excess is dropped with **one** line naming the count and the + section stays `available: true`; `files` is the main config then the documents actually merged, + spelled the way the config spells them (`/etc/nginx/conf.d/http.conf`, never the path under + `--root`, which is also what lets the parser match its own `include` lines); a literal include + naming a file this host does not have costs nothing and no line; and an `include` inside an + included file is not followed at all — `root=False` makes that structural, not a convention. + **The same add-on's `sockets` section (phase 28 task 03)** is the live half: `read_proc_table` + reads `/proc/net/tcp` and `/proc/net/tcp6` (resolved under `--root`, the config rule) through at + most `PROC_NET_MAX_LINES` = **4096 lines per table** — `/proc`'s own header among them, so 4095 + sockets — because a gateway's file is one line per socket and the read must not cost the poll + somebody else's connection count; the excess is reported in **one** line naming the bound and the + section stays `available: true`. `parse_proc_net_tcp(text, family)` keeps two kernel states only: + `0A` LISTEN → `{family, addr, port}` and `01` ESTABLISHED → + `{family, local: {addr, port}, remote: {addr, port}}`; every other state is a socket in transit + (`TIME_WAIT` above all) and is skipped in silence, since drawing it would put two pipes on one + connection and bury the live pairs. `_proc_net_addr` unrolls `/proc`'s per-32-bit-word + little-endian spelling through `socket.inet_ntop`, so a wildcard keeps its literal `0.0.0.0` / + `::` — the value the inbound join compares against — and a word that is not an address is `addr: + null` with the port kept, never a guess and never `0`. The two tables are read **independently**: + `available` is `true` when either answered (a kernel built without IPv6 has no `tcp6`, which is + absence, so no line — the nginx include rule), `null` with one combined `sockets:` line when + neither did, and a table that is there and denied costs one line even beside a good one. This + section has **no `false` arm on purpose** — `/proc/net/tcp` is not something a host leaves + uninstalled, and `false` beside an empty list would read as "no listeners", which is rule 2 — and + **no join with the config**: a socket does not know its domain, so phase 29's page joins these + ports to the `haproxy`/`nginx` sections and an unclaimed socket stays visible as one. + **Task 04 made the three sections a versioned contract with captured ground truth:** + `AGENT_VERSION` is **1.2.0** (`tests/test_agent.py` pins the constant and asserts the + `GET /schema` block names `haproxy`/`nginx`/`sockets` — each with its `available` flag — plus + `--no-proxy`, `--haproxy-config` and `--nginx-config`), and `scripts/capture-gateway-config.sh` + re-records the six fixtures the parsers are pinned to over `GATEWAY_SSH` (`gateway-ops`): the + haproxy config, the nginx main config **plus every document its own glob `include` names** (so + the fixture tree cannot drift from the config's tree while the parser still follows one level), + and `/proc/net/tcp{,6}`. Three rules it exists to keep. (a) **No privilege and no copy tool**: one + `ssh cat ` per document, because `/proc/net/tcp` reports size 0 and `scp` would copy + an empty file, and every path it reads is world-readable — the same read the agent performs as its + own user, so a capture that needed sudo would be recording something the agent cannot see. + (b) **A bad read is never a fixture**: a nonzero `cat`/`ls`, an empty body or a missing glob include + exits non-zero having written nothing, and a half-written file is removed — the pin is that the + recorded bytes survive a dead alias and an empty read both. (c) **Configs verbatim, socket tables + identifier-free**: the configs keep every `server_name`, `proxy_pass` and key path (a domain is the + map the page draws and a key path is a filename, not a key, while an agent-only bundle carries no + key at all), but a globally routable address in a socket table becomes a documentation one (RFC + 5737 `203.0.113.0/24`, RFC 3849 `2001:db8::/32`) through an idempotent column substitution — ports, + states, every other column and every space are verbatim, loopback/private/link-local/wildcard + addresses are kept, and the IPv4-mapped `::ffff:` spelling survives — because a global IPv6 on + Linux is EUI-64 (derived from the NIC MAC) and an inbound visitor's address is not this repo to + publish: the same rule that scrubs a drive's serial (invariant 6). Those two tables are volatile by + nature, so `tests/test_agent.py` asserts their **shape**, the ports the recorded configs claim, and + the absence of a globally routable endpoint — never a particular row, which a re-run may lawfully + change. What phase 28 stops at is the wire: `sources/host.py` passes **none** of the three into + `/api/system/snapshot` and `store.py` records no series for a proxy or a socket, so the sections + live in the agent's own document and reach no page until phase 29's. What the dashboard does read + today is their **error lines**, through `collectors.ADDON_ERROR_PREFIXES`, so a host with no proxy + config stays out of `/system`'s `degraded` (invariant 3). - `config.py` — env only, no config file format, no CLI flags. `.env` is a *seed*; a real env var always wins. Defaults are safe except the sources, which default to unset. The auth block (phase 11) is `DASH_ADMIN_PASSWORD` (the **seed**, and `_env()` strips, so `VAR=` means @@ -301,11 +410,17 @@ Responsibilities, one line each: same sentence on both pages and only the `timeout` differs (the disks poll has to outlast the agent's smartctl pass, and `None` is translated into httpx's sentinel because a literal `None` means *no timeout at all*). Agent `errors` - become the payload's `degraded` **except** the `smart:` add-on lines — a host without - smartmontools or device read access is unconfigured, not broken (invariant 3); they stay - in `agent.errors` for the `/disks` page to explain — which since phase 22 is only the - device-read case, because a host whose agent said `smartctl` is *missing* is not listed on - that page at all (`api._disk_visible`). + become the payload's `degraded` **except** the add-on lines named by + `ADDON_ERROR_PREFIXES` (`smart`, `haproxy`, `nginx`, `sockets`) — a host without smartmontools, + without device read access, without a proxy config, or whose kernel socket table did not answer + is unconfigured, not broken (invariant 3), and `degraded` is `alerts.py`'s input, so a fleet-wide + agent upgrade would otherwise put an overview WARN on every box that never ran haproxy or nginx. + Those lines stay verbatim in `agent.errors` for the page that owns the add-on — `/disks` explains + the `smart:` ones (and since phase 22 a host whose agent said `smartctl` is *missing* is not + listed on that page at all, `api._disk_visible`), phase 29's gateway page reads the + `haproxy:`/`nginx:`/`sockets:` ones the same way. + **A new agent add-on section adds its prefix here in the same commit**, or its absence-report + becomes a red banner on a page that has nothing to say about it. - `sources/host.py` — snapshot → payload + samples; reboot detection; GPU series per card (`gpu.busy_pct@`). `agent_tls_fields()` is the phase-15 passthrough: a TLS agent's `tls`/`bundle_rev` ride into the payload's existing `agent` block, and a token-mode snapshot @@ -979,7 +1094,7 @@ That scheme is the history API's key — treat it as a public interface. | ----------------- | ------------------ | -------------------------------------------------------------------- | | `driveripper` | `driveripper-ops` | Threadripper 24c/48t + Radeon AI PRO R9700, vLLM `:8000`, **upsd**, the dashboards | | `deskwork` | `deskwork-ops` | Strix Halo (Radeon 8060S iGPU), gufo `:8731` | -| `gateway` | `gateway-ops` | router, no GPU | +| `gateway` | `gateway-ops` | router, no GPU, the fleet's ingress: haproxy (`/etc/haproxy/haproxy.cfg`, mode `0644`: six `frontend`s on `::: v4v6` → six `backend`s, one `server : check inter 5s` each — the file the agent's `haproxy` section parses and `tests/fixtures/gateway_haproxy.cfg` captures) and nginx (`/etc/nginx/nginx.conf` plus `conf.d/http.conf` vhosts on `4443`/`8443` and `stream.d/stream.conf`'s `map $ssl_preread_server_name $targetBackend` → `proxy_pass $targetBackend` on `443` — the tree the agent's `nginx` section parses) | - **Transport is direct dial: dev is the prod topology, repo-wide (phase 10 task 05).** The designed shape is one `dashboards-agent` **systemd service per host**, installed by `ops` @@ -1023,14 +1138,41 @@ That scheme is the history API's key — treat it as a public interface. `host:gateway`, `disks:driveripper`, `disks:deskwork`, `disks:gateway`, `nut`, `llm:gufo`, `llm:vllm`. **The fleet is mTLS since 2026-10-21 (phase 24 task 03), so the bearer-token transport is now - history rather than the designed shape.** All three hosts run `agent/host_agent.py` **1.1.0** - from an installed bundle, every unit carries `--cert-bundle /etc/dashboards-agent.certs` (plus + history rather than the designed shape.** `gateway` runs `agent/host_agent.py` **1.2.0** (phase 28 + task 05, 2026-10-22) and `deskwork`/`driveripper` still run **1.1.0**; the spread costs nothing, + because 1.2.0 is purely additive — the three proxy-config sections are new keys and a 1.1.0 + snapshot simply has none — so those two move at their next re-stage rather than on a flag day. + The control that proves "purely additive" is `data/logs/verify-28-final.py`: it stashes the whole + working tree, imports both `host_agent.py` files side by side, calls both `collect()`s against + the same `--root`, and demands the 1.2.0 document equal the 1.1.0 document key-for-key with + `haproxy`/`nginx`/`sockets` added — and 1.2.0 with `--no-proxy` equal it exactly. + All three are installed from a bundle, every unit carries `--cert-bundle /etc/dashboards-agent.certs` (plus `RuntimeDirectory=dashboards-agent`, and still its `--token-file` line), and `HOST_TARGETS` is `"driveripper=https://10.1.0.126:9120 deskwork=https://10.1.0.51:9120 gateway=https://10.1.1.27:9120"` — **no `http://` entry and no userinfo anywhere**: the certificate is the credential (the pre-flip `.env.dev` is backed up in `data/env-backup/`). `agent.tls: true` + `agent.bundle_rev: 2` in every payload, `deployed_bundle_rev == bundle_rev` for all three on `/certs`, and - `./scripts/dev-server.sh status` reads the nine sources `ok`. The per-host order was the safety: + `./scripts/dev-server.sh status` reads the nine sources `ok`. **Phase 28 moved the gateway to + 1.2.0 on 2026-10-22 and only the agent file moved.** Task 04 read it read-only first + (`data/logs/verify-28-task04.py` → `28-task04-probe.out`): 1.1.0, `agent.tls: true` + `bundle_rev 2`, + every pre-1.2.0 section answering with live numbers and **no** `haproxy`/`nginx`/`sockets` key — + the control that made the install provably additive. Task 05 then ran the phase-24 procedure on + that one host (re-stage `/home/ops/agent-install/` first with `*-20261022-backup` snapshots, gap 10; + `sudo /home/ops/agent-install/deploy/install-agent.sh --bind 10.1.1.27 --port 9120` with the + arguments read back off `systemctl cat`; `systemctl restart dashboards-agent`) and verified it two + ways: the CA-only `--status` probe answers `version 1.2.0` on `/health` and still refuses + `/snapshot` with `client certificate required`, and the dashboard's own mTLS dial + (`data/logs/verify-28-task05.py` → `28-task05-verify.out`) reads `agent_version 1.2.0` with all + three sections `available: true` (6 frontends all routed / 6 backends, 11 nginx servers and the + `$targetBackend` map, 17 listeners including 443) beside every pre-1.2.0 section intact and + `host:gateway`/`disks:gateway` `ok`. That listener count is a reading, not a fact: the same probe + re-run on 2026-10-22 for this phase's completion pass (`data/logs/verify-28-final.py` → + `28-finalpass-live.out`) answered **19** listeners with the config halves unchanged (6/6, 11 + servers) — `/proc/net/tcp` is what a busy gateway writes every second, which is exactly why no + test asserts a row of it. The one fact from that transcript worth carrying to the next + host: the bundle **on disk** — not the `--cert-bundle` flag — is what puts the TLS lines into the + unit render, so an upgrade re-run that omits the flag is the silent path (no guard refusal) and + `systemctl cat` came back byte-identical (`26bd812…`); only `host_agent.py` changed. The per-host order was the safety: install agent 1.1.0 + bundle → `systemctl restart` → the CA-only `--status` probe answers → *then* that host's URL flips, so no source was red longer than its own host's restart. `data/logs/NOTES-24-rollout.md` is the transcript (sha256 of what was staged where, the unit @@ -1112,8 +1254,9 @@ That scheme is the history API's key — treat it as a public interface. filter removal, nor the re-run guard — running it would have replaced driveripper's `User=root` + `CapabilityBoundingSet` unit with the snapshot's capability-less one, **silently**. That trap is now closed on all three hosts: `/home/ops/agent-install/` is `ops`-owned, so the repo's - `deploy/install-agent.sh` (sha256 `ffbcd91f…`) and `agent/host_agent.py` (**1.1.0**) were staged - over it without root, and each snapshot was left beside the grant as + `deploy/install-agent.sh` (sha256 `ffbcd91f…`) and `agent/host_agent.py` (**1.1.0**, re-staged as + **1.2.0** on `gateway` in phase 28) were staged over it without root, and each snapshot was left + beside the grant as `install-agent.sh.oct18-backup`, which sudo will not run. **Re-staging is therefore the first step of any install, not an optional one** — the grant is only as current as the file it points at. The `systemctl restart *` grant on those two hosts is the other half of an unattended @@ -1233,9 +1376,9 @@ uv run python scripts/dev-certs-export.py --url http://127.0.0.1:8090 \ # GET /api/certs//bundle and writes # data/certs/.bundle at 0600 — mints nothing -uv run pytest # hermetic: 1549 passed + 15 skipped (11 live, +uv run pytest # hermetic: 1650 passed + 15 skipped (11 live, # 4 store_perf), no network -uv run pytest --cov # coverage gate (98.94%; floor 94%) +uv run pytest --cov # coverage gate (98.97%; floor 94%) uv run pytest -m live --live # opt-in: real fleet / engines / upsd uv run pytest -m store_perf --store-perf # opt-in: the store hot-path timings over the seeded # store, priced against `store_perf.BUDGETS` (the cached @@ -1270,6 +1413,11 @@ uv run ruff check src tests agent && uv run pyright SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh # ... plus smartctl per physical disk (needs device read # access: plain smartctl, else `sudo -n`; scrubs serials) +./scripts/capture-gateway-config.sh # ... plus the gateway's proxy tree and socket tables: + # haproxy.cfg, nginx.conf + its glob includes, and + # /proc/net/tcp{,6} over $GATEWAY_SSH (gateway-ops); + # world-readable paths only, and a failed or empty read + # never replaces a good fixture bash .agents/validate.sh # the phase gate: the four checks above (pytest # --cov --cov-fail-under=94, ruff, pyright, plus a @@ -2342,7 +2490,9 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl (phase 15 tasks 01-05 landed 2026-10-20; phase 24 task 03 rolled the fleet out 2026-10-21). The rollout is done and verified: gateway, deskwork and driveripper each run `agent/host_agent.py` **1.1.0** with `/etc/dashboards-agent.certs` installed `0640 root:dashboards-agent`, their units - carry `--cert-bundle` + `RuntimeDirectory=dashboards-agent` (and keep `--token-file`), + carry `--cert-bundle` + `RuntimeDirectory=dashboards-agent` (and keep `--token-file`) — the + gateway has since moved to **1.2.0** for phase 28's proxy-config sections on the same unit and + the same bundle, see the fleet record above — `HOST_TARGETS` is `https://` for all three with no userinfo, `agent.bundle_rev` reads `2` on every host and equals what `/api/certs` issued, and the nine sources are `ok`. The transcript is `data/logs/NOTES-24-rollout.md`, and task 04 re-read the whole end state **read-only** afterwards diff --git a/Containerfile b/Containerfile index db1812f..e4cd740 100644 --- a/Containerfile +++ b/Containerfile @@ -1,4 +1,4 @@ -# Homelab dashboards — one image, three dashboards, one port. +# Homelab dashboards — one image, four dashboards and their overview, one port. # # podman build -t dashboards . # podman run --rm -p 8090:8090 --env-file dashboards.env \ diff --git a/README.md b/README.md index ded04ee..54de8ea 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Homelab Dashboards -Four dashboards, one domain, one process, one SQLite file: +Four dashboards and the overview page that points at them — one domain, one process, one SQLite +file: | Page | What it answers | Source | | ----------- | ------------------------------------------------------------------------------- | -------------------------------------------------- | @@ -90,7 +91,8 @@ that exact line. upsd is dialled at ## The host agent `agent/host_agent.py` is one file, **stdlib only**, Python 3.10+. It reads `/proc`, -`/sys/block`, `/sys/class/drm` and `statvfs` and answers `GET /snapshot` (everything, one +`/sys/block`, `/sys/class/drm` and `statvfs` — and, on the host that fronts the fleet, the +haproxy/nginx configs and `/proc/net/tcp{,6}` — and answers `GET /snapshot` (everything, one JSON document; `?top=N` trims the process lists), `GET /info` (slow-changing identity: OS, kernel, CPU model, GPU list), `GET /schema` (the schema version plus this document) and `GET /health` (the one route that needs no credential) as JSON, with a plain HTML status @@ -147,6 +149,81 @@ one installed before phase 09 has no SMART pass to run. Opt out silently with `- The drive truth this is all developed against lives in `tests/fixtures/smartctl_driveripper_*.txt`: 16 real drives, recorded with serials scrubbed, each one parsed by a test. +The `haproxy` section is the same kind of best-effort add-on, for the host that fronts the fleet. +The agent reads `/etc/haproxy/haproxy.cfg` (`--haproxy-config` names another path) and reports +what the config says: every `frontend` block with its name, each `bind` address:port exactly as +written (`:::22` is the address `::` on port `22`) and the backend names it routes to +(`default_backend`, plus every named `use_backend`), and every `backend` block with each `server`'s +name, target and port — `port: null` when the line names none, never `0`. It interprets nothing: +the routing between the two lists is the **names**, so joining them is the page's job (the gateway +page is phase 29) and the agent never evaluates a rule. `available` answers three questions the way `smartctl_available` does — +`true` read and parsed, `false` no config at that path (this host runs no haproxy), `null` a file +is there but could not be read or understood — and each non-`true` answer costs exactly **one** +`errors` line for the pass. Like the SMART lines, those lines never make the *system* page amber: +a host that never installed haproxy is unconfigured, not broken, and the page that owns the add-on +reads the sentence out of `agent.errors`. A failing config read cannot fail `/snapshot` or +`/health`, and `--no-proxy` drops the section and the line both. The parser is pinned to +`tests/fixtures/gateway_haproxy.cfg`, the verbatim capture from gateway. Neither path is an env +var and `deploy/install-agent.sh` passes neither flag — the unit it writes runs on the defaults — +so a box whose configs live elsewhere means adding the flag to `$UNIT` after installing. + +The `nginx` section is that add-on's other half, and it reads more than one file: the main config +(`/etc/nginx/nginx.conf`, `--nginx-config` names another) **plus the files its own `include` lines +name**, one level deep, because that is where nginx keeps its vhosts +(`include /etc/nginx/conf.d/*.conf`) and its SNI routing (`/etc/nginx/stream.d/*.conf`). It +reports every `server` block with its `listen` address:port pairs (`[::]:8443` is the address `::` +on port `8443`; a bare `listen 80` has no address, and the `ssl`/`proxy_protocol` flags are not +this section's business), its `server_name` list (the catch-all `_` included verbatim — "any +domain" is what the page says about it), and every `proxy_pass`; every `upstream` block with its +`server` targets; and every `map` block with its keys. A `proxy_pass $var` is served with the +variable's name and `target: null`: nginx resolves that per connection from the `map` that defines +it, so the agent reports both facts **unmerged** and leaves the joining to a page — a merged target +would be a number the wire never published. `files` names the main config and the included +documents actually merged, and the include pass is bounded (8 files / 1 MB): what it could not +take is said in one `errors` line naming the count, and the section still reads `true`. The same +three-way `available`, the same one line per arm, the same rule that a missing, unreadable or +unparseable config degrades only this section — and `--no-proxy` drops the whole family with it, +in silence. + +The `sockets` section is that add-on's live half: what is actually open and carrying traffic, as +opposed to what a config claims. The agent reads `/proc/net/tcp` and `/proc/net/tcp6` and keeps +only the two states a gateway page can draw pipes from — kernel state `0A` LISTEN into `listens` +(`{family, addr, port}`) and `01` ESTABLISHED into `established` (`{family, local, remote}`, each +end an `{addr, port}`). Every other state is a socket in transit: a `TIME_WAIT` row is a +connection that stopped being true before the page painted, and a proxy accumulates thousands of +them, so they are skipped without a word. Each table is read through **at most 4096 lines** +(`/proc`'s own header among them, so 4095 sockets per table); a longer one reports the rows it read +and costs one `errors` line naming the bound, with `available` still `true`. `addr` is the address +itself, unrolled from `/proc`'s per-32-bit-word little-endian spelling — `0.0.0.0` and `::` keep +the literal wildcard form, because "open on every address" is the fact the inbound join keys on — +and a word that is not an address is `null`, never a guess. The tables are read independently: a +kernel without IPv6 has no `tcp6` file, which costs nothing and no line, while a table that is +there and denied costs one. `available` is `true` when either table answered and `null` when +neither did, with one line saying why; there is no `false`, because a table that did not answer is +"no answer" and not "this host has no sockets". Nothing here is joined to a config: a socket does +not know its domain or its backend, so phase 29's gateway page will be what joins these ports to +the `haproxy` and `nginx` sections — where a socket no frontend claims stays visible as one. Until +that page exists the three sections live only in the agent's own `/snapshot` document: no `/api` +payload grows them, no `sample` row is stored for them, and nothing charts them. + +All three parsers are pinned to captures rather than to descriptions, and +`./scripts/capture-gateway-config.sh` re-records them over the `GATEWAY_SSH` alias (default +`gateway-ops`): `/etc/haproxy/haproxy.cfg`, `/etc/nginx/nginx.conf` plus every document its own +glob `include` lines name (`gateway_nginx_conf_d_*.conf`, `gateway_nginx_stream_d_*.conf`), and +`/proc/net/tcp{,6}`. No sudo — those paths are world-readable, which is exactly what the agent +reads as itself — and a read that fails, or answers empty, leaves the existing fixture alone and +exits 1, so a laptop with no route to the gateway cannot turn six captures into six empty files. +The configs are not rewritten (their domains are the ingress map, and key material appears only +as the paths nginx is told to read); the socket tables lose only their identifiers, in line with +the rule that scrubs a drive's serial: a globally routable address becomes a documentation one +(RFC 5737 / RFC 3849), because a global IPv6 on Linux is EUI-64 and an inbound visitor's address +is not ours to publish. Every port, state, column and width in those tables is verbatim, and +those files are volatile by nature — a proxy in service has a different connection set every +second — so the tests assert shape and the ports the configs claim, never a particular row. +`AGENT_VERSION` is **1.2.0**, the version that ships all three sections; a host still running +1.1.0 answers `GET /schema` with a block that names none of them, and its snapshot simply has no +such keys. + `deploy/install-agent.sh --fetch-bundle --cert-name --session ` is the same TLS install with the download done inside the registration run: it asks `GET /api/certs//bundle`, writes the answer to a temp file, and hands it to the @@ -338,6 +415,20 @@ for a bundle install, so step 3 is a copy rather than a guess. A host left on `h working over its bearer token — the rollout is per host, and one revoked or expired certificate never blinks the others. +**Upgrading an already-registered host to a newer agent is one file, not a rollout.** Re-stage +`agent/host_agent.py` into `/home/ops/agent-install/agent/` (the snapshot-first rule applies: a path +grant is only as current as the file it names, so copy what you are replacing to +`host_agent.py--backup` first — a name the sudo grant does not name, so sudo will not run it), +run the granted path with **the unit's own arguments and nothing else** +(`sudo /home/ops/agent-install/deploy/install-agent.sh --bind --port 9120`), and restart. +No `--cert-bundle`, no `--fetch-bundle`, no `--force`: the bundle already on disk is what the unit +render reads — the flag only replaces the file — so the render equals the installed unit, the re-run +guard prints nothing, and `/etc/dashboards-agent.certs` and the token file keep their bytes. Then +the same two checks the registration uses: the CA-only `--status` probe (with `--bind`) answers +`{"ok": true, …, "version": "1.2.0"}` on `/health`, and the dashboard's own `https://` dial answers +whatever the new version added — `agent_version` plus the `haproxy` / `nginx` / `sockets` sections +for a 1.2.0 agent — beside the sections it always had. + Two things the first full rollout learned, both cheap to hit and expensive to guess at: * **A CA row older than the identifier fix must be rotated before any bundle is installed.** A @@ -953,6 +1044,11 @@ uv run ruff check src tests agent && uv run pyright # lint + types SMART_HOSTS="host-a-ops host-b-ops" ./scripts/capture-fixtures.sh # ... and smartctl per physical disk: plain smartctl, # `sudo -n` if that is denied; scrubs serials/WWNs +./scripts/capture-gateway-config.sh # ... and the gateway's proxy tree + socket tables: + # haproxy.cfg, nginx.conf + its glob includes, + # /proc/net/tcp{,6} over $GATEWAY_SSH (gateway-ops). + # World-readable paths only; a failed read never + # replaces a good fixture bash .agents/validate.sh # all of the above + `node --check` on every page # script + repo hygiene, with the 94% coverage floor ``` diff --git a/agent/host_agent.py b/agent/host_agent.py index eeed7e2..8a73ba1 100644 --- a/agent/host_agent.py +++ b/agent/host_agent.py @@ -32,6 +32,41 @@ Design rules fails the snapshot or `/health`. Absence is a documented state, not a red panel: install `smartmontools` and give the agent user read access to the block devices (typically the `disk` group), or pass `--no-smart` to opt out silently. +* **The proxy configs are a best-effort add-on too, with the same three-way answer.** The + `haproxy` section reads the host's haproxy config (`--haproxy-config`, default + /etc/haproxy/haproxy.cfg) and reports the `frontend` blocks (name, each `bind` + address:port, `default_backend`/`use_backend` references) and the `backend` blocks (name, + each `server` line's name and target host:port). The `nginx` section reads the main nginx + config (`--nginx-config`, default /etc/nginx/nginx.conf) **plus the files its own `include` + lines name, one level deep** (`include /etc/nginx/conf.d/*.conf`), and reports the `server` + blocks (each `listen` address:port, `server_name`, every `proxy_pass`), the `upstream` + blocks and the `map` blocks — a `proxy_pass $var` keeps the variable's name and a + `target: null`, because nginx picks that address per connection from the map and a merged + answer would be a number this wire never published. `available` answers three questions, the + way SMART's `smartctl_available` does: `true` the file was read and at least one block + parsed, `false` there is no such file (this host does not run haproxy or nginx — one `errors` + line for the whole pass, saying the file is missing), `null` a file is there but could not be + read or understood (also one line). Whichever arm fired, the snapshot and `/health` are + unaffected and the other sections are intact; `--no-proxy` removes the sections with no + errors line at all, exactly as `--no-smart` leaves every disk's `smart` null in silence. + A host with no proxy on it is unconfigured, not broken. +* **The live socket table** is the `sockets` section: `/proc/net/tcp` and `/proc/net/tcp6`, + read at source and reported as the two states a gateway page can draw pipes from — `0A` + LISTEN (the frontends that are open) and `01` ESTABLISHED (the pairs flowing right now). + Every other state is a socket in transit and is skipped in silence, because an ocean of + `TIME_WAIT` rows would bury the pairs the animation shows. Each table is read through + **at most** `PROC_NET_MAX_LINES` lines (4096, `/proc`'s own header among them, so 4095 + sockets per table): a busy proxy writes tens of thousands of rows and a snapshot that + copied them every poll would spend the request on a list no page can draw. A longer table + reports the rows it read and says it was truncated; `available` stays `true`. Addresses are + unrolled from `/proc`'s per-32-bit-word little-endian spelling into the address itself + (`0.0.0.0` and `::` keep their literal wildcard form, because the wildcard *is* the fact the + dashboard's inbound join keys on), and there is **no join with the config here**: a socket + does not know its domain or its backend, so phase 29's page joins these ports to the + `haproxy`/`nginx` sections instead. It is best-effort the way the configs are: neither table + answering is `available: null` with one `sockets:` line and an otherwise intact snapshot, + and `--no-proxy` removes it with the rest of the family, because a box with no proxy has + nothing to watch from either side. * **The state** is a small pid->cpu-time cache so `processes.top_cpu` can answer a question /proc cannot (CPU needs two samples), bounded and cleared when stale, plus the short-TTL SMART result cache above. @@ -80,6 +115,49 @@ SCHEMA (GET /snapshot) temperatures [{kind: hwmon|thermal, chip, label, celsius}] processes {total, top_rss: [{pid, name, rss_bytes}], top_cpu: [{pid, name, cpu_pct}]} + haproxy {available: true|false|null, config: str, frontends: [{name, binds: + [{addr, port}], default_backend: str|null, backends: [str]}], + backends: [{name, servers: [{name, target, port}]}]} + # parsed from --haproxy-config; additive (phase 28) and absent entirely + # with --no-proxy. `addr` is the bind's address as written (`:::22` -> + # `::`), `port` null when a target names none (never 0), and the + # frontend->backend routing is the names themselves: the dashboard + # joins them, the agent reports only what the config says. + nginx {available: true|false|null, config: str, files: [str], + servers: [{listen: [{addr, port}], server_names: [str], proxy_passes: + [{location: str|null, scheme: str|null, + target: {host, port}|null, variable: str|null}]}], + upstreams: [{name, servers: [{host, port}]}], + maps: [{variable, entries: [{key, target: {host, port}|null}]}]} + # parsed from --nginx-config plus the files its own `include` lines name + # (one level, at most 8 files / 1 MB of included text; `files` is the + # main config then every include actually merged); additive (phase 28) + # and absent entirely with --no-proxy. `listen` keeps the address as + # written (`[::]:8443` -> `"::"`, a bare `80` -> a null addr, never `*`), + # `server_names` keeps the catch-all `_` verbatim, `location` is null for + # a proxy_pass written in the server block itself (the stream form), and + # a `proxy_pass $var` is served with `variable` (the `$` dropped) and + # `target: null`: the map whose variable matches decides per connection, + # so both facts travel unmerged and the dashboard joins them. + sockets {available: true|null, listens: [{family, addr, port}], + established: [{family, local: {addr, port}, remote: {addr, port}}]} + # /proc/net/tcp and /proc/net/tcp6 read at source: kernel state 0A LISTEN + # into `listens`, 01 ESTABLISHED into `established`, every other state + # skipped (a socket in transit is not a pipe); additive (phase 28) and + # absent entirely with --no-proxy. Each table is read through at most + # PROC_NET_MAX_LINES (4096) lines including its own header, so a longer + # one is reported as read-with-truncation in one errors line with + # `available` still true. `family` is "v4"/"v6" and `addr` is that + # family's address with /proc's per-word little-endian spelling unrolled + # (`0.0.0.0` and `::` keep the literal wildcard form the inbound join + # keys on); `addr` is null for a word that is not an address and `port` + # null for a word that is not a number, never 0. `available` is true when + # either table answered and null when neither did, with one `sockets:` + # line saying why: there is no `false` arm, because a table that did not + # answer is "no answer" and not "this host has no sockets", and + # --no-proxy already reports "did not look" by removing the key. No + # config join: a socket does not know its domain, so the gateway page + # joins these ports to `haproxy` and `nginx` itself. errors [str] reads that failed; non-empty means the snapshot is partial Deployment: see deploy/install-agent.sh (systemd unit, binds 127.0.0.1 unless you pass @@ -95,6 +173,8 @@ from __future__ import annotations import argparse import contextlib +import fnmatch +import glob import json import os import re @@ -108,9 +188,9 @@ import time from collections.abc import Callable from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from typing import Any +from typing import Any, cast -AGENT_VERSION = "1.1.0" +AGENT_VERSION = "1.2.0" SCHEMA_VERSION = 1 PAGE_SIZE = 0 # resolved from sysconf at runtime @@ -170,6 +250,92 @@ _SMART_ATTR_FIELDS = { "Percentage_Lifetime_Use": "percentage_used", } +# --------------------------------------------------------------------------- # +# haproxy config (the additive `haproxy` section) +# --------------------------------------------------------------------------- # + +#: Where haproxy keeps its config on this fleet's gateway host, and the default for +#: `--haproxy-config`. A host that does not run haproxy has no file at this path, which is +#: the `available: false` arm rather than a fault worth more than one `errors` line. +HAPROXY_CONFIG_PATH = "/etc/haproxy/haproxy.cfg" + +#: A haproxy config is a few kilobytes of hand-written text, and this file is read on every +#: snapshot. The bound is what keeps a `--haproxy-config` pointed at a device node or a log +#: from costing the request a read that never ends — the lesson the SMART bounds exist for. +HAPROXY_CONFIG_MAX_BYTES = 1_000_000 + +#: The words that open a haproxy *section*. Used only to tell "a config with no proxies" +#: apart from "text that is not a haproxy config": a file whose top level names none of +#: these is the second fact, and the section says so instead of serving an empty `true`. +HAPROXY_KEYWORDS = frozenset({ + "global", "defaults", "frontend", "backend", "listen", "resolvers", "peers", + "mailers", "userlist", "cache", "ring", "programmatic", "extend", +}) + +# --------------------------------------------------------------------------- # +# nginx config (the additive `nginx` section) +# --------------------------------------------------------------------------- # + +#: Where nginx keeps its main config on this fleet's gateway host, and the default for +#: `--nginx-config`. A host that does not run nginx has no file at this path, which is the +#: `available: false` arm rather than a fault worth more than one `errors` line. +NGINX_CONFIG_PATH = "/etc/nginx/nginx.conf" + +#: The main config is read on every snapshot, so it gets the same size bound the haproxy one +#: has: a `--nginx-config` aimed at a device node must not cost the request a long read. +NGINX_CONFIG_MAX_BYTES = 1_000_000 + +#: Include-following is **one level and bounded**, which is what makes the section cheap +#: enough to read on every snapshot. The captured gateway tree is one level deep +#: (`nginx.conf` -> `conf.d/*.conf` + `stream.d/*.conf`), so 8 files covers a host that +#: splits one vhost per file with room to spare, and 1 MB of included text is two orders of +#: magnitude above the 10 KB this fleet's whole ingress occupies. Exceeding a bound drops the +#: excess with one `errors` line naming how much was left out — the SMART pass-budget shape: +#: the pass reports what it did not cover instead of quietly serving a shorter list. +NGINX_INCLUDE_MAX_FILES = 8 +NGINX_INCLUDE_MAX_BYTES = 1_000_000 + +#: The words that open an nginx *context* or name an unmistakably main-level directive. Used +#: only to tell "a config with no proxies" apart from "text that is not an nginx config": a +#: file whose top level names none of these is the second fact, and the section says so +#: instead of serving an empty `true`. +NGINX_CONTEXTS = frozenset({ + "http", "stream", "mail", "events", "server", "upstream", "map", "geo", "types", + "match", "split_clients", "limit_req_zone", "limit_conn_zone", "perl", "js_import", + "user", "worker_processes", "worker_cpu_affinity", "worker_rlimit_nofile", "env", + "include", "load_module", "pid", "error_log", "thread_pool", "pcre_jit", +}) + +#: The characters that make an `include` argument a glob rather than a path. nginx uses the +#: C library's `glob()`; `glob.glob` is the stdlib's, and this is the same test it applies. +_GLOB_SPECIALS = "*?[" + +# --------------------------------------------------------------------------- # +# live sockets (the additive `sockets` section) +# --------------------------------------------------------------------------- # + +#: The two kernel tables the section reads, each with the family label its rows carry. They +#: are read independently because a kernel built without IPv6 has no `/proc/net/tcp6` at all: +#: one table answering is the host telling us what it has, not a failure. +PROC_NET_TABLES: tuple[tuple[str, str], ...] = (("v4", "proc/net/tcp"), + ("v6", "proc/net/tcp6")) + +#: The read bound, in lines **per table**, `/proc`'s own header among them — so a truncated +#: pass reports at most `PROC_NET_MAX_LINES - 1` sockets. A gateway forwards every connection +#: its owner makes, and `/proc/net/tcp` is one line per socket including the `TIME_WAIT` ones +#: nobody can see: the bound is the cost of the section, not a display choice. The excess is +#: reported as what it is in one `errors` line (the SMART pass-budget and nginx include-bound +#: shape) instead of quietly serving a shorter list. +PROC_NET_MAX_LINES = 4096 + +#: The two TCP states the gateway page draws pipes from, as `/proc` spells them: `0A` LISTEN +#: (an open frontend) and `01` ESTABLISHED (a pair flowing right now). Every other state is a +#: socket in transit — `TIME_WAIT`, `CLOSE_WAIT`, the SYN queue — and an ocean of those would +#: bury the pairs the animation shows, so they are skipped without a word. +PROC_NET_STATE_LISTEN = "0A" +PROC_NET_STATE_ESTABLISHED = "01" + + # pci slot -> human model, filled on first look (lspci output cannot change at runtime). _GPU_MODEL_CACHE: dict[str, str | None] = {} @@ -238,12 +404,22 @@ class Collector: def __init__(self, root: str | os.PathLike[str] = "/", *, top_processes: int = 5, net_all: bool = False, smartctl: str = "smartctl", - smart: bool = True) -> None: + smart: bool = True, proxy: bool = True, + haproxy_config: str = HAPROXY_CONFIG_PATH, + nginx_config: str = NGINX_CONFIG_PATH) -> None: self.root = Path(root) self.top_processes = top_processes self.net_all = net_all self.smartctl = smartctl self.smart_enabled = smart + # The proxy add-on: the two configs and the live socket table. One flag for the family, + # because `--no-proxy` is one operator decision about a host that serves nothing — and a + # box with no proxy has no pipes to watch from either side. Both config paths are + # absolute on purpose: they are resolved under `root`, so the fake-tree tests read + # `root/etc/haproxy/haproxy.cfg`. + self.proxy_enabled = proxy + self.haproxy_config = haproxy_config + self.nginx_config = nginx_config self.smart_timeout_seconds = SMARTCTL_TIMEOUT_SECONDS self.smart_budget_seconds = SMARTCTL_BUDGET_SECONDS self.smart_cache_seconds = SMARTCTL_CACHE_SECONDS @@ -272,6 +448,48 @@ class Collector: text = self.read_text(*parts) return text.splitlines() if text else [] + # -- config-file reads (the proxy-config add-on) ------------------------- # + def _config_path(self, config: str) -> Path: + """Resolve a config path the way every other path here is resolved. + + An absolute path — the default of both `--haproxy-config` and `--nginx-config`, and what + a deployment writes — is read **under `--root`**, so the fake-tree tests read the fake + tree like every other collector here. A relative one is taken as written: haproxy's own + `-f` and nginx's `-c` both resolve it against the working directory, and a path an + operator handed this agent means what it means on that host, not under a chroot nobody + asked for. + """ + return Path(config) if not config.startswith("/") else self._path(config.lstrip("/")) + + def read_config(self, config: str, max_bytes: int) -> str: + """One size-bounded read of a config file, for the proxy add-on's best-effort sections. + + Raises rather than deciding what to report, because the two failures are two different + operator actions: `FileNotFoundError` is "nothing installed here" (the section's + `available: false` arm) and any other `OSError`/`ValueError` is "there and unreadable" + (the `null` arm, including a file over `max_bytes` — the bound that keeps a path aimed + at a log or a device node from costing the snapshot an unbounded read). + """ + path = self._config_path(config) + if path.stat().st_size > max_bytes: + raise ValueError(f"larger than {max_bytes} bytes") + return path.read_text(encoding="utf-8", errors="replace") + + def _config_label(self, arg: str, resolved: str) -> str: + """The path a config would write for a file this agent read under `--root`. + + `files` and the include join speak the config's paths (`/etc/nginx/conf.d/http.conf`), + because that is the sentence an operator can check against `nginx -t`. Only an absolute + argument is rooted, so only it comes back un-rooted; a relative one was taken as written + and is labelled as written. + """ + if not arg.startswith("/"): + return resolved + try: + return f"/{Path(resolved).relative_to(self.root)}" + except ValueError: + return resolved + def read_number(self, *parts: str) -> float | None: """First number in the file. @@ -653,6 +871,242 @@ class Collector: smart["model"] = device.get("model") device["smart"] = smart + # -- proxy configs (haproxy) ---------------------------------------------- # + def haproxy(self, errors: list[str]) -> dict[str, Any]: + """Read and parse the host's haproxy config — never raises, never fails the snapshot. + + `available` is the three-way answer `attach_smart`'s `smartctl_available` established, + because the three states say three different things and only one of them is a fault: + `true` the file was read and parsed, `false` there is no config at this path (this host + runs no haproxy — unconfigured, not broken), `null` a file is there but could not be + read or understood. Each non-`true` arm writes exactly **one** `haproxy: ` line for the + whole pass; the missing one says the file is **not found** and names the path, which is + what lets a dashboard tell "no haproxy here" from "haproxy config unreadable" without a + second probe, and what `tests/test_agent.py` pins. + + Everything else in the section is the config's own text: a bind keeps the address it was + written with (`:::22` splits into `"::"` and `22`), a `server` that names no port keeps + `port: null` rather than a made-up `0`, and the frontend to backend routing stays the + **names**, so the dashboard joins them and this file never interprets a rule. + """ + config = self.haproxy_config + section: dict[str, Any] = {"available": None, "config": config, + "frontends": [], "backends": []} + # Path resolution and the size bound live in `read_config`, the one place both + # proxy-config sections decide how a config file is read. + try: + text = self.read_config(config, HAPROXY_CONFIG_MAX_BYTES) + except FileNotFoundError: + section["available"] = False + errors.append(f"haproxy: {config} not found — haproxy is not installed here, or " + "its config lives elsewhere (--haproxy-config); --no-proxy opts out") + return section + except (OSError, ValueError) as exc: + errors.append(f"haproxy: {config} is there but could not be read: {exc}") + return section + + parsed = parse_haproxy_config(text) + if not parsed["recognised"]: + errors.append(f"haproxy: {config} holds no haproxy section (global, defaults, " + "frontend, backend, listen) — not a haproxy config") + return section + section["available"] = True + section["frontends"] = parsed["frontends"] + section["backends"] = parsed["backends"] + return section + + # -- proxy configs (nginx) ------------------------------------------------ # + def nginx(self, errors: list[str]) -> dict[str, Any]: + """Read and parse the host's nginx configs — never raises, never fails the snapshot. + + The same three-way answer as `haproxy()`, for the other half of the gateway's ingress: + `true` the main config was read and understood, `false` there is no config at this path + (this host runs no nginx — one `nginx: ` line saying the file is **not found**), `null` + a file is there but could not be read, could not be parsed, or holds no nginx context + (one line either way). The + wording is the cross-module contract `collectors.ADDON_ERROR_PREFIXES` exists for: the + dashboard reads "not found" to answer "is nginx installed here?", and the line never + makes the *system* page amber for a box that never had nginx. + + The main config is not the whole picture — nginx keeps its vhosts in the files its + `include` lines name — so those are followed **one level** and bounded + (`NGINX_INCLUDE_MAX_FILES` / `NGINX_INCLUDE_MAX_BYTES`, in `nginx_include_files`), and + `files` names the main config plus every included document that was actually merged. What + is left out is said out loud: each reason the pass could not take an include it found + costs one more line, never one per file. + """ + config = self.nginx_config + section: dict[str, Any] = {"available": None, "config": config, "files": [], + "servers": [], "upstreams": [], "maps": []} + try: + text = self.read_config(config, NGINX_CONFIG_MAX_BYTES) + except FileNotFoundError: + section["available"] = False + errors.append(f"nginx: {config} not found — nginx is not installed here, or its " + "config lives elsewhere (--nginx-config); --no-proxy opts out") + return section + except (OSError, ValueError) as exc: + errors.append(f"nginx: {config} is there but could not be read: {exc}") + return section + + # The two extra return values are already spoken about in `errors`; the section itself + # carries no counters, because "which files were read" is what `files` is for. + included, _dropped, _unreadable = self.nginx_include_files(nginx_include_paths(text), + errors) + parsed = parse_nginx_config(text, included) + if parsed["unparsed"]: + errors.append(f"nginx: {len(parsed['unparsed'])} included file(s) could not be " + f"parsed: {parsed['unparsed'][0]}") + if parsed["error"] is not None: + errors.append(f"nginx: {config} is there but could not be parsed: {parsed['error']}") + return section + if not parsed["recognised"]: + errors.append(f"nginx: {config} holds no nginx context (http, stream, server, " + "upstream, map) — not an nginx config") + return section + section["available"] = True + # The main config first: `files` is the answer to "where do these servers come from", + # and an operator comparing it with `nginx -t` should see the file nginx was started on. + section["files"] = [config, *parsed["files"]] + section["servers"] = parsed["servers"] + section["upstreams"] = parsed["upstreams"] + section["maps"] = parsed["maps"] + return section + + def nginx_include_files(self, include_args: list[str], + errors: list[str]) -> tuple[dict[str, str], int, list[str]]: + """Read the files the main config's `include` lines name. Returns (documents, dropped, unreadable). + + One level, bounded by `NGINX_INCLUDE_MAX_FILES` / `NGINX_INCLUDE_MAX_BYTES`: this runs on + every snapshot, and an `include /etc/nginx/**` written by someone who liked globs must + not turn a 2 KB read into a directory walk. A glob is expanded under `--root` like every + other path here and sorted, so the same tree always produces the same order; a literal + path that does not exist contributes nothing and **no line** (an `include` of a file this + host does not have is absence, the `available: false` arm one level down), while the two + facts that *are* worth a sentence get exactly one line each — the bound that dropped + `dropped` files, and the `unreadable` count with its first path. + + Each document is keyed by the path the **config** writes (`_config_label`), not by the + path under `--root` this host happens to have: that is what lets `parse_nginx_config` + match its own `include` lines against the documents, and what makes `files` readable to + an operator holding `nginx -t` output instead of a temp directory. + """ + documents: dict[str, str] = {} + unreadable: list[str] = [] + seen: set[str] = set() + used_bytes = 0 + dropped = 0 + for arg in include_args: + resolved = str(self._config_path(arg)) + candidates = sorted(glob.glob(resolved)) if any(c in arg for c in _GLOB_SPECIALS) \ + else [resolved] + for real in candidates: + label = self._config_label(arg, real) + if label in seen: + continue # two include lines naming one file read it once + seen.add(label) + if len(documents) >= NGINX_INCLUDE_MAX_FILES: + dropped += 1 + continue + path = Path(real) + try: + size = path.stat().st_size + if used_bytes + size > NGINX_INCLUDE_MAX_BYTES: + dropped += 1 + continue + documents[label] = path.read_text(encoding="utf-8", errors="replace") + except FileNotFoundError: + continue + except (OSError, ValueError): + unreadable.append(label) + continue + used_bytes += size + if dropped: + errors.append(f"nginx: {dropped} included file(s) beyond the " + f"{NGINX_INCLUDE_MAX_FILES}-file / {NGINX_INCLUDE_MAX_BYTES}-byte " + "include bound were not read — the section covers what it could read") + if unreadable: + errors.append(f"nginx: {len(unreadable)} included file(s) matched an include line but " + f"could not be read: {unreadable[0]}") + return documents, dropped, unreadable + + # -- live sockets -------------------------------------------------------- # + def read_proc_table(self, relative: str, + max_lines: int = PROC_NET_MAX_LINES) -> tuple[list[str], bool]: + """Read at most `max_lines` lines of a kernel table. Returns (lines, truncated). + + Raises `FileNotFoundError` for "this kernel publishes no such table" and any other + `OSError` for "it is there and I could not read it", because the section has to tell + those two apart and `read_text`/`read_lines` both swallow the difference into a value — + `read_lines` answering `[]` would make an unreadable table indistinguishable from a + host with no sockets, which is rule 2 with a helper's name on it. + + Stopping at the bound is the point of the method: the read never walks a table longer + than the section reports, so the cost of `sockets` belongs to this constant and not to + whatever connection count the box happens to be carrying. + """ + lines: list[str] = [] + with self._path(relative).open(encoding="utf-8", errors="replace") as handle: + for line in handle: + if len(lines) >= max_lines: + return lines, True + lines.append(line) + return lines, False + + def sockets(self, errors: list[str]) -> dict[str, Any]: + """Read the kernel's two TCP tables. Never raises, never fails the snapshot. + + `{available, listens, established}`: every LISTEN socket and every ESTABLISHED pair in + `/proc/net/tcp` and `/proc/net/tcp6`, each row carrying the family it came from, the + addresses unrolled from `/proc`'s per-word byte order, and **no join with the config** + — a socket does not know its domain or its backend, so phase 29's page joins these + ports to the `haproxy`/`nginx` sections and an unclaimed socket stays visible as one. + + The tables are read independently and either one answering makes `available: true`, + because a kernel built without IPv6 has no `tcp6` file at all: the host told us what it + has. An absent table therefore costs **no line** when the other answered (the nginx + include rule — absence is not a fault), while a present-but-denied one costs exactly + one, because that is the operator's fix. Neither answering is `available: null` with one + line naming both paths; nothing here can zero a snapshot. + + There is deliberately no `available: false` arm: `/proc/net/tcp` is not something a host + can leave uninstalled, so the arm the two config sections use for "nothing installed + here" has nothing to name, and `false` beside an empty list would read as "this host has + no listeners" — a fact this wire never published. + """ + section: dict[str, Any] = {"available": None, "listens": [], "established": []} + listens: list[dict[str, Any]] = [] + established: list[dict[str, Any]] = [] + missing: list[str] = [] + unreadable: list[tuple[str, str]] = [] + truncated: list[str] = [] + for family, relative in PROC_NET_TABLES: + label = f"/{relative}" + try: + lines, cut = self.read_proc_table(relative) + except FileNotFoundError: + missing.append(label) + continue + except (OSError, ValueError) as exc: + unreadable.append((label, str(exc))) + continue + if cut: + truncated.append(label) + table_listens, table_established = parse_proc_net_tcp("\n".join(lines), family) + listens.extend(table_listens) + established.extend(table_established) + answered = len(missing) + len(unreadable) < len(PROC_NET_TABLES) + # an absent table when the other answered is the kernel not having IPv6, not a fault, so + # it costs no line; being denied one does, and nothing answering always does. + if unreadable or not answered: + errors.append(socket_gap_line(missing, unreadable, not answered)) + if truncated: + errors.append(socket_truncation_line(truncated)) + section["listens"] = listens + section["established"] = established + section["available"] = True if answered else None + return section + # -- network ------------------------------------------------------------- # def network(self) -> dict[str, Any]: interfaces: list[dict[str, Any]] = [] @@ -1001,7 +1455,7 @@ class Collector: # best-effort, after the cheap reads: a wedged disk can cost the pass budget, not # the snapshot (see the module docstring and smart() for the bounds) self._safe(lambda: self.attach_smart(disk_io, errors), "smart", errors) - return { + document: dict[str, Any] = { "schema": SCHEMA_VERSION, "agent_version": AGENT_VERSION, "ts": time.time(), @@ -1016,8 +1470,22 @@ class Collector: "temperatures": temps, "cpu_temperature_celsius": self.cpu_temperature(temps), "processes": processes, - "errors": errors, } + # The proxy add-on (phase 28) — the two configs and the live socket table — gated on + # `--no-proxy`: disabled means the keys are **absent** ("not collected"), not empty, and + # a failure inside one is an `available: null` section plus its own one line — never a + # failed snapshot. Inserted before `errors` because the sections write into that list. + if self.proxy_enabled: + document["haproxy"] = self._safe(lambda: self.haproxy(errors), "haproxy", errors) \ + or {"available": None, "config": self.haproxy_config, + "frontends": [], "backends": []} + document["nginx"] = self._safe(lambda: self.nginx(errors), "nginx", errors) \ + or {"available": None, "config": self.nginx_config, "files": [], + "servers": [], "upstreams": [], "maps": []} + document["sockets"] = self._safe(lambda: self.sockets(errors), "sockets", errors) \ + or {"available": None, "listens": [], "established": []} + document["errors"] = errors + return document def _safe(self, fn: Callable[[], Any], name: str, errors: list[str]) -> Any: """One failing collector degrades its own section to empty; never the snapshot.""" @@ -1147,6 +1615,588 @@ def _split_stat(stat: str) -> list[str] | None: return [match.group("comm"), *match.group("rest").split()] +_HAPROXY_ADDR_PORT = re.compile(r"^(?P.*):(?P[0-9]+)$") + + +def _haproxy_addr_port(token: str) -> tuple[str | None, int] | None: + """`:::22` → `("::", 22)`, `[::1]:8080` → `("[::1]", 8080)`, `:80` → `(None, 80)`. + + Split at the **last** colon, because that is where haproxy's own address grammar puts the + port: an IPv6 bind survives it (`:::22` is the address `::` on port `22`) and a bracketed one + survives it, while splitting at the first colon would call the address of `[::1]:8080` just + `[`. A token whose tail is not digits has no port for this parser — haproxy allows bind forms + with none (`unix@/run/sock.sock`) and port ranges (`:8000-8100`) — and the caller skips those + rather than invent one. An address the config left empty (`bind :8080`) is `None`: "every + address" is what haproxy means there, and `*` is a spelling it did not use. + """ + match = _HAPROXY_ADDR_PORT.match(token) + if match is None: + return None + return (match.group("addr") or None), int(match.group("port")) + + +def _haproxy_backend_name(words: list[str]) -> str | None: + """The backend a `default_backend`/`use_backend` line names, or None when it names none. + + haproxy also lets the target be computed per request (`use_backend %[req.hdr(host), + map_dom(...)]`, `use_backend { bk1 } if ...`). That is not a name, and reporting the + expression as one would put a fetch expression where the dashboard expects a backend, so it + contributes nothing: the config does not say which backend serves that request. + """ + if len(words) < 2: + return None + name = words[1] + return None if name.startswith(("%", "{", '"')) else name + + +def _scan_haproxy_config(text: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]], bool]: + """One pass over a haproxy config: (frontends, backends, whether it is one at all). + + A haproxy config is a list of sections whose bodies are **indented**, so the indent is the + block structure and the first word of a top-level line is the section's kind. That is the + whole parser: a top-level line switches the current section (`frontend`, `backend`, + `listen`) or ends it (anything else, which is how `global`, `defaults` and every section this + page does not model are skipped), and an indented line is a directive of the current section — + `bind`, `default_backend`, `use_backend` on a frontend, `server` on a backend. Everything else + is ignored with no error: haproxy has hundreds of directives and this section is about what + listens and where it sends, not about what `timeout client` says. + """ + frontends: list[dict[str, Any]] = [] + backends: list[dict[str, Any]] = [] + recognised = False + current_fe: dict[str, Any] | None = None + current_be: dict[str, Any] | None = None + for line in text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue # a blank line keeps its section open; a comment holds no directive + words = stripped.split() + directive = words[0] + if not line[:1].isspace(): + current_fe = current_be = None # a top-level line ends the previous body + if directive in HAPROXY_KEYWORDS: + recognised = True + if len(words) < 2: + continue + name = words[1] + if directive in ("frontend", "listen"): + current_fe = {"name": name, "binds": [], "default_backend": None, "backends": []} + frontends.append(current_fe) + if directive in ("backend", "listen"): + current_be = {"name": name, "servers": []} + backends.append(current_be) + if directive == "listen": + # haproxy's own shorthand: a `listen` section *is* a frontend and a backend + # sharing one name, so the pair is reported as such and the routing name is + # the section's — the config's structure, not an inference about a request. + current_fe["default_backend"] = name + current_fe["backends"] = [name] + continue + if current_fe is not None: + if directive == "bind": + for token in words[1:]: + found = _haproxy_addr_port(token) + if found is not None: + current_fe["binds"].append({"addr": found[0], "port": found[1]}) + break # haproxy writes the address first; whatever follows is options + elif directive == "default_backend": + current_fe["default_backend"] = _haproxy_backend_name(words) + elif directive == "use_backend": + backend = _haproxy_backend_name(words) + if backend is not None and backend not in current_fe["backends"]: + current_fe["backends"].append(backend) + if current_be is not None and directive == "server" and len(words) > 2: + target = _haproxy_addr_port(words[2]) + current_be["servers"].append({ + "name": words[1], + # a target with no parsable port keeps the text the config wrote and a null + # port — "absent is never 0", on the one field haproxy makes optional + "target": words[2] if target is None else target[0], + "port": None if target is None else target[1], + }) + return frontends, backends, recognised + + +def parse_haproxy_config(text: str) -> dict[str, Any]: + """Parse a haproxy config's **text** into the `haproxy` section's data. Pure, never raises. + + `{frontends: [{name, binds: [{addr, port}], default_backend, backends}], backends: + [{name, servers: [{name, target, port}]}], recognised: bool}` — the frontend→backend routing + is the names themselves (`default_backend` plus every `use_backend`), so the dashboard joins + them and this file interprets no rule. + + `recognised` answers the separate question "is this text a haproxy config at all" (does its + top level name a section keyword?) and is deliberately **not** part of the served section: + `Collector.haproxy` turns it into `available`, the only verdict taken here. + """ + frontends, backends, recognised = _scan_haproxy_config(text) + return {"frontends": frontends, "backends": backends, "recognised": recognised} + + +# --------------------------------------------------------------------------- # +# nginx config: the tokenizer, the token parsers, the block walk +# --------------------------------------------------------------------------- # + + +def _nginx_tokens(text: str) -> list[tuple[str, str]]: + """Split nginx config text into `("w", word)`, `("{", "")`, `("}", "")`, `(";", "")`. + + nginx's grammar is words closed by `;` or opened by `{`, with `#` comments and quoted + strings that may contain any of those three — the captured gateway config writes + `add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;`, whose + value holds a semicolon and a space. So the three cases are handled before any split: + cutting on `;` alone would halve every `add_header`, and cutting on whitespace would turn + one quoted value into three words. Raises `ValueError` only for an unterminated quote; + every other shape is somebody's config and comes back as tokens. + + A `#` opens a comment only where a word would have started, which is where nginx's own + lexer reads one: `proxy_pass http://host/#/app` keeps its fragment, and the + `# include /etc/nginx/old.conf;` line an operator commented out contributes nothing. + """ + tokens: list[tuple[str, str]] = [] + word: list[str] = [] + started = False # an empty quoted string is still a word: the `''` key of a map + i, n = 0, len(text) + while i < n: + char = text[i] + if char == "#" and not started and (i == 0 or text[i - 1] in " \t\r\n"): + while i < n and text[i] != "\n": + i += 1 + continue + if char in "\"'": + quote = char + i += 1 + while i < n: + if text[i] == "\\" and i + 1 < n: + word.append(text[i + 1]) + i += 2 + continue + if text[i] == quote: + break + word.append(text[i]) + i += 1 + else: + raise ValueError("a quoted string is never closed") + started = True + i += 1 # past the closing quote + continue + if char in "{};": + if started: + tokens.append(("w", "".join(word))) + word, started = [], False + tokens.append((char, "")) + i += 1 + continue + if char.isspace(): + if started: + tokens.append(("w", "".join(word))) + word, started = [], False + i += 1 + continue + word.append(char) + started = True + i += 1 + if started: + tokens.append(("w", "".join(word))) + return tokens + + +def _nginx_statements(text: str) -> list[tuple[Any, ...]]: + """The brace-block scanner: tokens in, nested statements out. Raises on an unbalanced file. + + A statement is `("d", name, args)` for a directive or `("b", name, args, children)` for a + block. That is the whole difference between the two config languages this agent reads: + haproxy's structure is its indentation, nginx's is its braces, so nginx gets a scanner + instead of a line loop — and with it the two shapes that make the language what it is, a + `server { … location { … } … }` nested as deep as it goes and an `upstream` body whose + `server` lines are *targets* while an `http` body's `server {` is a *block*. + + `ValueError` here means "nginx itself would refuse to start on this file": a stray brace or + a directive that runs into the end of the file without its `;`. The caller turns that into + the section's `null` arm — a config that cannot parse is a fact about the config, and the + snapshot never learns the difference. + """ + root: list[tuple[Any, ...]] = [] + containers: list[list[tuple[Any, ...]]] = [root] + words: list[str] = [] + for kind, value in _nginx_tokens(text): + if kind == "w": + words.append(value) + continue + if kind == ";": + if words: # a bare `;` is whitespace to nginx + containers[-1].append(("d", words[0], words[1:])) + words = [] + continue + if kind == "{": + if not words: + raise ValueError("a '{' with no block name before it") + block: tuple[Any, ...] = ("b", words[0], words[1:], []) + containers[-1].append(block) + containers.append(cast("list[tuple[Any, ...]]", block[3])) + words = [] + continue + if words: + raise ValueError(f"a closing brace ends the block while {words[0]!r} is still open") + if len(containers) == 1: + raise ValueError("a closing brace with no opening one before it") + containers.pop() + if words: + raise ValueError(f"the file ends inside the directive {words[0]!r}, expecting ';' " + "or a closing brace") + if len(containers) != 1: + raise ValueError("an opening brace is never closed") + return root + + +#: The shape `_nginx_statements` returns, spelled once so the walkers below say what they take. +_NginxStatements = list[tuple[Any, ...]] + + +def _nginx_addr_port(token: str) -> tuple[str | None, int] | None: + """`80` → `(None, 80)`, `[::]:8443` → `("::", 8443)`, `10.1.0.9:8080` → itself. + + The three forms `listen` writes, in the spelling the config used: a bare port leaves the + address `None` (nginx's own "every address", and `*` is a spelling it did not use), and a + bracketed IPv6 address loses its brackets because the brackets are nginx's disambiguation + of the colon, not part of the address. A token with no digits at the end has no port here — + `unix:/run/nginx.sock`, a port range, a `default_server` written where an address belongs — + and the caller skips it rather than inventing one. + """ + if token.startswith("["): + host, closed, tail = token[1:].partition("]") + if not closed or not tail.startswith(":"): + return None + addr, port_text = host, tail[1:] + elif ":" in token: + addr, _, port_text = token.rpartition(":") + else: + addr, port_text = "", token + if not port_text.isdigit(): + return None + return (addr or None), int(port_text) + + +def _nginx_variable(token: str) -> str: + """The name of a nginx variable, without its `$` — the one spelling both halves agree on. + + The config writes `$targetBackend` in the `map` line and again in `proxy_pass`, and the + dashboard joins the two. Serving one with the sigil and one without would make that join a + string fix-up in the page, so the `$` — nginx's syntax for "this is a variable", not part of + its name — is dropped here, once. + """ + return token[1:] if token.startswith("$") else token + + +def _nginx_proxy_pass(token: str, location: str | None) -> dict[str, Any]: + """One `proxy_pass` line: the address it names, the variable it names, or neither. + + `proxy_pass http://nextcloud.example.com:11000;` is an address, and `proxy_pass + $targetBackend;` is not one — nginx resolves the variable per connection from the `map` + block that defines it — so the second is served with `variable` set and `target: null`. The + agent reports the two facts the config states and never merges them: a merged target would + be a number this wire never published (and the map's `default` is not "the" backend, it is + what a name nobody mapped gets). A trailing path (`/api/`) is dropped with the same reason + as haproxy's `check inter 5s`: it changes what is requested, not where the pipe lands. + + `location` is the enclosing `location` path, or `None` for the stream form, where the + directive is written in the `server` block itself and there is no path to name. + """ + entry: dict[str, Any] = {"location": location, "scheme": None, "target": None, + "variable": None} + if token.startswith("$"): + entry["variable"] = _nginx_variable(token) + return entry + scheme, sep, rest = token.partition("://") + if not sep: + # Not `scheme://…`: a unix socket (`unix:/run/x.sock`) is not a host and a bare word is + # not an address, so the line is reported as a proxy_pass with nothing resolved instead + # of guessing which of the two it is. + return entry + entry["scheme"] = scheme + authority = rest.split("/", 1)[0].split("?", 1)[0].split("#", 1)[0] + found = _nginx_addr_port(authority) + if found is not None: + entry["target"] = {"host": found[0], "port": found[1]} + elif ":" not in authority and authority: + # `proxy_pass http://backend_pool` names an `upstream` block, which is a host with no + # port: the name is the fact, and the port it does not state stays null. + entry["target"] = {"host": authority, "port": None} + return entry + + +def _collect_nginx(statements: _NginxStatements, out: dict[str, Any], *, root: bool, + server: dict[str, Any] | None = None, location: str | None = None, + upstream: dict[str, Any] | None = None, + map_ctx: dict[str, Any] | None = None) -> None: + """Walk one statement list, filling `out`. Pure over the parsed tree; interprets nothing. + + What is *not* here matters as much as what is: no `include` is followed below the level the + caller passed (that is the collector's bounded job, and `root=False` for an included file is + what makes one level structural rather than a promise), no `listen` flag is read (`ssl`, + `proxy_protocol`, `http2` change how the bytes travel, not what the page's pipes show), and + a directive outside a `server` block is ignored instead of being promoted into a synthetic + one — there is no such thing as a proxy_pass with no server. + """ + for statement in statements: + kind, name, args = statement[0], statement[1], statement[2] + if kind == "d": + if name == "include" and args: + if root: + out["includes"].append(args[0]) + continue + if upstream is not None: + if name == "server" and args: + found = _nginx_addr_port(args[0]) + upstream["servers"].append({ + # an upstream body's `server` line *is* a host, so a target with no port + # keeps its name and a null port rather than losing the line entirely + "host": args[0] if found is None else found[0], + "port": None if found is None else found[1], + }) + continue + if map_ctx is not None: + # Inside a `map` body the *directive name* is the key and its first argument is + # the value — `map $ssl_preread_server_name $targetBackend { a.example.com + # backend.internal:443; }` is one entry, not a directive called `a.example.com`. + found = _nginx_addr_port(args[0]) if args else None + map_ctx["entries"].append({ + "key": name, + # only an address becomes a target: the `$connection_upgrade` map's + # `upgrade` / `close` values are header words, and calling one a host + # would be this section inventing a fact to fill a field + "target": None if found is None else {"host": found[0], "port": found[1]}, + }) + continue + if server is not None: + if name == "listen" and args: + found = _nginx_addr_port(args[0]) + if found is not None: + server["listen"].append({"addr": found[0], "port": found[1]}) + elif name == "server_name": + # every name on the line, and the catch-all `_` kept exactly as written — + # "any domain" is what the page says about it, not what the agent decides + server["server_names"].extend(args) + elif name == "proxy_pass" and args: + server["proxy_passes"].append(_nginx_proxy_pass(args[0], location)) + continue + children = cast("_NginxStatements", statement[3]) + if name == "upstream" and args: + opened: dict[str, Any] = {"name": args[0], "servers": []} + out["upstreams"].append(opened) + _collect_nginx(children, out, root=root, upstream=opened) + elif name == "map" and len(args) >= 2: + opened_map: dict[str, Any] = {"variable": _nginx_variable(args[1]), "entries": []} + out["maps"].append(opened_map) + _collect_nginx(children, out, root=root, map_ctx=opened_map) + elif name == "server": + opened_server: dict[str, Any] = {"listen": [], "server_names": [], "proxy_passes": []} + out["servers"].append(opened_server) + _collect_nginx(children, out, root=root, server=opened_server) + elif name == "location": + _collect_nginx(children, out, root=root, server=server, + location=(" ".join(args) or None)) + else: + _collect_nginx(children, out, root=root, server=server, location=location) + + +def nginx_include_paths(text: str) -> list[str]: + """Every `include` argument the main config names, in document order. Never raises. + + The collector needs the list *before* it can read anything, and a config that will not + tokenize has no include lines to ask about — it comes back empty, and the parse failure is + reported once, by `parse_nginx_config`, rather than twice by two functions that disagree. + Includes nested inside `http { }` / `stream { }` count: that is exactly where the captured + gateway config keeps the two lines that hold its whole ingress map. + """ + try: + statements = _nginx_statements(text) + except ValueError: + return [] + out: dict[str, Any] = {"servers": [], "upstreams": [], "maps": [], "includes": []} + _collect_nginx(statements, out, root=True) + return cast("list[str]", out["includes"]) + + +def _nginx_include_matches(arg: str, included: dict[str, str]) -> list[str]: + """Which of the supplied documents this `include` argument names. + + Exact for a literal path, `fnmatch` for a glob, in the order the documents were handed over + (the collector reads them sorted, so that order is stable). `fnmatch`'s `*` crosses `/` + where the C library's `glob()` does not, which costs nothing here: the collector did the + real globbing and this only decides which side of the join a document belongs to. + """ + if any(c in arg for c in _GLOB_SPECIALS): + return [path for path in included if fnmatch.fnmatchcase(path, arg)] + return [path for path in included if path == arg] + + +def parse_nginx_config(main_text: str, + included: dict[str, str] | None = None) -> dict[str, Any]: + """Parse an nginx main config plus the documents its includes name. Pure, never raises. + + `{servers: [{listen: [{addr, port}], server_names: [str], proxy_passes: [{location, scheme, + target, variable}]}], upstreams: [{name, servers: [{host, port}]}], + maps: [{variable, entries: [{key, target}]}], files: [str], includes: [str], unparsed: + [str], recognised: bool, error: str|null}`. + + Three of those keys are the ones `Collector.nginx` does not serve, the way + `parse_haproxy_config`'s `recognised` is not: `error` is why the main text could not be + parsed, `unparsed` names an included document that could not, and `includes` is the raw + argument list (a caller that has no filesystem — every unit test — still gets to see what a + config *would* have read). `files` is the included documents that were merged, in order; + the collector puts the main config in front of it to make the section's `files`. + + A document that fails to parse is dropped and named in `unparsed`; it never drops the others + and never raises. The main text failing to parse is the one case that yields an empty result, + because there is nothing to merge it with — and even then the answer is this dict with + `error` set, not an exception on a snapshot path. + """ + out: dict[str, Any] = {"servers": [], "upstreams": [], "maps": [], "files": [], + "includes": [], "unparsed": [], "recognised": False, "error": None} + try: + main = _nginx_statements(main_text) + except ValueError as exc: + out["error"] = str(exc) + return out + out["recognised"] = any(statement[1] in NGINX_CONTEXTS for statement in main) + _collect_nginx(main, out, root=True) + documents = included or {} + merged: list[str] = [] + seen: set[str] = set() + for arg in cast("list[str]", out["includes"]): + for path in _nginx_include_matches(arg, documents): + if path in seen: + continue + seen.add(path) + try: + statements = _nginx_statements(documents[path]) + except ValueError: + out["unparsed"].append(path) + continue + merged.append(path) + # root=False: an include line inside an included file is a second level, and this + # pass is one level deep by design (see NGINX_INCLUDE_MAX_FILES). + _collect_nginx(statements, out, root=False) + out["files"] = merged + return out + + +# --------------------------------------------------------------------------- # +# live sockets (the additive `sockets` section) +# --------------------------------------------------------------------------- # + +def _proc_net_addr(raw: str, family: str) -> str | None: + """One `/proc/net/tcp` address word as the address it actually is. + + `/proc` writes the address as one 32-bit word per group, each group little-endian, in + hex: `0100007F` is 127.0.0.1 and v6 gets four of those groups. Unrolling therefore means + reversing the bytes **inside every group of four**, which is what `socket.inet_ntop` then + renders in the family's own spelling — so a wildcard stays the literal `0.0.0.0` / `::` the + dashboard's inbound join keys on and a v6 address arrives in the form the nginx config + writes it in (`[::]:8443`), not a hex blob the page would have to understand twice. A word + of the wrong length or with non-hex in it is `None`: an address nobody published is worse + than a gap beside a real socket. + """ + if len(raw) != (32 if family == "v6" else 8): + return None + try: + packed = bytes.fromhex(raw) + except ValueError: + return None + unrolled = b"".join(packed[i:i + 4][::-1] for i in range(0, len(packed), 4)) + try: + return socket.inet_ntop(socket.AF_INET6 if family == "v6" else socket.AF_INET, + unrolled) + except (OSError, ValueError): + return None + + +def _proc_net_endpoint(token: str, family: str) -> tuple[str | None, int | None]: + """One `ADDRESS:PORT` column into `(address, port)`; either half may be `None`. + + The port is hex (`221B` is 8731) and travels after the last colon, because a v6 address + has colons of its own in the canonical spelling `/proc` would have used. A token with no + colon is not an endpoint, and a port that is not hex is `None` rather than 0 — the haproxy + `server` line with no port keeps a null port for the same reason. + """ + addr_hex, sep, port_hex = token.rpartition(":") + if not sep: + return None, None + port: int | None + try: + port = int(port_hex, 16) + except ValueError: + port = None + return _proc_net_addr(addr_hex, family), port + + +def parse_proc_net_tcp(text: str, family: str) -> tuple[list[dict[str, Any]], + list[dict[str, Any]]]: + """Parse one `/proc/net/tcp{,6}` table into `(listens, established)`. Pure, never raises. + + `family` is the label the rows carry ("v4"/"v6") and decides how the address word is + unrolled. Only the two states a gateway page can use are returned: `0A` LISTEN becomes + `{family, addr, port}` and `01` ESTABLISHED becomes + `{family, local: {addr, port}, remote: {addr, port}}`. Every other state — `06` TIME_WAIT, + `08` CLOSE_WAIT, the SYN queue — is a socket in transit, and reporting it would bury the + live pairs in rows that stopped being true before the page drew them; skipping one costs no + `errors` line because the kernel is not broken, it is busy. + + `/proc`'s own header line costs nothing special: its state column reads `st`, which is not + a state, so the filter above drops it. A row whose endpoint will not decode is still + reported, with `addr`/`port` null — the socket exists, and the gap is the honest fact about + the word, not about the connection. + """ + listens: list[dict[str, Any]] = [] + established: list[dict[str, Any]] = [] + for line in text.splitlines(): + fields = line.split() + if len(fields) < 4: + continue + state = fields[3].upper() + if state == PROC_NET_STATE_LISTEN: + addr, port = _proc_net_endpoint(fields[1], family) + listens.append({"family": family, "addr": addr, "port": port}) + elif state == PROC_NET_STATE_ESTABLISHED: + local_addr, local_port = _proc_net_endpoint(fields[1], family) + remote_addr, remote_port = _proc_net_endpoint(fields[2], family) + established.append({ + "family": family, + "local": {"addr": local_addr, "port": local_port}, + "remote": {"addr": remote_addr, "port": remote_port}, + }) + return listens, established + + +def socket_gap_line(missing: list[str], unreadable: list[tuple[str, str]], + nothing_answered: bool) -> str: + """The one `sockets:` line for the tables that did not answer, whichever arms they are. + + A missing table and a denied one are two sentences — the first is `/proc` as shipped (a + kernel without IPv6 has no `tcp6`), the second is a read this process was not allowed — and + the line names each path with its own reason. When nothing answered the line says the whole + section is empty and names `--no-proxy`, the way the two config sections do, so the operator + reading one line knows whether to fix a mount or turn the section off. + """ + parts = [f"{path} not found" for path in missing] + parts.extend(f"{path}: {reason}" for path, reason in unreadable) + tail = (" — this host publishes no kernel socket table; --no-proxy opts out" + if nothing_answered else " — the tables that did answer are reported in full") + return f"sockets: {', '.join(parts)}{tail}" + + +def socket_truncation_line(paths: list[str]) -> str: + """The one `sockets:` line for a table longer than the read bound. + + It names the bound, because that is the number an operator would want to raise (in the + source, since there is no flag for it) and the difference between "the gateway has 4096 + sockets" and "the gateway has more sockets than this read takes". + """ + return (f"sockets: {', '.join(paths)} hold more than the {PROC_NET_MAX_LINES} lines this " + "read takes — the sockets past the bound are not in this section") + + # --------------------------------------------------------------------------- # # TLS bundle: what the dashboard signs, what the installer places # --------------------------------------------------------------------------- # @@ -1612,11 +2662,25 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--no-smart", action="store_true", help="skip the smartctl pass entirely (disks report smart: null, " "no error entries)") + parser.add_argument("--no-proxy", action="store_true", + help="skip the proxy add-on entirely (no `haproxy`, `nginx` or " + "`sockets` key, no error entries)") + parser.add_argument("--haproxy-config", default=HAPROXY_CONFIG_PATH, + help="haproxy config to read; a missing file is the documented " + "degradation (available=false, one error entry), not a failure") + parser.add_argument("--nginx-config", default=NGINX_CONFIG_PATH, + help=f"nginx main config to read, together with the files its own " + f"include lines name (one level, {NGINX_INCLUDE_MAX_FILES} files / " + f"{NGINX_INCLUDE_MAX_BYTES // 1000} KB); a missing file is the " + f"documented degradation (available=false, one error entry), " + f"not a failure") parser.add_argument("--once", action="store_true", help="print one snapshot as JSON and exit") args = parser.parse_args(argv) collector = Collector(args.root, top_processes=args.top, net_all=args.net_all, - smartctl=args.smartctl, smart=not args.no_smart) + smartctl=args.smartctl, smart=not args.no_smart, + proxy=not args.no_proxy, haproxy_config=args.haproxy_config, + nginx_config=args.nginx_config) if args.once: print(json.dumps({"ok": True, **collector.snapshot()}, indent=2, default=str)) return 0 diff --git a/scripts/capture-gateway-config.sh b/scripts/capture-gateway-config.sh new file mode 100755 index 0000000..084cddf --- /dev/null +++ b/scripts/capture-gateway-config.sh @@ -0,0 +1,352 @@ +#!/usr/bin/env bash +# Re-record the gateway's proxy configs and kernel socket tables into tests/fixtures/. +# +# ./scripts/capture-gateway-config.sh # dials the `gateway-ops` ssh alias +# GATEWAY_SSH=lab-gw ./scripts/capture-gateway-config.sh +# +# What lands where — one fixture per remote document, in `tests/fixtures/`: +# +# gateway_haproxy.cfg cat /etc/haproxy/haproxy.cfg +# gateway_nginx_main.conf cat /etc/nginx/nginx.conf +# gateway_nginx_conf_d_.conf cat /etc/nginx/conf.d/*.conf +# gateway_nginx_stream_d_.conf cat /etc/nginx/stream.d/*.conf +# gateway_proc_net_tcp.txt cat /proc/net/tcp +# gateway_proc_net_tcp6.txt cat /proc/net/tcp6 +# +# These documents are the ground truth behind `agent/host_agent.py`'s `haproxy`, `nginx` and +# `sockets` sections (phase 28) the way `tests/fixtures/nut_.vars` pins the UPS wire and +# `smartctl_driveripper_.txt` pins the drives. The include globs are read **out of the +# captured main config**, so a gateway that moves its vhosts to a third directory grows a third +# fixture on the next run instead of keeping a fixture set that quietly stopped being the tree +# the agent merges. +# +# Four rules, three of them `scripts/capture-fixtures.sh`'s: +# +# * **Verbatim, never hand-written** (AGENTS.md invariant 7). A fixture you typed is a test of +# your memory. Each file carries a `#` provenance block naming the remote path it came from. +# The header is comment-shaped because `#` is a comment in haproxy's config and in nginx's, +# and a `/proc` line whose 4th field is not a kernel state is skipped by the socket parser — +# provenance that costs no parser anything, which `tests/test_agent.py` pins by parsing the +# fixtures exactly as they sit on disk. +# * **The configs are not rewritten at all.** A domain is not an identifier to hide here: a +# `server_name` and the `proxy_pass` target beside it *are* the ingress map the gateway page +# exists to draw, and a scrubbed fixture is a fixture that tests a config this gateway does +# not run. Key material appears in these files only as the paths the proxy is told to read +# (`ssl_certificate_key /etc/letsencrypt/…`), which is a filename and not a key. No text +# filter of any kind sits between the read and a config fixture — +# `tests/test_deploy_scripts.py` runs this script against a fake remote holding a domain and +# demands the bytes back. +# * **A socket table keeps its shape and loses its identifiers** (AGENTS.md invariant 6, and +# the rule `capture-fixtures.sh` already applies to a drive: "Identifiers never go in the +# repo" — Serial, LU WWN, and the NVMe EUI-64 *which encodes the serial*). A Linux global +# IPv6 is EUI-64: `2603:…:5054:ff:fe84:b2e0` is this box's NIC address, and the other thing +# `/proc/net/tcp` carries is the public address of whoever was connected at the moment of +# the capture, which is not ours to publish. So any address `ipaddress` calls globally +# routable becomes a documentation address (RFC 5737 / RFC 3849) in first-seen order, +# re-encoded through the kernel's own per-32-bit-word little-endian spelling. Loopback, +# private, link-local, wildcard and already-documentation addresses are left exactly as they +# were, so the run is idempotent and the fixture still meets every shape the parser has to +# survive — `0.0.0.0`, `::`, `::ffff:a.b.c.d`, 32-hex-digit v6 words. Every port, state, +# queue, uid, inode and column width is byte-for-byte what the kernel answered, and only the +# two address *columns* are even looked at (a tx/rx queue pair is spelled like an address +# and is not one). The tables are volatile anyway — a proxy in service has a different +# connection set every second — so the tests assert shape, the two states kept, and the +# ports the captured configs claim to bind. Never a particular row. +# * **A bad read never replaces a good fixture.** Empty output, an ssh failure or a permission +# denial prints one line naming the file and the reason, leaves whatever is on disk alone, +# and exits 1 — the `slots_gufo.json` rule, so a laptop with no route to the gateway cannot +# turn six verbatim captures into six empty files. +# +# No sudo, deliberately: haproxy's config is 0644, nginx's tree is 0644 and `/proc/net/tcp{,6}` +# is world-readable, which is exactly what `dashboards-agent` reads as itself. A capture that +# needed root would record more than the agent can see — the mistake the smartctl pass in +# `capture-fixtures.sh` documents in the other direction. +# +# A file the host stops having is left in place rather than deleted: when the config moves, +# remove the orphan fixture in the same commit that updates the parser tests naming it. +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +FIX="$ROOT/tests/fixtures" +GATEWAY_SSH="${GATEWAY_SSH:-gateway-ops}" +mkdir -p "$FIX" + +# `-n` matters: `capture_included` walks the remote listing on stdin, and an ssh that reads +# stdin would eat the rest of that list and quietly capture one vhost file instead of all of them. +SSH_OPTS=(-n -o BatchMode=yes -o ConnectTimeout=8) +failed=0 + +usage() { + cat <<'TXT' +Usage: ./scripts/capture-gateway-config.sh [-h] + +Re-records the gateway's proxy configs and kernel socket tables into tests/fixtures/: +/etc/haproxy/haproxy.cfg, /etc/nginx/nginx.conf plus the documents its own glob includes +name, and /proc/net/tcp{,6}. Every read goes over the `$GATEWAY_SSH` ssh alias (default +`gateway-ops`) of a world-readable path, and a read that fails leaves the existing fixture +alone and exits 1. The configs land unedited; in the socket tables a globally routable +address becomes a documentation one and nothing else moves. See the block comment. +TXT +} + +case "${1:-}" in + -h|--help) usage; exit 0 ;; +esac + +# The socket tables' one edit, and the only place in this script that touches a captured byte. +# Kept as a here-doc in the house style (capture-fixtures.sh embeds two); it is pinned by +# running this script against a fake remote in tests/test_deploy_scripts.py. +SOCKET_IDENTIFIERS_OUT=$(cat <<'PY' +"""Move the globally-routable addresses in a /proc/net/tcp{,6} table to documentation space. + +Every column, every port and every byte of spacing is the kernel's. What changes is an address +`ipaddress` calls global, which becomes a documentation address in first-seen order (RFC 5737 +for v4, RFC 3849 for v6) re-encoded through the kernel's own per-32-bit-word little-endian +spelling — so the replacement is exactly as wide as the original and an IPv4-mapped v6 row stays +an IPv4-mapped v6 row. AGENTS.md invariant 6 is the reason: this repo's capture rule is that +identifiers never go in tracked files, and a global IPv6 on Linux is EUI-64 (it encodes the +NIC's MAC) while a visitor's public address belongs to the visitor. Loopback, private, +link-local, multicast, wildcard and already-documentation addresses are returned untouched, +which is what makes re-running this over its own output change nothing. +""" + +import ipaddress +import re +import sys + +DOC_V4 = int(ipaddress.IPv4Address("203.0.113.0")) +DOC_V6 = int(ipaddress.IPv6Address("2001:db8::")) + + +def unroll(word: str) -> bytes: + """`/proc`'s per-word little-endian spelling -> the address's own bytes.""" + raw = bytes.fromhex(word) + return b"".join(raw[i:i + 4][::-1] for i in range(0, len(raw), 4)) + + +def roll(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> str: + """The inverse, in the kernel's uppercase.""" + packed = addr.packed + return "".join(packed[i:i + 4][::-1].hex().upper() for i in range(0, len(packed), 4)) + + +class Replacer: + def __init__(self) -> None: + self.counters: dict[int, int] = {4: 0, 6: 0} + self.mapping: dict[str, str] = {} + self.replacements = 0 + + def documentation(self, version: int) -> str: + self.counters[version] += 1 + if version == 4: + return str(ipaddress.IPv4Address(DOC_V4 + self.counters[version])) + return str(ipaddress.IPv6Address(DOC_V6 + self.counters[version])) + + def column(self, token: str) -> str: + """One `:` column: rewrite the word, keep the port and its spelling.""" + word, _, port = token.rpartition(":") + if len(word) not in (8, 32) or any(c not in "0123456789abcdefABCDEF" for c in word): + return token + try: + addr = ipaddress.ip_address(unroll(word)) + except ValueError: + return token + # Only IPv6Address has `ipv4_mapped`; a v4 word is its own subject. + mapped = getattr(addr, "ipv4_mapped", None) + subject = mapped if mapped is not None else addr + if not subject.is_global: + return token + key = str(subject) + if key not in self.mapping: + self.mapping[key] = self.documentation(subject.version) + replacement = self.mapping[key] + self.replacements += 1 + # A v4-mapped row keeps its mapped spelling, so the row that exercises that branch of + # the agent's decoder still exercises it after the capture. + if mapped is not None: + new: ipaddress.IPv4Address | ipaddress.IPv6Address = ipaddress.IPv6Address( + f"::ffff:{replacement}") + else: + new = ipaddress.ip_address(replacement) + return f"{roll(new)}:{port}" + + def line(self, text: str) -> str: + """Rewrite columns 1 and 2 only — the two address columns of a socket table.""" + spans = [(m.start(), m.end(), m.group(0)) for m in re.finditer(r"\S+", text)] + if len(spans) < 4: + return text + out = [] + last = 0 + for index, (start, end, token) in enumerate(spans): + out.append(text[last:start]) + out.append(self.column(token) if index in (1, 2) else token) + last = end + out.append(text[last:]) + return "".join(out) + + +def main(argv: list[str]) -> int: + replacer = Replacer() + with open(argv[1], encoding="utf-8", errors="replace") as handle: + for line in handle: + sys.stdout.write(replacer.line(line)) + if replacer.replacements: + print(f" ({replacer.replacements} globally-routable address column(s) moved to " + f"documentation space; {len(replacer.mapping)} distinct endpoint(s))", + file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) +PY +) + +# header — the provenance block every fixture carries. +header() { + local kind="$1" path="$2" + echo "# Verbatim $path from $GATEWAY_SSH, read read-only over the ssh key by" + echo "# ./scripts/capture-gateway-config.sh (AGENTS.md invariant 7: verify against the" + echo "# deployed upstream, never guess its shape)." + case "$kind" in + config) + echo "# Nothing here is rewritten: it is this fleet's ingress map, and the domains in it are" + echo "# the data — an inbound name and the target it routes to are what these config sections" + echo "# exist to report. Key material appears only as the file paths the proxy reads." + ;; + sockets) + echo "# One line per TCP socket, in the kernel's own spelling. The only edit is the repo's" + echo "# identifier rule (invariant 6, the drive-serial scrub in capture-fixtures.sh): a" + echo "# globally routable address became a documentation one (RFC 5737 / RFC 3849), because a" + echo "# global IPv6 here is EUI-64 and a visitor's address is not ours to publish. Every port," + echo "# state, queue, uid, inode and column width is verbatim, as are the loopback, private" + echo "# and wildcard addresses the gateway's own frontends use." + echo "# Volatile by nature: a proxy in service has a different connection set every second, so" + echo "# a diff here on re-capture is expected. The tests assert shape, the two states the agent" + echo "# keeps, and the ports the captured configs claim to bind — never a particular row." + ;; + esac +} + +# capture — one ssh call, one document, written only if the read +# answered something. The remote command is a single `cat`: `scp` cannot read /proc at all (a +# procfs file reports size 0, so scp copies an empty file), and one mechanism for six fixtures +# is one mechanism to pin. +capture() { + local fixture="$1" kind="$2" path="$3" + local tmp="$FIX/.$fixture.tmp" err="$FIX/.$fixture.err" + local rc=0 + ssh "${SSH_OPTS[@]}" "$GATEWAY_SSH" "cat $path" >"$tmp" 2>"$err" || rc=$? + if [ "$rc" -ne 0 ] || [ ! -s "$tmp" ]; then + local why + why="$(head -n 1 "$err" 2>/dev/null)" + [ -n "$why" ] || { [ "$rc" -eq 0 ] && why="answered an empty file" || why="ssh exited $rc"; } + rm -f "$tmp" "$err" + echo " ✗ $fixture — $GATEWAY_SSH:$path unreadable ($why); the fixture on disk is left alone" + failed=1 + return 1 + fi + rm -f "$err" + { + header "$kind" "$path" + case "$kind" in + sockets) python3 -c "$SOCKET_IDENTIFIERS_OUT" "$tmp" ;; + *) cat "$tmp" ;; + esac + } >"$FIX/$fixture" + rc=0 + rm -f "$tmp" + echo " ✓ $fixture ($(wc -l <"$FIX/$fixture") lines from $GATEWAY_SSH:$path)" +} + +# capture_included — every `*.conf` the main config's include line +# names, one fixture each. The prefix names the directory the way the parser spells it +# (`gateway_nginx_conf_d_` for /etc/nginx/conf.d), the rest is the file's own name: +# /etc/nginx/conf.d/http.conf → gateway_nginx_conf_d_http.conf. +capture_included() { + local prefix="$1" dir="$2" + local listing="$FIX/.$prefix.ls" err="$FIX/.$prefix.ls.err" + local rc=0 + ssh "${SSH_OPTS[@]}" "$GATEWAY_SSH" "ls $dir/*.conf" >"$listing" 2>"$err" || rc=$? + if [ "$rc" -ne 0 ]; then + local why + why="$(head -n 1 "$err" 2>/dev/null)" + [ -n "$why" ] || { [ "$rc" -eq 0 ] && why="answered an empty list" || why="ssh exited $rc"; } + rm -f "$listing" "$err" + echo " ✗ ${prefix}* — $GATEWAY_SSH:$dir could not be listed ($why); the fixtures on disk are left alone" + failed=1 + return 1 + fi + rm -f "$err" + local path base + while IFS= read -r path; do + [ -n "$path" ] || continue + base="${path##*/}" + # Parameter expansion and nothing else builds the name, so there is no text filter of any + # kind between what the host said and what the fixture holds. + capture "${prefix}${base%.conf}.conf" config "$path" + done <"$listing" + rm -f "$listing" +} + +# --- haproxy ----------------------------------------------------------------- +# The file `parse_haproxy_config` is pinned to: frontends on `::: v4v6`, one +# `server : check inter 5s` per backend, and a `use_backend` whose target +# is a fetch expression — which is why a name is reported only where the config names one. +capture gateway_haproxy.cfg config /etc/haproxy/haproxy.cfg + +# --- nginx ------------------------------------------------------------------- +# The main config, then the documents its own include lines name one level deep: the same set +# `Collector.nginx` merges. A literal include naming a file this host does not have (this +# gateway's `include /etc/nginx/mime.types;`) is neither captured nor an error — absence of a +# document the parser needs nothing from is the shape the agent already reports as "costs +# nothing and no line". +if capture gateway_nginx_main.conf config /etc/nginx/nginx.conf; then + globs=() + while IFS= read -r line; do + probe="${line#"${line%%[![:space:]]*}"}" # leading whitespace off, for the two tests below + case "$probe" in ''|'#'*) continue ;; esac + case "$probe" in include*) ;; *) continue ;; esac + spec="${probe#include}" + spec="${spec%%;*}" + spec="${spec//[[:space:]]/}" + # Anything with a `*` is a glob include, and the agent merges it — so it gets a fixture, or + # the report below says the script has no name for it. Silent skipping is what would let the + # fixture set stop being the tree the agent reads. + case "$spec" in *"*)"*) continue ;; esac # not a directive then: a fetch expression + case "$spec" in *"*"*) globs+=("$spec") ;; esac + done <"$FIX/gateway_nginx_main.conf" + if [ ${#globs[@]} -eq 0 ]; then + echo " ✗ gateway_nginx_*.conf — the captured main config names no glob include, so the" + echo " vhost fixtures were not refreshed: has the tree the agent merges moved?" + failed=1 + fi + for spec in "${globs[@]}"; do + case "$spec" in + /etc/nginx/conf.d/*.conf) capture_included gateway_nginx_conf_d_ "${spec%/*}" ;; + /etc/nginx/stream.d/*.conf) capture_included gateway_nginx_stream_d_ "${spec%/*}" ;; + *) + echo " ✗ $spec — an include directory this script has no fixture name for" + echo " (add a case above and a fixture-pinned test in tests/test_agent.py)" + failed=1 + ;; + esac + done +fi + +# --- the live sockets -------------------------------------------------------- +# Both tables, always: `available` is decided across them, because a kernel built without IPv6 +# answers tcp and not tcp6 and that is absence rather than a fault (no line, the include rule). +# Keeping only LISTEN and ESTABLISHED is what makes a capture of a busy gateway usable — the +# TIME_WAIT rows in here are the ones the parser must skip, which is exactly what +# tests/test_agent.py asserts against this file. +capture gateway_proc_net_tcp.txt sockets /proc/net/tcp +capture gateway_proc_net_tcp6.txt sockets /proc/net/tcp6 + +if [ "$failed" -ne 0 ]; then + echo + echo "capture FAILED — every fixture named above is unchanged on disk. Is $GATEWAY_SSH an ssh" + echo "alias that answers (ssh $GATEWAY_SSH true), and are the paths still where they were?" + exit 1 +fi +echo "done — read the diff before committing the fixtures." diff --git a/src/dashboards/collectors.py b/src/dashboards/collectors.py index 355b4ff..d324311 100644 --- a/src/dashboards/collectors.py +++ b/src/dashboards/collectors.py @@ -36,6 +36,22 @@ class UpstreamError(Exception): self.status = status +#: Agent add-on sections that report their own absence on purpose. Their `errors` lines are +#: facts about a source this page does not draw — `smart:` belongs to `/disks`, `haproxy:` and +#: `nginx:` to the gateway page (phase 29) — so they never make the **system** payload +#: `degraded`, which is the overview's alert input. Without this a fleet-wide agent upgrade +#: would put a red "agent degraded" banner and an overview WARN on every host that has no +#: haproxy or nginx installed, which is invariant 3's failure mode with an upgrade's clothes on. +#: The page that owns the add-on still reads the lines: they stay verbatim in `agent.errors`. +#: +#: **A new agent add-on section adds its prefix here in the same commit.** The list the agent's +#: own tests filter by is pinned equal to this one +#: (`tests/test_agent.py::test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters`), +#: so a section that ships its absence-report without a prefix here fails instead of turning +#: every host amber. +ADDON_ERROR_PREFIXES: tuple[str, ...] = ("smart", "haproxy", "nginx", "sockets") + + # --------------------------------------------------------------------------- # # System hosts # --------------------------------------------------------------------------- # @@ -221,12 +237,15 @@ async def fetch_host_snapshot(client: httpx.AsyncClient, target: Target, if body.get("errors"): # The agent degraded a section (a /proc read failed) but still answered. Say so # in the payload rather than presenting a partial snapshot as a complete one — - # except the SMART add-on: a host without smartmontools or without device read - # access is *unconfigured, not broken* (invariant 3), and its `smart:` lines must - # not hold the system page amber forever. They stay in the snapshot payload - # (`agent.errors`) where the /disks page explains them in its own panel. + # except the add-on sections: a host without smartmontools or without device read + # access, without a proxy config on it, or whose kernel published no socket table, is + # *unconfigured, not broken* (invariant 3), and those lines must not hold the system page + # amber forever. They stay in the snapshot payload (`agent.errors`) where the page that + # owns the add-on explains them — `/disks` reads the `smart:` ones, phase 29's gateway + # page reads the `haproxy:`/`nginx:`/`sockets:` ones. errors = body["errors"] - core = ([line for line in errors if not str(line).startswith("smart")] + core = ([line for line in errors + if not str(line).lstrip().lower().startswith(ADDON_ERROR_PREFIXES)] if isinstance(errors, list) else errors) if core: result.payload["degraded"] = core diff --git a/tests/fixtures/gateway_haproxy.cfg b/tests/fixtures/gateway_haproxy.cfg new file mode 100644 index 0000000..cd448b0 --- /dev/null +++ b/tests/fixtures/gateway_haproxy.cfg @@ -0,0 +1,57 @@ +# Verbatim /etc/haproxy/haproxy.cfg from gateway-ops, read read-only over the ssh key by +# ./scripts/capture-gateway-config.sh (AGENTS.md invariant 7: verify against the +# deployed upstream, never guess its shape). +# Nothing here is rewritten: it is this fleet's ingress map, and the domains in it are +# the data — an inbound name and the target it routes to are what these config sections +# exist to report. Key material appears only as the file paths the proxy reads. +defaults + log stdout local0 info + mode tcp + option tcplog + timeout connect 10s + timeout client 30s + timeout server 30s + timeout tunnel 3600s + default-server init-addr last,libc,none + +frontend haproxy-22 + bind :::22 v4v6 + default_backend gitea-2222 + +frontend haproxy-2000 + bind :::2000 v4v6 + default_backend sec-gitea-2000 + +frontend haproxy-5201 + bind :::5201 v4v6 + default_backend iperf3-5201 + +frontend haproxy-8080 + bind :::8080 v4v6 + default_backend unifi-external-8080 + +frontend haproxy-25565 + bind :::25565 v4v6 + default_backend testcraft-25565 + +frontend haproxy-25566 + bind :::25566 v4v6 + default_backend nimcraft-25566 + +backend gitea-2222 + server gitea reeseapps-gitea.reeselink.com:2222 check inter 5s + +backend sec-gitea-2000 + server sec-gitea stackexpected-gitea.reeselink.com:2222 check inter 5s + +backend iperf3-5201 + server iperf3 web.reeselink.com:5201 check inter 5s + +backend unifi-external-8080 + server unifi unifi-external.reeselink.com:8080 check inter 5s + +backend testcraft-25565 + server mc1 minecraft.reeselink.com:25565 check inter 5s + +backend nimcraft-25566 + server mc2 minecraft.reeselink.com:25566 check inter 5s diff --git a/tests/fixtures/gateway_nginx_conf_d_http.conf b/tests/fixtures/gateway_nginx_conf_d_http.conf new file mode 100644 index 0000000..a381ee5 --- /dev/null +++ b/tests/fixtures/gateway_nginx_conf_d_http.conf @@ -0,0 +1,214 @@ +# Verbatim /etc/nginx/conf.d/http.conf from gateway-ops, read read-only over the ssh key by +# ./scripts/capture-gateway-config.sh (AGENTS.md invariant 7: verify against the +# deployed upstream, never guess its shape). +# Nothing here is rewritten: it is this fleet's ingress map, and the domains in it are +# the data — an inbound name and the target it routes to are what these config sections +# exist to report. Key material appears only as the file paths the proxy reads. +server { + listen 4443 ssl proxy_protocol; + listen [::]:4443 ssl proxy_protocol; + server_name nextcloud.reeseapps.com; + client_max_body_size 0; + + ssl_certificate /etc/letsencrypt/live/nextcloud.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/nextcloud.reeseapps.com/privkey.pem; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + location / { + proxy_pass http://nextcloud.reeselink.com:11000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} + +server { + listen 8443 ssl; + listen [::]:8443 ssl; + server_name nextcloud.reeseapps.com; + client_max_body_size 0; + + ssl_certificate /etc/letsencrypt/live/nextcloud.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/nextcloud.reeseapps.com/privkey.pem; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + location / { + proxy_pass http://nextcloud.reeselink.com:11001; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} + +server { + listen 4443 ssl proxy_protocol; + listen [::]:4443 ssl proxy_protocol; + server_name gitea.reeseapps.com; + client_max_body_size 0; + + ssl_certificate /etc/letsencrypt/live/gitea.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/gitea.reeseapps.com/privkey.pem; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + location / { + proxy_pass http://gitea.reeselink.com:3000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} + +server { + listen 4443 ssl proxy_protocol; + listen [::]:4443 ssl proxy_protocol; + server_name jellyfin.reeseapps.com; + client_max_body_size 0; + + ssl_certificate /etc/letsencrypt/live/jellyfin.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/jellyfin.reeseapps.com/privkey.pem; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + location / { + proxy_pass http://jellyfin.reeselink.com:8096; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} + +server { + listen 8443 ssl; + listen [::]:8443 ssl; + server_name alicia-ai-terminology.reeseapps.com; + + ssl_certificate /etc/letsencrypt/live/alicia-ai-terminology.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/alicia-ai-terminology.reeseapps.com/privkey.pem; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + location / { + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} + +server { + listen 4443 ssl proxy_protocol; + listen [::]:4443 ssl proxy_protocol; + server_name connors-legos.reeseapps.com; + + ssl_certificate /etc/letsencrypt/live/connors-legos.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/connors-legos.reeseapps.com/privkey.pem; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + location / { + proxy_pass http://bricktracker.reeselink.com:3333; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} + +server { + listen 4443 ssl proxy_protocol; + listen [::]:4443 ssl proxy_protocol; + server_name keycloak.reeseapps.com; + client_max_body_size 0; + + ssl_certificate /etc/letsencrypt/live/keycloak.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/keycloak.reeseapps.com/privkey.pem; + + location / { + proxy_pass http://keycloak.reeselink.com:8080; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} + +server { + listen 8443 ssl; + listen [::]:8443 ssl; + server_name admin.keycloak.reeseapps.com; + client_max_body_size 0; + + ssl_certificate /etc/letsencrypt/live/admin.keycloak.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/admin.keycloak.reeseapps.com/privkey.pem; + + location / { + proxy_pass http://keycloak.reeselink.com:8080; + proxy_set_header Host $host:$server_port; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-Host $host:$server_port; + } +} + +server { + listen 4443 ssl proxy_protocol; + listen [::]:4443 ssl proxy_protocol; + server_name emmaleaf.reeseapps.com; + client_max_body_size 10m; + + ssl_certificate /etc/letsencrypt/live/emmaleaf.reeseapps.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/emmaleaf.reeseapps.com/privkey.pem; + + location / { + proxy_pass http://emmaleaf.reeselink.com:80; + proxy_ssl_verify off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} diff --git a/tests/fixtures/gateway_nginx_main.conf b/tests/fixtures/gateway_nginx_main.conf new file mode 100644 index 0000000..23396c9 --- /dev/null +++ b/tests/fixtures/gateway_nginx_main.conf @@ -0,0 +1,63 @@ +# Verbatim /etc/nginx/nginx.conf from gateway-ops, read read-only over the ssh key by +# ./scripts/capture-gateway-config.sh (AGENTS.md invariant 7: verify against the +# deployed upstream, never guess its shape). +# Nothing here is rewritten: it is this fleet's ingress map, and the domains in it are +# the data — an inbound name and the target it routes to are what these config sections +# exist to report. Key material appears only as the file paths the proxy reads. +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /run/nginx.pid; + + +events { + worker_connections 1024; +} + +stream { + include /etc/nginx/stream.d/*.conf; +} + +http { + real_ip_header proxy_protocol; + set_real_ip_from 127.0.0.1; + set_real_ip_from 10.0.0.0/8; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format detailed '$remote_addr:$remote_port [$time_local] "$server_name" "$request" ' + '$status "$body_bytes_sent" "$http_referer" "$http_user_agent" ' + '$request_time "$upstream_addr" "$upstream_response_time" ' + '$ssl_protocol/$ssl_cipher'; + + access_log /var/log/nginx/access.log detailed; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + + keepalive_timeout 65; + + gzip on; + + server_tokens off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + server { + listen 80; + listen [::]:80; + server_name _; + return 301 https://$host$request_uri; + } + + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/tests/fixtures/gateway_nginx_stream_d_stream.conf b/tests/fixtures/gateway_nginx_stream_d_stream.conf new file mode 100644 index 0000000..feec053 --- /dev/null +++ b/tests/fixtures/gateway_nginx_stream_d_stream.conf @@ -0,0 +1,40 @@ +# Verbatim /etc/nginx/stream.d/stream.conf from gateway-ops, read read-only over the ssh key by +# ./scripts/capture-gateway-config.sh (AGENTS.md invariant 7: verify against the +# deployed upstream, never guess its shape). +# Nothing here is rewritten: it is this fleet's ingress map, and the domains in it are +# the data — an inbound name and the target it routes to are what these config sections +# exist to report. Key material appears only as the file paths the proxy reads. +map $ssl_preread_server_name $targetBackend { + default 127.0.0.1:4443; + homepage.reeseapps.com reeseapps-homepage.reeselink.com:443; + ntfy.reeseapps.com reeseapps-ntfy.reeselink.com:443; + gitea.reeseapps.com reeseapps-gitea.reeselink.com:443; + aipi.reeseapps.com reeseapps-litellm.reeselink.com:443; + chatreesept.reeseapps.com reeseapps-openwebui.reeselink.com:443; + keycloak.reeseapps.com reeseapps-keycloak.reeselink.com:443; + forgejo.reeseapps.com reeseapps-forgejo.reeselink.com:443; + brain.reeseapps.com reeseapps-brain-of-reese.reeselink.com:443; + brain-of-code.reeseapps.com reeseapps-brain-of-code.reeselink.com:443; + immich.reeseapps.com reeseapps-immich.reeselink.com:443; + + stackexpected.com stackexpected.reeselink.com:443; + gitea.stackexpected.com stackexpected-gitea.reeselink.com:443; + keycloak.stackexpected.com stackexpected-keycloak.reeselink.com:443; + nextcloud.stackexpected.com stackexpected-nextcloud.reeselink.com:443; + hedgedoc.stackexpected.com stackexpected-hedgedoc.reeselink.com:443; + wellsfamilycounseling.reeseapps.com wfc.reeselink.com:443; +} + +server { + listen 443; + listen [::]:443; + + proxy_connect_timeout 1800s; + proxy_timeout 1800s; + proxy_protocol on; + + proxy_pass $targetBackend; + ssl_preread on; + + resolver 10.1.0.1; +} diff --git a/tests/fixtures/gateway_proc_net_tcp.txt b/tests/fixtures/gateway_proc_net_tcp.txt new file mode 100644 index 0000000..0d1a58f --- /dev/null +++ b/tests/fixtures/gateway_proc_net_tcp.txt @@ -0,0 +1,58 @@ +# Verbatim /proc/net/tcp from gateway-ops, read read-only over the ssh key by +# ./scripts/capture-gateway-config.sh (AGENTS.md invariant 7: verify against the +# deployed upstream, never guess its shape). +# One line per TCP socket, in the kernel's own spelling. The only edit is the repo's +# identifier rule (invariant 6, the drive-serial scrub in capture-fixtures.sh): a +# globally routable address became a documentation one (RFC 5737 / RFC 3849), because a +# global IPv6 here is EUI-64 and a visitor's address is not ours to publish. Every port, +# state, queue, uid, inode and column width is verbatim, as are the loopback, private +# and wildcard addresses the gateway's own frontends use. +# Volatile by nature: a proxy in service has a different connection set every second, so +# a diff here on re-capture is expected. The tests assert shape, the two states the agent +# keeps, and the ports the captured configs claim to bind — never a particular row. + sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 00000000:20FB 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9978 1 0000000000000000 100 0 0 10 0 + 1: 00000000:08AE 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 7797 1 0000000000000000 100 0 0 10 0 + 2: 00000000:0050 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9974 1 0000000000000000 100 0 0 10 0 + 3: 00000000:01BB 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9972 1 0000000000000000 100 0 0 10 0 + 4: 00000000:115B 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9976 1 0000000000000000 100 0 0 10 0 + 5: 1B01010A:23A0 00000000:0000 0A 00000000:00000000 00:00000000 00000000 994 0 9479 1 0000000000000000 100 0 0 10 0 + 6: 1B01010A:BE2A B001010A:01BB 06 00000000:00000000 03:000013F5 00000000 0 0 0 3 0000000000000000 + 7: 0100007F:115B 0100007F:818C 01 00000000:00000000 00:00000000 00000000 101 0 62860 1 0000000000000000 20 4 30 10 -1 + 8: 1B01010A:BE2E B001010A:01BB 06 00000000:00000000 03:00001407 00000000 0 0 0 3 0000000000000000 + 9: 1B01010A:23A0 0204A8C0:C626 06 00000000:00000000 03:00000820 00000000 0 0 0 3 0000000000000000 + 10: 0100007F:115B 0100007F:C096 01 00000000:00000000 00:00000000 00000000 101 0 49558 1 0000000000000000 20 4 26 10 -1 + 11: 0100007F:115B 0100007F:B502 06 00000000:00000000 03:00000841 00000000 0 0 0 3 0000000000000000 + 12: 1B01010A:8290 A401010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 11850 1 0000000000000000 20 4 28 10 -1 + 13: 1B01010A:01BB 017100CB:90FA 01 00000000:00000000 00:00000000 00000000 101 0 21476 1 0000000000000000 35 4 31 10 -1 + 14: 1B01010A:23A0 0204A8C0:992E 06 00000000:00000000 03:00000C0A 00000000 0 0 0 3 0000000000000000 + 15: 1B01010A:CB4C B001010A:01BB 06 00000000:00000000 03:00000ECB 00000000 0 0 0 3 0000000000000000 + 16: 1B01010A:9C42 2700010A:01BB 06 00000000:00000000 03:00001259 00000000 0 0 0 3 0000000000000000 + 17: 1B01010A:01BB 0100010A:B2AC 01 00000000:00000000 00:00000000 00000000 101 0 77631 1 0000000000000000 20 4 31 10 -1 + 18: 1B01010A:01BB E600020A:BBE8 06 00000000:00000000 03:00000992 00000000 0 0 0 3 0000000000000000 + 19: 0100007F:C99E 0100007F:115B 06 00000000:00000000 03:000016BE 00000000 0 0 0 3 0000000000000000 + 20: 1B01010A:C1FE 2700010A:01BB 06 00000000:00000000 03:00000662 00000000 0 0 0 3 0000000000000000 + 21: 1B01010A:01BB E600020A:B5C4 06 00000000:00000000 03:000005EE 00000000 0 0 0 3 0000000000000000 + 22: 1B01010A:840E B001010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 21477 1 0000000000000000 20 4 30 10 -1 + 23: 1B01010A:D22A 4E01010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 77632 1 0000000000000000 21 4 30 10 -1 + 24: 1B01010A:01BB 017100CB:956A 01 00000000:00000000 00:00000000 00000000 101 0 62858 1 0000000000000000 29 4 30 10 -1 + 25: 0100007F:115B 0100007F:B4F6 06 00000000:00000000 03:000005ED 00000000 0 0 0 3 0000000000000000 + 26: 1B01010A:23A0 0204A8C0:8EE6 06 00000000:00000000 03:00001408 00000000 0 0 0 3 0000000000000000 + 27: 1B01010A:01BB EC01010A:94DE 01 00000000:00000000 00:00000000 00000000 101 0 11849 1 0000000000000000 20 4 31 10 -1 + 28: 0100007F:818C 0100007F:115B 01 00000000:00000000 00:00000000 00000000 101 0 62859 1 0000000000000000 21 4 24 10 -1 + 29: 1B01010A:23A0 0204A8C0:D5DE 06 00000000:00000000 03:0000020F 00000000 0 0 0 3 0000000000000000 + 30: 0100007F:115B 0100007F:B4F8 06 00000000:00000000 03:000005F6 00000000 0 0 0 3 0000000000000000 + 31: 1B01010A:A828 7600010A:63DD 06 00000000:00000000 03:00000F7F 00000000 0 0 0 3 0000000000000000 + 32: 0100007F:C096 0100007F:115B 01 00000000:00000000 00:00000000 00000000 101 0 49557 1 0000000000000000 20 4 24 10 -1 + 33: 1B01010A:D444 FE00010A:2AF8 01 00000000:00000000 00:00000000 00000000 101 0 49559 1 0000000000000000 20 4 30 10 -1 + 34: 1B01010A:9032 B001010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 11954 1 0000000000000000 20 4 30 10 -1 + 35: 1B01010A:977E 2700010A:01BB 06 00000000:00000000 03:00000C72 00000000 0 0 0 3 0000000000000000 + 36: 0100007F:115B 0100007F:B4EA 06 00000000:00000000 03:000005F0 00000000 0 0 0 3 0000000000000000 + 37: 1B01010A:08AE 0204A8C0:8290 01 00000048:00000000 01:00000017 00000000 0 0 86478 4 0000000000000000 24 8 15 10 -1 + 38: 1B01010A:9ED0 2700010A:01BB 06 00000000:00000000 03:0000170F 00000000 0 0 0 3 0000000000000000 + 39: 1B01010A:0050 027100CB:843E 01 0000018C:00000000 01:00000E57 00000007 101 0 84306 2 0000000000000000 4864 4 31 1 7 + 40: 1B01010A:CB5C B001010A:01BB 06 00000000:00000000 03:00000ED6 00000000 0 0 0 3 0000000000000000 + 41: 1B01010A:9776 2700010A:01BB 06 00000000:00000000 03:00000875 00000000 0 0 0 3 0000000000000000 + 42: 1B01010A:C59C FE00010A:2AF8 01 00000000:00000000 00:00000000 00000000 101 0 62861 1 0000000000000000 21 4 30 10 -1 + 43: 1B01010A:9154 2700010A:01BB 06 00000000:00000000 03:00001572 00000000 0 0 0 3 0000000000000000 + 44: 1B01010A:23A0 0204A8C0:9C42 06 00000000:00000000 03:00000FF3 00000000 0 0 0 3 0000000000000000 diff --git a/tests/fixtures/gateway_proc_net_tcp6.txt b/tests/fixtures/gateway_proc_net_tcp6.txt new file mode 100644 index 0000000..2f7ec95 --- /dev/null +++ b/tests/fixtures/gateway_proc_net_tcp6.txt @@ -0,0 +1,26 @@ +# Verbatim /proc/net/tcp6 from gateway-ops, read read-only over the ssh key by +# ./scripts/capture-gateway-config.sh (AGENTS.md invariant 7: verify against the +# deployed upstream, never guess its shape). +# One line per TCP socket, in the kernel's own spelling. The only edit is the repo's +# identifier rule (invariant 6, the drive-serial scrub in capture-fixtures.sh): a +# globally routable address became a documentation one (RFC 5737 / RFC 3849), because a +# global IPv6 here is EUI-64 and a visitor's address is not ours to publish. Every port, +# state, queue, uid, inode and column width is verbatim, as are the loopback, private +# and wildcard addresses the gateway's own frontends use. +# Volatile by nature: a proxy in service has a different connection set every second, so +# a diff here on re-capture is expected. The tests assert shape, the two states the agent +# keeps, and the ports the captured configs claim to bind — never a particular row. + sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 00000000000000000000000000000000:20FB 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9979 1 0000000000000000 100 0 0 10 0 + 1: 00000000000000000000000000000000:08AE 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 7799 1 0000000000000000 100 0 0 10 0 + 2: 00000000000000000000000000000000:1451 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9181 1 0000000000000000 100 0 0 10 0 + 3: 00000000000000000000000000000000:0050 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9975 1 0000000000000000 100 0 0 10 0 + 4: 00000000000000000000000000000000:0016 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9179 1 0000000000000000 100 0 0 10 0 + 5: 00000000000000000000000000000000:01BB 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9973 1 0000000000000000 100 0 0 10 0 + 6: 00000000000000000000000000000000:115B 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9977 1 0000000000000000 100 0 0 10 0 + 7: 00000000000000000000000000000000:63DE 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9184 1 0000000000000000 100 0 0 10 0 + 8: 00000000000000000000000000000000:63DD 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9183 1 0000000000000000 100 0 0 10 0 + 9: 00000000000000000000000000000000:07D0 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9180 1 0000000000000000 100 0 0 10 0 + 10: 00000000000000000000000000000000:1F90 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9182 1 0000000000000000 100 0 0 10 0 + 11: B80D0120000000000000000001000000:01BB 2377D3FDF349E0970000000003000000:C0AC 01 00000000:00000000 00:00000000 00000000 101 0 49556 1 0000000000000000 80 4 26 10 -1 + 12: B80D0120000000000000000001000000:01BB 2377D3FDF349E0970000000003000000:A120 01 00000000:00000000 00:00000000 00000000 101 0 11953 1 0000000000000000 59 4 30 7 7 diff --git a/tests/test_agent.py b/tests/test_agent.py index 8b691d7..1890682 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -14,7 +14,9 @@ from __future__ import annotations import http.client import importlib.util +import ipaddress import json +import os import re import socket import ssl @@ -43,7 +45,7 @@ from cryptography.hazmat.primitives.serialization import ( ) from cryptography.x509.oid import NameOID -from dashboards import certs +from dashboards import certs, collectors ROOT = Path(__file__).resolve().parents[1] AGENT_PATH = ROOT / "agent" / "host_agent.py" @@ -329,6 +331,32 @@ def write_fake_disk(root: Path, name: str, model: str | None = None) -> None: write_file(root, f"sys/block/{name}/device/model", f"{model}\n") +#: The add-on sections report their own absence in `errors` (`smart:`, `haproxy:`), because +#: "this box has no smartmontools / no haproxy" is a fact the page that owns the add-on has to +#: be able to print. Tests that assert a snapshot is otherwise clean filter the same prefixes +#: the dashboard does — `collectors.ADDON_ERROR_PREFIXES`, pinned equal to this list by +#: `test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters`, so neither side can +#: quietly grow a fourth opinion. +ADDON_ERROR_PREFIXES = ("smart", "haproxy", "nginx", "sockets") + + +def test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters() -> None: + """One list, two readers. The agent writes `smart:` / `haproxy:` / `nginx:` lines, this suite + filters them to decide a snapshot is otherwise clean, and `collectors.py` filters the same + prefixes to decide whether the **system** payload is `degraded` — which is what an overview + alert is built from. A new add-on section that forgets the prefix on the dashboard side puts + a WARN on every host that never installed it (invariant 3), and this is the test that makes + that a red build instead of a fleet-wide false alarm. + """ + assert collectors.ADDON_ERROR_PREFIXES == ADDON_ERROR_PREFIXES + + +def core_errors(snapshot: dict[str, Any]) -> list[str]: + """`errors` minus the add-ons' own "not here" lines: the failures that matter to a test.""" + return [line for line in snapshot["errors"] + if not str(line).lstrip().lower().startswith(ADDON_ERROR_PREFIXES)] + + def fake_calls(tmp_path: Path) -> list[str]: called = tmp_path / "smart-out" / "_called" return called.read_text().split() if called.exists() else [] @@ -347,7 +375,7 @@ def test_smart_pass_attaches_parsed_health_per_disk(tmp_path, monkeypatch) -> No "sdb": read_fixture(ATA_HDD_CAPTURE), })) snap = collector.snapshot() - assert snap["errors"] == [], snap["errors"] + assert core_errors(snap) == [], snap["errors"] by_name = {device["name"]: device for device in snap["disk_io"]["devices"]} assert by_name["nvme0n1"]["smart"]["model"] == "EXAMPLE NVMe SSD 2TB" assert by_name["nvme0n1"]["smart"]["power_on_hours"] == 8760.0 @@ -662,7 +690,7 @@ def test_no_smart_flag_stays_silent_and_null(tmp_path, monkeypatch) -> None: collector = host_agent.Collector(str(root), smartctl=make_fake_smartctl(tmp_path, {}), smart=False) snap = collector.snapshot() - assert snap["errors"] == [] + assert core_errors(snap) == [] assert snap["disk_io"]["devices"][0]["smart"] is None assert fake_calls(tmp_path) == [] @@ -687,6 +715,1335 @@ def test_smartctl_captures_carry_no_unscrubbed_serials() -> None: assert "SCRUBBED" in line, f"{path.name}: unscrubbed identifier: {line!r}" +# ---- haproxy config: the additive `haproxy` section (phase 28) --------------- # +# +# The text below is this fleet's gateway config as read (read-only) from +# `gateway:/etc/haproxy/haproxy.cfg` on 2026-10-21: `global` and `defaults`, then six +# `frontend` blocks of exactly `bind ::: v4v6` + `default_backend `, then six +# `backend` blocks of exactly `server : check inter 5s`. The structure, the +# indentation and the option spelling are verbatim; the service names and domains are synthetic, +# which is the `smartctl_reference_*` precedent — the shape is what the parser has to match, and +# a tracked file holding this fleet's whole ingress map is a disclosure the parser does not need. +# `tests/fixtures/gateway_haproxy.cfg` is the **verbatim** capture of that file (read-only over +# the `gateway-ops` key, 2026-10-21, no credential in it because haproxy configs hold none), and +# `test_the_recorded_gateway_haproxy_config_parses_too` pins the real service names against it. +# `scripts/capture-gateway-config.sh` (task 04) re-records the same bytes. + +GATEWAY_HAPROXY_SHAPE = """global + log /dev/log local0 + maxconn 4096 + +defaults + mode tcp + timeout connect 5s + timeout client 30s + timeout server 30s + +frontend haproxy-22 + bind :::22 v4v6 + default_backend svc-2222 + +frontend haproxy-2000 + bind :::2000 v4v6 + default_backend svc-2000 + +frontend haproxy-5201 + bind :::5201 v4v6 + default_backend svc-5201 + +frontend haproxy-8080 + bind :::8080 v4v6 + default_backend svc-8080 + +frontend haproxy-25565 + bind :::25565 v4v6 + default_backend svc-25565 + +frontend haproxy-25566 + bind :::25566 v4v6 + default_backend svc-25566 + +backend svc-2222 + server svc2222 svc2222.example.com:2222 check inter 5s + +backend svc-2000 + server svc2000 svc2000.example.com:2000 check inter 5s + +backend svc-5201 + server svc5201 svc5201.example.com:5201 check inter 5s + +backend svc-8080 + server svc8080 svc8080.example.com:8080 check inter 5s + +backend svc-25565 + server svc25565 svc25565.example.com:25565 check inter 5s + +backend svc-25566 + server svc25566 svc25566.example.com:25566 check inter 5s +""" + + +def haproxy_section(tmp_path: Path, text: str | None = None, **kwargs: Any) -> dict[str, Any]: + """One collector's `haproxy` section over a tree that holds (or does not) a config.""" + if text is not None: + write_file(tmp_path / "root", "etc/haproxy/haproxy.cfg", text) + errors: list[str] = [] + section = host_agent.Collector(str(tmp_path / "root"), **kwargs).haproxy(errors) + section["_errors"] = errors # the pass writes into the caller's list; tests read both + return section + + +def test_the_captured_haproxy_shape_parses_into_frontends_and_backends() -> None: + """The whole point of the section: six listening pipes and six targets, as the config says.""" + parsed = host_agent.parse_haproxy_config(GATEWAY_HAPROXY_SHAPE) + assert parsed["recognised"] is True + assert [frontend["name"] for frontend in parsed["frontends"]] == [ + "haproxy-22", "haproxy-2000", "haproxy-5201", "haproxy-8080", + "haproxy-25565", "haproxy-25566"] + # the split the dashboard's whole page depends on: `:::22` is the address `::` on port 22, + # not the address `:::` and not port `0` (an IPv6 bind is the common case on this host) + assert [frontend["binds"] for frontend in parsed["frontends"]] == [ + [{"addr": "::", "port": port}] for port in (22, 2000, 5201, 8080, 25565, 25566)] + assert [frontend["default_backend"] for frontend in parsed["frontends"]] == [ + "svc-2222", "svc-2000", "svc-5201", "svc-8080", "svc-25565", "svc-25566"] + assert [backend["servers"] for backend in parsed["backends"]] == [ + [{"name": f"svc{port}", "target": f"svc{port}.example.com", "port": port}] + for port in (2222, 2000, 5201, 8080, 25565, 25566)] + # routing is the names themselves, and the config joins: every frontend's default_backend is + # one of the backends listed beside it (phase 29 draws its pipes from exactly this) + names = {backend["name"] for backend in parsed["backends"]} + assert {frontend["default_backend"] for frontend in parsed["frontends"]} <= names + # `global`/`defaults` are a section each, not a frontend and not a backend + assert len(parsed["backends"]) == 6 + + +def test_the_recorded_gateway_haproxy_config_parses_too() -> None: + """The real file, pinned: the six pipes the gateway actually routes on, and the six targets + they land in. A re-capture that drifts — a frontend removed, a bind rewritten, a `server` + line re-flowed — fails here before it fails phase 29's page.""" + parsed = host_agent.parse_haproxy_config((FIXTURES / "gateway_haproxy.cfg").read_text()) + assert parsed["recognised"] is True + assert [frontend["name"] for frontend in parsed["frontends"]] == [ + "haproxy-22", "haproxy-2000", "haproxy-5201", "haproxy-8080", + "haproxy-25565", "haproxy-25566"] + assert [frontend["binds"] for frontend in parsed["frontends"]] == [ + [{"addr": "::", "port": port}] for port in (22, 2000, 5201, 8080, 25565, 25566)] + assert [frontend["default_backend"] for frontend in parsed["frontends"]] == [ + "gitea-2222", "sec-gitea-2000", "iperf3-5201", "unifi-external-8080", + "testcraft-25565", "nimcraft-25566"] + assert [(backend["name"], backend["servers"]) for backend in parsed["backends"]][:2] == [ + ("gitea-2222", [{"name": "gitea", "target": "reeseapps-gitea.reeselink.com", + "port": 2222}]), + ("sec-gitea-2000", [{"name": "sec-gitea", "target": "stackexpected-gitea.reeselink.com", + "port": 2222}]), + ] + # every frontend's target exists beside it — the join phase 29 draws its pipes from + names = {backend["name"] for backend in parsed["backends"]} + assert {frontend["default_backend"] for frontend in parsed["frontends"]} <= names + for backend in parsed["backends"]: + for server in backend["servers"]: + assert server["name"] and server["target"] and server["port"] is not None + + +def test_a_bind_splits_at_the_last_colon_and_one_without_a_port_is_skipped() -> None: + parsed = host_agent.parse_haproxy_config( + "frontend a\n" + " bind :::8443 v4v6\n" + " bind *:80 ssl crt /etc/haproxy/certs/example.com.pem\n" + " bind :8080\n" + " bind [::1]:9090\n" + " bind unix@/run/haproxy/admin.sock\n" + " bind 10.1.0.1:8000-8100\n" + " bind 127.0.0.1\n" + ) + assert parsed["frontends"][0]["binds"] == [ + {"addr": "::", "port": 8443}, + {"addr": "*", "port": 80}, + {"addr": None, "port": 8080}, # haproxy's "every address", spelled by absence + {"addr": "[::1]", "port": 9090}, + ] + assert parsed["recognised"] is True + + +def test_a_server_that_names_no_port_keeps_null_and_unknown_lines_are_ignored() -> None: + """Rule 2 on the one optional field, and the reason a real config cannot amber the page.""" + parsed = host_agent.parse_haproxy_config( + "backend bare\n" + " balance roundrobin\n" + " default-server check inter 5s\n" + " server withport 10.1.0.9:8080 check\n" + " server noport 10.1.0.10\n" + " server servicename host.example.com:http check\n" + " server broken\n" + " server-template tpl 4 pool.example.com:8080\n" + " option httpchk GET /healthz\n" + " http-request deny deny_status 403\n" + ) + assert parsed["backends"][0]["servers"] == [ + {"name": "withport", "target": "10.1.0.9", "port": 8080}, + {"name": "noport", "target": "10.1.0.10", "port": None}, + {"name": "servicename", "target": "host.example.com:http", "port": None}, + ] + # a `backend` body never grows frontend fields, however many directives it holds + assert "binds" not in parsed["backends"][0] + + +def test_a_listen_section_is_reported_as_the_pair_haproxy_says_it_is() -> None: + """haproxy's shorthand *is* a frontend and a backend of the same name; modelling it keeps a + config that uses it from reading as "this gateway routes nothing".""" + parsed = host_agent.parse_haproxy_config( + "listen stats\n" + " bind :::8404 v4v6\n" + " stats uri /stats\n" + " server own 127.0.0.1:8405\n" + ) + assert parsed["frontends"] == [{"name": "stats", "binds": [{"addr": "::", "port": 8404}], + "default_backend": "stats", "backends": ["stats"]}] + assert parsed["backends"] == [{"name": "stats", + "servers": [{"name": "own", "target": "127.0.0.1", + "port": 8405}]}] + + +def test_a_backend_computed_per_request_contributes_no_name() -> None: + """`use_backend %[req.hdr(host),map_dom(...)]` has no name in it, and putting the expression + where the dashboard expects a backend would be a wrong answer, not a partial one.""" + parsed = host_agent.parse_haproxy_config( + "frontend mapped\n" + " bind :::443 v4v6\n" + " use_backend %[req.hdr(host),lower,map_dom(/etc/haproxy/maps/domains.map)]\n" + " use_backend canary if is_canary\n" + " use_backend canary if is_other_canary\n" + ) + assert parsed["frontends"][0]["default_backend"] is None + assert parsed["frontends"][0]["backends"] == ["canary"] + + +def test_a_haproxy_config_path_is_an_argument(tmp_path: Path) -> None: + """The path is a flag because a package or a config management tree may keep it elsewhere; + it resolves under `--root`, so the fake-tree tests read the fake tree.""" + write_file(tmp_path / "root", "opt/haproxy/etc/haproxy.cfg", + "frontend f\n bind :::1234 v4v6\n") + errors: list[str] = [] + section = host_agent.Collector( + str(tmp_path / "root"), haproxy_config="/opt/haproxy/etc/haproxy.cfg").haproxy(errors) + assert section["available"] is True + assert section["config"] == "/opt/haproxy/etc/haproxy.cfg" + assert section["frontends"][0]["binds"] == [{"addr": "::", "port": 1234}] + assert errors == [] + + +def test_a_relative_haproxy_config_path_is_taken_as_written(tmp_path: Path, monkeypatch) -> None: + """`--smartctl` and haproxy's own `-f` both resolve a relative path against the working + directory, so this one does too — only the absolute default is read under `--root`.""" + (tmp_path / "haproxy.cfg").write_text("frontend f\n bind :::7 v4v6\n") + monkeypatch.chdir(tmp_path) + errors: list[str] = [] + section = host_agent.Collector("/", haproxy_config="haproxy.cfg").haproxy(errors) + assert section["available"] is True + assert section["frontends"][0]["binds"] == [{"addr": "::", "port": 7}] + assert errors == [] + + +def test_a_missing_haproxy_config_is_available_false_with_one_errors_line(tmp_path: Path) -> None: + """A host without haproxy is unconfigured, not broken: one line for the pass, and the other + sections still there.""" + section = haproxy_section(tmp_path) + assert section["available"] is False + assert section["config"] == host_agent.HAPROXY_CONFIG_PATH + assert section["frontends"] == [] and section["backends"] == [] + haproxy_errors = [line for line in section["_errors"] if line.startswith("haproxy:")] + assert len(haproxy_errors) == 1, "one missing file is one fact, not one per section" + assert "not found" in haproxy_errors[0] + # cross-module contract, the `smartctl: not found` pin's shape: the phase-29 gateway source + # reads this word to answer "is haproxy installed?", and `collectors` keeps the line out of + # the system page's `degraded`. Reword it here and that test fails instead of a page + # losing its explanation — and the line must name the path it looked at. + assert host_agent.HAPROXY_CONFIG_PATH in haproxy_errors[0] + assert "--no-proxy" in haproxy_errors[0], "the line has to say how to stop asking" + + +def test_the_haproxy_pass_says_nothing_when_the_config_reads_cleanly(tmp_path: Path) -> None: + section = haproxy_section(tmp_path, GATEWAY_HAPROXY_SHAPE) + assert section["available"] is True + assert section["_errors"] == [], "a read that worked writes no line" + assert len(section["frontends"]) == 6 + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads any mode, so a 000 file is unreadable only below it") +def test_an_unreadable_haproxy_config_is_available_null_with_one_errors_line( + tmp_path: Path) -> None: + """The file is there and the agent user cannot read it — a different operator action from + "no haproxy here", so a different `available` and a different sentence.""" + path = tmp_path / "root" / "etc" / "haproxy" / "haproxy.cfg" + path.parent.mkdir(parents=True) + path.write_text(GATEWAY_HAPROXY_SHAPE) + path.chmod(0o000) + try: + section = haproxy_section(tmp_path) + finally: + path.chmod(0o644) + assert section["available"] is None + assert len(section["_errors"]) == 1 and section["_errors"][0].startswith("haproxy:") + assert "could not be read" in section["_errors"][0] + + +def test_a_directory_where_the_config_should_be_is_available_null(tmp_path: Path) -> None: + """Root-proof arm of the same fact: the path exists and is not a readable file.""" + (tmp_path / "root" / "etc" / "haproxy" / "haproxy.cfg").mkdir(parents=True) + section = haproxy_section(tmp_path) + assert section["available"] is None + assert len(section["_errors"]) == 1 and "could not be read" in section["_errors"][0] + + +def test_an_oversized_haproxy_config_is_refused_rather_than_read( + tmp_path: Path, monkeypatch) -> None: + """The bound exists so a `--haproxy-config` aimed at a log or a device node cannot cost the + snapshot an unbounded read.""" + monkeypatch.setattr(host_agent, "HAPROXY_CONFIG_MAX_BYTES", 64) + section = haproxy_section(tmp_path, "frontend f\n" + " bind :::1 v4v6\n" * 40) + assert section["available"] is None + assert len(section["_errors"]) == 1 and "could not be read" in section["_errors"][0] + + +def test_a_config_that_is_not_haproxy_degrades_only_its_own_section(tmp_path: Path) -> None: + """Garbage text is `available: null` plus one line — and the snapshot it lives in is intact, + which is the invariant-4 half of this add-on.""" + root = tmp_path / "root" + write_file(root, "proc/stat", + "cpu 100 0 50 800 0 0 0 0 0 0\ncpu0 100 0 50 800 0 0 0 0 0 0\n") + write_file(root, "proc/meminfo", "MemTotal: 8000000 kB\nMemFree: 1000000 kB\n" + "MemAvailable: 4000000 kB\n") + write_file(root, "etc/haproxy/haproxy.cfg", "\x00\x01garbage\nnot a haproxy config at all\n") + snap = host_agent.Collector(str(root)).snapshot() + assert snap["haproxy"]["available"] is None + assert snap["haproxy"]["frontends"] == [] + haproxy_errors = [line for line in snap["errors"] if line.startswith("haproxy:")] + assert len(haproxy_errors) == 1, snap["errors"] + assert "no haproxy section" in haproxy_errors[0] + # the sections that are not this add-on still answered + assert snap["cpu"]["modes"]["total"] > 0 + assert snap["memory"]["total_bytes"] == 8000000 * 1024.0 + assert core_errors(snap) == [], snap["errors"] + + +def test_no_proxy_removes_the_haproxy_section_and_says_nothing(tmp_path: Path) -> None: + """The `--no-smart` precedent: opting out is silent, and "not collected" stays + distinguishable from "collected and empty" because the key is absent.""" + root = tmp_path / "root" + write_file(root, "etc/haproxy/haproxy.cfg", GATEWAY_HAPROXY_SHAPE) + snap = host_agent.Collector(str(root), proxy=False).snapshot() + assert "haproxy" not in snap, "--no-proxy removes the key, not just its contents" + assert core_errors(snap) == [] + + +def test_the_no_proxy_flag_reaches_the_collector_from_the_command_line(capsys) -> None: + """A flag nobody wired is a flag that does nothing: `--once` proves the CLI path, and one + flag covers the whole add-on family (`haproxy`, `nginx` and `sockets` leave together).""" + assert host_agent.main(["--once", "--no-proxy"]) == 0 + snap = json.loads(capsys.readouterr().out) + assert snap["ok"] is True + assert "haproxy" not in snap and "nginx" not in snap and "sockets" not in snap + assert host_agent.main(["--once"]) == 0 + out = json.loads(capsys.readouterr().out) + assert "haproxy" in out and "nginx" in out and "sockets" in out + + +def test_the_haproxy_section_is_documented_in_the_schema() -> None: + """`GET /schema` is how an install documents itself, so the section an operator is looking at + has to be named there with every key it carries — the `tls`/`bundle_rev` precedent.""" + assert schema_documents("haproxy") + doc = host_agent.__doc__ or "" + block = doc.split(" haproxy ", 1)[-1].split(" errors ", 1)[0] + assert block, "the haproxy section has no SCHEMA block to document it" + for key in ("available", "config", "frontends", "binds", "addr", "port", "default_backend", + "backends", "servers", "target"): + assert key in block, f"haproxy.{key} missing from the SCHEMA block" + assert "--no-proxy" in block and "--haproxy-config" in block, \ + "the two flags that decide whether this section exists belong in its documentation" + + +# ---- nginx config: the additive `nginx` section (phase 28) ------------------ # +# +# The three texts below are this fleet's gateway nginx tree as read (read-only) from +# `gateway:/etc/nginx/nginx.conf`, `/etc/nginx/conf.d/http.conf` and +# `/etc/nginx/stream.d/stream.conf` on 2026-10-21: a main config whose top level is `events` + +# `stream { include …/stream.d/*.conf; }` + `http { … include …/conf.d/*.conf; }` with one +# catch-all `server` on port 80; one `conf.d` file of `listen 4443 ssl proxy_protocol` / +# `listen 8443 ssl` vhosts each with a `server_name ` and a `location / { proxy_pass +# http://:; }`; one `stream.d` file with a +# `map $ssl_preread_server_name $targetBackend { … }` block and a `server` that ends in +# `proxy_pass $targetBackend; ssl_preread on;`. The structure, the indentation and the flag +# spelling are verbatim; the domains and targets are the `smartctl_reference_*` precedent — the +# shape is what the parser must match, and this fleet's ingress map is not a tracked file's +# business. `tests/fixtures/gateway_nginx_*.conf` (task 04) is the verbatim capture, and the +# re-record script is `scripts/capture-gateway-config.sh`. + +GATEWAY_NGINX_MAIN = """user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /run/nginx.pid; + +events { + worker_connections 1024; +} + +stream { + include /etc/nginx/stream.d/*.conf; +} + +http { + real_ip_header proxy_protocol; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 80; + listen [::]:80; + server_name _; + return 301 https://$host$request_uri; + } + + include /etc/nginx/conf.d/*.conf; +} +""" + +GATEWAY_NGINX_HTTP = """server { + listen 4443 ssl proxy_protocol; + listen [::]:4443 ssl proxy_protocol; + server_name nextcloud.example.com; + client_max_body_size 0; + + ssl_certificate /etc/letsencrypt/live/nextcloud.example.com/fullchain.pem; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + + location / { + proxy_pass http://nextcloud.internal.test:11000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} + +server { + listen 8443 ssl; + listen [::]:8443 ssl; + server_name alicia.example.com; + + location / { + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + } +} +""" + +GATEWAY_NGINX_STREAM = """map $ssl_preread_server_name $targetBackend { + default 127.0.0.1:4443; + brain.example.com brain-of-reese.internal.test:443; +} + +server { + listen 443; + listen [::]:443; + + proxy_connect_timeout 1800s; + proxy_timeout 1800s; + proxy_protocol on; + + proxy_pass $targetBackend; + ssl_preread on; + + resolver 10.1.0.1; +} +""" + + +def nginx_tree(root: Path, *, main: str | None = GATEWAY_NGINX_MAIN, + http: str | None = GATEWAY_NGINX_HTTP, stream: str | None = GATEWAY_NGINX_STREAM, + extra: dict[str, str] | None = None) -> Path: + """Write the captured gateway tree (or any part of it) under a fake `--root`.""" + for relative, text in (("etc/nginx/nginx.conf", main), + ("etc/nginx/conf.d/http.conf", http), + ("etc/nginx/stream.d/stream.conf", stream)): + if text is not None: + write_file(root, relative, text) + for relative, text in (extra or {}).items(): + write_file(root, relative, text) + return root + + +def nginx_section(tmp_path: Path, *, tree: bool = True, **kwargs: Any) -> dict[str, Any]: + """One collector's `nginx` section over a tree that holds (or does not) the configs.""" + root = tmp_path / "root" + if tree: + nginx_tree(root) + errors: list[str] = [] + section = host_agent.Collector(str(root), **kwargs).nginx(errors) + section["_errors"] = errors # the pass writes into the caller's list; tests read both + return section + + +def by_variable(maps: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """The `maps` list keyed by variable name, because that is the key the dashboard joins on.""" + return {entry["variable"]: entry for entry in maps} + + +def test_the_captured_nginx_shape_parses_into_servers_maps_and_targets() -> None: + """The whole point of the section: every pipe nginx listens on and where each one lands.""" + parsed = host_agent.parse_nginx_config(GATEWAY_NGINX_MAIN, { + "/etc/nginx/conf.d/http.conf": GATEWAY_NGINX_HTTP, + "/etc/nginx/stream.d/stream.conf": GATEWAY_NGINX_STREAM, + }) + assert parsed["recognised"] is True + assert parsed["error"] is None and parsed["unparsed"] == [] + # document order: the main config's own server, then stream.d (included first), then conf.d + assert [server["server_names"] for server in parsed["servers"]] == [ + ["_"], [], ["nextcloud.example.com"], ["alicia.example.com"]] + # `listen` keeps the address the config wrote: a bare `80` has no address (nginx's own "every + # address"), `[::]:4443` is the address `::` and not the string `[::]`, and the `ssl + # proxy_protocol` flags are not this section's business + assert [server["listen"] for server in parsed["servers"]] == [ + [{"addr": None, "port": 80}, {"addr": "::", "port": 80}], + [{"addr": None, "port": 443}, {"addr": "::", "port": 443}], + [{"addr": None, "port": 4443}, {"addr": "::", "port": 4443}], + [{"addr": None, "port": 8443}, {"addr": "::", "port": 8443}], + ] + assert [server["proxy_passes"] for server in parsed["servers"]] == [ + [], + # the stream form: no `location` exists, and the target is a variable, not an address + [{"location": None, "scheme": None, "target": None, "variable": "targetBackend"}], + [{"location": "/", "scheme": "http", "variable": None, + "target": {"host": "nextcloud.internal.test", "port": 11000}}], + [{"location": "/", "scheme": "http", "variable": None, + "target": {"host": "127.0.0.1", "port": 8081}}], + ] + maps = by_variable(parsed["maps"]) + assert set(maps) == {"connection_upgrade", "targetBackend"} + # the variable name is spelled one way on both sides of the join, and the `$` is nginx's + # syntax rather than part of the name — so it is dropped here once, not in the page + assert maps["targetBackend"]["entries"] == [ + {"key": "default", "target": {"host": "127.0.0.1", "port": 4443}}, + {"key": "brain.example.com", "target": {"host": "brain-of-reese.internal.test", + "port": 443}}, + ] + # …and the `proxy_pass $targetBackend` above still carries `target: null`: the map decides + # per connection, so merging it here would be a number the wire never published + assert parsed["servers"][1]["proxy_passes"][0]["target"] is None + # `$connection_upgrade`'s values are header words, not hosts: keys reported, targets null + assert maps["connection_upgrade"]["entries"] == [ + {"key": "default", "target": None}, {"key": "", "target": None}] + # the include lines, as written, in document order — including the one file this tree has no + # opinion about because it is not a config (`mime.types`) + assert parsed["includes"] == ["/etc/nginx/stream.d/*.conf", "/etc/nginx/mime.types", + "/etc/nginx/conf.d/*.conf"] + assert parsed["files"] == ["/etc/nginx/stream.d/stream.conf", "/etc/nginx/conf.d/http.conf"] + + +def test_the_section_names_the_files_the_numbers_came_from(tmp_path: Path) -> None: + """`files` is the answer to "which files is this page drawing?", main config first, and it + speaks the config's paths rather than the ones under `--root`.""" + section = nginx_section(tmp_path) + assert section["available"] is True + assert section["files"] == ["/etc/nginx/nginx.conf", + "/etc/nginx/stream.d/stream.conf", + "/etc/nginx/conf.d/http.conf"] + assert section["_errors"] == [], "a read that worked writes no line" + assert len(section["servers"]) == 4 + assert len(section["maps"]) == 2 + + +def test_a_listen_token_splits_the_three_forms_nginx_writes() -> None: + parsed = host_agent.parse_nginx_config( + "http { server {\n" + " listen 80;\n" + " listen [::]:4443 ssl;\n" + " listen 10.1.0.5:9000;\n" + " listen 8080 default_server;\n" + " listen unix:/run/nginx.sock;\n" + " listen [::1;\n" # a bracket that never closes is not an address + " listen [::1];\n" # an address with no port is not one either + " listen;\n" + " } }" + ) + assert parsed["servers"][0]["listen"] == [ + {"addr": None, "port": 80}, + {"addr": "::", "port": 4443}, + {"addr": "10.1.0.5", "port": 9000}, + {"addr": None, "port": 8080}, + ] + + +def test_a_proxy_pass_reports_the_address_the_variable_or_neither() -> None: + """Three spellings of the same directive, and the agent never resolves one of them.""" + parsed = host_agent.parse_nginx_config( + "http { server {\n" + " location /a/ { proxy_pass http://h1.test:8080/api; }\n" + " location /b/ { proxy_pass http://pool_name; }\n" + " location /c/ { proxy_pass $backend; }\n" + " location /d/ { proxy_pass unix:/run/app.sock; }\n" + " location /e/ { proxy_pass http://host:http; }\n" + " } }" + ) + assert parsed["servers"][0]["proxy_passes"] == [ + # the trailing path is dropped the way haproxy's `check inter 5s` is: it is not a place + {"location": "/a/", "scheme": "http", "target": {"host": "h1.test", "port": 8080}, + "variable": None}, + # an `upstream` name is a host with no port — the name is the fact, the port is not + {"location": "/b/", "scheme": "http", "target": {"host": "pool_name", "port": None}, + "variable": None}, + {"location": "/c/", "scheme": None, "target": None, "variable": "backend"}, + # a unix socket is neither a host nor a variable: the line is reported with nulls + {"location": "/d/", "scheme": None, "target": None, "variable": None}, + # a colon that is not a port either: the scheme is a fact, the host would be a guess + {"location": "/e/", "scheme": "http", "target": None, "variable": None}, + ] + + +@pytest.mark.parametrize(("text", "why"), [ + ("http { server { listen 80;", "an opening brace is never closed"), + ("http { server { listen 80", "the file ends inside the directive"), + ("} http { }", "no opening one before it"), + ("http { server { listen 80 } }", "is still open"), + ("http { { listen 80; } }", "with no block name"), + ('http { add_header X "unclosed; }', "quoted string is never closed"), +]) +def test_a_config_nginx_would_refuse_to_start_on_fails_the_scanner(text: str, + why: str) -> None: + """Six malformed files, one verdict each, and no exception on a snapshot path. + + The sentences matter because they are what the section says: this is the arm that answers + "the file is there and I could not read it as nginx", and a scanner that raised instead of + reporting would fail the whole `/snapshot` over one half-written vhost. + """ + parsed = host_agent.parse_nginx_config(text) + assert parsed["error"] is not None and why in parsed["error"], parsed + assert parsed["recognised"] is False and parsed["servers"] == [] + assert host_agent.nginx_include_paths(text) == [], "no includes out of a file that is not a tree" + + +def test_an_upstream_block_keeps_its_server_lines_as_targets() -> None: + """`server` means two different things in nginx, and the scanner tells them apart.""" + parsed = host_agent.parse_nginx_config( + "upstream pool_name {\n" + " server 10.1.0.9:8080 max_fails=3 fail_timeout=30s;\n" + " server backend.test;\n" + " server [::1]:8080;\n" + " keepalive 32;\n" + "}\n" + "http { server { listen 80; } }" + ) + assert parsed["upstreams"] == [{"name": "pool_name", "servers": [ + {"host": "10.1.0.9", "port": 8080}, + {"host": "backend.test", "port": None}, + {"host": "::1", "port": 8080}, + ]}] + # an upstream's `server` lines are not server blocks, however many of them there are + assert len(parsed["servers"]) == 1 + + +def test_a_quoted_value_holding_a_semicolon_is_one_word() -> None: + """The real config writes `add_header … "max-age=31536000; includeSubDomains" always;`. A + `;` inside quotes is data, and cutting on it would halve the file's block structure — same + for the `{`, and an escaped quote inside the value stays inside it. A bare `;` with nothing + before it is whitespace, not an empty directive.""" + parsed = host_agent.parse_nginx_config( + 'http { server { listen 80; ; add_header X "a;b { \\"c\\"" always; return 404; } }' + ) + assert parsed["servers"][0]["listen"] == [{"addr": None, "port": 80}] + assert parsed["error"] is None + # the stray `;` did not open an empty directive and the escaped quote did not end the value + assert parsed["recognised"] is True and len(parsed["servers"]) == 1 + + +def test_a_commented_include_line_is_not_an_include() -> None: + """An operator commented the old vhost file out; following it would report a pipe that is + not there, and reading it would cost a file the config took out of service.""" + parsed = host_agent.parse_nginx_config( + "http {\n # include /etc/nginx/conf.d/*.conf;\n server { listen 80; }\n}", + {"/etc/nginx/conf.d/http.conf": 'server { listen 9999; server_name ghost.test; }'}, + ) + assert parsed["includes"] == [] and parsed["files"] == [] + assert [server["listen"] for server in parsed["servers"]] == [[{"addr": None, "port": 80}]] + + +def test_includes_are_followed_one_level_and_no_deeper(tmp_path: Path) -> None: + """The captured tree is one level deep. Going deeper would mean reading files the pass was + never bounded for, so an include inside an included file contributes nothing — and no + error, because the config is not wrong, the pass is deliberately shallow.""" + root = tmp_path / "root" + nginx_tree(root, stream=None, http=None, extra={ + # deliberately outside conf.d, so the main config's own glob cannot be what read it + "etc/nginx/hidden/deep.conf": 'server { listen 7777; server_name deep.test; }\n', + }) + write_file(root, "etc/nginx/conf.d/http.conf", + GATEWAY_NGINX_HTTP + "\ninclude /etc/nginx/hidden/deep.conf;\n") + errors: list[str] = [] + section = host_agent.Collector(str(root)).nginx(errors) + assert section["available"] is True + assert section["files"] == ["/etc/nginx/nginx.conf", "/etc/nginx/conf.d/http.conf"] + assert 7777 not in [entry["port"] for server in section["servers"] for entry + in server["listen"]], "the second level is not this pass's job" + assert errors == [] + + +def test_a_literal_include_is_read_when_this_host_has_it(tmp_path: Path) -> None: + """The non-glob arm of the include line. `mime.types` is the captured case: it exists on a + real host, holds a `types { … }` block this section does not model, and costs one file.""" + root = tmp_path / "root" + nginx_tree(root, http=None, stream=None, extra={ + "etc/nginx/mime.types": "types {\n text/html html htm;\n}\n", + }) + errors: list[str] = [] + section = host_agent.Collector(str(root)).nginx(errors) + assert section["available"] is True + assert section["files"] == ["/etc/nginx/nginx.conf", "/etc/nginx/mime.types"] + assert section["servers"][0]["listen"] == [{"addr": None, "port": 80}, + {"addr": "::", "port": 80}] + assert errors == [] + + +def test_an_include_naming_a_file_this_host_does_not_have_says_nothing(tmp_path: Path) -> None: + """`include /etc/nginx/mime.types;` is in every main config and in no fake tree. Absence is + not a fault, and a line per absent file would bury the arms that are.""" + section = nginx_section(tmp_path) + assert "/etc/nginx/mime.types" not in section["files"] + assert section["_errors"] == [], "absence is the `available: false` arm one level down" + + +def test_the_include_file_bound_drops_the_excess_with_one_line_naming_the_count( + tmp_path: Path) -> None: + """The SMART pass-budget shape: cover what you can, say what you could not, keep the section + `available: true` because the config was read.""" + root = tmp_path / "root" + nginx_tree(root, http=None, stream=None) + for index in range(10): + write_file(root, f"etc/nginx/conf.d/vhost{index}.conf", + f"server {{ listen {8100 + index}; server_name h{index}.test; }}\n") + errors: list[str] = [] + section = host_agent.Collector(str(root)).nginx(errors) + assert section["available"] is True + limit = host_agent.NGINX_INCLUDE_MAX_FILES + assert section["files"] == ["/etc/nginx/nginx.conf"] + [ + f"/etc/nginx/conf.d/vhost{index}.conf" for index in range(limit)] + assert len(section["servers"]) == limit + 1 # main's own server plus the 8 that were read + nginx_errors = [line for line in errors if line.startswith("nginx:")] + assert len(nginx_errors) == 1, "one bound is one fact, not one per dropped file" + assert f"{10 - limit} included file(s)" in nginx_errors[0], nginx_errors + assert f"{limit}-file" in nginx_errors[0], "the line names the bound it hit" + + +def test_the_include_byte_bound_drops_the_file_that_would_cross_it(tmp_path: Path, + monkeypatch) -> None: + """The other half of the bound: one huge include must not turn a snapshot into a file copy.""" + monkeypatch.setattr(host_agent, "NGINX_INCLUDE_MAX_BYTES", 120) + root = tmp_path / "root" + nginx_tree(root, stream=None) + # a padding comment, so the file is still valid nginx: it has to be dropped for its size and + # not because it is the half-written vhost the next test is about + write_file(root, "etc/nginx/conf.d/http.conf", f"# {'x' * 100}\n") + write_file(root, "etc/nginx/conf.d/second.conf", "server { listen 8123; }\n") + errors: list[str] = [] + section = host_agent.Collector(str(root)).nginx(errors) + assert section["available"] is True + assert section["files"] == ["/etc/nginx/nginx.conf", "/etc/nginx/conf.d/http.conf"] + nginx_errors = [line for line in errors if line.startswith("nginx:")] + assert len(nginx_errors) == 1 and "1 included file(s)" in nginx_errors[0] + assert "120-byte" in nginx_errors[0], "the line names the bound it hit, in the unit it is set in" + + +def test_an_oversized_main_config_is_refused_rather_than_read(tmp_path: Path, + monkeypatch) -> None: + """The bound exists so a `--nginx-config` aimed at a log or a device node cannot cost the + snapshot an unbounded read — the haproxy rule, on the other half of the add-on.""" + monkeypatch.setattr(host_agent, "NGINX_CONFIG_MAX_BYTES", 64) + section = nginx_section(tmp_path) + assert section["available"] is None + assert len(section["_errors"]) == 1 and "could not be read" in section["_errors"][0] + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads any mode, so a 000 file is unreadable only below it") +def test_an_unreadable_main_config_is_available_null_with_one_errors_line( + tmp_path: Path) -> None: + """The file is there and the agent user cannot read it — a different operator action from + "no nginx here", so a different `available` and a different sentence.""" + root = tmp_path / "root" + nginx_tree(root) + path = root / "etc" / "nginx" / "nginx.conf" + path.chmod(0o000) + try: + section = nginx_section(tmp_path, tree=False) # do not rewrite the tree over the mode + finally: + path.chmod(0o644) + assert section["available"] is None + assert len(section["_errors"]) == 1 and section["_errors"][0].startswith("nginx:") + assert "could not be read" in section["_errors"][0] + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads any mode, so a 000 file is unreadable only below it") +def test_an_unreadable_included_file_is_reported_once_and_the_rest_still_lands( + tmp_path: Path) -> None: + """One denied vhost file must not cost the whole section — and must not pass in silence + either, because the page would draw four pipes where nginx has five.""" + root = tmp_path / "root" + nginx_tree(root, main=None) + (root / "etc" / "nginx").mkdir(parents=True, exist_ok=True) + write_file(root, "etc/nginx/nginx.conf", "http { include /etc/nginx/conf.d/*.conf; }\n") + write_file(root, "etc/nginx/conf.d/second.conf", "server { listen 8124; }\n") + denied = root / "etc" / "nginx" / "conf.d" / "http.conf" + denied.chmod(0o000) + try: + errors: list[str] = [] + section = host_agent.Collector(str(root)).nginx(errors) + finally: + denied.chmod(0o644) + assert section["available"] is True + assert [entry["port"] for server in section["servers"] for entry in server["listen"]] == [8124] + nginx_errors = [line for line in errors if line.startswith("nginx:")] + assert len(nginx_errors) == 1, "one denied file is one line, not one per include line" + assert "could not be read" in nginx_errors[0] + assert "/etc/nginx/conf.d/http.conf" in nginx_errors[0], "it names what it could not read" + + +def test_an_included_file_that_will_not_parse_is_dropped_and_named(tmp_path: Path) -> None: + """A half-written vhost file is that file's problem: the section keeps every other server + and says which document it could not read, because the main config still answered.""" + root = tmp_path / "root" + nginx_tree(root, stream=None, http="server { listen 4443;\n") + errors: list[str] = [] + section = host_agent.Collector(str(root)).nginx(errors) + assert section["available"] is True + assert section["files"] == ["/etc/nginx/nginx.conf"] + nginx_errors = [line for line in errors if line.startswith("nginx:")] + assert len(nginx_errors) == 1 + assert "could not be parsed" in nginx_errors[0] + assert "/etc/nginx/conf.d/http.conf" in nginx_errors[0] + # the main config's own server is still there: one bad include degrades, it does not blank + assert [entry["port"] for entry in section["servers"][0]["listen"]] == [80, 80] + + +def test_a_missing_nginx_config_is_available_false_with_one_errors_line(tmp_path: Path) -> None: + """A host without nginx is unconfigured, not broken: one line for the pass, and the other + sections still there.""" + section = nginx_section(tmp_path, tree=False) + assert section["available"] is False + assert section["config"] == host_agent.NGINX_CONFIG_PATH + assert section["files"] == [] and section["servers"] == [] + nginx_errors = [line for line in section["_errors"] if line.startswith("nginx:")] + assert len(nginx_errors) == 1, "one missing file is one fact, not one per section" + assert "not found" in nginx_errors[0] + # cross-module contract, the `smartctl: not found` and `haproxy: … not found` pins' shape: + # the phase-29 gateway source reads this word to answer "is nginx installed?", and + # `collectors` keeps the line out of the system page's `degraded`. + assert host_agent.NGINX_CONFIG_PATH in nginx_errors[0] + assert "--nginx-config" in nginx_errors[0], "the line names the flag that moves the path" + assert "--no-proxy" in nginx_errors[0], "the line has to say how to stop asking" + + +def test_text_that_is_not_an_nginx_config_degrades_only_its_own_section(tmp_path: Path) -> None: + """Valid text that names no nginx context is a different fact from an unreadable file, and + it is still one line and still only this section's problem.""" + root = tmp_path / "root" + nginx_tree(root, http=None, stream=None) + write_file(root, "etc/nginx/nginx.conf", "some-other-daemon on;\nopts 3;\n") + errors: list[str] = [] + section = host_agent.Collector(str(root)).nginx(errors) + assert section["available"] is None + assert section["servers"] == [] and section["files"] == [] + nginx_errors = [line for line in errors if line.startswith("nginx:")] + assert len(nginx_errors) == 1, errors + assert "no nginx context" in nginx_errors[0] + + +def test_a_corrupt_nginx_config_leaves_the_rest_of_the_snapshot_intact(tmp_path: Path) -> None: + """The invariant-4 half of this add-on: an unparseable main config is `available: null` plus + one line, and `/snapshot` still answers with its cpu and memory sections.""" + root = tmp_path / "root" + write_file(root, "proc/stat", + "cpu 100 0 50 800 0 0 0 0 0 0\ncpu0 100 0 50 800 0 0 0 0 0 0\n") + write_file(root, "proc/meminfo", "MemTotal: 8000000 kB\nMemFree: 1000000 kB\n" + "MemAvailable: 4000000 kB\n") + write_file(root, "etc/nginx/nginx.conf", "http { server { listen 80;\n") + snap = host_agent.Collector(str(root)).snapshot() + assert snap["nginx"]["available"] is None + assert snap["nginx"]["servers"] == [] + nginx_errors = [line for line in snap["errors"] if line.startswith("nginx:")] + assert len(nginx_errors) == 1, snap["errors"] + assert "could not be parsed" in nginx_errors[0] + assert snap["cpu"]["modes"]["total"] > 0 + assert snap["memory"]["total_bytes"] == 8000000 * 1024.0 + assert core_errors(snap) == [], snap["errors"] + + +def test_an_nginx_failure_inside_the_collector_still_yields_a_section(tmp_path: Path, + monkeypatch) -> None: + """Whatever raises in here, the document keeps the key with `available: null` — a snapshot + that loses the key would read as a pre-1.2.0 agent, not as a failed add-on.""" + def explode(*args: object) -> dict[str, Any]: + raise RuntimeError("boom") + + monkeypatch.setattr(host_agent.Collector, "nginx", explode) + snap = host_agent.Collector(str(tmp_path / "root")).snapshot() + assert snap["nginx"] == {"available": None, "config": host_agent.NGINX_CONFIG_PATH, + "files": [], "servers": [], "upstreams": [], "maps": []} + assert any(line.startswith("nginx:") for line in snap["errors"]) + assert core_errors(snap) == [], "the add-on's own failure line is not a system degradation" + + +def test_an_nginx_config_path_is_an_argument(tmp_path: Path) -> None: + """Packaged and containerised nginx trees keep the main config in different places; the path + resolves under `--root`, so the fake-tree tests read the fake tree.""" + write_file(tmp_path / "root", "opt/nginx/conf/nginx.conf", + 'http { server { listen 7777; server_name arg.test; } }\n') + errors: list[str] = [] + section = host_agent.Collector( + str(tmp_path / "root"), nginx_config="/opt/nginx/conf/nginx.conf").nginx(errors) + assert section["available"] is True + assert section["config"] == "/opt/nginx/conf/nginx.conf" + assert section["files"] == ["/opt/nginx/conf/nginx.conf"] + assert section["servers"][0]["listen"] == [{"addr": None, "port": 7777}] + assert errors == [] + + +def test_a_relative_include_line_is_resolved_and_labelled_as_written(tmp_path: Path, + monkeypatch) -> None: + """The relative arm of both halves: an include argument written without a leading `/` is + resolved against the working directory (where nginx's own prefix would resolve it) and stays + spelled that way in `files`, because that is how the config wrote it.""" + (tmp_path / "nginx.conf").write_text("http { include conf.d/http.conf; }\n") + write_file(tmp_path, "conf.d/http.conf", "server { listen 8123; server_name rel.test; }\n") + monkeypatch.chdir(tmp_path) + errors: list[str] = [] + section = host_agent.Collector("/", nginx_config="nginx.conf").nginx(errors) + assert section["available"] is True + assert section["files"] == ["nginx.conf", "conf.d/http.conf"] + assert section["servers"][0]["listen"] == [{"addr": None, "port": 8123}] + assert errors == [] + + +def test_a_relative_nginx_config_path_is_taken_as_written(tmp_path: Path, monkeypatch) -> None: + """nginx's own `-c` resolves a relative path against the working directory, so this one does + too — only the absolute default is read under `--root`.""" + (tmp_path / "nginx.conf").write_text("http { server { listen 7; } }\n") + monkeypatch.chdir(tmp_path) + errors: list[str] = [] + section = host_agent.Collector("/", nginx_config="nginx.conf").nginx(errors) + assert section["available"] is True + assert section["servers"][0]["listen"] == [{"addr": None, "port": 7}] + assert errors == [] + + +def test_two_include_lines_naming_one_file_read_it_once(tmp_path: Path) -> None: + """`include conf.d/http.conf;` and `include conf.d/*.conf;` name the same file, and reading + it twice would double every server block it holds.""" + root = tmp_path / "root" + nginx_tree(root, stream=None, main="http {\n" + " include /etc/nginx/conf.d/http.conf;\n" + " include /etc/nginx/conf.d/*.conf;\n" + "}\n") + errors: list[str] = [] + section = host_agent.Collector(str(root)).nginx(errors) + assert section["available"] is True + assert section["files"] == ["/etc/nginx/nginx.conf", "/etc/nginx/conf.d/http.conf"] + # two vhosts in http.conf, and not four + assert [entry["port"] for server in section["servers"] for entry in server["listen"]] == [ + 4443, 4443, 8443, 8443] + assert errors == [] + + +def test_the_include_paths_of_a_config_that_will_not_parse_are_simply_none() -> None: + """The collector asks this before it can report anything, so a broken file must answer + empty rather than raise — and the parse failure is then said once, by the parser.""" + assert host_agent.nginx_include_paths("http { server { listen 80;\n") == [] + assert host_agent.nginx_include_paths("http { include /a/*.conf; }") == ["/a/*.conf"] + + +def test_no_proxy_removes_the_nginx_section_and_says_nothing(tmp_path: Path) -> None: + """The `--no-smart` precedent, on the second section: opting out is silent, and "not + collected" stays distinguishable from "collected and empty" because the key is absent.""" + root = tmp_path / "root" + nginx_tree(root) + snap = host_agent.Collector(str(root), proxy=False).snapshot() + assert "nginx" not in snap, "--no-proxy removes the key, not just its contents" + assert core_errors(snap) == [] + + +def test_the_nginx_section_reaches_the_command_line(tmp_path: Path, capsys) -> None: + """A flag nobody wired is a flag that does nothing: `--once` proves the CLI path for both + the path argument and the opt-out.""" + config = tmp_path / "nginx.conf" + config.write_text('http { server { listen 7778; server_name cli.test; } }\n') + assert host_agent.main(["--once", "--nginx-config", str(config)]) == 0 + snap = json.loads(capsys.readouterr().out) + assert snap["ok"] is True + assert snap["nginx"]["available"] is True + assert snap["nginx"]["servers"][0]["listen"] == [{"addr": None, "port": 7778}] + assert host_agent.main(["--once", "--no-proxy"]) == 0 + assert "nginx" not in json.loads(capsys.readouterr().out) + + +def test_the_nginx_section_is_documented_in_the_schema() -> None: + """`GET /schema` is how an install documents itself, so the section an operator is looking at + has to be named there with every key it carries — the `haproxy` block's precedent.""" + assert schema_documents("nginx") + doc = host_agent.__doc__ or "" + block = doc.split(" nginx ", 1)[-1].split(" errors ", 1)[0] + assert block, "the nginx section has no SCHEMA block to document it" + for key in ("available", "config", "files", "servers", "listen", "addr", "port", + "server_names", "proxy_passes", "location", "scheme", "target", "variable", + "upstreams", "maps", "entries", "key"): + assert key in block, f"nginx.{key} missing from the SCHEMA block" + assert "--no-proxy" in block and "--nginx-config" in block, \ + "the two flags that decide whether this section exists belong in its documentation" + + +# ---- live sockets: the additive `sockets` section (phase 28) ---------------- # + +#: A fragment of `/proc/net/tcp` as the deployed kernel writes it: the header verbatim, the +#: column order and spacing verbatim, the addresses documentation space (RFC 5737) and the +#: hexadecimals computed from them the way `/proc` computes them — one 32-bit word, printed +#: little-endian. `tests/fixtures/gateway_proc_net_tcp{,6}.txt` is the verbatim capture; this is +#: the pinned shape with the values a test can assert, the `smartctl_reference_*` precedent. +#: Rows: LISTEN 0.0.0.0:80, ESTABLISHED 10.1.0.5:443 <- 203.0.113.7:50000, that same pair in +#: TIME_WAIT, and LISTEN 127.0.0.1:8082. +REFERENCE_PROC_NET_TCP = """\ + sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 00000000:0050 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 4314 1 0000000000000000 100 0 0 10 0 + 1: 0500010A:01BB 077100CB:C350 01 00000000:00000000 00:00000000 00000000 1000 0 5122 1 00000000d22dedba 20 4 28 10 -1 + 2: 0500010A:01BB 077100CB:C350 06 00000000:00000000 00:00000000 00000000 0 0 0 3 0000000000000000 20 0 0 10 0 + 3: 0100007F:1F92 00000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 4389 1 0000000000000000 100 0 0 10 0 +""" + +#: The v6 half, same shape: LISTEN `::`:443 (what haproxy writes as `bind :::443 v4v6`), an +#: ESTABLISHED pair between two documentation-space v6 addresses (RFC 3849), the same pair in +#: TIME_WAIT, and LISTEN `::ffff:127.0.0.1:8082` — the IPv4-mapped spelling a v6 socket uses, +#: which stays a v6 row instead of being folded into the v4 table. +REFERENCE_PROC_NET_TCP6 = """\ + sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 00000000000000000000000000000000:01BB 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 4315 1 0000000000000000 100 0 0 10 0 + 1: B80D0120000000000000000005000000:20FB B80D0120000001000000000009000000:CE41 01 00000000:00000000 00:00000000 00000000 1000 0 5130 1 00000000d22dedba 20 4 28 10 -1 + 2: B80D0120000000000000000005000000:20FB B80D0120000001000000000009000000:CE41 06 00000000:00000000 00:00000000 00000000 0 0 0 3 0000000000000000 20 0 0 10 0 + 3: 0000000000000000FFFF00000100007F:1F92 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 4390 1 0000000000000000 100 0 0 10 0 +""" + + +def sockets_tree(root: Path, tcp: str = REFERENCE_PROC_NET_TCP, + tcp6: str | None = REFERENCE_PROC_NET_TCP6) -> Path: + """A proc tree whose only interesting content is the two socket tables.""" + write_file(root, "proc/stat", "cpu 9000 0 1000 80000\n") + write_file(root, "proc/meminfo", "MemTotal: 8000000 kB\nMemAvailable: 5000000 kB\n") + write_file(root, "proc/net/tcp", tcp) + if tcp6 is not None: + write_file(root, "proc/net/tcp6", tcp6) + return root + + +def test_a_captured_proc_net_tcp_fragment_parses_into_listens_and_established() -> None: + """The v4 table's two facts, exactly, with `/proc`'s own hexadecimals.""" + listens, established = host_agent.parse_proc_net_tcp(REFERENCE_PROC_NET_TCP, "v4") + assert listens == [ + {"family": "v4", "addr": "0.0.0.0", "port": 80}, + {"family": "v4", "addr": "127.0.0.1", "port": 8082}, + ] + assert established == [ + {"family": "v4", + "local": {"addr": "10.1.0.5", "port": 443}, + "remote": {"addr": "203.0.113.7", "port": 50000}}, + ] + + +@pytest.mark.parametrize("state", ["02", "03", "04", "05", "06", "07", "08", "09", "0B", "0C"]) +def test_a_socket_in_transit_is_never_reported_as_a_pipe(state: str) -> None: + """Every kernel state but LISTEN and ESTABLISHED is a socket that is not a pipe. + + `06` is TIME_WAIT, which is what a proxy's finished client connections turn into by the + thousand: reporting them would draw two pipes for one connection and bury the live pairs + under rows that stopped being true before the page painted. The kernel's own list + (`net/tcp_states.h`) is what this walks, so a state added upstream is skipped too. + """ + line = (f" 0: 0500010A:01BB 077100CB:C350 {state} 00000000:00000000 00:00000000" + " 00:00000000 00000000 1000 0 5122 1 00000000d22dedba 20 4 28 10 -1") + assert host_agent.parse_proc_net_tcp(line, "v4") == ([], []) + + +def test_the_captured_fragment_actually_holds_the_row_it_omits() -> None: + """The TIME_WAIT assertion above is only worth anything if the capture has one to omit.""" + states = [line.split()[3] for line in REFERENCE_PROC_NET_TCP.splitlines()[1:] + if len(line.split()) > 3] + assert states == ["0A", "01", "06", "0A"], "the v4 fragment lost its socket in transit" + v6_states = [line.split()[3] for line in REFERENCE_PROC_NET_TCP6.splitlines()[1:] + if len(line.split()) > 3] + assert v6_states == ["0A", "01", "06", "0A"] + listens, established = host_agent.parse_proc_net_tcp(REFERENCE_PROC_NET_TCP, "v4") + assert len(established) == 1, "the TIME_WAIT row leaked into the established list" + assert len(listens) == 2 + + +def test_a_captured_proc_net_tcp6_fragment_unrolls_its_little_endian_words() -> None: + """v6 is four 32-bit words, each printed little-endian; the parser must unroll each group. + + A parser that read the hex as written would print `b80d:120:…` — a plausible-looking address + that is not the one the kernel reported, which is worse than a gap beside a real socket. + """ + listens, established = host_agent.parse_proc_net_tcp(REFERENCE_PROC_NET_TCP6, "v6") + assert listens == [ + {"family": "v6", "addr": "::", "port": 443}, + {"family": "v6", "addr": "::ffff:127.0.0.1", "port": 8082}, + ] + assert established == [ + {"family": "v6", + "local": {"addr": "2001:db8::5", "port": 8443}, + "remote": {"addr": "2001:db8:1::9", "port": 52801}}, + ] + + +def test_the_wildcard_listeners_keep_their_literal_wildcard_addresses() -> None: + """The wildcard **is** the fact the inbound join keys on, so it is never nullled or starred. + + haproxy's `bind :::80 v4v6` and nginx's bare `listen 80` both mean "any address on this + port", and the page reaches that fact by comparing the socket's address to the config's; a + `*`, a `null` or a dropped field would each lose every wildcard frontend on the gateway. + """ + v4_listens, _ = host_agent.parse_proc_net_tcp(REFERENCE_PROC_NET_TCP, "v4") + v6_listens, _ = host_agent.parse_proc_net_tcp(REFERENCE_PROC_NET_TCP6, "v6") + assert v4_listens[0]["addr"] == "0.0.0.0" + assert v6_listens[0]["addr"] == "::" + assert "*" not in str(v4_listens) and "*" not in str(v6_listens) + + +def test_an_endpoint_that_is_not_an_address_is_null_and_never_a_guess() -> None: + """A short word, a non-hex word and a non-numeric port each cost a field, not a row.""" + assert host_agent._proc_net_endpoint("0100007F:1F92", "v4") == ("127.0.0.1", 8082) + assert host_agent._proc_net_endpoint("0100007:1F92", "v4") == (None, 8082) + assert host_agent._proc_net_endpoint("010000ZZ:1F92", "v4") == (None, 8082) + assert host_agent._proc_net_endpoint("0100007F:nope", "v4") == ("127.0.0.1", None) + assert host_agent._proc_net_endpoint("local_address", "v4") == (None, None) + listens, _ = host_agent.parse_proc_net_tcp( + " 0: local_address rem_address st rest\n" + " 1: 00000000:0050 00000000:0000 0A rest\n", "v4") + assert listens == [{"family": "v4", "addr": "0.0.0.0", "port": 80}], \ + "/proc's own header line must not become a socket" + + +def test_an_address_this_platform_cannot_render_is_a_gap_and_not_a_crash( + monkeypatch: pytest.MonkeyPatch) -> None: + """`socket.inet_ntop` is the one call here the agent does not control. + + It exists on every Linux the agent runs on, and this pins what happens if it ever refuses: + the address becomes `null` and the socket row stays, because a snapshot path may not raise + over a rendering question, and the port — the half the join actually needs — is still true. + """ + def refuse(_family: int, _bytes: bytes) -> str: + raise OSError("no inet_ntop here") + + monkeypatch.setattr(host_agent.socket, "inet_ntop", refuse) + listens, _ = host_agent.parse_proc_net_tcp(REFERENCE_PROC_NET_TCP, "v4") + assert listens[0] == {"family": "v4", "addr": None, "port": 80} + + +def test_the_socket_tables_are_read_from_the_proc_tree(tmp_path: Path) -> None: + """The section over a tree, not just over text: `--root` resolves `/proc/net/tcp`.""" + root = tmp_path / "root" + sockets_tree(root) + errors: list[str] = [] + section = host_agent.Collector(str(root)).sockets(errors) + assert section["available"] is True + assert section["listens"] == [ + {"family": "v4", "addr": "0.0.0.0", "port": 80}, + {"family": "v4", "addr": "127.0.0.1", "port": 8082}, + {"family": "v6", "addr": "::", "port": 443}, + {"family": "v6", "addr": "::ffff:127.0.0.1", "port": 8082}, + ], "the v4 rows come first and the family label says which table each came from" + assert len(section["established"]) == 2 + + +def test_a_tree_without_the_socket_tables_is_null_with_one_errors_line(tmp_path: Path) -> None: + """Neither table answering is the section's own `null` plus **one** line, nothing else. + + A snapshot is not the place to discover that `/proc` is missing, and one line is the whole + report: the other sections still have to be readable and the system source still has to be + `ok` (the `sockets` prefix is in the dashboard's add-on list for exactly that). + """ + root = tmp_path / "root" + sockets_tree(root, tcp="", tcp6=None) + (root / "proc" / "net" / "tcp").unlink() + errors: list[str] = [] + section = host_agent.Collector(str(root)).sockets(errors) + assert section == {"available": None, "listens": [], "established": []} + assert len(errors) == 1, errors + assert errors[0].startswith("sockets:") + assert "/proc/net/tcp" in errors[0] and "/proc/net/tcp6" in errors[0] + assert "--no-proxy" in errors[0] + + +@pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="running as root") +def test_the_socket_tables_reach_a_snapshot_and_nothing_else_moves(tmp_path: Path) -> None: + """The section is additive in the strongest sense: adding the tables changes no other field. + + A snapshot is a dozen independent reads, and the cheapest proof that a new one is a guest + rather than a renovation is to take the same tree twice — once with the socket tables, once + without — and demand every other key byte-identical. `ts` and `duration_ms` are the two + fields a snapshot owns, so they are the only ones allowed to differ besides `sockets`. + """ + with_tables = tmp_path / "with" + sockets_tree(with_tables) + without = tmp_path / "without" + sockets_tree(without) + (without / "proc" / "net" / "tcp").unlink() + (without / "proc" / "net" / "tcp6").unlink() + first = host_agent.Collector(str(with_tables)).snapshot() + second = host_agent.Collector(str(without)).snapshot() + assert first["sockets"]["available"] is True + assert second["sockets"]["available"] is None + for key in first: + if key in ("ts", "duration_ms", "sockets", "errors"): + continue + assert first[key] == second[key], f"{key} moved when the socket section appeared" + # the gap line is the add-on's own voice: it reaches `errors` and nothing else + assert any(line.startswith("sockets:") for line in second["errors"]) + assert not any(line.startswith("sockets:") for line in first["errors"]) + + +def test_an_unreadable_socket_table_costs_one_line_while_the_other_still_reports( + tmp_path: Path) -> None: + """Partial coverage is `available: true` plus one line: the host said something worth saying.""" + root = tmp_path / "root" + sockets_tree(root) + (root / "proc" / "net" / "tcp6").chmod(0) + errors: list[str] = [] + section = host_agent.Collector(str(root)).sockets(errors) + assert section["available"] is True + assert [row["family"] for row in section["listens"]] == ["v4", "v4"] + assert len(errors) == 1, errors + assert errors[0].startswith("sockets:") and "/proc/net/tcp6" in errors[0] + assert "not found" not in errors[0], "a denied file is not a missing one" + + +def test_a_missing_second_table_costs_nothing_when_the_first_answered(tmp_path: Path) -> None: + """A kernel built without IPv6 has no `/proc/net/tcp6` at all. + + That is the host telling us what it has, so the section is `available: true` and the pass is + silent — the nginx rule that an include naming a file this host does not have contributes + nothing and no line, and the reason a fleet-wide agent rollout does not put a line in every + IPv4-only container's snapshot. + """ + root = tmp_path / "root" + sockets_tree(root, tcp6=None) + errors: list[str] = [] + section = host_agent.Collector(str(root)).sockets(errors) + assert section["available"] is True + assert [row["family"] for row in section["listens"]] == ["v4", "v4"] + assert errors == [] + + +def test_the_socket_read_is_bounded_and_says_so(tmp_path: Path) -> None: + """A busy gateway writes one line per socket; the bound is what keeps that out of the poll. + + Truncation is reported, not hidden: `available` stays `true`, the first rows are the rows the + section carries, and one line names the bound so the difference between "4096 sockets" and + "more sockets than this read takes" is on the wire (the SMART pass-budget shape). + """ + root = tmp_path / "root" + rows = [" sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt" + " uid timeout inode"] + rows += [f" {n}: 00000000:{n:04X} 00000000:0000 0A 00000000:00000000 00:00000000" + " 00000000 0 0 1 1 0000000000000000 100 0 0 10 0" + for n in range(1, 5000)] + sockets_tree(root, tcp="\n".join(rows) + "\n", tcp6=None) + errors: list[str] = [] + section = host_agent.Collector(str(root)).sockets(errors) + assert section["available"] is True + assert len(section["listens"]) == host_agent.PROC_NET_MAX_LINES - 1, \ + "the bound counts the lines read, /proc's own header among them" + assert [row["port"] for row in section["listens"][:2]] == [1, 2] + assert section["listens"][-1]["port"] == host_agent.PROC_NET_MAX_LINES - 1 + assert len(errors) == 1, errors + assert errors[0].startswith("sockets:") and "/proc/net/tcp" in errors[0] + assert str(host_agent.PROC_NET_MAX_LINES) in errors[0] + + +def test_no_proxy_removes_the_sockets_section_and_says_nothing(tmp_path: Path) -> None: + """A box with no proxy has nothing to watch from either side, so opting out is silent.""" + root = tmp_path / "root" + sockets_tree(root) + snap = host_agent.Collector(str(root), proxy=False).snapshot() + assert "sockets" not in snap, "--no-proxy removes the key, not just its contents" + assert core_errors(snap) == [] + + +def test_a_sockets_failure_inside_the_collector_still_yields_a_section( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The `_safe` wrapper is the section's last line of defence, like the two configs'""" + root = tmp_path / "root" + sockets_tree(root) + collector = host_agent.Collector(str(root)) + + def explode(errors: list[str]) -> dict: + raise RuntimeError("kaboom") + + monkeypatch.setattr(collector, "sockets", explode) + snap = collector.snapshot() + assert snap["sockets"] == {"available": None, "listens": [], "established": []} + assert any(line.startswith("sockets:") for line in snap["errors"]) + + +def test_the_sockets_section_reaches_the_command_line(tmp_path: Path, capsys) -> None: + """`--once` proves the CLI path, including the opt-out the operator actually uses.""" + root = tmp_path / "root" + sockets_tree(root) + assert host_agent.main(["--once", "--root", str(root)]) == 0 + snap = json.loads(capsys.readouterr().out) + assert snap["ok"] is True + assert snap["sockets"]["available"] is True + assert {"family": "v6", "addr": "::", "port": 443} in snap["sockets"]["listens"] + assert host_agent.main(["--once", "--root", str(root), "--no-proxy"]) == 0 + assert "sockets" not in json.loads(capsys.readouterr().out) + + +def test_the_sockets_section_is_documented_in_the_schema() -> None: + """`GET /schema` is how an install documents itself; the two configs set the precedent.""" + assert schema_documents("sockets") + doc = host_agent.__doc__ or "" + block = doc.split(" sockets ", 1)[-1].split(" errors ", 1)[0] + assert block, "the sockets section has no SCHEMA block to document it" + for key in ("available", "listens", "established", "family", "addr", "port", + "local", "remote"): + assert key in block, f"sockets.{key} missing from the SCHEMA block" + assert "--no-proxy" in block, "the flag that decides whether this section exists belongs here" + for table in ("/proc/net/tcp", "/proc/net/tcp6"): + assert table in block, f"{table} is what the section reads; name it in the schema" + + +@linux_only +def test_the_real_proc_net_tcp_parses_on_the_box_running_pytest() -> None: + """The parser against the kernel that is actually here. + + The shape is what this pins: every row carries its family, a port that parsed is a plausible + one, and a wildcard listener is still a wildcard. Whether this machine has any listeners at + all is not a claim a test may make, so it does not. + """ + collector = host_agent.Collector("/") + errors: list[str] = [] + section = collector.sockets(errors) + assert section["available"] is True, "a Linux box with /proc mounted must answer for sockets" + assert errors == [] + for row in section["listens"]: + assert row["family"] in ("v4", "v6") + assert row["port"] is None or 0 <= row["port"] <= 65535 + if row["port"] is not None: + assert row["addr"] is not None, "a decoded port from an undecodable address is drift" + for row in section["established"]: + assert set(row) == {"family", "local", "remote"} + assert set(row["local"]) == set(row["remote"]) == {"addr", "port"} + + # ---- the real collector --------------------------------------------------- # @linux_only @@ -696,7 +2053,8 @@ def test_snapshot_against_the_live_system_is_complete() -> None: collector = host_agent.Collector("/") snap = collector.snapshot() for key in ("schema", "ts", "host", "cpu", "memory", "filesystems", "disk_io", - "network", "gpus", "temperatures", "processes", "errors"): + "network", "gpus", "temperatures", "processes", "haproxy", "nginx", + "sockets", "errors"): assert key in snap, f"{key} missing from a real snapshot" assert snap["host"]["hostname"] assert snap["cpu"]["modes"]["total"] > 0 @@ -705,8 +2063,27 @@ def test_snapshot_against_the_live_system_is_complete() -> None: assert snap["filesystems"], "the box it runs on has at least one mounted filesystem" # SMART notes are allowed here: the box running pytest may have no smartctl or no # device read access, and both are documented degradations, never snapshot failures. - hard_errors = [e for e in snap["errors"] if not e.startswith("smart")] + # So are the proxy ones — most boxes have no haproxy, and saying so is the add-on's + # `available: false` arm, not a degraded section. So is `sockets:` on a box whose /proc has + # no net tables mounted, which is the same "unconfigured, not broken" sentence. + hard_errors = core_errors(snap) assert hard_errors == [], f"degraded sections must be empty on a healthy box: {hard_errors}" + # `files` is the config's own paths: the main config first when it was read, empty when not + if snap["nginx"]["available"] is True: + assert snap["nginx"]["files"][0] == snap["nginx"]["config"] + else: + assert snap["nginx"]["files"] == [] + # the add-on is present by default, and its own verdict is one of the three arms + assert snap["haproxy"]["available"] in (True, False, None) + assert snap["haproxy"]["config"] == host_agent.HAPROXY_CONFIG_PATH + assert snap["nginx"]["available"] in (True, False, None) + assert snap["nginx"]["config"] == host_agent.NGINX_CONFIG_PATH + # the socket table is present by default too, and answers for itself: /proc/net/tcp is not + # something a host can leave uninstalled, so the arms here are "read" and "did not read" + assert snap["sockets"]["available"] in (True, None) + for row in snap["sockets"]["listens"]: + assert row["family"] in ("v4", "v6") + assert row["port"] is None or 0 <= row["port"] <= 65535 def test_captured_fixture_has_the_shape_the_dashboard_depends_on(agent_snapshot) -> None: @@ -875,9 +2252,11 @@ def test_agent_is_dependency_free() -> None: top = {name.split(".")[0] for name in imports} third_party = top - set(sys.stdlib_module_names) - {"host_agent"} assert not third_party, f"agent grew a non-stdlib import: {third_party}" - assert top <= {"__future__", "argparse", "collections", "contextlib", "dataclasses", "http", - "json", "os", "pathlib", "re", "shutil", "socket", "ssl", "subprocess", - "tempfile", "threading", "time", "typing"}, ( + # `fnmatch` + `glob` (phase 28) join the list on purpose: an nginx `include` line is a glob, + # and both are as old as the 3.10 floor and as dependency-free as `re`. + assert top <= {"__future__", "argparse", "collections", "contextlib", "dataclasses", "fnmatch", + "glob", "http", "json", "os", "pathlib", "re", "shutil", "socket", "ssl", + "subprocess", "tempfile", "threading", "time", "typing"}, ( f"agent import footprint grew: {sorted(top)}") @@ -1378,9 +2757,281 @@ def schema_documents(field: str) -> bool: return re.search(rf"^\s+{field}\s", schema, re.M) is not None -def test_the_agent_version_and_schema_document_the_new_fields() -> None: - """1.1.0 because the payload grew, and the docstring is what `GET /schema` answers.""" - assert host_agent.AGENT_VERSION == "1.1.0" +def test_the_tls_fields_are_documented_in_the_schema() -> None: + """The phase-15 fields are documented in the docstring, which is what `GET /schema` answers. + + The version constant itself is pinned by the newest phase's test below — one place names the + number, so a bump moves one assertion and not a file's worth of history. + """ assert schema_documents("tls") and schema_documents("bundle_rev") assert "--cert-bundle" in (host_agent.__doc__ or "") + +def test_1_2_0_is_the_version_that_ships_the_three_proxy_sections() -> None: + """The version bump is the fleet's way of saying "this file has the gateway sections". + + Phase 28 added three sections and one flag to the payload, so an install that predates them + is distinguishable by one string: `--status`, `/health` and this test all read the same + constant. The SCHEMA block is `GET /schema`'s body, so it names all three, each with the + three-way `available` the dashboard reads, and `--no-proxy`, the flag that removes the family. + """ + assert host_agent.AGENT_VERSION == "1.2.0" + doc = host_agent.__doc__ or "" + for section in ("haproxy", "nginx", "sockets"): + assert schema_documents(section), f"{section} is not documented in the SCHEMA block" + block = doc.split(f" {section} ", 1)[-1] + assert "available" in block, f"{section}'s block does not document its three-way answer" + assert "--no-proxy" in doc and "--haproxy-config" in doc and "--nginx-config" in doc + + +# ---- the recorded gateway: the captures themselves (phase 28 task 04) -------------------- # + +#: Every document `scripts/capture-gateway-config.sh` records, at the tree path the agent reads +#: it from and with the fixture it came from. This is the phase's ground truth: the synthetic +#: shapes above are edge cases with pinned labels, and these are the box (invariant 7). +GATEWAY_CAPTURES = { + "etc/haproxy/haproxy.cfg": "gateway_haproxy.cfg", + "etc/nginx/nginx.conf": "gateway_nginx_main.conf", + "etc/nginx/conf.d/http.conf": "gateway_nginx_conf_d_http.conf", + "etc/nginx/stream.d/stream.conf": "gateway_nginx_stream_d_stream.conf", + "proc/net/tcp": "gateway_proc_net_tcp.txt", + "proc/net/tcp6": "gateway_proc_net_tcp6.txt", +} + +#: What each captured document was read from, so a test can say which file to re-record. +GATEWAY_SOURCES = { + "gateway_haproxy.cfg": "/etc/haproxy/haproxy.cfg", + "gateway_nginx_main.conf": "/etc/nginx/nginx.conf", + "gateway_nginx_conf_d_http.conf": "/etc/nginx/conf.d/http.conf", + "gateway_nginx_stream_d_stream.conf": "/etc/nginx/stream.d/stream.conf", + "gateway_proc_net_tcp.txt": "/proc/net/tcp", + "gateway_proc_net_tcp6.txt": "/proc/net/tcp6", +} + + +def gateway_tree(root: Path) -> Path: + """A fake root carrying the recorded gateway: the six captures at their real paths. + + `mime.types` is deliberately absent, because it is absent on the host too — the main config + includes it and the agent's answer for that is one include line that costs nothing. + """ + write_file(root, "proc/stat", + "cpu 100 0 50 800 0 0 0 0 0 0\ncpu0 100 0 50 800 0 0 0 0 0 0\n") + write_file(root, "proc/meminfo", "MemTotal: 8000000 kB\nMemFree: 1000000 kB\n" + "MemAvailable: 4000000 kB\n") + for tree_path, fixture in GATEWAY_CAPTURES.items(): + write_file(root, tree_path, (FIXTURES / fixture).read_text()) + return root + + +def gateway_collector(root: Path) -> Any: + return host_agent.Collector(str(root), smartctl="/nonexistent/smartctl") + + +def test_the_gateway_captures_are_captures_and_say_so() -> None: + """Each fixture names the remote path it came from and the script that read it. + + That header is the difference between a capture and a file somebody typed, and the parsers + skip it — `#` is a comment in both config formats, and a `/proc` line whose 4th field is not + a kernel state is not a socket row — so provenance costs nothing on either side. + """ + for fixture, source in GATEWAY_SOURCES.items(): + path = FIXTURES / fixture + assert path.is_file(), f"{fixture} is missing: run ./scripts/capture-gateway-config.sh" + head = path.read_text().splitlines()[:10] + assert head[0].startswith(f"# Verbatim {source} from "), head[:1] + assert any("capture-gateway-config.sh" in line for line in head), head + + +def test_the_recorded_gateway_nginx_tree_merges_the_documents_it_names(tmp_path: Path) -> None: + """The real main config plus the real vhosts, through the real include pass. + + `files` is the answer an operator can check against `nginx -t`, and the map is the SNI table + the whole point of the stream block: the map's *values* are the outbound targets, and the + server beside it has no target at all because the map decides per connection. + """ + errors: list[str] = [] + section = gateway_collector(gateway_tree(tmp_path / "root")).nginx(errors) + assert section["available"] is True, errors + assert errors == [], f"the recorded tree should cost no error line: {errors}" + assert section["files"] == ["/etc/nginx/nginx.conf", + "/etc/nginx/stream.d/stream.conf", + "/etc/nginx/conf.d/http.conf"], \ + "the merge set changed: re-run ./scripts/capture-gateway-config.sh" + assert len(section["servers"]) >= 10, "a gateway with a handful of vhosts is not this gateway" + + maps = {cast("dict[str, Any]", m)["variable"]: m for m in section["maps"]} + assert "targetBackend" in maps and "connection_upgrade" in maps, list(maps) + entries = {cast("str", cast("dict[str, Any]", e)["key"]): cast("dict[str, Any]", e) + for e in maps["targetBackend"]["entries"]} + brain = entries["brain.reeseapps.com"]["target"] + assert brain == {"host": "reeseapps-brain-of-reese.reeselink.com", "port": 443}, brain + assert entries["default"]["target"] == {"host": "127.0.0.1", "port": 4443}, entries["default"] + # `map $http_upgrade` answers header words, and a word is not a host (rule 2). + upgrade = {cast("str", cast("dict[str, Any]", e)["key"]): cast("dict[str, Any]", e)["target"] + for e in maps["connection_upgrade"]["entries"]} + assert upgrade == {"default": None, "": None}, upgrade + + +def test_the_recorded_gateway_routes_by_sni_without_inventing_a_target(tmp_path: Path) -> None: + """One vhost and the stream server, named: what the phase-29 page draws its pipes from.""" + errors: list[str] = [] + section = gateway_collector(gateway_tree(tmp_path / "root")).nginx(errors) + servers = [cast("dict[str, Any]", s) for s in section["servers"]] + + def ports(server: dict[str, Any]) -> set[int | None]: + return {cast("dict[str, Any]", listen)["port"] for listen in server["listen"]} + + nextcloud = [s for s in servers + if "nextcloud.reeseapps.com" in cast("list[str]", s["server_names"]) + and 4443 in ports(s)] + assert nextcloud, "the recorded vhost for nextcloud on 4443 is gone" + passes = [cast("dict[str, Any]", p) for p in nextcloud[0]["proxy_passes"]] + assert {"host": "nextcloud.reeselink.com", "port": 11000} in [ + cast("dict[str, Any]", p)["target"] for p in passes], passes + + stream = [s for s in servers + if ports(s) == {443} and not cast("list[str]", s["server_names"])] + assert stream, "the stream server that terminates :443 and routes by SNI is gone" + variable_pass = [cast("dict[str, Any]", p) for p in stream[0]["proxy_passes"] + if p["variable"] == "targetBackend"] + assert variable_pass and variable_pass[0]["target"] is None, \ + "`proxy_pass $targetBackend` must travel as the variable, never as a merged target" + + +def in_documentation_space(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + """RFC 5737 (`203.0.113.0/24`) or RFC 3849 (`2001:db8::/32`) — the ranges the capture uses.""" + return (address in ipaddress.ip_network("203.0.113.0/24") + or address in ipaddress.ip_network("2001:db8::/32")) + + +def proc_data_lines(text: str) -> list[list[str]]: + """The socket rows of a `/proc/net/tcp` capture: no kernel header, no provenance header. + + The kernel numbers its rows (`0:`, `1:` …), which is the one test that separates a socket from + a line of comment or a column title — both of which have four fields too. + """ + rows = [] + for line in text.splitlines(): + fields = line.split() + if len(fields) > 4 and fields[0].endswith(":") and fields[0][:-1].isdigit(): + rows.append(fields) + return rows + + +def test_the_recorded_socket_tables_decode_every_row_they_hold() -> None: + """Real `/proc` bytes, every address and port of them resolved. + + The agent's decoder is being read against a kernel's own output rather than a hand-written + fragment: a row whose address comes back `None` means the spelling changed under us, which is + exactly what a fixture is for. And the rows the parser must *not* report are here in volume — + a proxy's `TIME_WAIT` pile — so their absence from the answer is measured, not assumed. + """ + transit = 0 + for fixture, family in (("gateway_proc_net_tcp.txt", "v4"), ("gateway_proc_net_tcp6.txt", "v6")): + text = (FIXTURES / fixture).read_text() + rows = proc_data_lines(text) + assert rows, f"{fixture} holds no socket row at all" + listens, established = host_agent.parse_proc_net_tcp(text, family) + parsed = [*listens, *established] + assert parsed, f"{fixture} produced no socket at all" + broken = [row for row in parsed + if any(endpoint.get("addr") is None or endpoint.get("port") is None + for endpoint in ([row] if "addr" in row + else [row["local"], row["remote"]]))] + assert not broken, f"{fixture} has a row /proc's spelling does not explain: {broken[:3]}" + + states = [fields[3] for fields in rows] + kept = states.count(host_agent.PROC_NET_STATE_LISTEN) \ + + states.count(host_agent.PROC_NET_STATE_ESTABLISHED) + assert len(parsed) == kept, ( + f"{fixture}: the parser returned {len(parsed)} rows for {kept} LISTEN/ESTABLISHED " + f"sockets (states present: {sorted(set(states))})") + transit += len(states) - kept + + +def test_the_recorded_configs_and_the_recorded_sockets_agree(tmp_path: Path) -> None: + """The config claims frontends; the kernel agrees, and one listener belongs to neither. + + Two independent reads of one box at one moment: haproxy's `bind`s and nginx's `listen`s are + ports the gateway is answering on, and the agent's own port is a listener no proxy config + names. That last one is the case phase 29 must keep visible rather than drop — a socket with + no claim is a fact, not a join failure. + """ + collector = gateway_collector(gateway_tree(tmp_path / "root")) + errors: list[str] = [] + haproxy = collector.haproxy(errors) + nginx = collector.nginx(errors) + sockets = collector.sockets(errors) + assert errors == [], errors + assert haproxy["available"] is True and nginx["available"] is True + assert sockets["available"] is True + + claimed: set[int] = set() + for frontend in cast("list[dict[str, Any]]", haproxy["frontends"]): + claimed |= {cast("int", b["port"]) for b in cast("list[dict[str, Any]]", frontend["binds"])} + for server in cast("list[dict[str, Any]]", nginx["servers"]): + claimed |= {cast("int", ls["port"]) for ls in cast("list[dict[str, Any]]", server["listen"]) + if ls["port"] is not None} + listening = {cast("int", row["port"]) for row in + cast("list[dict[str, Any]]", sockets["listens"]) if row["port"] is not None} + missing = claimed - listening + assert not missing, ( + f"the configs claim ports {sorted(missing)} nothing is listening on: the two captures " + "were taken apart, or a service was down — re-run ./scripts/capture-gateway-config.sh") + assert 9120 in listening - claimed, \ + "the agent's own listener should be in the socket table and claimed by neither config" + + +def test_the_recorded_socket_tables_publish_no_globally_routable_endpoint() -> None: + """AGENTS.md invariant 6, on the artifacts: identifiers never go in the repo. + + The socket table is the one capture here that can hold somebody else's address, and on Linux + a global IPv6 is EUI-64 — it encodes the NIC's MAC, which is the identifier class + `capture-fixtures.sh` already refuses to record for a drive. Private, loopback, wildcard and + documentation addresses are all expected here; a globally routable one is a capture that got + away. + """ + offenders: list[str] = [] + documented = 0 + for fixture in ("gateway_proc_net_tcp.txt", "gateway_proc_net_tcp6.txt"): + for fields in proc_data_lines((FIXTURES / fixture).read_text()): + for column in (fields[1], fields[2]): + word = column.rpartition(":")[0] + packed = bytes.fromhex(word) + raw = b"".join(packed[i:i + 4][::-1] for i in range(0, len(packed), 4)) + address = ipaddress.ip_address(raw) + # Only an IPv6Address has `ipv4_mapped` (3.14 added `ipv6_mapped` beside it). + subject = getattr(address, "ipv4_mapped", None) or address + if subject.is_global: + offenders.append(f"{fixture}: {subject}") + if in_documentation_space(subject): + documented += 1 + assert not offenders, ("globally routable endpoints are in a tracked fixture: " + + ", ".join(offenders) + + " — scripts/capture-gateway-config.sh is the thing that stops them") + assert documented, ("no documentation-space address either, which means this fixture no " + "longer pins the substitution the capture makes") + + +def test_the_recorded_gateway_answers_all_three_sections_at_once(tmp_path: Path) -> None: + """The phase's end state in one harness: one 1.2.0-shaped read of the captured gateway. + + `haproxy` and `nginx` from the config tree, `sockets` from the kernel tables, all three in + one snapshot, nothing in `errors`, and the old sections exactly as they were — which is what + phase 29 builds its page on. + """ + snap = gateway_collector(gateway_tree(tmp_path / "root")).snapshot() + assert snap["agent_version"] == host_agent.AGENT_VERSION + for section in ("haproxy", "nginx", "sockets"): + assert snap[section]["available"] is True, snap["errors"] + assert len(snap["haproxy"]["frontends"]) == 6, "the recorded gateway has six frontends" + assert len(snap["haproxy"]["backends"]) == 6, "and one backend each" + assert snap["nginx"]["maps"], "the SNI map is the point of the nginx capture" + assert snap["sockets"]["listens"], "the socket capture must answer listeners" + assert core_errors(snap) == [], snap["errors"] + # The old sections are untouched by the add-on, which is the whole compatibility claim. + for section in ("host", "cpu", "memory", "filesystems", "disk_io", "network", + "temperatures", "processes", "errors"): + assert section in snap, f"the pre-1.2.0 payload lost {section}" + diff --git a/tests/test_collectors.py b/tests/test_collectors.py index fe83d56..7c751db 100644 --- a/tests/test_collectors.py +++ b/tests/test_collectors.py @@ -249,6 +249,55 @@ async def test_real_degradations_still_show_when_smart_noise_is_alongside() -> N assert result.payload["degraded"] == ["temperatures: no hwmon sensors"] +async def test_a_host_without_a_proxy_config_does_not_degrade_the_system_source() -> None: + """The proxy add-on's absence lines, same rule and same reason as `smart:` above: two of this + fleet's three hosts run no haproxy and no nginx, so an agent upgrade must not put a + `degraded` payload (and therefore an overview alert, `alerts.py` reads this key) on a box that + never had either. The lines survive in `agent.errors` for the page that owns them — phase 29's + gateway source reads them the way `/disks` reads `smart_errors`. + + `collectors.ADDON_ERROR_PREFIXES` is the one list of these prefixes; this asserts every + name by behaviour, so dropping any of them fails here, and `tests/test_agent.py` pins both + the wording of the lines the agent actually writes and the equality of the two lists. + """ + missing_haproxy = ("haproxy: /etc/haproxy/haproxy.cfg not found — haproxy is not installed " + "here, or its config lives elsewhere (--haproxy-config); --no-proxy " + "opts out") + missing_nginx = ("nginx: /etc/nginx/nginx.conf not found — nginx is not installed here, or " + "its config lives elsewhere (--nginx-config); --no-proxy opts out") + no_sockets = ("sockets: /proc/net/tcp not found, /proc/net/tcp6 not found — this host " + "publishes no kernel socket table; --no-proxy opts out") + body = agent_body() + body["errors"] = [missing_haproxy, missing_nginx, no_sockets, + "smart: smartctl 7.5 not found"] + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=body) + + async with client_for(handler) as client: + result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + assert "degraded" not in result.payload, "unconfigured is not broken (invariant 3)" + for prefix in ("haproxy:", "nginx:", "sockets:"): + assert any(prefix in str(e) for e in result.payload["agent"]["errors"]), \ + "the sentence survives for the page that explains it" + + +async def test_a_real_degradation_shows_alongside_the_proxy_addon_noise() -> None: + """Filtering the add-ons must not swallow a genuine half-snapshot beside them.""" + body = agent_body() + body["errors"] = ["nginx: /etc/nginx/nginx.conf is there but could not be read: boom", + "temperatures: no hwmon sensors"] + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=body) + + async with client_for(handler) as client: + result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + assert result.payload["degraded"] == ["temperatures: no hwmon sensors"] + + # --------------------------------------------------------------------------- # # physical disks (the /disks source's fetch half) # --------------------------------------------------------------------------- # diff --git a/tests/test_deploy_scripts.py b/tests/test_deploy_scripts.py index 87c6378..66a4108 100644 --- a/tests/test_deploy_scripts.py +++ b/tests/test_deploy_scripts.py @@ -1,4 +1,12 @@ -"""Contract tests for the scripts an operator runs: `deploy/` and `scripts/dev-*.sh`/`.py`. +"""Contract tests for the scripts an operator runs: `deploy/`, `scripts/dev-*.sh`/`.py`, and +`scripts/capture-gateway-config.sh`. + +The capture script gets the same treatment as the installer because it can do the same kind of +damage quietly: pointed at a gateway it cannot read, it is six empty files where six verbatim +captures were, and every parser test that reads them then passes against nothing. So it is run +here, against a stub `ssh` and a fake remote tree, and the assertions are about what lands on +disk — including the one edit it makes (a globally routable socket address becomes documentation +space) and the thing it never does (rewrite a config). There is no shell test framework here — and adding one would buy nothing a subprocess does not — but phase 10 rests on a sentence that lives in **two** files: the sudoers grant in @@ -46,6 +54,7 @@ from dashboards.store import SampleStore ROOT = Path(__file__).resolve().parent.parent DEPLOY = ROOT / "deploy" +SCRIPTS = ROOT / "scripts" INSTALLER = DEPLOY / "install-agent.sh" SUDOERS = DEPLOY / "agent-sudoers" BASH = shutil.which("bash") or "/bin/bash" @@ -2334,3 +2343,379 @@ def test_a_local_bundle_still_validates_with_its_own_flag_name(tmp_path: Path) - assert result.returncode == 2 assert result.stderr.startswith("--cert-bundle"), result.stderr assert "--fetch-bundle" not in result.stderr + + +# --- the gateway capture (phase 28 task 04) ------------------------------------------------ + +#: The script that records `tests/fixtures/gateway_*` — the ground truth the agent's `haproxy`, +#: `nginx` and `sockets` parsers are pinned to. It has no Python-side coverage to speak of (the +#: project's coverage source is `src/dashboards`), so the pins here **run it**, against a fake +#: remote reached through a stubbed `ssh`, the way the installer sandbox stubs `install`. +CAPTURE = SCRIPTS / "capture-gateway-config.sh" + +#: The remote documents, and the fixture each one becomes. A new include directory on the gateway +#: means a new entry here *and* a new `case` in the script — the script refuses silently-unknown +#: globs for exactly this reason. +CAPTURE_TARGETS = ( + "/etc/haproxy/haproxy.cfg", + "/etc/nginx/nginx.conf", + "/etc/nginx/conf.d/*.conf", + "/etc/nginx/stream.d/*.conf", + "/proc/net/tcp", + "/proc/net/tcp6", +) + +#: The stub `ssh`. It drops the options the script passes, records every call, and answers `cat` +#: and `ls` from `$FAKE_REMOTE` as if that directory were the remote filesystem's `/`. Nothing +#: here can reach a network: the point is to exercise the script's own branches (the include +#: globs, the refusal rule, the socket-table edit) with no gateway in sight. +CAPTURE_SSH_STUB = """#!/bin/sh +printf '%s\\n' "$*" >> "$STUB_LOG" +while [ $# -gt 0 ]; do + case "$1" in + -o) shift 2 ;; + -n) shift ;; + *) break ;; + esac +done +host="$1"; shift +command="$*" +if [ "${FAIL_SSH:-0}" = "1" ]; then + echo "ssh: connect to host $host port 22: Connection refused" >&2 + exit 255 +fi +case "$command" in + "cat "*) + path="${command#cat }" + file="$FAKE_REMOTE$path" + [ -f "$file" ] || { echo "cat: $path: No such file or directory" >&2; exit 1; } + cat "$file" + ;; + "ls "*) + spec="${command#ls }" + dir="${spec%/*}" + glob="${spec##*/}" + found=0 + for file in "$FAKE_REMOTE$dir"/$glob; do + [ -e "$file" ] || continue + found=1 + printf '%s\\n' "${file#"$FAKE_REMOTE"}" + done + [ "$found" = 1 ] || { echo "ls: cannot access '$spec': No such file or directory" >&2; exit 2; } + ;; + *) + echo "stub ssh: unsupported remote command: $command" >&2 + exit 2 + ;; +esac +""" + +#: A gateway tree small enough to assert on whole, with the two shapes the parsers exist for: a +#: `use_backend` whose target is a fetch expression, and a stream `proxy_pass` of a map variable. +CAPTURE_FAKE_GATEWAY: dict[str, str] = { + "/etc/haproxy/haproxy.cfg": ( + "global\n maxconn 4096\n" + "defaults\n mode tcp\n" + "frontend haproxy-22\n bind :::22 v4v6\n default_backend gitea-2222\n" + "frontend haproxy-8443\n bind :::8443 v4v6\n" + " use_backend %[req.hdr(host),lower,map_dom(/etc/haproxy/maps,b0)\n" + "backend gitea-2222\n server gitea gitea.reeselink.com:2222 check inter 5s\n" + ), + "/etc/nginx/nginx.conf": ( + "user nginx;\n" + "stream {\n include /etc/nginx/stream.d/*.conf;\n}\n" + "http {\n" + " include /etc/nginx/mime.types;\n" + " server { listen 80; listen [::]:80; server_name _; return 301 https://$host; }\n" + " include /etc/nginx/conf.d/*.conf;\n}\n" + ), + "/etc/nginx/conf.d/http.conf": ( + "server {\n" + " listen 4443 ssl proxy_protocol;\n" + " server_name nextcloud.reeseapps.com;\n" + " ssl_certificate_key /etc/letsencrypt/live/nextcloud.reeseapps.com/privkey.pem;\n" + " location / { proxy_pass http://nextcloud.reeselink.com:11000; }\n}\n" + ), + "/etc/nginx/conf.d/tls.conf": ( + "server {\n listen 8443 ssl;\n server_name brain.reeseapps.com;\n" + " location / { proxy_pass http://reeseapps-brain-of-reese.reeselink.com:443; }\n}\n" + ), + "/etc/nginx/stream.d/stream.conf": ( + "map $ssl_preread_server_name $targetBackend {\n" + " default 127.0.0.1:4443;\n" + " brain.example reeseapps-brain-of-reese.reeselink.com:443;\n}\n" + "server {\n listen 443;\n proxy_pass $targetBackend;\n}\n" + ), + # 8.8.8.8 and 2603:6013::1 are globally routable, `::ffff:8.8.8.8` is a visitor behind a v6 + # socket, and 203.0.113.7 is already documentation space. The tx/rx queue pair on the last row + # is spelled like an address column and is not one. + "/proc/net/tcp": ( + " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid" + " timeout inode\n" + " 0: 00000000:1EFF 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0" + " 0 1 1 0000000000000000 100 0 0 10 0\n" + " 1: 1B01010A:01BB 08080808:911A 01 00000000:00000000 00:00000000 00000000 101" + " 0 2 1 00000000d22dedba 20 4 28 10 -1\n" + " 2: 0100007F:23A0 017100CB:C0A8 06 08080808:08080808 00:00000000 00000000 0" + " 0 3 3 0000000000000000 20 0 0 10 0\n" + ), + "/proc/net/tcp6": ( + " sl local_address remote_address" + " st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" + " 0: 00000000000000000000000000000000:01BB 00000000000000000000000000000000:0000 0A" + " 00000000:00000000 00:00000000 00000000 0 0 4 1 0000000000000000 100 0 0 10 0\n" + " 1: 1360032601010031FF005450E0B284FE:01BB 0000000000000000FFFF000008080808:C0AC 01" + " 00000000:00000000 00:00000000 00000000 101 0 5 1 0000000000000000 63 4 26 10 -1\n" + ), +} + + +def _capture_sandbox( + tmp_path: Path, + gateway: dict[str, str] | None = None, + *, + recorded: dict[str, str] | None = None, + fail_ssh: bool = False, + alias: str | None = None, +) -> tuple[Path, Path, Path, dict[str, str]]: + """A copy of the capture script, a fake gateway, and a PATH whose `ssh` reaches only the fake. + + Returns (script, fixtures dir, stub call log, env). The script is copied into + `/scripts/` because it derives `tests/fixtures/` from its own location, which is what + keeps the repository's fixtures out of reach of a test. + """ + script = tmp_path / "scripts" / CAPTURE.name + script.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(CAPTURE, script) + + fake = tmp_path / "gateway" + for path, body in {**CAPTURE_FAKE_GATEWAY, **(gateway or {})}.items(): + target = fake / path.lstrip("/") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + + fixtures = tmp_path / "tests" / "fixtures" + fixtures.mkdir(parents=True, exist_ok=True) + for name, body in (recorded or {}).items(): + (fixtures / name).write_text(body, encoding="utf-8") + + stubs = tmp_path / "stubs" + stubs.mkdir() + stub = stubs / "ssh" + stub.write_text(CAPTURE_SSH_STUB, encoding="utf-8") + stub.chmod(0o755) + call_log = tmp_path / "ssh-calls.log" + call_log.write_text("", encoding="utf-8") + + env = {**os.environ, + "PATH": f"{stubs}{os.pathsep}{os.environ['PATH']}", + "STUB_LOG": str(call_log), + "FAKE_REMOTE": str(fake), + "FAIL_SSH": "1" if fail_ssh else "0"} + if alias is not None: + env["GATEWAY_SSH"] = alias + return script, fixtures, call_log, env + + +def _capture(tmp_path: Path, **kwargs: object) -> subprocess.CompletedProcess[str]: + script, _fixtures, _log, env = _capture_sandbox(tmp_path, **kwargs) # type: ignore[arg-type] + return subprocess.run([BASH, str(script)], capture_output=True, text=True, env=env, check=False) + + +def _fixture_body(path: Path) -> str: + """A fixture without its `#` provenance block — the bytes the host answered.""" + return "".join(line for line in path.read_text().splitlines(keepends=True) + if not line.startswith("#")) + + +def test_the_gateway_capture_exists_and_asks_for_no_privilege() -> None: + """Six world-readable reads over an ssh key: nothing here needs root or a rewrite tool.""" + assert CAPTURE.is_file(), "scripts/capture-gateway-config.sh is missing (phase 28 task 04)" + assert os.access(CAPTURE, os.X_OK) + text = CAPTURE.read_text() + for path in CAPTURE_TARGETS: + assert path in text, f"{path} is no longer captured" + code = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) + assert "sudo" not in code, ( + "the capture must read exactly what dashboards-agent can read; a pass that needed root " + "would record more than the agent can see (the smartctl note in capture-fixtures.sh)" + ) + for tool in ("sed ", "awk ", " tr ", "cut "): + assert tool not in text, f"{tool.strip()} would be a text filter between the read and a fixture" + + +def test_the_gateway_capture_dials_the_alias_and_nothing_else(tmp_path: Path) -> None: + """No hardcoded address: every remote command goes to `$GATEWAY_SSH`, default `gateway-ops`.""" + text = CAPTURE.read_text() + assert 'GATEWAY_SSH="${GATEWAY_SSH:-gateway-ops}"' in text + # Comments may name an address (`0.0.0.0` is a shape the parser has to survive); code may not. + code = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) + stray = [a for a in re.findall(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", code) + if not a.startswith("203.0.113.")] + assert not stray, f"the script dials or embeds an address: {stray}" + + _script, _fixtures, log, env = _capture_sandbox(tmp_path) + subprocess.run([BASH, str(_script)], capture_output=True, text=True, env=env, check=False) + assert _dialled_hosts(log) == {"gateway-ops"} + + other = _capture_sandbox(tmp_path / "alias", alias="lab-gw") + subprocess.run([BASH, str(other[0])], capture_output=True, text=True, env=other[3], check=False) + assert _dialled_hosts(other[2]) == {"lab-gw"} + + +def _dialled_hosts(log: Path) -> set[str]: + """The host argument of every ssh the stub was called for — options are not hosts.""" + hosts = set() + for line in log.read_text().splitlines(): + tokens = [token for token in line.split() if not token.startswith("-") and "=" not in token] + if tokens: + hosts.add(tokens[0]) + return hosts + + +def test_the_gateway_capture_records_every_document_of_the_tree_it_read(tmp_path: Path) -> None: + """The happy path, run: one fixture per remote document, include globs and all. + + Two files in `conf.d` is deliberate. The script reads the include globs out of the captured + main config rather than a hardcoded pair of directories, so a gateway that adds a vhost file + gets it captured, and `files` in the payload stays the tree the config actually merges. + """ + result = _capture(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + fixtures = tmp_path / "tests" / "fixtures" + expected = {"gateway_haproxy.cfg", "gateway_nginx_main.conf", + "gateway_nginx_conf_d_http.conf", "gateway_nginx_conf_d_tls.conf", + "gateway_nginx_stream_d_stream.conf", + "gateway_proc_net_tcp.txt", "gateway_proc_net_tcp6.txt"} + assert {p.name for p in fixtures.iterdir()} == expected, result.stdout + assert _fixture_body(fixtures / "gateway_haproxy.cfg") == \ + CAPTURE_FAKE_GATEWAY["/etc/haproxy/haproxy.cfg"] + assert _fixture_body(fixtures / "gateway_nginx_conf_d_tls.conf") == \ + CAPTURE_FAKE_GATEWAY["/etc/nginx/conf.d/tls.conf"] + # `mime.types` is included by the config and absent on the host: no fixture, no complaint — + # the same absence-the-agent-reports rule, one level up. + assert not (fixtures / "gateway_nginx_mime.types").exists() + + +def test_the_gateway_capture_writes_the_domains_it_read_verbatim(tmp_path: Path) -> None: + """Nothing is scrubbed from a config, and the domains are the proof. + + The `server_name` and the `proxy_pass` target beside it are the ingress map, and the + `ssl_certificate_key` is a path rather than a key — so the file the fixture holds is the file + the gateway runs, byte for byte. The one exception in this script is the socket tables' + globally routable addresses, which `capture-fixtures.sh`'s identifier rule requires. + """ + _capture(tmp_path) + vhosts = (tmp_path / "tests" / "fixtures" / "gateway_nginx_conf_d_http.conf").read_text() + assert "server_name nextcloud.reeseapps.com;" in vhosts + assert "proxy_pass http://nextcloud.reeselink.com:11000;" in vhosts + assert "ssl_certificate_key /etc/letsencrypt/live/nextcloud.reeseapps.com/privkey.pem;" in vhosts + assert "SCRUB" not in vhosts and "REDACTED" not in vhosts + assert next((line for line in vhosts.splitlines() + if line.startswith("# Verbatim /etc/nginx/conf.d/http.conf from gateway-ops")), + None), "every fixture says which remote document it is" + + +def test_a_dead_gateway_leaves_the_recorded_fixtures_alone(tmp_path: Path) -> None: + """The `slots_gufo.json` rule: a capture script must never turn six good fixtures into six + empty files, because then the parser tests pass against nothing (or fail for the wrong + reason) and the ground truth is gone.""" + keep = {"gateway_haproxy.cfg": "# old\nfrontend kept\n bind :::22 v4v6\n", + "gateway_proc_net_tcp.txt": " 0: 00000000:1EFF 00000000:0000 0A x\n"} + result = _capture(tmp_path, recorded=keep, fail_ssh=True) + assert result.returncode == 1, result.stdout + for name, body in keep.items(): + assert (tmp_path / "tests" / "fixtures" / name).read_text() == body, \ + f"{name} was overwritten by a read that failed" + for name in keep: + assert f"✗ {name}" in result.stdout, result.stdout + assert "Connection refused" in result.stdout, "one line names the reason, per fixture" + assert sorted(p.name for p in (tmp_path / "tests" / "fixtures").iterdir()) == sorted(keep), \ + "a failed capture leaves no temp files behind" + + +def test_an_empty_read_is_refused_the_way_a_failed_one_is(tmp_path: Path) -> None: + """`cat` exiting 0 on an empty file is still not a capture.""" + keep = {"gateway_proc_net_tcp.txt": " 0: 00000000:1EFF 00000000:0000 0A keep me\n"} + result = _capture(tmp_path, gateway={"/proc/net/tcp": ""}, recorded=keep) + assert result.returncode == 1, result.stdout + assert "gateway_proc_net_tcp.txt" in result.stdout + assert "empty" in result.stdout + assert (tmp_path / "tests" / "fixtures" / "gateway_proc_net_tcp.txt").read_text() == \ + next(iter(keep.values())) + + +def test_an_include_dir_with_no_fixture_name_is_reported_and_fails(tmp_path: Path) -> None: + """A config that moved its vhosts is a loud failure, not a fixture set that quietly stopped + being the tree the agent merges.""" + moved = {"/etc/nginx/nginx.conf": ( + "http {\n include /etc/nginx/sites-enabled/*.conf;\n}\n" + "stream {\n include /etc/nginx/stream.d/*.conf;\n}\n")} + result = _capture(tmp_path, gateway=moved) + assert result.returncode == 1, result.stdout + assert "/etc/nginx/sites-enabled/*.conf" in result.stdout + assert not (tmp_path / "tests" / "fixtures" / "gateway_nginx_sites_enabled_web.conf").exists() + + +def test_a_config_with_no_glob_include_says_the_vhost_fixtures_were_not_refreshed( + tmp_path: Path) -> None: + keep = {"gateway_nginx_conf_d_http.conf": "# keep\nserver {\n listen 4443;\n}\n"} + result = _capture(tmp_path, gateway={"/etc/nginx/nginx.conf": "user nginx;\nevents { }\n"}, + recorded=keep) + assert result.returncode == 1, result.stdout + assert "no glob include" in result.stdout + assert (tmp_path / "tests" / "fixtures" / "gateway_nginx_conf_d_http.conf").read_text() == \ + next(iter(keep.values())) + + +def test_the_socket_capture_publishes_no_globally_routable_endpoint(tmp_path: Path) -> None: + """The one edit the capture makes, run rather than asserted in prose. + + Two rules at once, which is what makes the socket fixtures both usable and publishable: a + globally routable address (including one behind `::ffff:`) becomes documentation space, and + everything else — every port, every private/loopback/wildcard word, every other column, and + the column widths themselves — is byte-for-byte what the kernel answered. A tx/rx queue pair + is spelled exactly like an address column and must survive untouched, which is why the edit + reads two columns and not the line. + """ + _capture(tmp_path) + fixtures = tmp_path / "tests" / "fixtures" + v4 = _fixture_body(fixtures / "gateway_proc_net_tcp.txt").splitlines() + v6 = _fixture_body(fixtures / "gateway_proc_net_tcp6.txt").splitlines() + raw_v4 = CAPTURE_FAKE_GATEWAY["/proc/net/tcp"].splitlines() + raw_v6 = CAPTURE_FAKE_GATEWAY["/proc/net/tcp6"].splitlines() + assert len(v4) == len(raw_v4) and len(v6) == len(raw_v6), "a row was added or lost" + + assert v4[1] == raw_v4[1], "a wildcard listener must survive untouched" + assert "08080808:911A" not in "".join(v4), "8.8.8.8 is still in the fixture" + assert "017100CB:911A" in v4[2], v4[2] # 203.0.113.1, the port kept + assert len(v4[2]) == len(raw_v4[2]), "the replacement must be exactly as wide as the original" + assert v4[3] == raw_v4[3], ( + "an already-documentation address and the tx/rx queue pair on that row both moved: only " + "the two address columns may be looked at") + assert "08080808:08080808" in v4[3], v4[3] + + assert "1360032601010031FF005450E0B284FE" not in "".join(v6), "a global v6 is left in" + assert "B80D0120000000000000000001000000:01BB" in v6[2], v6[2] # 2001:db8::1, port kept + assert "FFFF0000017100CB:C0AC" in v6[2] or "FFFF0000" in v6[2], \ + "a v4-mapped row must stay v4-mapped, or the parser's branch for it stops being exercised" + assert v6[1] == raw_v6[1], "the wildcard v6 listener moved" + assert len(v6[2]) == len(raw_v6[2]) + + +def test_the_socket_capture_is_idempotent_so_a_re_run_does_not_churn(tmp_path: Path) -> None: + """Re-recording the same moment must not diff, or every capture moves the fixture twice. + + Documentation space is not globally routable, so the second pass leaves it alone — the same + property that makes the drive-serial scrub re-runnable. + """ + once = tmp_path / "once" + _capture(once) + again = _capture_sandbox(tmp_path / "twice", + gateway={"/proc/net/tcp": _fixture_body( + once / "tests" / "fixtures" / "gateway_proc_net_tcp.txt")}) + result = subprocess.run([BASH, str(again[0])], capture_output=True, text=True, + env=again[3], check=False) + assert result.returncode == 0, result.stdout + result.stderr + assert _fixture_body(again[1] / "gateway_proc_net_tcp.txt") == _fixture_body( + once / "tests" / "fixtures" / "gateway_proc_net_tcp.txt")