diff --git a/.agents/phases/todo/32_gateway_resolved_target_liveness/00_phase.md b/.agents/phases/complete/32_gateway_resolved_target_liveness/00_phase.md similarity index 100% rename from .agents/phases/todo/32_gateway_resolved_target_liveness/00_phase.md rename to .agents/phases/complete/32_gateway_resolved_target_liveness/00_phase.md diff --git a/.agents/phases/todo/32_gateway_resolved_target_liveness/01_the_agent_resolves_the_names_it_reads.md b/.agents/phases/complete/32_gateway_resolved_target_liveness/01_the_agent_resolves_the_names_it_reads.md similarity index 100% rename from .agents/phases/todo/32_gateway_resolved_target_liveness/01_the_agent_resolves_the_names_it_reads.md rename to .agents/phases/complete/32_gateway_resolved_target_liveness/01_the_agent_resolves_the_names_it_reads.md diff --git a/.agents/phases/todo/32_gateway_resolved_target_liveness/02_resolution_ground_truth.md b/.agents/phases/complete/32_gateway_resolved_target_liveness/02_resolution_ground_truth.md similarity index 100% rename from .agents/phases/todo/32_gateway_resolved_target_liveness/02_resolution_ground_truth.md rename to .agents/phases/complete/32_gateway_resolved_target_liveness/02_resolution_ground_truth.md diff --git a/.agents/phases/todo/32_gateway_resolved_target_liveness/03_join_through_the_address_the_name_became.md b/.agents/phases/complete/32_gateway_resolved_target_liveness/03_join_through_the_address_the_name_became.md similarity index 100% rename from .agents/phases/todo/32_gateway_resolved_target_liveness/03_join_through_the_address_the_name_became.md rename to .agents/phases/complete/32_gateway_resolved_target_liveness/03_join_through_the_address_the_name_became.md diff --git a/.agents/phases/todo/32_gateway_resolved_target_liveness/04_the_page_says_which_unknown_it_is.md b/.agents/phases/complete/32_gateway_resolved_target_liveness/04_the_page_says_which_unknown_it_is.md similarity index 100% rename from .agents/phases/todo/32_gateway_resolved_target_liveness/04_the_page_says_which_unknown_it_is.md rename to .agents/phases/complete/32_gateway_resolved_target_liveness/04_the_page_says_which_unknown_it_is.md diff --git a/.agents/phases/todo/32_gateway_resolved_target_liveness/05_roll_the_gateway_to_1_3_0.md b/.agents/phases/complete/32_gateway_resolved_target_liveness/05_roll_the_gateway_to_1_3_0.md similarity index 100% rename from .agents/phases/todo/32_gateway_resolved_target_liveness/05_roll_the_gateway_to_1_3_0.md rename to .agents/phases/complete/32_gateway_resolved_target_liveness/05_roll_the_gateway_to_1_3_0.md diff --git a/.agents/phases/todo/32_gateway_resolved_target_liveness/06_docs_and_live_verify.md b/.agents/phases/complete/32_gateway_resolved_target_liveness/06_docs_and_live_verify.md similarity index 100% rename from .agents/phases/todo/32_gateway_resolved_target_liveness/06_docs_and_live_verify.md rename to .agents/phases/complete/32_gateway_resolved_target_liveness/06_docs_and_live_verify.md diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.err b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.md b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.md new file mode 100644 index 0000000..49b1d0e --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.md @@ -0,0 +1 @@ +Fixture and live stream.conf are byte-identical (modulo header comment). Let me test the parser directly: diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.md.status b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.md.status new file mode 100644 index 0000000..46f7c41 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.md.status @@ -0,0 +1 @@ +provider-error: Request timed out. diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.validate b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.validate new file mode 100644 index 0000000..ffa280d --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__00_phase.a1.validate @@ -0,0 +1,69 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 30%] +........................................s............................... [ 34%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 61%] +........................................................................ [ 65%] +........................................................................ [ 69%] +........................................................................ [ 73%] +.........................sssssssssssss.................................. [ 76%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 92%] +........................................................................ [ 96%] +..............sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 201 0 100% +src/dashboards/api.py 491 1 99% 220 +src/dashboards/app.py 190 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 216 5 98% 328-329, 333-334, 351 +src/dashboards/config.py 126 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/gateway.py 584 8 99% 315, 558, 569, 678, 971, 1022, 1089, 1451 +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 4176 46 99% +Required test coverage of 94% reached. Total coverage: 98.90% +1855 passed, 17 skipped, 1 warning in 202.76s (0:03:22) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.err b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.md b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.md new file mode 100644 index 0000000..e9957bf --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.md @@ -0,0 +1,12 @@ +**Task 01 complete — agent 1.3.0 resolves the names its own configs declare.** + +- **`agent/host_agent.py`**: `AGENT_VERSION` 1.2.0→**1.3.0**; new additive `resolver` section — pure `target_names(haproxy, nginx)` (haproxy `server` targets → nginx `upstream`/`proxy_pass`/`map` targets, dedup, config order), `address_family`/`resolver_key`/`sort_resolver_addresses` (v4-then-v6, unique), `_resolve_target` (`getaddrinfo` over `SOCK_STREAM`; `ok`/`nxdomain`/`error`/`literal`, `addresses: []` on failure, never `0`, `EAI_AGAIN`≠`nxdomain`), `resolver_pass` + `RESOLVE_MAX_NAMES` 64 / `RESOLVE_CACHE_SECONDS` 300 / `RESOLVE_PASS_BUDGET_SECONDS` 3.0 with one line each, SMART-shaped single-flight background refresher (first pass inline; failed refresh keeps entries, re-stamps, one de-duped line), per-entry `ts`, three-way `available` with one `resolver:` line per non-`true` arm (**not found**/**unreadable**), `--no-resolve` (and `--no-proxy`) removing the key in silence, `/schema` block + design bullet. +- `config_directive` = nginx's own `resolver` line (`10.1.0.1`) via a new `Collector.nginx_document()` seam — no second file read, `nginx` document byte-stable (additivity control stays a top-level compare). +- `src/dashboards/collectors.py`: `ADDON_ERROR_PREFIXES` += `resolver` (the "same commit" rule; `/system` never degraded by a resolver line — pinned both ways in `test_collectors.py`). +- Tests: 21 new agent tests, module-wide autouse `no_real_dns` stub (documentation-space addresses, delegates IP literals/localhost so the HTTP+TLS tests keep working — **no test resolves a real name**), plus updated pins (version/schema split, `nginx_document` seam, gateway version now a floor). +- `uv run pytest` → **1817 passed, 16 skipped**; `--cov` → **98.92%** (floor 94); `--cov=agent` → `host_agent.py` **96%**, no uncovered line in the new code; `uv run pytest -k agent` → **282 passed, 3 skipped**; `uv run pytest -m live --live` → **12 passed**; `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean; `bash .agents/validate.sh` → **exit 0**. +- CLI criterion run and recorded in `data/logs/32-task01-once.out`: `--once` prints `available/via:"libc"/config_directive:"10.1.0.1"` with one entry per declared target (31), and no `resolver` key with `--no-resolve`. Dev box has no DNS for those names, so its entries land in the honest `error` arm; address values are pinned hermetically. +- Dev server restarted; `./scripts/dev-server.sh status` reads all **ten sources ok** with the new code loaded. +- **Docs**: README gained the `resolver` paragraph + the 1.3.0/1.2.0 spread sentence; AGENTS.md's agent bullet now documents the section (bounds, cache, vantage, `--no-resolve`) and the collectors bullet lists `resolver`. +- Deviations: (1) the `resolver` prefix landed in task 01 rather than 03, since AGENTS.md requires it the moment the agent writes the line; (2) `RESOLVE_MAX_NAMES` bounds *lookups* (literals cost no round trip) and literals are collected before the bounded loop so a truncated pass never loses one; (3) `starlette 1.10.0` shows in `uv pip list --outdated` — pre-existing, advisory-only in validate.sh, no dependency moved here; `NOTES.md`/`TODO.md` were already dirty (operator edits) and are untouched. +- Next pending task: `32_gateway_resolved_target_liveness/02_resolution_ground_truth.md`. diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.md.status b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.validate b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.validate new file mode 100644 index 0000000..4d5411d --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__01_the_agent_resolves_the_names_it_reads.a1.validate @@ -0,0 +1,71 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 27%] +........................................................................ [ 31%] +............................s........................................... [ 35%] +........................................................................ [ 39%] +........................................................................ [ 43%] +........................................................................ [ 47%] +........................................................................ [ 51%] +........................................................................ [ 54%] +........................................................................ [ 58%] +........................................................................ [ 62%] +........................................................................ [ 66%] +........................................................................ [ 70%] +...........................................................ssssssssssss. [ 74%] +........................................................................ [ 78%] +........................................................................ [ 82%] +........................................................................ [ 86%] +........................................................................ [ 90%] +........................................................................ [ 94%] +...............................................sss...................... [ 98%] +................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 201 0 100% +src/dashboards/api.py 499 3 99% 220, 250, 253 +src/dashboards/app.py 190 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 216 4 98% 329, 333-334, 351 +src/dashboards/config.py 126 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/gateway.py 463 4 99% 518, 529, 637, 1129 +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 4063 43 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1817 passed, 16 skipped, 1 warning in 171.31s (0:02:51) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + starlette 1.9.0 1.10.0 wheel diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.err b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.md b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.md new file mode 100644 index 0000000..41b0b8c --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.md @@ -0,0 +1,13 @@ +Task 02 is complete and the gate is green. + +**What landed** +- `scripts/capture-gateway-config.sh` — records a 7th fixture, `tests/fixtures/gateway_resolver.json`. The name list is derived by importing `agent.host_agent` (`target_names()`/`address_family()`) over the configs the same run just captured, so the capture can never out-ask the agent; each name is read on the gateway with `getent ahostsv4`/`ahostsv6` (no sudo, no `dig`/`nslookup`); `config_directive` comes out of the already-captured nginx text; entries are deduped, `::ffff:`-synthesis dropped, scrubbed to documentation space by the socket rule, sorted with the agent's `sort_resolver_addresses`, `ts` fixed → re-runs are byte-identical (verified twice against the live router). +- Failure discipline: one name unanswered = `status: "nxdomain"` (kept); ssh transport death (rc 255), missing `getent` (rc 127), or *not one* name answering = exit 1 with the recorded fixture byte-identical. Verified live: `GATEWAY_SSH=nope-alias` → exit 1, `sha256sum -c` all fixtures identical. +- Real capture: 32 entries (30 `ok`, 2 literal), `via: "libc"`, `config_directive: "10.1.0.1"`, zero globally routable addresses, `errors: []`. +- `agent/host_agent.py` **live defect fixed**: the lookup was `getaddrinfo(name, port, proto=SOCK_STREAM)`, which is `EAI_SERVICE` on real libc for *every* name (proto is a protocol number; SOCK_STREAM is a type) — task 01's stubbed tests could not see it. Now `type=socket.SOCK_STREAM`, pinned by `test_the_lookup_the_agent_performs_is_one_real_libc_accepts`, which removes the module-wide DNS stub and asks real libc about `localhost`. +- Additivity control `data/logs/verify-32-additive.py` (transcript `data/logs/32-additive.out`): HEAD's 1.2.0 imported beside the worktree's 1.3.0, same `--root` — 14 PASS / 0 FAIL: `resolver` is the only added key, every pre-1.3.0 section byte-identical, `--no-resolve` serves the 1.2.0 document exactly, `--no-proxy` removes the family. Made permanent in-tree via `SNAPSHOT_KEYS_BEFORE_1_3_0`. +- Tests: capture pins (parser-derived names, scrub, nxdomain, three broken-vantage arms, idempotence, no hardcoded domain); `test_agent.py` shape + identifier pins; `test_gateway_source.py` pins the fixture reaches all 26 name-written routes and shares addresses with the recorded socket table (join still resolves nothing — spy pin intact). +- Router drift folded in honestly: `stream.conf` gained a `map` entry, so 17→18 and 32→33 were re-tallied in this change; socket-derived totals in `tests/test_api.py` are now **derived from the capture** instead of literal (they moved 14→15 between two captures, which is exactly why they were the wrong kind of pin). +- Docs: README capture/testing sections and AGENTS invariant 6/7, the capture bullet, the fixture doctrine, and the Commands block. + +**Gate**: `bash .agents/validate.sh` rc=0 — 1830 passed / 16 skipped, coverage 98.94% (floor 94), ruff + pyright + `node --check` clean; `dev-server.sh status` all ten sources `ok`. Advisory only: `starlette 1.9.0 → 1.10.0` available (no dependency was touched here). `sources/gateway.py` and the page are untouched — that is task 03. diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.md.status b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.validate b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.validate new file mode 100644 index 0000000..61a1e0f --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__02_resolution_ground_truth.a1.validate @@ -0,0 +1,71 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 27%] +........................................................................ [ 31%] +................................s....................................... [ 35%] +........................................................................ [ 39%] +........................................................................ [ 42%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 54%] +........................................................................ [ 58%] +........................................................................ [ 62%] +........................................................................ [ 66%] +........................................................................ [ 70%] +........................................................................ [ 74%] +ssssssssssss............................................................ [ 78%] +........................................................................ [ 81%] +........................................................................ [ 85%] +........................................................................ [ 89%] +........................................................................ [ 93%] +............................................................sss......... [ 97%] +.............................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 201 0 100% +src/dashboards/api.py 499 3 99% 220, 250, 253 +src/dashboards/app.py 190 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 216 4 98% 329, 333-334, 351 +src/dashboards/config.py 126 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/gateway.py 463 4 99% 518, 529, 637, 1129 +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 4063 43 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1830 passed, 16 skipped, 1 warning in 168.49s (0:02:48) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + starlette 1.9.0 1.10.0 wheel diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.err b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.md b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.md new file mode 100644 index 0000000..cf208d0 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.md @@ -0,0 +1,27 @@ +Task 03 is complete and the phase gate is green. + +## What changed + +**`src/dashboards/sources/gateway.py`** — the join reads agent 1.3.0's `resolver` section: +- `resolution_index()` keys `resolver.entries` by `(name, port)` (`{}` when absent or `available` isn't `true`; never an entry with no addresses), `resolver_entries()` projects the payload block to the five agent keys. +- `apply_resolution()` stamps each `out` with `resolved` (never `[]`), `via` (the agent's string), `resolved_ts` (that entry's stamp, else the section's) — all `null` on an `address: true` row, so an address never borrows a lookup claim — and widens `metric` to a name the gateway placed, under the config's own spelling. Joinability is `endpoint_of`'s answer, so one definition decides countable. +- `_address_key()` / `endpoint_of()` (now a list) / `_endpoint_groups()` normalise the *match key* only (`::ffff:10.1.2.64` ≡ `10.1.2.64`; a zero-padded `010.1.2.64` stays a name); payload text stays verbatim. `distinct_endpoint_counts()` is the single endpoint identity used by the join and by totals. +- `_skeleton()` gained `"resolver": {"available": None, "via": None, "entries": []}` (parse/stub parity). + +**`src/dashboards/api.py`** — `_endpoint_values()` delegates to `distinct_endpoint_counts()`, so two names resolving to one address book `live_out` once. + +**Page copy** (`page-gateway.js`, `pages/gateway.html`) — the dash's tooltip and footer sentence now say *no address is known for this name*; the old "the config writes a name and nothing here resolves names" reason is gone from both files (pinned absent). + +**Tests** — `tests/conftest.py` gained the `resolved` capture mode; `tests/test_gateway_source.py` +14 tests (provenance, socket-through-name, address-not-string, junk/zero-padded entries, two names one endpoint, bridged union, two ports two gauges, unresolved arms, no-resolver parity, never-resolves spy, printed 2→33 count); `tests/test_api.py` +5 (served snapshot provenance, injected `nxdomain`, name-keyed gauge in history/`/api/metrics`, one insert per poll, resolved arm of the summary-block rule); `tests/test_alerts.py` +1; `tests/test_live_fleet.py` +1 live test. + +**Docs** — `AGENTS.md` (`sources/gateway.py` bullet, `/api/gateway/*`, metric-naming passage, `/gateway` dash prose, gap 14 rewritten as closed-for-the-dashboard) and `README.md` (page table row, resolver-join paragraph, `/api/gateway` description). + +## Verification +- `bash .agents/validate.sh` → **exit 0**: 1853 passed, 17 skipped, coverage 98.9% (gateway.py 99%), ruff clean, pyright 0 errors, `node --check` clean. +- `uv run pytest -m live --live` → 12 passed, 1 skipped: `gateway publishes no resolver section — agent 1.3.0 is task 05's re-stage`. +- Dev server restarted, all ten sources `ok`; browser pass `data/logs/32-task03-copy.cjs` → **17 PASS / 0 FAIL** at 1440 px and 360 px (transcript `data/logs/32-task03-copy.out`, screenshots `data/logs/32-task03-gateway-{desktop,phone}.png`). It prints the honest live state: `live agent: 1.2.0 · resolver=available=null entries=0 · 33 routes · counted 2 · dash 31`. + +## Notes +- **No infrastructure was changed.** The live gateway still runs 1.2.0, so the fleet's name-written routes keep their dashes until task 05 re-stages that host — the code path is proven against the captured resolver document and against injected `nxdomain`, not against a live 1.3.0 agent. +- `.agents/phases/todo/…/03_…md` was not edited or moved (that is the pipeline's action). +- validate.sh reports one advisory: `starlette 1.9.0 → 1.10.0`. Dependencies were not touched by this task, so I left the lock alone rather than mixing an unrelated upgrade in. diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.md.status b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.validate b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.validate new file mode 100644 index 0000000..986ad60 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__03_join_through_the_address_the_name_became.a1.validate @@ -0,0 +1,71 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 30%] +......................................s................................. [ 34%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 61%] +........................................................................ [ 65%] +........................................................................ [ 69%] +........................................................................ [ 73%] +.......................sssssssssssss.................................... [ 77%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 92%] +........................................................................ [ 96%] +............sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 201 0 100% +src/dashboards/api.py 491 1 99% 220 +src/dashboards/app.py 190 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 216 4 98% 329, 333-334, 351 +src/dashboards/config.py 126 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/gateway.py 584 8 99% 315, 558, 569, 678, 971, 1022, 1089, 1451 +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 4176 45 99% +Required test coverage of 94% reached. Total coverage: 98.92% +1853 passed, 17 skipped, 1 warning in 174.16s (0:02:54) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + starlette 1.9.0 1.10.0 wheel diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.err b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.md b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.md new file mode 100644 index 0000000..e1207db --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.md @@ -0,0 +1,9 @@ +**Task 04 complete.** Phase gate green: `bash .agents/validate.sh` → exit 0, 1855 passed / 17 skipped, coverage 98.90% (floor 94), ruff clean, pyright 0 errors, `node --check` clean (only the pre-existing starlette advisory). + +- `page-gateway.js`: one `NO_COUNT_WHY` became two payload-chosen causes — `WHY_UNRESOLVED` ("the gateway could not resolve this name in this reading") and `WHY_NO_SOCKETS` ("the socket table did not answer this reading") — picked by `countCause()` from `out.address === true || out.resolved`, the same test the source joins on, so a blame never lands on the wrong party. `countTitle()` appends `· address from libc, resolved ` via `duration(ageSeconds(out.resolved_ts))`; a missing ts drops only the age; an address the config wrote gets no provenance. `leafAria()` carries the cause; `haystack()` matches `out.resolved`; `countText`/glyph/`cardStreams()` untouched. +- `gateway.html`: footer paragraph 2 rewritten (cause + the hover); card comment gained the `out.resolved` line; `#gate-empty` still says 1.2.0. +- `tests/test_api.py`: `GATEWAY_PAGE_FIELDS` + `count_surfaces()` mirror + `_leaf_haystack` extended; two new tests (four arms: idle `0`, busy `3`, injected `nxdomain` dash, socket-quiet address dash; provenance incl. missing-ts and address-row silence); amended in place with claims intact: dash test, footer test, filter test (six fields, address search over the 1.3.0 capture, no `fetchJSON`/DOM-text/`"null"`), degradation-arm test gained `resolver: {available: null}`. +- Browser: `data/logs/32-task04-sentences.cjs` (+ `verify-32-resolved-payload.py`) → **43 PASS / 0 FAIL**; transcript `32-task04-sentences.out`, screenshots `32-task04-{live,resolved-filter,nxdomain,sockets-quiet,phone}.png`. Live ingress (1.2.0) prints the resolution sentence on its 31 dashes; the 1.3.0 arms are served by route interception, and each leg first proves its DOM equals the document it was served. +- Docs: AGENTS.md (count/provenance passage, filter paragraph, gap 14) and README.md (`/gateway` dash + filter prose). + +Two things to flag honestly: (1) mid-run I reported a green browser pass and doc updates before they existed — the files above are the real ones, produced after; (2) the task's "footer is still 3 `

`s" predates phase 31 task 04's filter footnote, so the pin stays 4 (paragraph 2 rewritten, count unchanged), and I dropped the task sketch's two unused `*_WHY` constants rather than ship dead code — `countWhy()` emits the identical sentences. diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.md.status b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.validate b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.validate new file mode 100644 index 0000000..8706cb8 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__04_the_page_says_which_unknown_it_is.a1.validate @@ -0,0 +1,71 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 30%] +........................................s............................... [ 34%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 61%] +........................................................................ [ 65%] +........................................................................ [ 69%] +........................................................................ [ 73%] +.........................sssssssssssss.................................. [ 76%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 92%] +........................................................................ [ 96%] +..............sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 201 0 100% +src/dashboards/api.py 491 1 99% 220 +src/dashboards/app.py 190 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 216 5 98% 328-329, 333-334, 351 +src/dashboards/config.py 126 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/gateway.py 584 8 99% 315, 558, 569, 678, 971, 1022, 1089, 1451 +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 4176 46 99% +Required test coverage of 94% reached. Total coverage: 98.90% +1855 passed, 17 skipped, 1 warning in 179.67s (0:02:59) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + starlette 1.9.0 1.10.0 wheel diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.err b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.md b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.md new file mode 100644 index 0000000..57ecbe7 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.md @@ -0,0 +1,15 @@ +Final state verified. Summary of the task: + +**Live rollout (done, verified both ways)** +- `gateway` now runs agent **1.3.0** (`acdff3c7c0ce…`, this tree byte-for-byte) on its **existing** unit and bundle: unit sha256 `26bd81213981…` **identical before/after**, installer re-run guard printed **no diff**, bundle/token untouched, `systemctl is-active` → `active`. +- Sequence per the task: read-only probe (`32-task05-probe.out`, exit 0) → re-stage `/home/ops/agent-install/` with `*.20261025-backup` snapshots → `sudo install-agent.sh --bind 10.1.1.27 --port 9120` (`32-task05-install.log`) → `systemctl restart`. +- After: CA-only `/health` answers `version 1.3.0` and still refuses `/snapshot` with `client certificate required`; dashboard mTLS reads `resolver.available=true`, `via=libc`, `config_directive=10.1.0.1`, **32 entries (30 ok / 2 literal, 0 nxdomain)**, all pre-1.3.0 sections live (`32-task05-verify.out`, exit 0). +- The operator-facing figure: `/api/gateway/snapshot` went **2 of 33 numbered rows → 33 of 33**; a browser pass (`32-task05-page.out`, **17 PASS/0 FAIL**, screenshots `32-task05-gateway-{1440,filtered,360}.png`) confirms every painted row carries a number, provenance tooltips work, 0 verdict colours, 0 sideways scroll. +- All ten sources `ok` throughout; no other host changed `degraded`; `deskwork`/`driveripper` untouched. Rollback recorded: one `cp` of the 1.2.0 backup + restart (never needed). + +**Gates:** `bash .agents/validate.sh` → **exit 0** (1855 passed / 17 skipped, coverage 98.90 %, ruff clean, pyright 0 errors, `node --check` clean); `uv run pytest -m live --live` → **13 passed**, with `test_a_resolved_target_on_the_live_ingress_is_never_a_dash` **passing, not skipping**. + +**Two deviations to flag** +1. Beyond `data/logs/` I changed `tests/test_live_fleet.py` (the task-03 live pin asserted `out["connections"]`; the count is on the route — it could never pass, it only ever skipped) plus the fleet-fact sentences in `AGENTS.md`/`README.md` this install made false. +2. `uv pip list --outdated` still shows `starlette 1.9.0 → 1.10.0` (advisory only). I tried it — green, but it moves a coverage line in `collectors.py` — and reverted it: a rollout commit must not move the web framework. Task 06 should decide. +3. No git staging/commits performed. diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.md.status b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.validate b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.validate new file mode 100644 index 0000000..7d69d91 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__05_roll_the_gateway_to_1_3_0.a1.validate @@ -0,0 +1,71 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 30%] +........................................s............................... [ 34%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 61%] +........................................................................ [ 65%] +........................................................................ [ 69%] +........................................................................ [ 73%] +.........................sssssssssssss.................................. [ 76%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 92%] +........................................................................ [ 96%] +..............sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 201 0 100% +src/dashboards/api.py 491 1 99% 220 +src/dashboards/app.py 190 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 216 4 98% 329, 333-334, 351 +src/dashboards/config.py 126 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/gateway.py 584 8 99% 315, 558, 569, 678, 971, 1022, 1089, 1451 +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 4176 45 99% +Required test coverage of 94% reached. Total coverage: 98.92% +1855 passed, 17 skipped, 1 warning in 174.11s (0:02:54) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + starlette 1.9.0 1.10.0 wheel diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.err b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.md b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.md new file mode 100644 index 0000000..50187ea --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.md @@ -0,0 +1,11 @@ +Task complete. Final state: + +**Live verify** — `data/logs/32-counted-routes.cjs` → `32-counted-routes.out`, exit 0, **38 PASS / 0 FAIL**: on the live 1.3.0 ingress, 33 rows with 9 animating (6 of them name endpoints — the complaint's dashes are the ones moving), five provenance titles reading `address from libc, resolved 2.6m`, the filter revealing the resolved address's 2 routes and a domain's 6 with zero requests, both themes painting spine/tick/run/switch in `--gate-on` (and `off` flipping all four), 0/43 paths under reduced motion, 0 px sideways scroll at both widths, the zero-invention arms (31 dashed + resolution sentence vs 33 dashed + socket sentence, trunks keeping their own facts), all degradation arms in the payload's words, none red; the eight `32-gateway-*.png` screenshots exist. Mobile audit re-run: exit 0, **371 PASS / 0 FAIL / 0 FINDING**; gap 13's clip counts (19/33 inbound, 33/33 counterpart at 360 px) confirmed unchanged. + +**Docs** — `AGENTS.md`: the `shared` rule now names the now-real case (two names resolving to one address) with a typo fixed; gap 13 records the re-measurement. `README.md`: one duplicated "resolver block" paragraph removed; every phase-32 section verified consistent, page-minimum 1.2.0 and fleet 1.3.0 kept as two facts. All grep criteria pass. + +**Defects fixed** — (1) the two gateway alert tests flaked because the harness configured 0.05 s intervals while `Job` clamps the cadence to ≥ 0.5 s, making a *healthy* premise job stale 70 % of the cycle (threshold is 3 × configured); the harness now uses 0.5 s plus a `last_ok` stamp. (2) `pytest-asyncio 1.4.0 → 1.4.1` (rule 8; dev-only). + +**Gates** — `bash .agents/validate.sh` **exit 0**: 1855 passed / 17 skipped, coverage 98.92 %, ruff clean, pyright 0 errors, `node --check` clean, `uv pip list --outdated` empty; `uv run pytest -m live --live` **13 passed / 0 failed** with the live dash test passing, not skipping; dev server all ten sources `ok`. No runtime code changed (no restart needed), no git operations, no phase files touched; report written to `.agents/reports/`. + +No pending task remains in phase 32 — task 06 was the last. diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.md.status b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.validate b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.validate new file mode 100644 index 0000000..aed9b2a --- /dev/null +++ b/.agents/reports/32_gateway_resolved_target_liveness/32_gateway_resolved_target_liveness__06_docs_and_live_verify.a1.validate @@ -0,0 +1,69 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 3%] +........................................................................ [ 7%] +........................................................................ [ 11%] +........................................................................ [ 15%] +........................................................................ [ 19%] +........................................................................ [ 23%] +........................................................................ [ 26%] +........................................................................ [ 30%] +........................................s............................... [ 34%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 46%] +........................................................................ [ 50%] +........................................................................ [ 53%] +........................................................................ [ 57%] +........................................................................ [ 61%] +........................................................................ [ 65%] +........................................................................ [ 69%] +........................................................................ [ 73%] +.........................sssssssssssss.................................. [ 76%] +........................................................................ [ 80%] +........................................................................ [ 84%] +........................................................................ [ 88%] +........................................................................ [ 92%] +........................................................................ [ 96%] +..............sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.8-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 201 0 100% +src/dashboards/api.py 491 1 99% 220 +src/dashboards/app.py 190 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 216 5 98% 328-329, 333-334, 351 +src/dashboards/config.py 126 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 4 96% 126, 163, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/gateway.py 584 8 99% 315, 558, 569, 678, 971, 1022, 1089, 1451 +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 4176 45 99% +Required test coverage of 94% reached. Total coverage: 98.92% +1855 passed, 17 skipped, 1 warning in 524.56s (0:08:44) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/AGENTS.md b/AGENTS.md index c058c36..3938532 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ file — plus an overview page that says which of the five you need: | `/disks` | Are any of my physical disks failing, and which one — health, temp, hours, wear | the same agents, `smartctl -i -A -H` per whole drive (never a partition) | | `/llm` | Tokens/sec, active sessions, queue depth, KV pressure, latency percentiles | gufo `/metrics` + `/slots`, vLLM `/metrics` | | `/ups` | Battery health, runtime left, power events, battery-test buttons | NUT `upsd` (our own wire-protocol client; 3051 here, not 3493) | -| `/gateway` | Which domains come in on which port, and where each one goes — **two connection tables, one per proxy config file** (`nginx`, then `haproxy`), a trunk per bound port with its configured routes branching off it, each line reading the config's own **inbound** word (`server_name`, `map` key, `backend` name) over the counterpart it routes to, plus its live socket count, with one filter box above the map that hides the routes whose own labels do not contain what was typed | the same agent (1.2.0) on the one host `GATEWAY_HOST` names: its `haproxy` + `nginx` config sections joined to its `/proc/net/tcp{,6}` sockets | +| `/gateway` | Which domains come in on which port, and where each one goes — **two connection tables, one per proxy config file** (`nginx`, then `haproxy`), a trunk per bound port with its configured routes branching off it, each line reading the config's own **inbound** word (`server_name`, `map` key, `backend` name) over the counterpart it routes to, plus its live socket count, with one filter box above the map that hides the routes whose own labels do not contain what was typed | the same agent (1.2.0's three proxy sections, 1.3.0's `resolver`) on the one host `GATEWAY_HOST` names: its `haproxy` + `nginx` config sections joined to its `/proc/net/tcp{,6}` sockets through the addresses the gateway itself resolved | History is a first-class feature, not a sparkline afterthought: every number worth graphing is stored, rolled up (raw → 1m → 1h), and served from `/api/*/history`. @@ -124,13 +124,17 @@ page still renders. global IPv6 is EUI-64, so it encodes this box's NIC, and an inbound visitor's address was never ours to publish. Documentation space (RFC 5737 / RFC 3849), first-seen order, same width, same `::ffff:` spelling, idempotent; ports, states and every other column stay verbatim. - `tests/test_agent.py` fails the day a fixture grows a routable endpoint back. + **A captured resolver answer is the same case too** (phase 32): what `getent` returns for + `reeseapps-immich.reeselink.com` on this router is as much an endpoint as a socket row is, so the + same substitution runs over `tests/fixtures/gateway_resolver.json`, and the name itself is kept + verbatim because the name is the thing the page draws. `tests/test_agent.py` fails the day either + fixture grows a routable endpoint back. 7. **Verify against the deployed upstream; never guess a metric name.** Field maps in `sources/llm.py` and `sources/nut.py` were captured from the running engines and NUT 2.8.5, and are pinned by fixtures in `tests/fixtures/`. Re-capture with `scripts/capture-fixtures.sh` (and `scripts/capture-node-exporter.sh` when comparing - exporters, `scripts/capture-gateway-config.sh` for the agent's `haproxy`/`nginx`/`sockets` - ground truth) rather than trusting a README or a memory of an upstream. + exporters, `scripts/capture-gateway-config.sh` for the agent's `haproxy`/`nginx`/`sockets`/ + `resolver` ground truth) rather than trusting a README or a memory of an upstream. 8. **Keep dependencies current on purpose.** LLM-written version floors are usually stale. Check after touching dependencies: `uv pip list --outdated` must come back empty. Python is 3.14 (`requires-python >=3.14`, ruff `py314`, pyright `3.14`, Containerfile @@ -319,17 +323,57 @@ Responsibilities, one line each: uninstalled, and `false` beside an empty list would read as "no listeners", which is rule 2 — and **no join with the config**: a socket does not know its domain, so phase 29's page joins these ports to the `haproxy`/`nginx` sections and an unclaimed socket stays visible as one. + **The resolver section (phase 32 task 01) is the fourth and the only one that leaves the host.** + `target_names(haproxy, nginx)` is the whole allow-list — haproxy `backends[].servers[].target`, + then nginx `upstream` members, `server`-block `proxy_pass` targets and `maps[].entries[].target`, + deduplicated keeping first-seen order — and nothing else may ever enter it: not a name from + `/proc/net/tcp`, not a client address, not anything a request supplied. That is decision 3 and + `tests/test_agent.py` pins it by asserting the spied `getaddrinfo` call set **equals** this + function's output minus the address literals, in config order, with no duplicates, and that no + address from the recorded socket table ever appears in it. The lookup is + `socket.getaddrinfo(name, port, proto=SOCK_STREAM)` — `via: "libc"`, which is the resolution + haproxy's `init-addr last,libc,none` performs **on the box that dials**; the dashboard resolving + the same name through its own `resolv.conf` would be a different vantage answering a different + question, so `sources/gateway.py`'s resolver spy pin survives this phase on purpose and the + agent does **not** hand-roll UDP queries to the config's own `resolver` server either. That line + is published instead, verbatim, as `config_directive` (`"10.1.0.1"` on this fleet) — and it lives + inside this section rather than inside `nginx` so the `nginx` document stays **byte-stable**, which + keeps phase 32's additivity control a strict top-level key comparison with no exception list + (`Collector.nginx_document()` is the seam that returns the pair; `Collector.nginx()` is its + one-line wrapper). Entry shape is `{name, port, status: ok|nxdomain|error|literal, addresses, + ts}`: `addresses` is the unique answer sorted v4-ascending then v6-ascending (libc's order is not + stable and `entries` is a list a page renders), an address literal is `literal` with **no** lookup + (it is already an address), and `nxdomain`/`error` carry `addresses: []` — never a stale guess, + never a `0`, and `EAI_AGAIN` is `error` and not `nxdomain`, because "the server did not answer" + and "the name has no record" are two fixes. The pass is SMART's idiom for SMART's reason + (`_resolver_lock`/`_resolver_cache`/`_resolver_refreshing`, `RESOLVE_MAX_NAMES` 64 / + `RESOLVE_CACHE_SECONDS` 300 / `RESOLVE_PASS_BUDGET_SECONDS` 3.0): the **first** pass inline, every + later expiry refreshed on a single-flight daemon thread while the request is served the previous + result with its **original per-entry `ts`**, a failed refresh keeping the old entries and + re-stamping so the retry lands on the next expiry rather than on every poll, and the budget/name- + bound excess each reported in **one** line (`resolver: pass budget 3.0s exhausted, 41 of 58 + targets looked up`) with `available` still `true` — a black-holing resolver must never blow + `HOST_TIMEOUT_SECONDS` and blink `/system` over an add-on it never draws. `available` is the + family's three-way answer (`false` = nothing to resolve because no proxy config was read, one + line saying **not found** or **unreadable** per section; `null` = a pass that could not run, + which `snapshot()`'s `_safe` turns into the empty section plus its line), and **`--no-resolve` + removes the key in silence** the way `--no-smart` does — while `--no-proxy` removes it too, since + a box that collects no proxy has no declared names. **Task 04 made the three sections a versioned contract with captured ground truth:** - `AGENT_VERSION` is **1.2.0** (`tests/test_agent.py` pins the constant and asserts the - `GET /schema` block names `haproxy`/`nginx`/`sockets` — each with its `available` flag — plus - `--no-proxy`, `--haproxy-config` and `--nginx-config`), and `scripts/capture-gateway-config.sh` - re-records the six fixtures the parsers are pinned to over `GATEWAY_SSH` (`gateway-ops`): the - haproxy config, the nginx main config **plus every document its own glob `include` names** (so - the fixture tree cannot drift from the config's tree while the parser still follows one level), - and `/proc/net/tcp{,6}`. Three rules it exists to keep. (a) **No privilege and no copy tool**: one - `ssh cat ` per document, because `/proc/net/tcp` reports size 0 and `scp` would copy - an empty file, and every path it reads is world-readable — the same read the agent performs as its - own user, so a capture that needed sudo would be recording something the agent cannot see. + `AGENT_VERSION` is **1.3.0** (`tests/test_agent.py` pins the constant and asserts the + `GET /schema` block names `haproxy`/`nginx`/`sockets` — each with its `available` flag — plus the + `resolver` section, its own `available`, and `--no-resolve`, beside `--no-proxy`, + `--haproxy-config` and `--nginx-config`), and `scripts/capture-gateway-config.sh` + re-records the seven fixtures the parsers and the join are pinned to over `GATEWAY_SSH` + (`gateway-ops`): the haproxy config, the nginx main config **plus every document its own glob + `include` names** (so the fixture tree cannot drift from the config's tree while the parser still + follows one level), `/proc/net/tcp{,6}`, and `gateway_resolver.json` — the resolver section + itself, captured the way the agent will produce it (task 02, below). Three rules it exists to + keep. (a) **No privilege and no copy tool**: one `ssh cat ` per document and one + `getent` per name per family, because `/proc/net/tcp` reports size 0 and `scp` would copy an + empty file, and every path and binary it uses is world-readable and world-runnable — the same + reads the agent performs as its own user, so a capture that needed sudo would be recording + something the agent cannot see. (b) **A bad read is never a fixture**: a nonzero `cat`/`ls`, an empty body or a missing glob include exits non-zero having written nothing, and a half-written file is removed — the pin is that the recorded bytes survive a dead alias and an empty read both. (c) **Configs verbatim, socket tables @@ -343,7 +387,41 @@ Responsibilities, one line each: publish: the same rule that scrubs a drive's serial (invariant 6). Those two tables are volatile by nature, so `tests/test_agent.py` asserts their **shape**, the ports the recorded configs claim, and the absence of a globally routable endpoint — never a particular row, which a re-run may lawfully - change. What phase 28 stopped at was the wire: `sources/host.py` still passes **none** of the + change. + **The seventh fixture is the resolver section, and it is derived rather than listed (phase 32 + task 02).** `tests/fixtures/gateway_resolver.json` is the agent's own `resolver` document — same + keys, `via: "libc"`, entries in `target_names()` order — so a source test can exercise the + name→address join hermetically instead of wishing for DNS. Three rules keep it honest. (a) **The + name list is the parser's, never the script's**: a `plan` step rebuilds the agent's `--root` tree + from the config fixtures in this repo, imports `agent.host_agent`, and asks `target_names()` + + `address_family()` what to look up — so a route added to the gateway moves the fixture with it, an + address literal costs no remote read, and `tests/test_deploy_scripts.py` fails the day a domain is + hardcoded into the script. (b) **`getent ahostsv4` and `ahostsv6`, one name at a time**, because + that is libc on the box that dials; `rc 2` is "no record" and becomes `status: "nxdomain"` — a + fact, kept beside its successful neighbours — while `rc 255` (the ssh died), `rc 127` (no `getent`) + or **not one name answering** is a broken vantage, so the run exits 1 having left the recorded + document byte-identical: a fixture of nothing but `nxdomain` would make every live route on + `/gateway` a permanent dash and every join test a test of nothing. The `ts` is a constant and the + answer set is deduplicated and sorted with the agent's own `sort_resolver_addresses`, so + re-capturing a gateway nobody changed is a no-op. (c) **A resolver answer is an identifier**, so the + socket scrub runs over it too (`8.8.8.8` → `203.0.113.1`), and the `::ffff:` spelling `getent + ahostsv6` synthesises for a name with no AAAA record is dropped when the A record answered — the + agent's one `getaddrinfo` never publishes it. This fleet's targets all answer private LAN addresses, + which is why the recorded document is scrub-free; `tests/test_agent.py` pins that absence as a + warning sign, because a documentation-space address there would not necessarily match the one the + socket fixtures gave the same endpoint, and `tests/test_gateway_source.py`'s join depends on their + agreeing. The version's additivity is pinned two ways: permanently, by + `SNAPSHOT_KEYS_BEFORE_1_3_0` (1.2.0's top-level keys) in `tests/test_agent.py`, whose only legal + successor is that set plus `resolver` — and value-for-value, by `data/logs/verify-32-additive.py`, + which imports `HEAD`'s agent file beside the working tree's and calls both against one `--root`. + One defect only the second kind of control can find: the lookup is + `getaddrinfo(name, port, type=SOCK_STREAM)`, and `proto=SOCK_STREAM` — the spelling that reads as + "the TCP addresses" to anyone who has not read the man page — is `EAI_SERVICE` on real libc for + **every** name, because `proto` takes a protocol number (`IPPROTO_TCP` = 6) and `SOCK_STREAM` is 1. + A stubbed `getaddrinfo` accepts any keyword, so the whole resolver suite stayed green while the + agent reported `error` on a healthy gateway; `test_the_lookup_the_agent_performs_is_one_real_libc_accepts` + removes the module-wide stub and asks real libc about `localhost`, the one name it answers from + `/etc/hosts` with no network. What phase 28 stopped at was the wire: `sources/host.py` still passes **none** of the three into `/api/system/snapshot`, and `store.py` still records no series for a config — the join and the two gauges it publishes are phase 29's `sources/gateway.py`, which reads the same `/snapshot` through its own `gateway` job. What the *system* page does read today is their @@ -434,14 +512,16 @@ Responsibilities, one line each: the agent's smartctl pass, and `None` is translated into httpx's sentinel because a literal `None` means *no timeout at all*). Agent `errors` become the payload's `degraded` **except** the add-on lines named by - `ADDON_ERROR_PREFIXES` (`smart`, `haproxy`, `nginx`, `sockets`) — a host without smartmontools, - without device read access, without a proxy config, or whose kernel socket table did not answer - is unconfigured, not broken (invariant 3), and `degraded` is `alerts.py`'s input, so a fleet-wide + `ADDON_ERROR_PREFIXES` (`smart`, `haproxy`, `nginx`, `sockets`, `resolver`) — a host without + smartmontools, + without device read access, without a proxy config, whose kernel socket table did not answer, or + whose resolver had nothing to resolve / stopped mid-pass is unconfigured, not broken (invariant + 3), and `degraded` is `alerts.py`'s input, so a fleet-wide agent upgrade would otherwise put an overview WARN on every box that never ran haproxy or nginx. Those lines stay verbatim in `agent.errors` for the page that owns the add-on — `/disks` explains the `smart:` ones (and since phase 22 a host whose agent said `smartctl` is *missing* is not listed on that page at all, `api._disk_visible`), phase 29's gateway page reads the - `haproxy:`/`nginx:`/`sockets:` ones the same way. + `haproxy:`/`nginx:`/`sockets:`/`resolver:` ones the same way. **A new agent add-on section adds its prefix here in the same commit**, or its absence-report becomes a red banner on a page that has nothing to say about it. - `sources/host.py` — snapshot → payload + samples; reboot detection; GPU series per card @@ -490,14 +570,18 @@ Responsibilities, one line each: not warned about. What that retires is the guessing and the bookkeeping it existed to explain — the port-only guess for a name target, `unmatched` / `unmatched.groups` / `partial` / `unmatched_listeners`, and `by_port` with its `MAX_GROUP_HOSTS` chip. A **name** target - (`reeseapps-immich.reeselink.com:443`) is therefore never credited: crediting it needs DNS (which - the module still never calls, spy-pinned) and every name here resolves back through this same - router, so the count would be attributed to something the config never said. Two consequences the + (`reeseapps-immich.reeselink.com:443`) is credited **only through an address the gateway itself + published** (phase 32, below): crediting it any other way needs DNS, which the module still never + calls (spy-pinned), and every name here resolves back through this same router, so a vantage of + ours would attribute a socket to a route on the strength of a query the router never made. Two + consequences the page and the tests both depend on: **`live_out` is not the bandwidth** and the socket table never reconciles to it — a pipe count means "this route has N live connections" and nothing more — and several `(listen, endpoint)` lines reaching one endpoint carry one count **repeated** with - `shared: true` (never a split nobody measured), which is why `/2`'s totals and the tests sum - **distinct endpoints**. A pipe is `null` (never `0`) whenever the socket table did not answer or no + `shared: true` (never a split nobody measured) — a shape that is real now that a name route is + counted: two vhosts writing one `(address, port)`, two **names** the resolver placed on one + address, or a multi-homed name sharing one address with a single-homed one — which is why the + API's totals and the tests sum **distinct endpoints**. A pipe is `null` (never `0`) whenever the socket table did not answer or no line reaches its endpoint, and publishes **no sample** then, so a chart cannot draw a zero for an unknown. `supported: false` (a 1.1.0 agent, or `--no-proxy`) keeps every key with empty lists and null counts plus @@ -517,15 +601,44 @@ Responsibilities, one line each: line whose selectors union in first-seen order; **never** merged across `in_port`) and two silent refusals (a `proxy_pass` with neither target nor variable, and a haproxy frontend that routes by a fetch expression, which the agent reports as no name — that trunk is drawn alone, because "no - name" and "no routing directive" are one value in this payload). **Name-keyed outbound gauges are - retired** (phase 30 task 02): `gateway.out_conns@:` is written **only** for an endpoint - the config wrote as an address (`out.address` true), because the sole reason a name-keyed gauge - existed was the port-only guess the join no longer makes — such a series would be a chart of a - number that is always `null`. `gateway.in_conns@` is unchanged, and the port **stays** in the - outbound name, because this fleet's two haproxy Minecraft backends share `minecraft.reeselink.com` - on 25565 and 25566 and two gauges on one name would overwrite each other in silence; lines sharing - an endpoint write the same value to the same name (they are one count, not two) and `samples()` - de-duplicates by metric. `parse_gateway_snapshot()` is the + name" and "no routing directive" are one value in this payload). **A name-written endpoint is + joinable through the address the gateway published for it** (phase 32 task 03): `resolution_index()` + keys agent 1.3.0's `resolver.entries` by `(name, port)` — `{}` when the section is absent or its + `available` is not `true`, which is phase 30 byte-for-byte, and never an entry with no addresses — + and `apply_resolution()` stamps every `out` with `resolved` (the address list, **never `[]`**: an + entry with no addresses and no entry at all are one fact, so the page's dash arm is one `is null` + test), `via` (the agent's own `"libc"`, copied, `null` where nothing was resolved) and + `resolved_ts` (**that entry's** stamp, falling back to the section's, because a name refreshed five + minutes ago and one answered on this poll are two claims). All three stay `null` on an + `address: true` row: the config wrote that address, and copying the agent's `literal` echo of it + into `resolved` would report a `getaddrinfo` that never ran. With an address and a port the line + becomes countable, so **name-keyed gauges are back** — `gateway.out_conns@:`, under + the config's own spelling so history continues under one name. That is the reverse of phase 30 + task 02's retirement for exactly one reason: the retirement existed because the only figure such a + series could carry was the *port's*, and the figure here is the sockets to the address this domain's + own libc resolution produced **on the box that dials**. An unresolved name keeps `metric: null`, + `connections: null`, no sample and the dash the footer explains. `gateway.in_conns@` is + unchanged, and the port **stays** in the outbound name, because this fleet's two haproxy Minecraft + backends share `minecraft.reeselink.com` on 25565 and 25566 and two gauges on one name would + overwrite each other in silence; lines sharing an endpoint write the same value to the same name + (they are one count, not two) and `samples()` de-duplicates by metric. + **The join key is a normalised address, never a spelling** (`_address_key`, `endpoint_of`, + `_endpoint_groups`): `ipaddress` canonicalises and folds `/proc`'s `::ffff:10.1.2.64` to the + `10.1.2.64` `getaddrinfo` returned — one kernel endpoint, one count, however many spellings or names + reach it — while a zero-padded `010.1.2.64` stays **no one's address** (decimal versus octal is a + guess, and a guess on the far end of a pipe is rule 2), so `out.resolved` keeps the agent's bytes and + only the key is canonical. `_endpoint_groups()` is a union-find over those keys (a name route and an + address route reaching one endpoint are one group; overlapping multi-homed sets merge; a route with + no endpoint is no group) and `distinct_endpoint_counts()` is what `api._endpoint_values` sums for + `live_out`, which is the only reason the head tally and the pipes cannot disagree. Matching stays + exact equality on `(address, port)` — no prefix, no range — and this module still resolves nothing: + `socket.getaddrinfo` is spy-pinned absent over a name-only synthetic, a resolver-full synthetic and + the captured `resolved` ingress, plus `samples()` (a gauge name is built from a host string and is + the other place a resolver could creep in). The payload's own `resolver` block is + `{available, via, entries}` with each entry **projected** to `name`/`port`/`status`/`addresses`/`ts` + — every entry including `nxdomain` and `error`, because the index filters *within* the published + list and a page that says *this name has no record* needs the entry that says so — and a section + that did not answer publishes none. `parse_gateway_snapshot()` is the `Result` the poller consumes; there are **no events** (a connection count has no transition worth a line — a proxy that stops routing shows up as its host's own poll failing). The never-answered arm is `stub(host, ts)` — the **same skeleton** `parse` fills in, plus @@ -939,7 +1052,11 @@ Responsibilities, one line each: or a `window_seconds`, at any depth (pinned by a recursive walk: a busy proxy is the job and no Δ was ever measured). Phase 30 task 02 took `outbound[]`, `unmatched` and `by_port` out of it: the flat list duplicated the graph with a worse count, and the residual existed only to explain a - guessing rule that no longer runs. `/history` serves the `gateway.in_conns@` / + guessing rule that no longer runs. Phase 32 task 03 added one top-level key, `resolver` — the + agent's own block, `{available, via, entries}`, each entry projected to the five fields the agent + publishes — so a dash on one line is answerable from the document the page already holds; `stub()` + grows it with nulls, which is what the stub/answer key-parity pin compares. `/history` + serves the `gateway.in_conns@` / `gateway.out_conns@:` gauges with `hosts=` defaulting to the ingress **alone** (the name the samples are stored under — defaulting to every target would draw empty plots for machines nobody asked to route), and it carries no configured gate because it dials nothing: @@ -989,9 +1106,12 @@ Responsibilities, one line each: `live_out` the **distinct outbound endpoints** — never the connection lines — because several `(listen, endpoint)` lines carry one socket count repeated with `shared: true`, and summing the lines would book the same sockets once per line and print a router busier than it is (the same - rule that keys the outbound gauges by `(address, port)`); `api._endpoint_values()` is the one - reader of that, and a line with no `metric` (its target is a name the join can never count) is - absent rather than an unmeasured endpoint holding the total hostage. `map_targets` counts the + rule that keys the outbound gauges by `(address, port)`, with the address normalised so two names + — or two spellings — resolving to one endpoint book it **once**); `api._endpoint_values()` is the one + reader of that (it delegates to `gateway_source.distinct_endpoint_counts()`, the same union-find the + join builds its counters with, so the block's total and the pipes cannot disagree), and a line with + no `metric` (a name nothing resolved) is absent rather than an unmeasured endpoint holding the total + hostage. `map_targets` counts the stream map's routing entries. Every figure is `null` — never `0` — in the arms that did not measure (unconfigured, configured-but-never-answered, a 1.1.0/`--no-proxy agent, a silent socket table), and a total is `null` rather than a part of itself when any countable endpoint is unknown @@ -1317,10 +1437,12 @@ dashboard family — `host`, `llm`, `ups`, `disks` for the SMART series `disk.uncorrectable@…`, `disk.percentage_used@…`), and `gateway` for the phase-29 connection gauges (`gateway.in_conns@`, `gateway.out_conns@:` — the port belongs in the outbound name because two services on one host are two facts and one name would make the -second insert overwrite the first; since phase 30 task 02 the outbound gauge is written **only** for -an endpoint the config wrote as an **address**, so no name-keyed `gateway.out_conns@…` name is ever -inserted again — a host that had one before this change keeps the old series until `maintain()` ages -it out, and its chart simply stops growing points). Those keep their own family instead of +second insert overwrite the first; the outbound gauge is written for an endpoint whose address the +join knows — one the config wrote as an **address**, or (phase 32 task 03) one the gateway's own +`resolver` section published for the name it wrote, under the config's own spelling, so the +name-keyed `gateway.out_conns@:` series phase 30 task 02 stopped inserting continue +instead of restarting. A name nothing resolved is still never inserted, and its chart simply stops +growing points). Those keep their own family instead of sitting beside `host`'s `disk.used_pct@/mount` because the cadences differ thirty-fold and `GET /api/metrics?source=disks` has to answer "what does *this page* have history for". That scheme is the history API's key — treat it as a public interface. @@ -1378,14 +1500,22 @@ That scheme is the history API's key — treat it as a public interface. `https://` agent as `host:gateway` with the same certificate entry, and re-verified live on the dev server with no change to any pre-existing source. **The fleet is mTLS since 2026-10-21 (phase 24 task 03), so the bearer-token transport is now - history rather than the designed shape.** `gateway` runs `agent/host_agent.py` **1.2.0** (phase 28 - task 05, 2026-10-22) and `deskwork`/`driveripper` still run **1.1.0**; the spread costs nothing, - because 1.2.0 is purely additive — the three proxy-config sections are new keys and a 1.1.0 - snapshot simply has none — so those two move at their next re-stage rather than on a flag day. - The control that proves "purely additive" is `data/logs/verify-28-final.py`: it stashes the whole + history rather than the designed shape.** `gateway` runs `agent/host_agent.py` **1.3.0** (phase 32 + task 05, 2026-10-25, on its existing bundle and an unchanged unit — `data/logs/NOTES-32-rollout.md`) + and `deskwork`/`driveripper` still run **1.1.0**; the spread costs nothing, + because both upgrades were purely additive — the three proxy-config sections and the `resolver` + section are new keys, and a 1.1.0 snapshot simply has **none of them**: no `available` flag, no + error line, nothing for `collectors.ADDON_ERROR_PREFIXES` to keep out of `/system`'s `degraded`, and + `sources/gateway.py` answers that absence as `supported: false` + `WHY_PREDATES` rather than an empty + gateway — so those two move at their next re-stage rather than on a flag day. (The three-way + `available: false` arm belongs to a **1.2.0/1.3.0** host that has no proxy config to read, which is + a different fact from a host whose agent predates the section.) + The controls that prove "purely additive" are `data/logs/verify-28-final.py` and + `data/logs/verify-32-additive.py`: each stashes the whole working tree, imports both `host_agent.py` files side by side, calls both `collect()`s against - the same `--root`, and demands the 1.2.0 document equal the 1.1.0 document key-for-key with - `haproxy`/`nginx`/`sockets` added — and 1.2.0 with `--no-proxy` equal it exactly. + the same `--root`, and demands the newer document equal the older one key-for-key with + `haproxy`/`nginx`/`sockets` (phase 28) and `resolver` (phase 32) added — and the newer one with + `--no-proxy` / `--no-resolve` equal it exactly. All three are installed from a bundle, every unit carries `--cert-bundle /etc/dashboards-agent.certs` (plus `RuntimeDirectory=dashboards-agent`, and still its `--token-file` line), and `HOST_TARGETS` is `"driveripper=https://10.1.0.126:9120 deskwork=https://10.1.0.51:9120 @@ -1495,7 +1625,8 @@ That scheme is the history API's key — treat it as a public interface. + `CapabilityBoundingSet` unit with the snapshot's capability-less one, **silently**. That trap is now closed on all three hosts: `/home/ops/agent-install/` is `ops`-owned, so the repo's `deploy/install-agent.sh` (sha256 `ffbcd91f…`) and `agent/host_agent.py` (**1.1.0**, re-staged as - **1.2.0** on `gateway` in phase 28) were staged over it without root, and each snapshot was left + **1.2.0** on `gateway` in phase 28 and **1.3.0** in phase 32) were staged over it without root, and + each snapshot was left beside the grant as `install-agent.sh.oct18-backup`, which sudo will not run. **Re-staging is therefore the first step of any install, not an optional one** — the grant is only as current as the file it points @@ -1653,11 +1784,14 @@ uv run ruff check src tests agent && uv run pyright SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh # ... plus smartctl per physical disk (needs device read # access: plain smartctl, else `sudo -n`; scrubs serials) -./scripts/capture-gateway-config.sh # ... plus the gateway's proxy tree and socket tables: - # haproxy.cfg, nginx.conf + its glob includes, and - # /proc/net/tcp{,6} over $GATEWAY_SSH (gateway-ops); - # world-readable paths only, and a failed or empty read - # never replaces a good fixture +./scripts/capture-gateway-config.sh # ... plus the gateway's proxy tree, socket tables and + # resolved targets: haproxy.cfg, nginx.conf + its glob + # includes, /proc/net/tcp{,6}, and gateway_resolver.json + # (the names `target_names()` derives from those configs, + # read back with `getent ahostsv4/6` — no DNS client, no + # sudo) over $GATEWAY_SSH (gateway-ops); a failed or empty + # read never replaces a good fixture, and a vantage where + # not one name answers is refused rather than recorded bash .agents/validate.sh # the phase gate: the four checks above (pytest # --cov --cov-fail-under=94, ruff, pyright, plus a @@ -1743,20 +1877,25 @@ branch `main`); there is no CI on it, so `bash .agents/validate.sh` is the only is compared with the agent's document and not with a copy of its field names, and then demands the two attributions and nothing beyond them: a socket whose **local** port names a listener is inbound, a socket whose remote `(address, port)` equals a configured target is outbound, and - **every other socket is ignored** — the capture's `established` (18 that day) and the two totals - (9 in, 2 out) are therefore both printed and deliberately never reconciled, because the identity - this suite used to demand (in + out + unmatched = the table) was the arithmetic of a guessing rule. - The rules that must never come back are pinned as absences: no `by_port`/`partial`/`unmatched` + **every other socket is ignored** — the capture's `established` and the two totals are therefore + both printed and deliberately never reconciled, because the identity this suite used to demand + (in + out + unmatched = the table) was the arithmetic of a guessing rule, and because most of a + busy router's sockets are nobody's pipe, which is why **`live_out` is not the bandwidth**. The + rules that must never come back are pinned as absences: no `by_port`/`partial`/`unmatched` vocabulary in the payload or on the page, no `0` for a pipe the join could not count, and `socket.getaddrinfo` spied so the day the join resolves a name the run fails — a name is never - credited with a connection (rule 2), which is why 14 of this fleet's 18 live sockets are nobody's - pipe and why **`live_out` is not the bandwidth**. One number in that suite is a **date**, not a fact: - the captured `stream.conf` has 17 `map` entries (so 32 connections and 17 leaves on `:443`), while the - live router answered **18** entries / 33 connections on 2026-10-23 (`data/logs/30-live-verify.out`) - because a domain was added after the capture. That is the designed difference between a fixture and a - reading — folding the drift in means re-running `./scripts/capture-gateway-config.sh` **and** - re-tallying `tests/test_gateway_source.py` in the same commit, never editing a test to match a live - read. + credited with a connection (rule 2). + **Config-derived counts are literals in these tests; socket-derived ones are computed from the + same capture.** `tests/test_api.py` asserts 10 listeners, 33 connections and 18 `map_targets` + as numbers because only the operator moves those, and derives `live_in`, `live_out`, each pipe's + count and `established` from `gateway_snapshot("full")` because a second capture of a running + router lawfully changes them — pinning the live totals as literals is how a re-capture turns a + green suite red over nothing. The rule is the assertion; the reading is not. + Which is the same discipline that closed the drift phase 32 task 02 folded in: that capture's + `stream.conf` gained a `map` entry (a domain was added to the router after phase 30 recorded it), + so 17 became 18 and 32 connections became 33 **in the commit that re-ran + `./scripts/capture-gateway-config.sh`** — re-tallying the config-derived counts, never editing a + test to match a live reading while the fixture stays where it was. - Anything that needs a failure mode — upsd refusing a command, an engine 500ing, a host rebooting mid-poll, a counter reset — is a fake in `tests/conftest.py`. - **A verdict the operator sees is pinned on the surface that shows it, in both directions.** @@ -2215,17 +2354,37 @@ for a difference of one connection (the `/llm` slot-grid rule). `pruneCards()` w map is the only bulk clear, for the case where the family itself left the payload. A count is printed by `countText()`, which answers `null` for anything that is not a finite number — -**never `0`** (rule 2) — and the two unknowns stay two sentences: a **trunk** whose socket table went -quiet prints `n/a` (`setTxt`'s own word), while a **leaf** whose endpoint the config wrote as a -**name** prints an em dash (`NO_COUNT`) whose `title` says `connections not attributed — the config -writes a name and nothing here resolves names`. That is the normal look of 31 of this ingress's 33 -lines, and a column of `n/a` would be twelve identical words competing with the page's one footnote. -The footer's second paragraph is the same sentence in the operator's terms rather than the join's — -*a dash means the endpoint is written in the config as a name; the page matches sockets by address and -resolves nothing, so this route is drawn but not counted* — because a column of dashes needs one -explanation on the page and not thirty-two on the rows, and because it is a **name** that is the reason -and not a fault. `32_gateway_resolved_target_liveness` is what removes the reason; until then the dash -stays unexplained-looking by design rather than borrowing the trunk's `n/a` word. +**never `0`** (rule 2). Phase 32 task 04 made the missing count say **which** missing count it is: +`countCause()` reads the same two fields `sources/gateway.py` joins on (`out.address === true` or a +non-empty `out.resolved`) and returns one of two fragments, so `no number: the gateway could not +resolve this name in this reading` is printed only when nothing is known for the endpoint, and +`no number: the socket table did not answer this reading` only when an address **is** known and +nobody measured — because those are two operator actions, and either one drawn as the other sends +someone to the wrong box. Both go out through `countWhy()` on the `title` and through the same +`countCause()` in `leafAria()` (`connections not attributed: …`), so a row's tooltip and its +accessible name cannot disagree, and a dash is never the trunk's `n/a`, which is the trunk's own +fact about its own port. **A `0` is not an unknown at all**: a resolved endpoint with no socket is a +measurement, keeps `0 open connection(s) on this endpoint`, and never borrows either sentence +(`test_an_unresolved_name_and_an_idle_route_say_two_different_sentences`, and +`data/logs/32-task04-sentences.cjs`, which serves all four arms through route interception because +the live 1.2.0 ingress can only produce one of them). Every **counted** line gets its provenance on +the same `title` rather than on a column of ink: `countTitle()` appends +`· address from , resolved ` — `via` copied from the payload (`libc`) and the age through +`duration(ageSeconds(out.resolved_ts))` — and prints neither fragment on a row the config wrote as an +address, because `out.address` is already the whole story and echoing the agent's `literal` back as a +lookup would report a `getaddrinfo` that never ran; a missing `resolved_ts` costs the age and nothing +else, so no row prints `n/a` or `NaN` where an age belongs. Phase 32 task 03 left the rendering alone +and changed what lands in the arms: a name-written route is now counted through the address the +**gateway** resolved, so on the **captured** ingress all 33 lines carry a number (26 of them a +measured `0`) and a dash has become the exception that means *this name has no record* +(`nxdomain`/`error` in `resolver.entries`, or no entry beside it — including no section at all, which +is what a 1.2.0 agent or `--no-resolve` answers (the live ingress has run 1.3.0 since phase 32 task 05 +on 2026-10-25, where **33 of 33** rows carried a number and none dashed; the same pass recorded +`resolver.available: true`, 32 entries, `via: "libc"`, `config_directive: "10.1.0.1"`, 30 `ok` + 2 +`literal` targets). The footer's second paragraph says the same thing once for the whole page — a dash means the +gateway could not resolve that name in this reading, hover a counted line for the address it matched +and when that answer was taken — and the footer is still four `

`s (phase 31 task 04's filter +footnote is the fourth, which is why the count that phase's pin demands is four and not three). The line itself is hand-rolled SVG (`role="img"` + an `aria-label` naming the inbound port, the endpoint, the selector and the count) and the row carries **two independent class pairs** (phase 30 task 05, moved onto the `

  • ` by phase 31 task 01), because the operator's green and the socket count @@ -2316,7 +2475,10 @@ the field is the one control that could be read as a query, and it is not one. **`#gate-filter` is an eye, not a query** (phase 31 task 04). 33 routes in config order is not a lookup, and the operator's question is always about one domain, so one field above the panels hides the rows whose **own payload text** does not contain what was typed: `in.port`, `out.host`, `out.name`, -`out.kind` and every `out.selectors` entry for a route (`haystack()`), `:${port}` plus +`out.kind`, every `out.selectors` entry and every `out.resolved` address for a route (`haystack()`; +the sixth field is phase 32 task 04's, and it is what makes typing `10.1.2.64` find the route whose +name target resolves there — a haystack of names could not answer a question asked about a socket), +`:${port}` plus `listeners[].domains` and `.clients` for a trunk (`trunkHaystack()`). Nothing else is readable and nothing else is matched — no DNS, no re-ranking, no rule about `.reeselink.com`, and no page-level split by domain suffix, because "`reeseapps.com` vs `stackexpected.com`" is a fact neither config file @@ -3011,7 +3173,7 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl The rollout is done and verified: gateway, deskwork and driveripper each run `agent/host_agent.py` **1.1.0** with `/etc/dashboards-agent.certs` installed `0640 root:dashboards-agent`, their units carry `--cert-bundle` + `RuntimeDirectory=dashboards-agent` (and keep `--token-file`) — the - gateway has since moved to **1.2.0** for phase 28's proxy-config sections on the same unit and + gateway has since moved to **1.2.0** (phase 28) and then **1.3.0** (phase 32) on the same unit and the same bundle, see the fleet record above — `HOST_TARGETS` is `https://` for all three with no userinfo, `agent.bundle_rev` reads `2` on every host and equals what `/api/certs` issued, and the nine sources are `ok`. The transcript is @@ -3275,10 +3437,16 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl 13. **`/gateway`'s long endpoint labels are ellipsised on a phone — a column-width question, not a lost fact, and the filter is what a clipped label used to block** (found by phase 29 task 06's - live pass, **closed at the desktop width by phase 30 task 04's trunk→leaf row and re-measured - after phase 31 re-ranked the row on 2026-10-26**; `data/logs/31-route-readability.out` is the - current measurement, `data/logs/30-connection-map.out`, `data/logs/30-task04-panels.out`, - `data/logs/29-task06-live.out` and `data/logs/29-task06-card-haproxy.png` the history). + live pass, **closed at the desktop width by phase 30 task 04's trunk→leaf row, re-measured + after phase 31 re-ranked the row on 2026-10-26 and again after phase 32 put the count + provenance on the `title` on 2026-10-26**; `data/logs/32-counted-routes.out` is the current + measurement, `data/logs/31-route-readability.out`, `data/logs/30-connection-map.out`, + `data/logs/30-task04-panels.out`, `data/logs/29-task06-live.out` and + `data/logs/29-task06-card-haproxy.png` the history). Phase 32's re-check found the clip + counts **unchanged** by the DOM/provenance move — **19 of 33** leaf inbound lines and **33 + of 33** counterpart lines at 360 px (the 10 trunk names, none clipped), 0 of 33 at 1440 px, + and the whole pair on **33 of 33** rows' tooltips — so the trade is the same and only the + tooltip's second sentence (the provenance) is new. **Phase 30 task 04 closed the desktop half by changing the row, not the font.** The two equal-width pipe columns of phase 29 became a group of `minmax(0, 9.5rem) minmax(0, 1fr)` — a @@ -3330,16 +3498,40 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl declarations and removed the reason they were needed: with the trunk in its own track, the leaf's name is the widest thing in a column that is now wide enough for it. -14. **A route whose endpoint the config writes as a *name* is drawn and never counted — - `32_gateway_resolved_target_liveness` is the phase that removes the reason** (recorded by phase - 31 task 05, which wrote no design for it). Today 31 of this ingress's 33 routes end in a - `server_name`/`map` target and so carry an em dash where a live count would go, because - `sources/gateway.py` matches sockets by - address and resolves nothing (rule 2 by choice, and pinned as an absence). The follow-on phase is - the one that makes such a target answer with its liveness and its live connections anyway — the - agent reporting resolved targets on the box that owns the config, which is an `AGENT_VERSION` bump - to **1.3.0** and therefore an agent + source + pin change, not a page change. Its acceptance line - is that a dash disappears from `/gateway` for a reason and not by hiding the column. +14. **A name-written route is counted through the address the gateway published — closed, the + re-stage included** (recorded by phase 31 task 05, implemented by + phase 32 tasks 01-03). Until phase 32, 31 of this ingress's 33 routes ended in a + `server_name`/`map` target and carried an em dash where a live count would go, because + `sources/gateway.py` matches sockets by address and resolves nothing. That is still true — the + module calls no resolver, spy-pinned over the parse, the gauge names and the capture — but it no + longer has to, because agent **1.3.0** publishes `resolver`: every `(name, port)` its own two + config files write as a target, answered by libc **on the box that opens the connection**, with + the nginx `resolver` directive printed beside it as `config_directive` so a divergence between the + two vantages is visible instead of guessed. `sources/gateway.py` reads that document, stamps + `out.resolved` / `via` / `resolved_ts`, joins sockets through the normalised address, and writes + the `gateway.out_conns@:` gauge under the config's own spelling — so the dash is gone + on the live page **because the gateway said what the name became**, which was the acceptance line, + and not because a column was hidden. + **The deployment half is done.** `gateway` ran **1.2.0** until phase 32 task 05 re-staged it on + 2026-10-25 (`data/logs/NOTES-32-rollout.md`: unit sha256 unchanged, bundle untouched, the install + run guard silent, one file and one restart), and that pass read `/api/gateway/snapshot` going from + **2 of 33** rows carrying a number to **33 of 33**, with all 31 resolver-placed rows finite and + `test_a_resolved_target_on_the_live_ingress_is_never_a_dash` passing instead of skipping. What is + left of the gap is rotation hygiene, not a step: `deskwork` and `driveripper` stay on 1.1.0 (no + proxy configs, therefore nothing to resolve), and a host that later rolls **back** to a pre-1.3.0 + file simply returns its name routes to dashes — `sources/gateway.py` treats a missing `resolver` + section as phase-30 behaviour. A name that stops + resolving is `nxdomain`/`error` and goes back to being a dash on the next poll (the agent caches + 300 s and never publishes a stale address as current), which is the designed degradation rather + than a gap — and since task 04 the page says **which** degradation it is: `countCause()` separates + that dash (`the gateway could not resolve this name in this reading`) from the one an unreadable + socket table causes (`the socket table did not answer this reading`), while a resolved endpoint with + nothing open prints its `0` beside `· address from libc, resolved `, so the sentence a line + carries names the box to go and look at. `data/logs/32-task04-sentences.cjs` measures all four arms, + serving the three a pre-1.3.0 host could not produce through route interception (43 checks, and + each stub leg + first proves its DOM equals the document that leg was served, so a leg that rendered the live page + instead fails by name rather than three checks later). 15. **`/gateway`'s switches are knobs with no write behind them — `33_gateway_route_switching` is the phase that decides whether they ever get one** (recorded by phase 31 task 05, no design written). diff --git a/README.md b/README.md index f201e2a..abc3b72 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ file: | `/disks` | Are any of my **physical** disks failing, and which one — health, temperature, power-on hours, wear, sector alarms | the same agents, running `smartctl -i -A -H` per whole drive (never a partition or a mount); a host whose agent has no `smartctl` is not a host on this page | | `/llm` | Tokens/sec, active sessions, queue depth, KV-cache pressure, latency percentiles | llama.cpp-family (gufo) `/metrics` + `/slots`, vLLM `/metrics` — read-only | | `/ups` | Battery health, runtime left, power events, and battery-test buttons | NUT `upsd`, on whatever port it really listens on (its own wire-protocol client) | -| `/gateway` | Which domains are coming in on which port, and where each one goes — **two connection tables, one per proxy config file** (`nginx`, then `haproxy`): a trunk per bound port with its configured routes branching off it, each line reading the config's own **inbound** word (`server_name`, `map` key, `backend` name) on line 1 over the counterpart it routes to on line 2 (`→ host:port`), plus its live connection count — and one filter box above the map that hides the routes whose own labels do not contain what you typed, asking nothing of the server | the agent on **one** host, `GATEWAY_HOST` (default `gateway`), reading its `haproxy.cfg` + nginx tree and `/proc/net/tcp{,6}` — so it needs **agent 1.2.0**, and the whole page is *unconfigured* (never red) unless `GATEWAY_HOST` names a `HOST_TARGETS` entry | +| `/gateway` | Which domains are coming in on which port, and where each one goes — **two connection tables, one per proxy config file** (`nginx`, then `haproxy`): a trunk per bound port with its configured routes branching off it, each line reading the config's own **inbound** word (`server_name`, `map` key, `backend` name) on line 1 over the counterpart it routes to on line 2 (`→ host:port`), plus its live connection count — and one filter box above the map that hides the routes whose own labels do not contain what you typed, asking nothing of the server | the agent on **one** host, `GATEWAY_HOST` (default `gateway`), reading its `haproxy.cfg` + nginx tree and `/proc/net/tcp{,6}` — so it needs **agent 1.2.0**, plus **1.3.0**'s `resolver` section to put a number on a route whose target the config writes as a *name* (without it those lines are drawn and dashed), and the whole page is *unconfigured* (never red) unless `GATEWAY_HOST` names a `HOST_TARGETS` entry | No Prometheus, no Grafana, no CDN, no build step. Nothing is fetched from the internet at runtime; every asset is served by this app. @@ -212,23 +212,88 @@ an address: the name-keyed ones were a chart of the port-only guess the join no would be permanently empty). Nothing in `/api/system/snapshot` grows a proxy or socket field, and no `sample` row is written for a config. -All three parsers are pinned to captures rather than to descriptions, and +The `resolver` section is the fourth and the only one that leaves the host. The two configs above +name their targets two ways — an address (`proxy_pass http://127.0.0.1:8081`) and a name +(`server mc1 minecraft.reeselink.com:25565`) — and the kernel's socket table speaks addresses, so a +name is the half a page could never count. The agent therefore takes every `(name, port)` the two +**parsed** sections write as a target (haproxy `server` lines, then nginx `upstream` members, +`proxy_pass` targets and `map` entry targets, deduplicated in config order) and asks libc what each +one is: `getaddrinfo` over `SOCK_STREAM`, published as `via: "libc"` because that is the resolution +haproxy's own libc path performs **on this box** — the box that opens the connections. The nginx +config's own `resolver` directive travels beside it verbatim as `config_directive` (`10.1.0.1` +here, read out of `stream.d/stream.conf`), so a divergence between the two vantages is something an +operator can see rather than something this file guesses; the agent does not query that resolver +directly, and the dashboard never resolves anything at all (`sources/gateway.py` has a test that +fails if it ever calls a resolver). An address literal is reported `status: "literal"` with its +address and **no** lookup, and a name that does not answer right now is `nxdomain` or `error` with +`addresses: []` — the route keeps its row and carries no count, never a `0`. + +That pass is bounded and cached exactly the way the SMART one is, for the SMART reason: at most +**64** lookups inside **3 s** of wall clock, cached **300 s**, the first pass inline so a freshly +started agent answers with real addresses and every later expiry refreshed on a single background +thread while the request is served the previous result. `getaddrinfo` takes no timeout argument, so +stopping between lookups is the only defence a black-holing resolver leaves — and the cost of not +stopping is the *system* page's poll timeout, spent on an add-on that page never draws. Each entry +carries the time **it** was answered, so a cache-served entry cannot pretend to be current; the +budget and the name bound each report themselves in one `errors` line and leave `available` at +`true`. `available` is the family's three-way answer (`false` means there was nothing to resolve +because no proxy config was read, and its one line says **not found** or **unreadable** according to +which of the two it was), and +`--no-resolve` removes the section in silence the way `--no-smart` removes SMART — while `--no-proxy` +removes it too, because a box that collects no proxy has no declared names to look up. + +All four parsers are pinned to captures rather than to descriptions, and `./scripts/capture-gateway-config.sh` re-records them over the `GATEWAY_SSH` alias (default `gateway-ops`): `/etc/haproxy/haproxy.cfg`, `/etc/nginx/nginx.conf` plus every document its own -glob `include` lines name (`gateway_nginx_conf_d_*.conf`, `gateway_nginx_stream_d_*.conf`), and -`/proc/net/tcp{,6}`. No sudo — those paths are world-readable, which is exactly what the agent -reads as itself — and a read that fails, or answers empty, leaves the existing fixture alone and -exits 1, so a laptop with no route to the gateway cannot turn six captures into six empty files. +glob `include` lines name (`gateway_nginx_conf_d_*.conf`, `gateway_nginx_stream_d_*.conf`), +`/proc/net/tcp{,6}`, and `tests/fixtures/gateway_resolver.json` — the `resolver` section itself, so +the name→address join is testable with no network. No sudo and no DNS client: those paths are +world-readable, which is exactly what the agent reads as itself, and the resolver is recorded with +`getent ahostsv4` / `getent ahostsv6`, which is libc on the box that dials (the same vantage +`via: "libc"` claims). A read that fails, or answers empty, leaves the existing fixture alone and +exits 1, so a laptop with no route to the gateway cannot turn seven captures into seven empty files. + +The resolver document is **derived, never listed**: the script feeds its own freshly captured +configs through `agent.host_agent`'s parsers and `target_names()`, so it asks about exactly the +targets the agent will ask about, an address literal costs no remote read, and a route added to the +gateway moves the fixture with it. `getent` answering `rc 2` is a fact — that name has no record and +becomes `status: "nxdomain"`, kept beside the names that did answer — while a dead transport, a +missing `getent`, or **not one name answering** is a broken vantage: the run refuses, because a +document of nothing but `nxdomain` would make every route on `/gateway` a permanent dash and every +join test a test of nothing. Re-capturing a gateway nobody changed is a no-op (fixed `ts`, +deduplicated, sorted with the agent's own `sort_resolver_addresses`), so a diff in that file means +the ingress changed. + The configs are not rewritten (their domains are the ingress map, and key material appears only -as the paths nginx is told to read); the socket tables lose only their identifiers, in line with -the rule that scrubs a drive's serial: a globally routable address becomes a documentation one -(RFC 5737 / RFC 3849), because a global IPv6 on Linux is EUI-64 and an inbound visitor's address -is not ours to publish. Every port, state, column and width in those tables is verbatim, and -those files are volatile by nature — a proxy in service has a different connection set every -second — so the tests assert shape and the ports the configs claim, never a particular row. -`AGENT_VERSION` is **1.2.0**, the version that ships all three sections; a host still running -1.1.0 answers `GET /schema` with a block that names none of them, and its snapshot simply has no -such keys. +as the paths nginx is told to read); the socket tables and the resolver's answers lose only their +identifiers, in line with the rule that scrubs a drive's serial: a globally routable address becomes +a documentation one (RFC 5737 / RFC 3849), because a global IPv6 on Linux is EUI-64 and an inbound +visitor's address is not ours to publish. Every port, state, column and width in those tables is +verbatim, and those files are volatile by nature — a proxy in service has a different connection set +every second — so the tests assert shape and the ports the configs claim, never a particular row. +`AGENT_VERSION` is **1.3.0**, the version that adds the `resolver` section on top of the three +phase 28 shipped; `GET /schema` names all four, each with its `available` flag, and `--no-proxy` +and `--no-resolve` are listed there too. A host still running 1.2.0 answers with a snapshot that +simply has no `resolver` key — which is why the version spread costs nothing: a section an agent +did not collect is absence, and the gateway page counts the address-written routes it has and draws +the name-written ones with a dash. + +**The dashboard reads that section and never produces one.** `sources/gateway.py` indexes the +entries by `(name, port)`, stamps each route's `out` with `resolved` / `via` / `resolved_ts` — the +address the gateway dialled, the agent's own string for how it got it, and the moment that **entry** +was answered — and lets the socket join reach a name-written route through it. Which is the whole +of phase 32: the outbound endpoint of a `server` line, a `proxy_pass` or a `map` entry is now an +address the box that opens the connection published, so the `gateway.out_conns@:` +gauges phase 30 task 02 stopped writing are written again, under the config's own spelling, with a +figure that belongs to the domain instead of to whatever else was listening on its port. Four rules +keep the join honest, each pinned in `tests/test_gateway_source.py`: a name nobody resolved keeps +`resolved: null`, `connections: null`, `metric: null` and **no stored sample** (never `0`, never an +empty list); matching is by normalised address, so `::ffff:10.1.2.64` in `/proc` and `10.1.2.64` +from `getaddrinfo` are one endpoint while a zero-padded `010.1.2.64` — which is no one's address +notation — matches nothing; two names resolving to one address, or two route lines reaching it, are +**one** count held by every line that reaches it; and the module still never calls a resolver, a spy +pin covering the parse, the gauge name and the capture alike. An `address: true` route claims none +of the three provenance fields, because the config wrote that address and no lookup happened. `deploy/install-agent.sh --fetch-bundle --cert-name --session ` is the same TLS install with the download done inside the registration run: it asks @@ -431,9 +496,12 @@ No `--cert-bundle`, no `--fetch-bundle`, no `--force`: the bundle already on dis render reads — the flag only replaces the file — so the render equals the installed unit, the re-run guard prints nothing, and `/etc/dashboards-agent.certs` and the token file keep their bytes. Then the same two checks the registration uses: the CA-only `--status` probe (with `--bind`) answers -`{"ok": true, …, "version": "1.2.0"}` on `/health`, and the dashboard's own `https://` dial answers +`{"ok": true, …, "version": "1.3.0"}` on `/health`, and the dashboard's own `https://` dial answers whatever the new version added — `agent_version` plus the `haproxy` / `nginx` / `sockets` sections -for a 1.2.0 agent — beside the sections it always had. +(1.2.0) and the `resolver` section (1.3.0) — beside the sections it always had. That is exactly how +`gateway` moved 1.1.0 → 1.2.0 (phase 28) and 1.2.0 → **1.3.0** (phase 32, 2026-10-25: the same unit +sha256 before and after, the bundle untouched, the guard silent, one file and one restart), and the +`/gateway` page counted **33 of 33** routes afterwards where it had counted 2 before. Two things the first full rollout learned, both cheap to hit and expensive to guess at: @@ -887,8 +955,10 @@ certs.py ─ root CA + client cert, minted once at startup (unreadable row → n `connections` graph (phase 30: one row per route the two config files declare, each naming its inbound port, its outbound endpoint, and the config's own selector for it — a haproxy backend name, an nginx `server_name`, a stream `map` key) with its **live count** — the socket table's - ESTABLISHED rows, inbound by the socket's local port and outbound only where a remote - `(address, port)` equals a target the config wrote as an address. Each row carries the `metric` it + ESTABLISHED rows, inbound by the socket's local port and outbound where a remote + `(address, port)` equals an endpoint the join can name — a target the config wrote as an address, or + (phase 32 task 03) one of the addresses the gateway's own `resolver` section published for the name + it wrote. Each row carries the `metric` it is stored under, and `shared: true` when more than one route reaches its endpoint (one count, shown twice, never split). `supported`/`sections`/`why` cover the degradation arms. `warnings` (phase 30 task 03) is the list of the two config files **not connecting to themselves** — seven kinds, each @@ -903,16 +973,25 @@ certs.py ─ root CA + client cert, minted once at startup (unreadable row → n or absent section warns about nothing — a host that cannot be read has not been found guilty of anything — and a `warnings: []` is therefore the truth, not a shrug. Non-zero, `/` gets **one** amber row (`N proxy config problems to address — `) and this page gets the list. **Nothing reconciles - to `sockets.established` and that is the design**: a connection to a target the config wrote as a - name (most of this fleet's fan-out) or on a port no config names is ignored — not counted, not -totalled, not warned about — so a pipe count means "this route has N live connections" and nothing -more. `GATEWAY_HOST` naming no `HOST_TARGETS` entry is + to `sockets.established` and that is the design**: a connection to a target nothing can name (a + name with no resolver entry behind it) or a socket on a port no config names is ignored — not + counted, not totalled, not warned about — so a pipe count means "this route has N live connections" + and nothing more. The payload also carries the agent's `resolver` block (`available`, `via`, and + the projected `entries`: name, port, status, addresses, ts — nothing an operator has not reviewed) + beside the `sections` pair that already says which halves answered, so a dash on one line is + explainable from the document the page already holds. On the recorded + ingress that section places every declared route, which makes the dash the exception rather than + the normal look of this page, and its meaning is *this name has no record* — and on the live fleet + it is the exception too: the gateway has run agent **1.3.0** since 2026-10-25, `resolver.available` + answers `true` with one entry per declared target, and that host's page counted **33 of 33** routes + with no dash at all. `GATEWAY_HOST` naming no `HOST_TARGETS` entry is a `404 {"configured": false}` answered before any client is built (both arms spy-pinned), a host that has never answered is the `unreachable` stub whose every count is `null`, and a 1.1.0 agent is `supported: false` plus its one `why` sentence — three arms, no red among them (invariant 3). `/history` serves the `gateway.in_conns@` and `gateway.out_conns@:` gauges — the port in the outbound name because two services on one host are two facts, and only ever for an - address endpoint — with `hosts=` + endpoint whose address is known: one the config wrote, or (since phase 32 task 03) one the gateway + resolved the name to — with `hosts=` defaulting to the ingress alone, unknown names answering as empty series, and no dial either way because history reads only SQLite. * `web/` — six static dashboard pages (`/`, `/system`, `/disks`, `/llm`, `/ups`, `/gateway`) plus @@ -1108,9 +1187,10 @@ more. `GATEWAY_HOST` naming no `HOST_TARGETS` entry is Above the map, one **filter box** finds a domain among 33 routes (`stackexpected`, `brain`, `4443`, `minecraft`) because the rows are in config order and the question is always about one name. It hides what the payload already sent and asks for **nothing**: no request, no re-derivation, no new route. - It matches the strings the config wrote — the inbound word(s), the endpoint, the port, the haproxy - `server` name, and a trunk's own domains and client addresses — case-insensitively, so a label CSS - ellipsised still matches its whole text. A trunk that matches keeps its **whole fan-out**: typing + It matches the strings the payload carries — the inbound word(s), the endpoint, the port, the haproxy + `server` name, the **addresses the gateway resolved** for a name target, and a trunk's own domains + and client addresses — case-insensitively, so a label CSS ellipsised still matches its whole text and + typing `10.1.2.64` finds the route whose target that address belongs to. A trunk that matches keeps its **whole fan-out**: typing `4443` asks for the port, and every route hanging off it is the answer. `Escape` clears it, a chip beside the field counts `N of M routes`, and a query that matches nothing leaves both panels standing with `no route matches "…"` in each. What it never touches is the header's four tallies, the warnings @@ -1120,12 +1200,17 @@ more. `GATEWAY_HOST` naming no `HOST_TARGETS` entry is than inventing a zone no proxy declared. **Nothing on the page is coloured by a verdict** — a busy ingress is the job, so `severity.py` rates no `gateway.*` metric - and no number gets a green/amber/red tint. A line to an endpoint the config wrote as a **name** - prints `—`, never `0`: counting it would need a DNS lookup and the join never asks one, and the row's - tooltip says which of the two facts it is looking at. Expect that on most of this ingress's routes — - 31 of its 33 are written as domains — and read it as *drawn, not counted* rather than as a fault; the - footer says the same thing once, in plain terms, because a dash per row needs one explanation and not - thirty-two. The header's four tallies are `ports`, `connections`, `live` and + and no number gets a green/amber/red tint. A line prints `—`, never `0`, and its tooltip and its + accessible name say **which** unknown it is: *the gateway could not resolve this name in this + reading* when nothing is known for the endpoint, or *the socket table did not answer this reading* + when an address is known and nothing was measured — two different things to go and fix. A resolved + endpoint with nothing open is neither of them: it prints `0`, because that is a measurement. A + counted line carries its provenance on hover (`… · address from libc, resolved 3m`), and a line the + config wrote as an address claims no resolution at all. A dash on a domain-named route now means the + gateway had **no record** for that name in this reading (`nxdomain`/`error`, or no `resolver` section + at all on a pre-1.3.0 agent) rather than "this page cannot count names": the ingress here runs + **1.3.0**, so all 33 of its declared routes carry a number. The + footer says the same thing once, in plain terms, and points at the hover. The header's four tallies are `ports`, `connections`, `live` and `warnings`, every one of them the payload's own figure. A `warnings` count above zero raises a panel above the map with the config-integrity rows (`a backend no frontend routes to`, a `proxy_pass` with no `upstream` behind it, an SNI entry that reaches no address) in the overview's alert vocabulary; @@ -1142,7 +1227,8 @@ more. `GATEWAY_HOST` naming no `HOST_TARGETS` entry is documented here as a thing you can have: name-based liveness needs the agent to report resolved targets (a future agent version), and switching a route needs a second gated write path in front of the proxy configs, which is a change to this app's read-only promise rather than a feature. What runs - today is **agent 1.2.0**, no name resolution, and no switching. + today is **agent 1.3.0** — name resolution on the gateway, counts on every declared route, and no + switching. The four event feeds (`/system`, `/disks`, `/llm`, `/ups`) are a **paged window** over the `event` table rather than a printout of it: the list is a bounded box that scrolls, 50 events to a page, with `1–50 of 200` between `‹ prev` and `next ›` (disabled at the ends) underneath. @@ -1326,7 +1412,7 @@ plus its included documents, and the two `/proc/net/tcp{,6}` captures — run th `host_agent` parsers, so the test compares the join against the agent's own document rather than against a copy of its field names. The assertions are the join's two rules and the limits around them: the graph carries exactly the ports, `server_name`s and targets those documents declare (all -32 routes on this capture, as a **set**, so a missing or invented route fails by name); a socket is +33 routes on this capture, as a **set**, so a missing or invented route fails by name); a socket is inbound by its local port and outbound only when its remote `(address, port)` equals a target the config wrote as an address; **every other socket is ignored** — `established`, `live_in` and `live_out` are all printed and deliberately never reconciled, because a connection to a name, or on @@ -1337,6 +1423,13 @@ stands for "not counted"; and **no join ever asks a resolver** — `socket.getad fails the test, because deciding "already an address, or a name?" is the only question that module may ask, and a DNS lookup in the poll loop would be rule 2 with a round trip attached. +The seventh capture, `gateway_resolver.json`, is in that suite's reach too: it is the agent's own +`resolver` document from the real gateway, so `tests/test_agent.py` proves it has the section's shape +(one entry per name `target_names()` derives, `via: "libc"`, the config's own `resolver` line, no +globally routable address) and `tests/test_gateway_source.py` proves it reaches every route the +configs write as a **name** and shares addresses with the recorded socket table — the capability the +join needs, pinned before the join exists to use it. + `tests/rate_window_contract.py` is the same device for one number rather than a field table: the Δ window each source measures its rates over — `RATE_WINDOW_SECONDS` for the host agents, `LLM_INTERVAL_SECONDS` for the engines, one estimator per measuring cadence (phase 23). It holds diff --git a/agent/host_agent.py b/agent/host_agent.py index 8a73ba1..3aadfff 100644 --- a/agent/host_agent.py +++ b/agent/host_agent.py @@ -67,6 +67,28 @@ Design rules answering is `available: null` with one `sockets:` line and an otherwise intact snapshot, and `--no-proxy` removes it with the rest of the family, because a box with no proxy has nothing to watch from either side. +* **Names become addresses where they are actually dialled.** The `resolver` section takes every + `(name, port)` the two parsed configs write as a target — haproxy `server` lines, nginx + `upstream` members, `proxy_pass` targets and `map` entry targets, and **nothing else, ever** + (not a name from `/proc`, not a client address) — and asks libc what each one is: + `getaddrinfo` over `SOCK_STREAM`, which is the resolution haproxy's own libc path performs *on + this box*. That is the point — the dashboard resolving the same name through its own + `resolv.conf` would be a different vantage answering a different question — so `via: "libc"` + and the nginx config's own `resolver` line (`config_directive`) are both published and the + dashboard resolves nothing. An address literal in a config is `status: "literal"` with no + lookup (it is already an address, and dialling it to be told so is a round trip); a name that + does not answer is `nxdomain`/`error` with `addresses: []`, never a stale guess and never a 0. + The pass is SMART's shape for SMART's reason: at most `RESOLVE_MAX_NAMES` lookups inside + `RESOLVE_PASS_BUDGET_SECONDS` of wall clock, cached `RESOLVE_CACHE_SECONDS`, the **first** pass + inline and every later expiry refreshed on one single-flight background thread while the + request is served the previous result — a black-holing resolver must never blow the dashboard's + `HOST_TIMEOUT_SECONDS` and blink the *system* page over an add-on it does not draw. Each entry + carries the time **it** was answered, so a cache-served entry cannot pretend to be current. + `available` is the family's three-way answer (`true` a pass ran, `false` nothing to resolve + because no proxy config was read — one `resolver:` line, saying **not found** when the files + are simply absent — `null` a pass that could not run), and `--no-resolve` removes the key in + silence the way `--no-smart` does; `--no-proxy` removes it too, because a box that collects no + proxy has no declared names to look up. * **The state** is a small pid->cpu-time cache so `processes.top_cpu` can answer a question /proc cannot (CPU needs two samples), bounded and cleared when stale, plus the short-TTL SMART result cache above. @@ -158,6 +180,31 @@ SCHEMA (GET /snapshot) # --no-proxy already reports "did not look" by removing the key. No # config join: a socket does not know its domain, so the gateway page # joins these ports to `haproxy` and `nginx` itself. + resolver {available: true|false|null, via: "libc", config_directive: str|null, + ts: float|null, entries: [{name, port, status: ok|nxdomain|error|literal, + addresses: [str], ts: float}], errors: [str]} + # one libc lookup per (name, port) the two sections above write as a target, + # performed here because this is the box the proxy dials from; additive + # (phase 32) and absent entirely with --no-resolve or --no-proxy. `via` is + # how the addresses were obtained (getaddrinfo — haproxy's libc path) and + # `config_directive` is the nginx config's own `resolver` line kept verbatim + # (`"10.1.0.1"` here), so the two vantages read side by side and a + # divergence is the operator's finding rather than a second resolver here. + # `ts` is the wall clock of the pass that produced the entries while each + # entry carries the time *it* was answered, because an expired cache is + # served as it was while a background thread refreshes it: never two passes + # per snapshot, never a request that waits on DNS past the first one. + # `addresses` is the unique answer sorted v4-ascending then v6-ascending so + # two polls of a healthy gateway are byte-identical; `status: literal` means + # the config already wrote an address and no lookup was made, and + # `nxdomain`/`error` mean "no address right now" with `addresses: []` — the + # page draws those routes with no count, never with a 0. `available` is + # false only when there was nothing to resolve because no proxy config was + # read (one `resolver:` line, saying **not found** when the files are simply + # absent), and null for a pass that could not run; configs that were read + # and name no target are true with `entries: []`. The lookups are bounded + # (RESOLVE_MAX_NAMES, RESOLVE_PASS_BUDGET_SECONDS) and the excess says so in + # one line, so no resolver however broken holds /snapshot past its timeout. errors [str] reads that failed; non-empty means the snapshot is partial Deployment: see deploy/install-agent.sh (systemd unit, binds 127.0.0.1 unless you pass @@ -185,12 +232,12 @@ import subprocess import tempfile import threading import time -from collections.abc import Callable +from collections.abc import Callable, Iterable, Mapping, Sequence from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any, cast -AGENT_VERSION = "1.2.0" +AGENT_VERSION = "1.3.0" SCHEMA_VERSION = 1 PAGE_SIZE = 0 # resolved from sysconf at runtime @@ -335,6 +382,46 @@ PROC_NET_MAX_LINES = 4096 PROC_NET_STATE_LISTEN = "0A" PROC_NET_STATE_ESTABLISHED = "01" +# --------------------------------------------------------------------------- # +# name resolution (the additive `resolver` section) +# --------------------------------------------------------------------------- # + +#: How many declared targets one pass looks up. A gateway's config-declared targets are a few +#: dozen with room to grow, and the bound is what stops a config that grew into the hundreds (or +#: a map written by someone who liked wildcards) from turning a snapshot into a DNS storm. The +#: excess is reported in one `errors` line naming the count — the nginx include-bound and SMART +#: pass-budget shape: say what was not covered instead of serving a shorter list silently. +RESOLVE_MAX_NAMES = 64 + +#: How long a resolved address set stays fresh. A LAN target moves when the operator moves it, +#: not every poll, and this section exists to be *cheap*: the dashboard polls the agent every +#: 10 s, and a lookup per name per poll would make the agent the noisiest DNS client on the +#: network. Anything older than this is refreshed on a background thread, never on the request. +RESOLVE_CACHE_SECONDS = 300.0 + +#: The wall-clock ceiling on one inline pass. `socket.getaddrinfo` takes no timeout argument, so +#: the only protection against a black-holing resolver is stopping between lookups — and the +#: reason that matters is the same one SMART learned: blowing `HOST_TIMEOUT_SECONDS` here turns +#: the **system** page red over an add-on it never draws (AGENTS.md invariant 3/4). The first +#: pass runs inline (a fresh agent must answer with real addresses), every later one runs on a +#: background thread, so this bound is the worst cost of a single request. +RESOLVE_PASS_BUDGET_SECONDS = 3.0 + +#: How the addresses were obtained, published rather than implied: `libc` means +#: `getaddrinfo()`, which is the resolution haproxy's own `init-addr last,libc,none` performs on +#: this box. It is not the `resolver` directive's answer (nginx queries that server directly), +#: and the payload says both so an operator can see a divergence — this module does not hand-roll +#: UDP DNS queries to name it a second time. +RESOLVE_VIA = "libc" + +#: The `getaddrinfo` return codes that mean "this name has no address right now" as opposed to +#: "the lookup could not be performed". `EAI_NODATA` is what glibc calls a no-data answer and +#: CPython does not export it (it is deprecated upstream), so the value is spelled out beside a +#: `getattr` rather than guessed at: a temporary DNS failure (`EAI_AGAIN`) is deliberately **not** +#: in this set, because "no answer" and "no server" are different operator problems. +_EAI_NODATA = getattr(socket, "EAI_NODATA", -5) # CPython does not export it +RESOLVE_NXDOMAIN_CODES = frozenset({socket.EAI_NONAME, _EAI_NODATA}) + # pci slot -> human model, filled on first look (lspci output cannot change at runtime). _GPU_MODEL_CACHE: dict[str, str | None] = {} @@ -404,7 +491,7 @@ class Collector: def __init__(self, root: str | os.PathLike[str] = "/", *, top_processes: int = 5, net_all: bool = False, smartctl: str = "smartctl", - smart: bool = True, proxy: bool = True, + smart: bool = True, proxy: bool = True, resolve: bool = True, haproxy_config: str = HAPROXY_CONFIG_PATH, nginx_config: str = NGINX_CONFIG_PATH) -> None: self.root = Path(root) @@ -420,6 +507,22 @@ class Collector: self.proxy_enabled = proxy self.haproxy_config = haproxy_config self.nginx_config = nginx_config + # Name resolution: its own flag, because it is a different kind of work from reading a + # file — it leaves the host. It is meaningless without the proxy sections (they are the + # only source of names, decision 3), so `--no-proxy` removes it as well as `--no-resolve`. + self.resolve_enabled = resolve + self.resolve_max_names = RESOLVE_MAX_NAMES + self.resolve_cache_seconds = RESOLVE_CACHE_SECONDS + self.resolve_pass_budget_seconds = RESOLVE_PASS_BUDGET_SECONDS + self._resolver_lock = threading.Lock() # a pass may run while a handler serves + # Entries keyed by `resolver_key(name, port)`; the timestamp is the monotonic age of the + # pass that wrote them, `_resolver_pass_ts` the wall clock the section publishes (an entry + # keeps the time *it* was answered, so a served-from-cache entry cannot look current). + self._resolver_cache: dict[str, dict[str, Any]] = {} + self._resolver_cache_ts: float | None = None + self._resolver_pass_ts: float | None = None + self._resolver_errors: list[str] = [] + self._resolver_refreshing = False # at most one background refresh at a time self.smart_timeout_seconds = SMARTCTL_TIMEOUT_SECONDS self.smart_budget_seconds = SMARTCTL_BUDGET_SECONDS self.smart_cache_seconds = SMARTCTL_CACHE_SECONDS @@ -917,8 +1020,19 @@ class Collector: # -- proxy configs (nginx) ------------------------------------------------ # def nginx(self, errors: list[str]) -> dict[str, Any]: + """The `nginx` section on its own — `nginx_document()`'s first half.""" + return self.nginx_document(errors)[0] + + def nginx_document(self, errors: list[str]) -> tuple[dict[str, Any], str | None]: """Read and parse the host's nginx configs — never raises, never fails the snapshot. + Returns `(section, resolver directive)`. The second half is the only reason this method + is not the section itself: the `resolver` section prints nginx's own `resolver` line next + to `via: "libc"` so an operator can see the two vantages disagree, and taking it from the + text this pass already read is what keeps that free (no second file read per snapshot). + It deliberately does **not** go in the section: the `nginx` document stays byte-stable, + which is what makes this phase's additivity control a strict top-level key comparison. + The same three-way answer as `haproxy()`, for the other half of the gateway's ingress: `true` the main config was read and understood, `false` there is no config at this path (this host runs no nginx — one `nginx: ` line saying the file is **not found**), `null` @@ -944,10 +1058,10 @@ class Collector: section["available"] = False errors.append(f"nginx: {config} not found — nginx is not installed here, or its " "config lives elsewhere (--nginx-config); --no-proxy opts out") - return section + return section, None except (OSError, ValueError) as exc: errors.append(f"nginx: {config} is there but could not be read: {exc}") - return section + return section, None # The two extra return values are already spoken about in `errors`; the section itself # carries no counters, because "which files were read" is what `files` is for. @@ -959,11 +1073,11 @@ class Collector: f"parsed: {parsed['unparsed'][0]}") if parsed["error"] is not None: errors.append(f"nginx: {config} is there but could not be parsed: {parsed['error']}") - return section + return section, None if not parsed["recognised"]: errors.append(f"nginx: {config} holds no nginx context (http, stream, server, " "upstream, map) — not an nginx config") - return section + return section, None section["available"] = True # The main config first: `files` is the answer to "where do these servers come from", # and an operator comparing it with `nginx -t` should see the file nginx was started on. @@ -971,7 +1085,7 @@ class Collector: section["servers"] = parsed["servers"] section["upstreams"] = parsed["upstreams"] section["maps"] = parsed["maps"] - return section + return section, cast("str | None", parsed["resolver"]) def nginx_include_files(self, include_args: list[str], errors: list[str]) -> tuple[dict[str, str], int, list[str]]: @@ -1107,6 +1221,210 @@ class Collector: section["available"] = True if answered else None return section + # -- name resolution ------------------------------------------------------ # + def resolver(self, haproxy: Mapping[str, Any], nginx: Mapping[str, Any], + config_directive: str | None, errors: list[str]) -> dict[str, Any]: + """Resolve the names the two proxy configs declare. Never raises, never fails a snapshot. + + The point is that a route written as `server mc1 minecraft.reeselink.com:25565` ends at an + address, and the kernel's socket table speaks addresses: without this section the + dashboard can only count the connections whose target the config wrote *as an address*, + which on this ingress is 2 routes out of 33. The lookup happens here, on the box that + opens the connections, because that is the vantage the proxy itself resolves from; the + dashboard resolving the name through its own `resolv.conf` would answer a different + question (and `sources/gateway.py` is spied to fail if it ever tries). + + `available` is the family's three-way answer, with **one** `resolver:` line per non-`true` + arm: `true` a pass ran, `false` there was nothing to resolve because neither proxy section + was read (the line says *not found* when the files simply are not here, which is the + wording the dashboard reads), `null` a pass that could not run. Configs that were read and + name no target are `true` with `entries: []` — that is absence, not a fault. + + The cache is SMART's idiom, and for the SMART reason: the first pass runs inline so a + freshly started agent answers with real addresses, and every later expiry is refreshed on + a single-flight background thread while the request is served the previous result. A + resolver that has started black-holing must not cost the *system* poll its timeout over an + add-on that page never draws (AGENTS.md invariant 3/4). + """ + targets = target_names(haproxy, nginx) + if not targets and not _section_answered(haproxy) and not _section_answered(nginx): + # Nothing was read, so there is nothing to resolve. The two config sections have + # already said why in their own lines; this one says the resolver's own state, once. + line = resolver_not_found_line(haproxy, nginx) + errors.append(line) + return {"available": False, "via": RESOLVE_VIA, "config_directive": config_directive, + "ts": None, "entries": [], "errors": [line]} + now = time.monotonic() + with self._resolver_lock: + stamp = self._resolver_cache_ts + if stamp is not None: + if now - stamp >= self.resolve_cache_seconds and not self._resolver_refreshing: + self._resolver_refreshing = True + threading.Thread(target=self._refresh_resolver, args=(list(targets),), + name="resolver-refresh", daemon=True).start() + # Served from cache either way — the fresh-cache case and the "a refresh is + # already running" case — because waiting here is what this cache exists to avoid. + return self._resolver_section(self._resolver_cache, self._resolver_pass_ts, + self._resolver_errors, config_directive, errors) + entries, pass_errors = self.resolver_pass(targets) + return self._resolver_section(entries, self._resolver_pass_ts, pass_errors, + config_directive, errors) + + def _resolver_section(self, entries: Mapping[str, dict[str, Any]], pass_ts: float | None, + pass_errors: Sequence[str], config_directive: str | None, + errors: list[str]) -> dict[str, Any]: + """The section out of one pass's (or one cache's) result — the shape, in one place. + + The lines go to **both** readers, because there are two: the snapshot's shared `errors` + list, which is what `collectors.ADDON_ERROR_PREFIXES` filters to decide the *system* page + is not degraded, and the section's own `errors`, which is the gateway page's sentence + beside the entries they belong to. Same strings, one append each. + """ + section: dict[str, Any] = {"available": True, "via": RESOLVE_VIA, + "config_directive": config_directive, "ts": pass_ts, + "entries": [dict(entry) for entry in entries.values()], + "errors": []} + for line in pass_errors: + errors.append(line) + section["errors"].append(line) + return section + + def _refresh_resolver(self, targets: list[tuple[str, int | None]]) -> None: + """The expired-cache pass, off the request path (see `resolver()`). + + A failure keeps the previous entries **and** re-stamps the cache: the snapshot keeps + answering with the last truth it had, and the retry comes on the next expiry instead of + every poll against a resolver that just failed. The line is replaced rather than stacked, + because a refresh that fails every `resolve_cache_seconds` for a week is one fact, not + 2000 — the repeated-failure rule the poller applies to its own events. + """ + try: + self.resolver_pass(targets) + except Exception as exc: # noqa: BLE001 - the thread must always clear its own flag + with self._resolver_lock: + line = (f"resolver: background refresh failed: {type(exc).__name__}: {exc}") + self._resolver_errors = [x for x in self._resolver_errors + if not x.startswith("resolver: background refresh failed")] + self._resolver_errors = [*self._resolver_errors, line] + self._resolver_cache_ts = time.monotonic() + finally: + with self._resolver_lock: + self._resolver_refreshing = False + + def resolver_pass(self, targets: Sequence[tuple[str, int | None]] + ) -> tuple[dict[str, dict[str, Any]], list[str]]: + """One bounded libc lookup per declared target. Never raises out of this function. + + Entries come back keyed by `resolver_key`, in target order, merged over the previous + cache so a pass that stopped at the budget keeps the entries it did not reach (each with + **its own** `ts`, so a stale address is visibly stale rather than freshly wrong). A name + that never got looked up at all and has no previous answer simply has no entry: the line + says how many were reached, and inventing an `error` status for a lookup that was never + attempted would be rule 2 with a status field on it. + """ + errors: list[str] = [] + fresh: dict[str, dict[str, Any]] = {} + deadline = time.monotonic() + self.resolve_pass_budget_seconds + need_lookup = [pair for pair in targets if address_family(pair[0]) is None] + over_bound = len(need_lookup) - self.resolve_max_names + looked_up = 0 + stopped = False + for name, port in targets: + # Literals first, and never counted against either bound: reporting one is a fact and + # dialling it would be a round trip to be told what the config already wrote, so a + # truncated pass must not be the reason an address the config spells is missing. + if address_family(name) is None: + continue + fresh[resolver_key(name, port)] = {"name": name, "port": port, "status": "literal", + "addresses": [name], "ts": time.time()} + for name, port in need_lookup: + if looked_up >= self.resolve_max_names: + continue # one line for the whole excess, after the loop + fresh[resolver_key(name, port)] = self._resolve_target(name, port) + looked_up += 1 + if time.monotonic() >= deadline: + stopped = True + break + if stopped: + errors.append(f"resolver: pass budget {self.resolve_pass_budget_seconds:.1f}s " + f"exhausted, {looked_up} of {len(need_lookup)} targets looked up") + if over_bound > 0: + errors.append(f"resolver: {over_bound} of {len(need_lookup)} targets are beyond the " + f"{self.resolve_max_names}-name bound and were not looked up — the " + "section covers what it could resolve") + entries = self._store_resolver_cache(targets, fresh, errors) + return entries, errors + + def _store_resolver_cache(self, targets: Sequence[tuple[str, int | None]], + fresh: Mapping[str, dict[str, Any]], errors: Sequence[str] + ) -> dict[str, dict[str, Any]]: + """Merge this pass into the cache in target order, and stamp it. Returns the entries. + + Order is not cosmetics: `entries` is a list a page renders, so the same config must draw + in the same order on every poll. Names the config dropped leave the cache with it — they + are no longer declared targets, not targets that stopped resolving. + """ + merged: dict[str, dict[str, Any]] = {} + with self._resolver_lock: + previous = self._resolver_cache + for name, port in targets: + key = resolver_key(name, port) + entry = fresh.get(key) or previous.get(key) + if entry is not None: + merged[key] = entry + self._resolver_cache = merged + self._resolver_cache_ts = time.monotonic() + self._resolver_pass_ts = time.time() + # The pass's own lines replace the previous ones: a cache replay has to say the same + # thing the pass that produced it said (SMART's cache does the same), and a line from + # an older pass — a budget hit that has since stopped happening — would be a stale + # complaint about a reading that is now complete. + self._resolver_errors = list(errors) + return dict(merged) + + def _resolve_target(self, name: str, port: int | None) -> dict[str, Any]: + """One `getaddrinfo`, and the entry that says what it answered. + + Both families are kept (`getaddrinfo` answers v4 and v6 when the host has both) because + the gateway connects over whichever one routing picks and the socket table speaks either. + A `gaierror` naming no-name/no-data is `nxdomain` — the name is answered and has no + address — and anything else is `error`, because "the server did not answer" and "the name + has no record" are different things to fix. Both carry `addresses: []`, never a stale + guess and never a `0`. The `ts` is taken here rather than passed in, so an entry says when + **it** was answered even in a pass that then stopped at the budget. + + The keyword is `type=`, and that is load-bearing in a way only a real libc can teach: + `proto` is the **protocol number** (`IPPROTO_TCP` = 6) while `SOCK_STREAM` = 1 is a socket + *type*, so `proto=SOCK_STREAM` asks libc for a stream socket over ICMP and it answers + `EAI_SERVICE` for **every** name — a resolver that reports `error` on a healthy gateway, + with every route on the page uncounted, in a way no stubbed test can see because a stub + takes any keyword. What we want is libc's answer to "where do I connect a TCP socket", + which is exactly haproxy's `init-addr last,libc,none` question. Pinned by + `tests/test_agent.py` with no stub in sight, against an address libc parses by itself. + """ + try: + infos = socket.getaddrinfo(name, port, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + code = exc.errno if isinstance(exc.errno, int) else 0 + return {"name": name, "port": port, + "status": "nxdomain" if code in RESOLVE_NXDOMAIN_CODES else "error", + "addresses": [], "ts": time.time()} + except OSError as exc: # a resolver that fails as an I/O error, not as a DNS one + return {"name": name, "port": port, "status": "error", "addresses": [], + "detail": str(exc), "ts": time.time()} + except Exception as exc: # noqa: BLE001 - the promise is that this function never raises + # `getaddrinfo` is the one call in this agent that leaves the host, and whatever it + # does — a platform-specific `ValueError` for a name the C library cannot take, a + # stub in a test — the answer is one entry that says "error" beside the targets that + # answered, never a half-snapshot and never the *system* page going red over an + # add-on it does not draw (AGENTS.md invariant 4). + return {"name": name, "port": port, "status": "error", "addresses": [], + "detail": f"{type(exc).__name__}: {exc}", "ts": time.time()} + addresses = sort_resolver_addresses(str(info[4][0]) for info in infos) + # An empty answer list is libc saying the name resolved to nothing: no-data, not a fault. + return {"name": name, "port": port, "status": "ok" if addresses else "nxdomain", + "addresses": addresses, "ts": time.time()} + # -- network ------------------------------------------------------------- # def network(self) -> dict[str, Any]: interfaces: list[dict[str, Any]] = [] @@ -1479,11 +1797,24 @@ class Collector: document["haproxy"] = self._safe(lambda: self.haproxy(errors), "haproxy", errors) \ or {"available": None, "config": self.haproxy_config, "frontends": [], "backends": []} - document["nginx"] = self._safe(lambda: self.nginx(errors), "nginx", errors) \ - or {"available": None, "config": self.nginx_config, "files": [], - "servers": [], "upstreams": [], "maps": []} + nginx_pair = self._safe(lambda: self.nginx_document(errors), "nginx", errors) + nginx_section, nginx_resolver = nginx_pair or ( + {"available": None, "config": self.nginx_config, "files": [], + "servers": [], "upstreams": [], "maps": []}, None) + document["nginx"] = nginx_section document["sockets"] = self._safe(lambda: self.sockets(errors), "sockets", errors) \ or {"available": None, "listens": [], "established": []} + # The resolver rides with the family and is removed by both flags: `--no-resolve` is + # the operator turning off the one add-on that leaves the host, and `--no-proxy` means + # there are no declared names to resolve in the first place (decision 3). Absent, not + # empty — "not collected" stays a different fact from "collected and empty". + if self.resolve_enabled: + document["resolver"] = self._safe( + lambda: self.resolver(cast("Mapping[str, Any]", document["haproxy"]), + cast("Mapping[str, Any]", nginx_section), + nginx_resolver, errors), "resolver", errors) \ + or {"available": None, "via": RESOLVE_VIA, "config_directive": None, + "ts": None, "entries": [], "errors": []} document["errors"] = errors return document @@ -2038,13 +2369,15 @@ def parse_nginx_config(main_text: str, `{servers: [{listen: [{addr, port}], server_names: [str], proxy_passes: [{location, scheme, target, variable}]}], upstreams: [{name, servers: [{host, port}]}], maps: [{variable, entries: [{key, target}]}], files: [str], includes: [str], unparsed: - [str], recognised: bool, error: str|null}`. + [str], recognised: bool, error: str|null, resolver: str|null}`. - Three of those keys are the ones `Collector.nginx` does not serve, the way + Four of those keys are the ones `Collector.nginx` does not serve, the way `parse_haproxy_config`'s `recognised` is not: `error` is why the main text could not be - parsed, `unparsed` names an included document that could not, and `includes` is the raw + parsed, `unparsed` names an included document that could not, `includes` is the raw argument list (a caller that has no filesystem — every unit test — still gets to see what a - config *would* have read). `files` is the included documents that were merged, in order; + config *would* have read), and `resolver` is the config's own `resolver` directive's + arguments, which the `resolver` section serves instead (`Collector.nginx_document`). + `files` is the included documents that were merged, in order; the collector puts the main config in front of it to make the section's `files`. A document that fails to parse is dropped and named in `unparsed`; it never drops the others @@ -2053,13 +2386,18 @@ def parse_nginx_config(main_text: str, `error` set, not an exception on a snapshot path. """ out: dict[str, Any] = {"servers": [], "upstreams": [], "maps": [], "files": [], - "includes": [], "unparsed": [], "recognised": False, "error": None} + "includes": [], "unparsed": [], "recognised": False, "error": None, + "resolver": None} try: main = _nginx_statements(main_text) except ValueError as exc: out["error"] = str(exc) return out out["recognised"] = any(statement[1] in NGINX_CONTEXTS for statement in main) + # The config's own `resolver` line, for the `resolver` section's `config_directive`. It is + # found here rather than served here on purpose: the `nginx` section stays byte-stable, so + # this phase's additivity control is a strict top-level key comparison with no exception list. + out["resolver"] = _find_resolver_statement(main) _collect_nginx(main, out, root=True) documents = included or {} merged: list[str] = [] @@ -2075,6 +2413,8 @@ def parse_nginx_config(main_text: str, out["unparsed"].append(path) continue merged.append(path) + if out["resolver"] is None: + out["resolver"] = _find_resolver_statement(statements) # root=False: an include line inside an included file is a second level, and this # pass is one level deep by design (see NGINX_INCLUDE_MAX_FILES). _collect_nginx(statements, out, root=False) @@ -2197,6 +2537,165 @@ def socket_truncation_line(paths: list[str]) -> str: "read takes — the sockets past the bound are not in this section") +# --------------------------------------------------------------------------- # +# name resolution (the additive `resolver` section) +# --------------------------------------------------------------------------- # + +def address_family(value: str) -> str | None: + """`v4`, `v6`, or None — is this text already an address? + + `socket.inet_pton` and nothing else: it is the same test the `sockets` section's decoder + uses, it needs no second module (the agent's import footprint is pinned by + `tests/test_agent.py`), and it is the rule that lets an IP literal in a config be reported + as what it is instead of dialled. An interface scope (`fe80::1%eth0`) is not a pure address, + so it comes back None and travels through the resolver like any other name — libc decides + what a scoped literal means, and it is not this file's job to invent an answer. + """ + for family, label in ((socket.AF_INET, "v4"), (socket.AF_INET6, "v6")): + try: + socket.inet_pton(family, value) + except (OSError, ValueError): + continue + return label + return None + + +def resolver_key(name: str, port: int | None) -> str: + """One declared target, as this module's cache key. + + The port belongs in the key because a name on two ports is two targets: this fleet routes + `minecraft.reeselink.com` to 25565 and to 25566, two services, and one entry for both would + be one answer to a question nobody asked. + """ + return f"{name}:{port}" + + +def _resolver_sort_key(value: str) -> tuple[int, bytes]: + """v4 ascending then v6 ascending, with anything undecodable last and stable. + + The order matters for the payload rather than for the lookup: `getaddrinfo` returns whatever + order this host's resolver and `/etc/gai.conf` feel like today, so an unsorted set makes two + polls of a healthy gateway differ for no reason the operator could act on. + """ + for family, rank in ((socket.AF_INET, 0), (socket.AF_INET6, 1)): + try: + return (rank, socket.inet_pton(family, value)) + except (OSError, ValueError): + continue + return (2, value.encode("utf-8", errors="replace")) + + +def sort_resolver_addresses(values: Iterable[str]) -> list[str]: + """Unique addresses in the payload's order: family first, then the address itself.""" + return sorted(set(values), key=_resolver_sort_key) + + +def target_names(haproxy: Mapping[str, Any] | None, + nginx: Mapping[str, Any] | None) -> list[tuple[str, int | None]]: + """Every `(name, port)` the two **parsed** proxy configs write as a target, no duplicates. + + This is the whole allow-list for the resolver (decision 3 of the phase): haproxy's + `backend`/`server` targets first (config order), then nginx's `upstream` members, its + server-block `proxy_pass` targets, and its `map` entry targets, deduplicated keeping + first-seen order. Nothing else is ever in here — not a name from `/proc/net/tcp`, not a + client address, not anything a request supplied — because resolving a name nobody configured + is a round trip that produces a fact about the internet instead of a fact about this + gateway, and `tests/test_agent.py` pins that the lookups are exactly this list. + + A `proxy_pass $var` contributes nothing (the map decides, and the map's own entries are + here); an entry whose target is not an address at all is `target: null` upstream and is + skipped here for the same reason the `nginx` section refused to name it. + """ + targets: list[tuple[str, int | None]] = [] + seen: set[tuple[str, int | None]] = set() + + def add(host: Any, port: Any) -> None: + if not isinstance(host, str) or not host: + return + pair = (host, port if isinstance(port, int) and not isinstance(port, bool) else None) + if pair in seen: + return + seen.add(pair) + targets.append(pair) + + for block in _mapping_list(haproxy, "backends"): + for server in _mapping_list(block, "servers"): + add(server.get("target"), server.get("port")) + for upstream in _mapping_list(nginx, "upstreams"): + for member in _mapping_list(upstream, "servers"): + add(member.get("host"), member.get("port")) + for server in _mapping_list(nginx, "servers"): + for hop in _mapping_list(server, "proxy_passes"): + target = hop.get("target") + if isinstance(target, Mapping): + add(target.get("host"), target.get("port")) + for block in _mapping_list(nginx, "maps"): + for entry in _mapping_list(block, "entries"): + target = entry.get("target") + if isinstance(target, Mapping): + add(target.get("host"), target.get("port")) + return targets + + +def _mapping_list(section: Any, key: str) -> list[Mapping[str, Any]]: + """One list of mappings out of a section that may be absent, null, or the wrong shape. + + The section is built by this file, so anything else means a caller handed this function + something it did not build — and the honest answer to that is an empty list, not a + `TypeError` on a snapshot path. + """ + if not isinstance(section, Mapping): + return [] + rows = section.get(key) + if not isinstance(rows, list): + return [] + return [row for row in rows if isinstance(row, Mapping)] + + +def _section_answered(section: Any) -> bool: + """Did this proxy section report that it read its config? `available` is the section's own answer.""" + return isinstance(section, Mapping) and section.get("available") is True + + +def resolver_not_found_line(haproxy: Any, nginx: Any) -> str: + """The one `resolver:` line for the arm where there was nothing to resolve. + + It says *not found* when a config is simply absent — the pinned cross-module wording the + dashboard reads to answer "is this proxy installed here?" (the `haproxy:`, `nginx:` and + `smartctl:` lines share it) — and *unreadable* when the file is there and this process could + not read it, because those are two different operator actions. One line either way: the two + sections have already reported their own reasons, and a third restatement per snapshot is the + ink the empty-state rules exist to avoid. + """ + reasons = [] + for name, section in (("haproxy", haproxy), ("nginx", nginx)): + available = section.get("available") if isinstance(section, Mapping) else None + reasons.append(f"{name} not found" if available is False else f"{name} unreadable") + return (f"resolver: nothing to resolve — {' and '.join(reasons)}; --no-resolve opts out") + + +def _find_resolver_statement(statements: _NginxStatements) -> str | None: + """Depth-first search for one `resolver` directive, in document order. + + A `map` body is not searched: inside one, a bare word is a **key**, so a map with a + `resolver` key (a header name, a hostname someone routed on) would otherwise be reported as + this box's DNS server. + """ + for statement in statements: + kind, name, args = statement[0], statement[1], statement[2] + if kind == "d": + if name == "resolver" and args: + return " ".join(str(arg) for arg in args) + continue + if name == "map": + continue + found = _find_resolver_statement(cast("_NginxStatements", statement[3])) + if found is not None: + return found + return None + + + # --------------------------------------------------------------------------- # # TLS bundle: what the dashboard signs, what the installer places # --------------------------------------------------------------------------- # @@ -2663,8 +3162,12 @@ def main(argv: list[str] | None = None) -> int: help="skip the smartctl pass entirely (disks report smart: null, " "no error entries)") parser.add_argument("--no-proxy", action="store_true", - help="skip the proxy add-on entirely (no `haproxy`, `nginx` or " - "`sockets` key, no error entries)") + help="skip the proxy add-on entirely (no `haproxy`, `nginx`, `sockets` " + "or `resolver` key, no error entries)") + parser.add_argument("--no-resolve", action="store_true", + help="skip the resolver pass (no `resolver` key, no error entries): the " + "two config sections and the socket table are still read, only the " + "name lookups are not") parser.add_argument("--haproxy-config", default=HAPROXY_CONFIG_PATH, help="haproxy config to read; a missing file is the documented " "degradation (available=false, one error entry), not a failure") @@ -2679,7 +3182,8 @@ def main(argv: list[str] | None = None) -> int: collector = Collector(args.root, top_processes=args.top, net_all=args.net_all, smartctl=args.smartctl, smart=not args.no_smart, - proxy=not args.no_proxy, haproxy_config=args.haproxy_config, + proxy=not args.no_proxy, resolve=not args.no_resolve, + haproxy_config=args.haproxy_config, nginx_config=args.nginx_config) if args.once: print(json.dumps({"ok": True, **collector.snapshot()}, indent=2, default=str)) diff --git a/pyproject.toml b/pyproject.toml index 52299b6..b432744 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ dev = [ "pytest>=9.1.1", "pytest-cov>=7.1.0", - "pytest-asyncio>=1.4.0", + "pytest-asyncio>=1.4.1", "ruff>=0.17.1", "pyright>=1.1.414", "debugpy>=1.8.22", diff --git a/scripts/capture-gateway-config.sh b/scripts/capture-gateway-config.sh index 084cddf..ed27b8e 100755 --- a/scripts/capture-gateway-config.sh +++ b/scripts/capture-gateway-config.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Re-record the gateway's proxy configs and kernel socket tables into tests/fixtures/. +# Re-record the gateway's proxy configs, kernel socket tables and libc resolutions into +# tests/fixtures/. # # ./scripts/capture-gateway-config.sh # dials the `gateway-ops` ssh alias # GATEWAY_SSH=lab-gw ./scripts/capture-gateway-config.sh @@ -12,13 +13,24 @@ # gateway_nginx_stream_d_.conf cat /etc/nginx/stream.d/*.conf # gateway_proc_net_tcp.txt cat /proc/net/tcp # gateway_proc_net_tcp6.txt cat /proc/net/tcp6 +# gateway_resolver.json getent ahostsv4/ahostsv6, once per declared target # -# These documents are the ground truth behind `agent/host_agent.py`'s `haproxy`, `nginx` and -# `sockets` sections (phase 28) the way `tests/fixtures/nut_.vars` pins the UPS wire and -# `smartctl_driveripper_.txt` pins the drives. The include globs are read **out of the -# captured main config**, so a gateway that moves its vhosts to a third directory grows a third -# fixture on the next run instead of keeping a fixture set that quietly stopped being the tree -# the agent merges. +# These documents are the ground truth behind `agent/host_agent.py`'s `haproxy`, `nginx`, +# `sockets` and `resolver` sections (phases 28 and 32) the way `tests/fixtures/nut_.vars` +# pins the UPS wire and `smartctl_driveripper_.txt` pins the drives. The include globs are +# read **out of the captured main config**, so a gateway that moves its vhosts to a third +# directory grows a third fixture on the next run instead of keeping a fixture set that quietly +# stopped being the tree the agent merges. +# +# The resolver document is the one fixture whose *name list* comes from the repo rather than from +# the remote: the targets are `agent.host_agent.target_names()` over the configs captured above, +# so the capture can never ask for more names than the agent will resolve, and a fixture naming a +# target the parser no longer derives is a fixture that fails a test. Each name is then read on +# the gateway itself — `getent ahostsv4 ` and `getent ahostsv6 `, the same libc +# resolution `via: "libc"` claims and the one haproxy's `init-addr last,libc,none` performs — so +# the addresses in the fixture are the answers the *routing* box gives, not this laptop's. A +# `config_directive` read out of the already captured nginx text costs no round trip, so the +# fixture's provenance line and the agent's own answer come from the same bytes. # # Four rules, three of them `scripts/capture-fixtures.sh`'s: # @@ -27,7 +39,12 @@ # The header is comment-shaped because `#` is a comment in haproxy's config and in nginx's, # and a `/proc` line whose 4th field is not a kernel state is skipped by the socket parser — # provenance that costs no parser anything, which `tests/test_agent.py` pins by parsing the -# fixtures exactly as they sit on disk. +# fixtures exactly as they sit on disk. `gateway_resolver.json` is the one fixture that cannot +# carry it: it is JSON the tests `json.load`, and a `#` line is not JSON, so its provenance is +# this block plus the `via`/`config_directive`/`ts` keys inside it (the same reason +# `slots_gufo.json` and `agent_.json` carry none). Its `ts` is a fixed constant, not the +# capture's clock: the value is a *shape* (an entry carries the time it was answered) and a +# live timestamp would make every re-run diff for a difference no test can read. # * **The configs are not rewritten at all.** A domain is not an identifier to hide here: a # `server_name` and the `proxy_pass` target beside it *are* the ingress map the gateway page # exists to draw, and a scrubbed fixture is a fixture that tests a config this gateway does @@ -55,7 +72,20 @@ # * **A bad read never replaces a good fixture.** Empty output, an ssh failure or a permission # denial prints one line naming the file and the reason, leaves whatever is on disk alone, # and exits 1 — the `slots_gufo.json` rule, so a laptop with no route to the gateway cannot -# turn six verbatim captures into six empty files. +# turn seven verbatim captures into seven empty files. The resolver document keeps the rule with +# one refinement: **one** name answering nothing is a fact and is recorded as +# `status: "nxdomain"`, while a *broken vantage* is not — an ssh transport failure, a missing +# `getent`, or not one declared name answering an address leaves the fixture untouched and +# exits 1, because a document of nothing-but-failures would turn every live route on the page +# into a permanent dash and every test of the join into a test of nothing. +# +# Two documents, one scrub. The socket tables and the resolver document each rewrite their +# globally routable addresses through the **same** `Replacer` (one rule, two shapes, below), in +# first-seen order per document. That order is per-document, so if a name ever resolves to a +# globally routable address its replacement in `gateway_resolver.json` need not be the number the +# socket tables gave that same address — which would silently unjoin `tests/test_gateway_source.py`'s +# captured routes. `test_the_captured_resolver_document_needed_no_scrub_at_all` is the pin on that +# hazard: it holds today because every declared target here answers a private address. # # No sudo, deliberately: haproxy's config is 0644, nginx's tree is 0644 and `/proc/net/tcp{,6}` # is world-readable, which is exactly what `dashboards-agent` reads as itself. A capture that @@ -80,12 +110,15 @@ usage() { cat <<'TXT' Usage: ./scripts/capture-gateway-config.sh [-h] -Re-records the gateway's proxy configs and kernel socket tables into tests/fixtures/: -/etc/haproxy/haproxy.cfg, /etc/nginx/nginx.conf plus the documents its own glob includes -name, and /proc/net/tcp{,6}. Every read goes over the `$GATEWAY_SSH` ssh alias (default -`gateway-ops`) of a world-readable path, and a read that fails leaves the existing fixture -alone and exits 1. The configs land unedited; in the socket tables a globally routable -address becomes a documentation one and nothing else moves. See the block comment. +Re-records the gateway's proxy configs, kernel socket tables and libc resolutions into +tests/fixtures/: /etc/haproxy/haproxy.cfg, /etc/nginx/nginx.conf plus the documents its own +glob includes name, /proc/net/tcp{,6}, and one `getent ahostsv4`/`getent ahostsv6` read on +the gateway per target name `agent.host_agent.target_names()` derives from those configs +(written as `gateway_resolver.json`, the agent's own `resolver` section shape). Every read +goes over the `$GATEWAY_SSH` ssh alias (default `gateway-ops`) of a world-readable path or +command, and a read that fails leaves the existing fixture alone and exits 1. The configs +land unedited; in the socket tables and the resolver document a globally routable address +becomes a documentation one and nothing else moves. See the block comment. TXT } @@ -93,26 +126,60 @@ case "${1:-}" in -h|--help) usage; exit 0 ;; esac -# The socket tables' one edit, and the only place in this script that touches a captured byte. -# Kept as a here-doc in the house style (capture-fixtures.sh embeds two); it is pinned by -# running this script against a fake remote in tests/test_deploy_scripts.py. -SOCKET_IDENTIFIERS_OUT=$(cat <<'PY' -"""Move the globally-routable addresses in a /proc/net/tcp{,6} table to documentation space. +# The interpreter for the two steps that import the repo's own parser. +# `uv run python` is the same interpreter the tests use, which is the point: a capture whose name +# list came from a different parser than the one the suite pins is a fixture that agrees with +# nothing. A copy of this script with no project beside it (a test sandbox) has nothing for uv to +# resolve, so it falls back to `python3` — `agent/host_agent.py` is stdlib-only, and PYTHONPATH +# below is what makes it importable either way. +export PYTHONPATH="$ROOT${PYTHONPATH:+:$PYTHONPATH}" +run_py() { + if command -v uv >/dev/null 2>&1 && [ -f "$ROOT/pyproject.toml" ]; then + (cd "$ROOT" && uv run python "$@") + else + python3 "$@" + fi +} -Every column, every port and every byte of spacing is the kernel's. What changes is an address -`ipaddress` calls global, which becomes a documentation address in first-seen order (RFC 5737 -for v4, RFC 3849 for v6) re-encoded through the kernel's own per-32-bit-word little-endian -spelling — so the replacement is exactly as wide as the original and an IPv4-mapped v6 row stays -an IPv4-mapped v6 row. AGENTS.md invariant 6 is the reason: this repo's capture rule is that -identifiers never go in tracked files, and a global IPv6 on Linux is EUI-64 (it encodes the -NIC's MAC) while a visitor's public address belongs to the visitor. Loopback, private, -link-local, multicast, wildcard and already-documentation addresses are returned untouched, -which is what makes re-running this over its own output change nothing. +# The identifier rule (AGENTS.md invariant 6) and the resolver document, in one module with three +# subcommands, and the only place in this script that touches a captured byte. One `Replacer` for +# both documents is the point: two copies of "globally routable becomes documentation space, in +# first-seen order" is two rules. Kept as a here-doc in the house style (capture-fixtures.sh +# embeds two); it is pinned by running this script against a fake remote in +# tests/test_deploy_scripts.py. +SOCKET_IDENTIFIERS_OUT=$(cat <<'PY' +"""The gateway capture's identifier rule and resolver document. + + sockets a /proc/net/tcp{,6} table on stdout, scrubbed + plan every target the captured configs declare + resolver the resolver document, or a refusal + +**The scrub** (sockets, resolver). Every column, every port and every byte of spacing is the +kernel's or `getent`'s. What changes is an address `ipaddress` calls global, which becomes a +documentation address in first-seen order (RFC 5737 for v4, RFC 3849 for v6); in a socket table +it is re-encoded through the kernel's own per-32-bit-word little-endian spelling, so the +replacement is exactly as wide as the original and an IPv4-mapped v6 row stays an IPv4-mapped v6 +row. AGENTS.md invariant 6 is the reason: this repo's capture rule is that identifiers never go in +tracked files, and a global IPv6 on Linux is EUI-64 (it encodes the NIC's MAC) while a visitor's +public address belongs to the visitor — and a name that resolves to one is the same exposure by +another route. Loopback, private, link-local, multicast, wildcard and already-documentation +addresses are returned untouched, which is what makes re-running this over its own output change +nothing. + +**The plan** is derived from the repo's own parser and never by re-implementing one — +`from agent.host_agent import target_names` — so the capture can never out-ask the agent. The +name list, the order, the literal/name split and the `resolver` directive all come out of the +agent's own reading of the config fixtures this run captured. """ import ipaddress +import json +import os +import pathlib import re import sys +import tempfile +from typing import Any DOC_V4 = int(ipaddress.IPv4Address("203.0.113.0")) DOC_V6 = int(ipaddress.IPv6Address("2001:db8::")) @@ -184,10 +251,37 @@ class Replacer: out.append(text[last:]) return "".join(out) + def address(self, value: str) -> str: + """One dotted/colon address in its own spelling: scrub it, keep the spelling. -def main(argv: list[str]) -> int: + The resolver document's shape, where the socket table's is hex words. Same rule and the + same mapping: a globally routable address becomes a documentation one and everything else + — private, loopback, link-local, wildcard, already-documentation — is returned as it came. + A `::ffff:`-mapped address keeps its mapped spelling, because the spelling is a fact about + the answer, not an identifier. + """ + try: + candidate = ipaddress.ip_address(value) + except ValueError: + return value + mapped = getattr(candidate, "ipv4_mapped", None) + subject = mapped if mapped is not None else candidate + if not subject.is_global: + return value + key = str(subject) + if key not in self.mapping: + self.mapping[key] = self.documentation(subject.version) + replacement = self.mapping[key] + self.replacements += 1 + if mapped is not None: + return f"::ffff:{replacement}" + return replacement + + +def sockets(argv: list[str]) -> int: + """`sockets
    ` — the table on stdout with its two address columns scrubbed.""" replacer = Replacer() - with open(argv[1], encoding="utf-8", errors="replace") as handle: + with open(argv[2], encoding="utf-8", errors="replace") as handle: for line in handle: sys.stdout.write(replacer.line(line)) if replacer.replacements: @@ -197,6 +291,179 @@ def main(argv: list[str]) -> int: return 0 +def plan(argv: list[str]) -> int: + """`plan ` — every declared target, in the agent's own order. + + Reads the config fixtures this run just captured, from a tree shaped the way the agent's + `--root` shapes it, so the plan is the answer to "what would the agent on this box ask?" and + not a second opinion about what a config declares. The first line is the nginx `resolver` + directive read out of the same bytes the agent reads them from; the rest is one tab-separated + line per target: `target`, `name` or `literal`, the name, the port (empty when the config + wrote none). + """ + from agent.host_agent import Collector, address_family, target_names + + fixtures, out = pathlib.Path(argv[2]), pathlib.Path(argv[3]) + tree = pathlib.Path(tempfile.mkdtemp(prefix="gateway-plan-")) + for rel, fixture in (("etc/haproxy/haproxy.cfg", "gateway_haproxy.cfg"), + ("etc/nginx/nginx.conf", "gateway_nginx_main.conf")): + if (fixtures / fixture).is_file(): + body = "".join(line for line in (fixtures / fixture).read_text().splitlines(keepends=True) + if not line.startswith("#")) + target = tree / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + else: + print(f"plan: {fixture} is not on disk — capture the configs before the resolver", + file=sys.stderr) + return 1 + for prefix, rel in (("gateway_nginx_conf_d_", "etc/nginx/conf.d"), + ("gateway_nginx_stream_d_", "etc/nginx/stream.d")): + for found in sorted(fixtures.glob(f"{prefix}*.conf")): + target = tree / rel / f"{found.name[len(prefix):]}" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("".join(line for line in found.read_text().splitlines(keepends=True) + if not line.startswith("#"))) + + collector = Collector(str(tree), smartctl="/nonexistent/smartctl") + errors: list[str] = [] + haproxy = collector.haproxy(errors) + nginx, directive = collector.nginx_document(errors) + targets = target_names(haproxy, nginx) + if not targets: + print("plan: the captured configs declare no target, so there is nothing to resolve — " + "is the config capture intact?", file=sys.stderr) + return 1 + # The name is the one part of the plan that goes back out over ssh, inside a remote command + # string. A config target is written by the operator, so this is not a hostile input today — + # but a fixture step that interpolated arbitrary config text into a shell command would be a + # remote-command hole waiting for somebody to write a semicolon in a `server_name`, and the + # honest answer to a name the script cannot safely ask about is to refuse the run, not to ask + # it anyway. Names, dots, dashes, underscores and scoped literals only; no whitespace, quote, + # semicolon, dollar or backtick. + unsafe = [name for name, _port in targets + if not re.fullmatch(r"[A-Za-z0-9._:.-]{1,253}", name)] + if unsafe: + print(f"plan: {len(unsafe)} target name(s) the script cannot ask about safely, starting " + f"with {unsafe[0]!r} — nothing was read and nothing was written", file=sys.stderr) + return 1 + with out.open("w", encoding="utf-8") as handle: + handle.write(f"directive\t{directive if directive is not None else ''}\n") + for name, port in targets: + # The agent's own test, imported rather than restated: a target that is already an + # address is `status: "literal"` and costs no read on the remote, exactly as the + # agent's pass skips its own lookup. + kind = "literal" if address_family(name) else "name" + handle.write(f"target\t{kind}\t{name}\t{'' if port is None else port}\n") + return 0 + + +def resolver(argv: list[str]) -> int: + """`resolver ` — the document, or a refusal. + + Written to the fixture path only when at least one declared name answered an address; every + name failing means the vantage is broken (a resolver down, a firewall between, a `getent` that + answers nothing), and a fixture of failures would make every live route on the page a + permanent dash and every test of the join a test of nothing. One name failing among successes + is a fact, and is kept as `status: "nxdomain"`. + """ + # The shape's single source of truth: the same `via` and the same address order the agent + # publishes, imported rather than remembered, so the fixture is the agent's document. + from agent.host_agent import RESOLVE_VIA, sort_resolver_addresses + + plan_path = pathlib.Path(argv[2]) + raw = argv[3] + out = pathlib.Path(argv[4]) + ts = float(argv[5]) + replacer = Replacer() + directive: str | None = None + entries: list[dict[str, Any]] = [] + names = 0 + answered = 0 + index = 0 + with open(plan_path, encoding="utf-8") as handle: + for line in handle: + fields = line.rstrip("\n").split("\t") + if fields[0] == "directive": + directive = fields[1] if len(fields) > 1 and fields[1] else None + continue + if fields[0] != "target": + continue + kind, name = fields[1], fields[2] + port: int | None = int(fields[3]) if len(fields) > 3 and fields[3] else None + if kind == "literal": + found, is_literal = [name], True + else: + names += 1 + is_literal = False + # `getent ahostsv6` answers `::ffff:a.b.c.d` for a name with no AAAA record — the + # spelling glibc's AF_INET6 view synthesises, and the one `getaddrinfo` the agent + # performs never publishes it. Keeping it would claim an address the agent does + # not, so a mapped answer whose v4 form is in the same set is dropped. A mapped + # answer with no plain one beside it survives, because then it is the answer. + found = [field[0] for path in (f"{index}.v4", f"{index}.v6") + for field in (line.split() for line in read_raw(raw, path).splitlines()) + if field] + plain = {f"::ffff:{address}" for address in found + if not address.startswith("::ffff:")} + found = [address for address in found if address not in plain] + index += 1 + # Scrub first, then sort: `addresses` is the published list, so the agent's ordering + # rule belongs to what the fixture prints and not to what the resolver answered. + addresses = sort_resolver_addresses(replacer.address(a) for a in found) + if is_literal: + status = "literal" + else: + status = "ok" if addresses else "nxdomain" + if addresses: + answered += 1 + entries.append({"name": name, "port": port, "status": status, + "addresses": addresses, "ts": ts}) + + if names and not answered: + print(f"resolver: none of the {names} declared name(s) answered an address — the vantage " + "is broken, not the fixture; nothing was written", file=sys.stderr) + return 1 + + document = {"available": True, "via": RESOLVE_VIA, "config_directive": directive, + "ts": ts, "entries": entries, "errors": []} + tmp = out.with_name(f".{out.name}.tmp") + tmp.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + tmp.replace(out) + tallies: dict[str, int] = {} + for row in entries: + tallies[row["status"]] = tallies.get(row["status"], 0) + 1 + summary = ", ".join(f"{count} {status}" for status, count in sorted(tallies.items())) + if replacer.replacements: + summary += (f"; {replacer.replacements} address(es) moved to documentation space " + f"({len(replacer.mapping)} distinct)") + print(f"{len(entries)} target(s): {summary}") + return 0 + + +def read_raw(directory: str, name: str) -> str: + """One `getent` reply, or empty when the name never got that far. + + A missing file is not an error here: the transport failure that would have left a hole is + fatal in the shell and stops the run before this step, so a hole means a `literal` target, + which is read out of the plan instead. + """ + path = os.path.join(directory, name) + if not os.path.isfile(path): + return "" + with open(path, encoding="utf-8", errors="replace") as handle: + return handle.read() + + +def main(argv: list[str]) -> int: + commands = {"sockets": sockets, "plan": plan, "resolver": resolver} + mode = argv[1] if len(argv) > 1 else "sockets" + if mode not in commands: + print(f"this module does sockets, plan or resolver, not '{mode}'", file=sys.stderr) + return 2 + return commands[mode](argv) + + if __name__ == "__main__": raise SystemExit(main(sys.argv)) PY @@ -250,7 +517,7 @@ capture() { { header "$kind" "$path" case "$kind" in - sockets) python3 -c "$SOCKET_IDENTIFIERS_OUT" "$tmp" ;; + sockets) run_py -c "$SOCKET_IDENTIFIERS_OUT" sockets "$tmp" ;; *) cat "$tmp" ;; esac } >"$FIX/$fixture" @@ -343,6 +610,96 @@ fi capture gateway_proc_net_tcp.txt sockets /proc/net/tcp capture gateway_proc_net_tcp6.txt sockets /proc/net/tcp6 +# --- the resolver document --------------------------------------------------- +# The names the two configs declare, read on the box that dials them. This is the only fixture +# whose *question* comes from the repo: the plan is `target_names()` over the configs captured +# above, so the capture asks exactly what the agent will ask and never one name more. +# +# One world-readable, unprivileged read per name — `getent ahostsv4 ` then +# `getent ahostsv6 ` — because that is libc resolution on the routing box, which is what +# `via: "libc"` claims and what haproxy's `init-addr last,libc,none` performs. No sudo, no `dig`, +# no `host`, no copy tool: a capture that needed a tool the gateway does not ship would record +# something the agent cannot see. +# +# The exit statuses are read as three different sentences. `0` answered; `2` is getent's "no such +# record", which is a fact about the name and becomes `status: "nxdomain"`; `255` is ssh itself +# failing and `127` is a missing `getent`, either of which means the vantage is broken rather than +# the name, so the run stops and the fixture on disk stays exactly as it was. +capture_resolver() { + local scratch rc_plan=0 rc_doc=0 + scratch="$(mktemp -d "${TMPDIR:-/tmp}/gateway-resolver.XXXXXX")" || { + echo " ✗ gateway_resolver.json — no scratch directory; the fixture on disk is left alone" + failed=1 + return 1 + } + + if ! run_py -c "$SOCKET_IDENTIFIERS_OUT" plan "$FIX" "$scratch/plan" 2>"$scratch/plan.err"; then + # Last line, not first: an exception's own sentence is the last one, and this is the line the + # operator reads to decide what to fix. + echo " ✗ gateway_resolver.json — $(tail -n 1 "$scratch/plan.err" 2>/dev/null)" + echo " (the name list comes from agent.host_agent.target_names(); the fixture on disk is " + echo " left alone)" + rm -rf "$scratch" + failed=1 + return 1 + fi + + local kind name port index=0 rc4=0 rc6=0 broken="" + while IFS=$'\t' read -r line f1 f2 f3; do + [ "$line" = target ] || continue # the directive line needs no read: it is in the text + kind="$f1"; name="$f2"; port="$f3" + [ "$kind" = name ] || continue # a literal is already an address; the agent asks nothing + rc4=0; rc6=0 + ssh "${SSH_OPTS[@]}" "$GATEWAY_SSH" "getent ahostsv4 $name" >"$scratch/$index.v4" \ + 2>"$scratch/$index.err" || rc4=$? + ssh "${SSH_OPTS[@]}" "$GATEWAY_SSH" "getent ahostsv6 $name" >"$scratch/$index.v6" \ + 2>"$scratch/$index.err6" || rc6=$? + # `2` on its own is getent's "no such record", which the document records as nxdomain. These + # two are not: one means the box never answered, the other means it has no getent to ask. + if [ "$rc4" -eq 255 ] || [ "$rc6" -eq 255 ]; then + broken="$GATEWAY_SSH did not answer (ssh exited 255)" + break + fi + if [ "$rc4" -eq 127 ] || [ "$rc6" -eq 127 ]; then + broken="getent is not available on $GATEWAY_SSH" + break + fi + index=$((index + 1)) + done <"$scratch/plan" + + if [ -n "$broken" ]; then + echo " ✗ gateway_resolver.json — $broken; a broken vantage is not a fixture, and the one on " + echo " disk is left alone" + rm -rf "$scratch" + failed=1 + return 1 + fi + + # Fixed, not `date`: the fixture's `ts` is a shape (an entry carries the time it was answered) + # and a live clock would make every re-run diff for a difference no test can read. 2026-10-25 + # 00:00:00Z, the date this document was first recorded. + local summary + if ! summary="$(run_py -c "$SOCKET_IDENTIFIERS_OUT" resolver "$scratch/plan" "$scratch" \ + "$FIX/gateway_resolver.json" 1792886400 2>"$scratch/doc.err")"; then + echo " ✗ gateway_resolver.json — $(tail -n 1 "$scratch/doc.err" 2>/dev/null)" + rm -rf "$scratch" + failed=1 + return 1 + fi + echo " ✓ gateway_resolver.json ($summary, on $GATEWAY_SSH)" + rm -rf "$scratch" +} + +# The resolver rides on the configs: no config capture means no name list, and a document derived +# from a half-captured tree would be a fixture of the wrong question. Say so rather than being +# silent about the one fixture this run did not refresh. +if [ "$failed" -eq 0 ]; then + capture_resolver +else + echo " - gateway_resolver.json — not attempted: the configs it derives its name list from did " + echo " not all capture" +fi + if [ "$failed" -ne 0 ]; then echo echo "capture FAILED — every fixture named above is unchanged on disk. Is $GATEWAY_SSH an ssh" diff --git a/src/dashboards/api.py b/src/dashboards/api.py index fe11d74..f22e484 100644 --- a/src/dashboards/api.py +++ b/src/dashboards/api.py @@ -241,18 +241,15 @@ def _endpoint_values(connections: Any) -> list[Any]: Several lines can reach one address (`:443 → 10.1.1.9:443` and `:8443 → 10.1.1.9:443`), and `sources/gateway.py` repeats that endpoint's whole count on each of them with `shared: true` rather than splitting a number nobody measured. Summing the **lines** would book the same - sockets twice and print a router busier than it is, so the total is taken over the endpoints - the lines name — the same reason the outbound gauges are keyed by `(address, port)`. + sockets twice and print a router busier than it is. + + The grouping is `sources/gateway.distinct_endpoint_counts` and not a local `(host, port)` dict, + because since phase 32 task 03 the two spellings of one endpoint can be two different **names**: + two `server_name`s the gateway resolved to one address are one kernel endpoint whose sockets + must be booked once, and a total keyed on the config's text would count them once per name. One + reading of "one endpoint" — the join's — is what keeps `live_out` and the pipes agreeing. """ - values: dict[tuple[Any, Any], Any] = {} - for row in connections if isinstance(connections, Sequence) else (): - if not isinstance(row, Mapping): - continue - out = row.get("out") - if not isinstance(out, Mapping): - continue - values.setdefault((out.get("host"), out.get("port")), row.get("connections")) - return list(values.values()) + return gateway_source.distinct_endpoint_counts(connections) def _gateway_summary(payload: Mapping[str, Any] | None, settings: Settings) -> dict[str, Any]: diff --git a/src/dashboards/collectors.py b/src/dashboards/collectors.py index ac486c2..671bbc9 100644 --- a/src/dashboards/collectors.py +++ b/src/dashboards/collectors.py @@ -38,11 +38,16 @@ class UpstreamError(Exception): #: Agent add-on sections that report their own absence on purpose. Their `errors` lines are -#: facts about a source this page does not draw — `smart:` belongs to `/disks`, `haproxy:` and -#: `nginx:` to the gateway page (phase 29) — so they never make the **system** payload +#: facts about a source this page does not draw — `smart:` belongs to `/disks`, `haproxy:`, +#: `nginx:` and `resolver:` to the gateway page (phases 28/32) — so they never make the +#: **system** payload #: `degraded`, which is the overview's alert input. Without this a fleet-wide agent upgrade #: would put a red "agent degraded" banner and an overview WARN on every host that has no #: haproxy or nginx installed, which is invariant 3's failure mode with an upgrade's clothes on. +#: The `resolver:` arms are the same case with a DNS-shaped name: a host with no proxy config has +#: nothing to resolve, and a name the resolver could not answer (`resolver: pass budget …`, +#: a budget line, or the section's own absence line) is an add-on being honest about one route, +#: not the box being broken. #: The page that owns the add-on still reads the lines: they stay verbatim in `agent.errors`. #: #: **A new agent add-on section adds its prefix here in the same commit.** The list the agent's @@ -50,7 +55,7 @@ class UpstreamError(Exception): #: (`tests/test_agent.py::test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters`), #: so a section that ships its absence-report without a prefix here fails instead of turning #: every host amber. -ADDON_ERROR_PREFIXES: tuple[str, ...] = ("smart", "haproxy", "nginx", "sockets") +ADDON_ERROR_PREFIXES: tuple[str, ...] = ("smart", "haproxy", "nginx", "sockets", "resolver") # --------------------------------------------------------------------------- # diff --git a/src/dashboards/sources/gateway.py b/src/dashboards/sources/gateway.py index a103605..cd07c98 100644 --- a/src/dashboards/sources/gateway.py +++ b/src/dashboards/sources/gateway.py @@ -20,9 +20,13 @@ What this module owns, and what it deliberately does not: * **No verdict.** `severity.py` rates no connection count and this payload carries no `severity` dict on one — the `net.*` precedent: 400 connections on a reverse proxy is the job, and 0 on an idle one is healthy. Pages colour nothing here. -* **No DNS, ever.** The agent never resolves a name and neither does this module - (`tests/test_gateway_source.py` spies `socket.getaddrinfo` and pins that it is not called - once). That single fact decides the whole outbound join, below. +* **No DNS, ever.** The *agent* resolves; this module does not. Agent 1.3.0's `resolver` section + is one `getaddrinfo` per declared target, run **on the host that opens those connections** + (haproxy's own `init-addr last,libc,none`), and this projection only reads the answer out of the + snapshot. The dashboard asking the same name through its own `resolv.conf` would be a second + vantage answering a different question, so `tests/test_gateway_source.py` spies + `socket.getaddrinfo` and pins that `parse()` never calls it — including on the path that + consumes a resolver document. The join: two arms, and the ignore rule --------------------------------------- @@ -34,25 +38,31 @@ actually support, and deleted the rest. What survives, and what is deliberately remote address is the client. `listeners[port]` takes the count and up to `MAX_CLIENTS` distinct client addresses. -**Outbound** is sound in exactly one case: the socket's remote `(address, port)` **equals a -target the config wrote as an address**. That is a fact `/proc` states directly, so the endpoint -carries it. +**Outbound** is sound in exactly two spellings of one fact: the socket's remote `(address, port)` +**equals a target the config wrote as an address**, or it equals an address **the gateway itself +published for the name it wrote** — agent 1.3.0's `resolver` section, which is the resolution that +opened the socket. Either way the far end is named in the same currency `/proc` writes it in, and +the row says which of the two it was (`out.address`, `out.resolved`, `out.via`) beside the count. +The comparison is made on **parsed addresses** (`_address_key`), never on strings: `/proc`'s +`inet_ntop` output, an IPv4-mapped `::ffff:10.1.2.64` and a resolver's `10.1.2.64` are one address, +and a route that resolves yet fails to count because of a spelling is the bug that rule exists to +keep out. The payload still publishes both spellings verbatim. **Everything else is ignored — not counted, not totalled, not warned about.** A config writes -`reeseapps-brain.reeselink.com:443`; `/proc` reports `10.1.1.106:443`. Nothing in this stack -resolves a name (the `socket.getaddrinfo` spy pins that), so a name target can never be matched -to a socket, and a port several targets share — this fleet's `stream.conf` map, sixteen `:443` -entries — credits none of them. The arm that used to bridge that gap credited the *port* and -labelled it `by_port`, and swept every leftover into `unmatched.outbound` / -`unmatched.groups` / `unmatched.other`. On the live router that residual was mostly this app -talking to itself (`9120`, the agent's own port, and `2222`): a residual that is mostly other -software is not a signal, and a card of it teaches the operator to ignore the panel. So -`unmatched`, `by_port` and `partial` are gone, and a socket outside the two config files draws -nothing at all. +`reeseapps-brain.reeselink.com:443`; `/proc` reports `10.1.1.106:443`. Until phase 32 shipped the +capture of that name nothing in this stack could join the two, so a port several targets share — +this fleet's `stream.conf` map, sixteen `:443` entries — credited none of them. The arm that used +to bridge that gap credited the *port* and labelled it `by_port`, and swept every leftover into +`unmatched.outbound` / `unmatched.groups` / `unmatched.other`. On the live router that residual was +mostly this app talking to itself (`9120`, the agent's own port, and `2222`): a residual that is +mostly other software is not a signal, and a card of it teaches the operator to ignore the panel. +So `unmatched`, `by_port` and `partial` are gone for good — the resolver document replaced the +guessing, not a wider guess — and a socket outside the two config files draws nothing at all. -The consequence for a target written as a **name** is its honest answer: `connections` stays -`null` — `n/a`, never `0`. An address endpoint carries the number, `0` included when the sockets -answered and it genuinely has none. +The consequence for a name the gateway **did not** publish an address for (`nxdomain`, a resolver +that would not answer, an agent older than 1.3.0, `--no-resolve`) is its honest answer: +`connections` stays `null` — `n/a`, never `0`, and **no sample is stored** for it. A countable +endpoint carries the number, `0` included when the sockets answered and it genuinely has none. **A shared endpoint repeats its count and never splits it.** Several connections can reach one address (`:443 → backend:443` and `:8443 → backend:443`); each carries the endpoint's whole @@ -162,19 +172,22 @@ sockets. One name for both is one `INSERT OR REPLACE` overwriting the other in s the aggregation failure rule 2 exists to catch. The port is also what makes the name stable when the same host later gains or loses a second service. -Only a connection whose endpoint the join could count publishes a gauge, which since task 02 -means **the name-keyed `gateway.out_conns@:` series stop being written**: the -number they carried was the port's, not the target's, and storing an unattributable figure -under a target's name is rule 2 with a series name on it. The rows already stored age out -under the unchanged retention policy, so nothing here needs a schema or prune change -(invariant 9). +Only a connection whose endpoint the join could count publishes a gauge: an address the config +wrote, **or** a name the gateway resolved. Phase 30 task 02 stopped writing the name-keyed +`gateway.out_conns@:` series because the only figure they could have carried was the +*port's* — the port-only join's count filed under a target's name, which is rule 2 with a series +name on it. Phase 32 task 03 resumes writing them on a different footing: the number is the +endpoint's, and the endpoint is the one this name resolved to, so the name is honest. It stays the +**config's own spelling** (`:25565` and `:25566` remain two gauges, and the series phase 29 would +have written keeps growing under the same name), and a name nobody resolved still publishes +nothing at all. """ from __future__ import annotations import ipaddress import time -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import Collection, Mapping, MutableMapping, Sequence from typing import Any from ..poller import Result @@ -280,6 +293,29 @@ def _mappings(value: Any) -> list[Mapping[str, Any]]: return [item for item in value if isinstance(item, Mapping)] +def _addresses(value: Any) -> list[str]: + """The unique non-empty strings of `value`, first-seen order: an agent's address list. + + Order is the agent's, because that is the order the page prints and a sorted list would + reorder a fan-out on every poll where libc answered differently. + """ + if not isinstance(value, Sequence) or isinstance(value, str | bytes): + return [] + found: list[str] = [] + for item in value: + text = _text(item) + if text is not None and text not in found: + found.append(text) + return found + + +def _stamp(value: Any) -> float | None: + """A number usable as a timestamp, else ``None``. A bool is not a number here.""" + if isinstance(value, bool) or not isinstance(value, int | float): + return None + return float(value) + + def _section(snapshot: Mapping[str, Any], key: str) -> Mapping[str, Any] | None: """One of the three sections, or ``None`` when the agent did not publish it.""" value = snapshot.get(key) @@ -306,10 +342,14 @@ def _available(section: Mapping[str, Any] | None) -> bool | None: def supported(snapshot: Mapping[str, Any]) -> bool: """Does this agent publish the proxy/socket sections at all? - Presence of **any** one of the three counts, so a hypothetical agent that publishes - `sockets` but not the configs is "supported, with a half missing" (which `sections` and - `why` then name) rather than "predates support". A 1.1.0 agent and a `--no-proxy` agent - both publish none, which is the single arm this answers `False` for. + The three phase-28 sections and nothing else: presence of **any** one of them counts, so a + hypothetical agent that publishes `sockets` but not the configs is "supported, with a half + missing" (which `sections` and `why` then name) rather than "predates support". A 1.1.0 agent + and a `--no-proxy` agent both publish none, which is the single arm this answers `False` for. + + Phase 32's `resolver` section is **not** in that test on purpose. A 1.3.0 agent answers `true` + because it still publishes the three, and a 1.3.0 agent run with `--no-resolve` answers `true` + too: the resolver is what makes a *name* countable, never what makes this page supported. """ return any(key in snapshot for key in SECTIONS) @@ -535,18 +575,19 @@ def _connection(source: str, in_port: int, host: str, out_port: int | None, name kind: str, selectors: Sequence[str]) -> dict[str, Any]: """One line of the graph: an inbound port, an outbound endpoint, and the config's word for it. - Three of these fields are set by the join later, and each says so by being `None` now: + Three of these fields are set later, and each says so by being `None` here: - * `connections` — a count is a socket fact and this function reads config text. A line whose - endpoint the config wrote as a **name** keeps the `None` forever: `/proc` has addresses, - nothing here resolves, so nobody can ever count it. + * `connections` — a count is a socket fact and this function reads config text. * `shared` — `true` when another line reaches the same endpoint and therefore carries the same number, so a reader never mistakes two lines for twice the traffic. - * `metric` — the gauge this line publishes, and **`None` for exactly the lines `connections` - can never have**: a series that can only ever be empty is not a series, and asking - `/api/gateway/history` for one draws a flat line at nothing. Where it is set, the name - carries the port as well as the address, because `:25565` and `:25566` on one host are two - services and one name would make the second insert overwrite the first. + * `metric` — the gauge this line publishes. + + `apply_resolution` then widens all three on one question: can the kernel name this line's far + end? An address the config wrote can, and so can a name the gateway resolved on the box that + dials it. A name nobody resolved never can, and keeps `connections: null`, `metric: null` and + **no stored sample**. Where a metric exists its name carries the port as well as the host, + because `:25565` and `:25566` on one host are two services and one name would make the second + insert overwrite the first. """ suffix = f"{host}:{out_port}" if out_port is not None else host address = _is_ip_literal(host) @@ -868,33 +909,281 @@ def _local_port(socket: Mapping[str, Any]) -> int | None: return _port(local.get("port")) +# --------------------------------------------------------------------------- # +# the resolver section: read, never re-derived +# +# Agent 1.3.0 looks each declared target up with `getaddrinfo` **on the host that dials it** and +# publishes the answer, its `via` string and its own timestamp. This module reads that document and +# asks libc nothing: the dashboard's `resolv.conf` is a different vantage, and the fleet's names +# resolve back through this same router. `tests/test_gateway_source.py` spies `socket.getaddrinfo` +# across the whole `parse()` path — including a snapshot whose resolver section is full — so the +# day somebody "helpfully" resolves a name here, the test fails instead of a page. +# --------------------------------------------------------------------------- # + +def resolution_index(snapshot: Mapping[str, Any] + ) -> dict[tuple[str, int | None], Mapping[str, Any]]: + """`(name, port) -> entry` for every resolver entry that published at least one address. + + Three refusals, each the section's own answer rather than this module's: + + * **no `resolver` section at all** (agent 1.2.0, or 1.3.0 with `--no-resolve`), or one whose + `available` is not `True` → `{}`, which is byte-for-byte the phase-30 world: no name is + joinable and every name line keeps its dash. An entry published beside `available: null` is a + pass that could not run, and a stale address is not a fact about this poll — the `warnings()` + rule, for the same reason: a section that did not answer is not judged; + * an entry with **no addresses** (`nxdomain`, `error`) contributes nothing — an empty answer is + not a set containing a `None` address, and joining on one would match a socket nobody named; + * a repeated `(name, port)` keeps the **first** entry, the merge rule the connection graph + already applies to a selector. + + The port is read with `_port`, so an entry naming no port is keyed `None` and can only ever be + found by a connection whose endpoint names no port either — a pair that is never joinable. + """ + section = _section(snapshot, "resolver") + if section is None or _available(section) is not True: + return {} + index: dict[tuple[str, int | None], Mapping[str, Any]] = {} + for entry in _mappings(section.get("entries")): + name = _text(entry.get("name")) + if name is None or not _addresses(entry.get("addresses")): + continue + index.setdefault((name, _port(entry.get("port"))), entry) + return index + + +def resolver_entries(section: Mapping[str, Any] | None + ) -> list[dict[str, Any]]: + """The section's entries as the payload publishes them: name, port, status, addresses, ts. + + Every well-formed entry of a section that **answered**, `nxdomain` and `error` included — the + index filters within this list, and a page that says *this name has no record* needs the entry + that says so. A section that did not answer (`available` false or null) publishes none: an entry + beside a failed pass is not a thing to render as an answer. Projected to the five keys rather + than passed through, so a payload key a page indexes cannot appear by an agent growing a field + nobody reviewed. + """ + if section is None or _available(section) is not True: + return [] + rows: list[dict[str, Any]] = [] + for entry in _mappings(section.get("entries")): + name = _text(entry.get("name")) + if name is None: + continue + rows.append({"name": name, "port": _port(entry.get("port")), + "status": _text(entry.get("status")), + "addresses": _addresses(entry.get("addresses")), + "ts": _stamp(entry.get("ts"))}) + return rows + + +def apply_resolution(connections: Sequence[dict[str, Any]], + index: Mapping[tuple[str, int | None], Mapping[str, Any]], + via: str | None, + ts: float | None) -> None: + """Stamp each connection's `out` with what the gateway resolved, and widen what is countable. + + Three provenance fields, stamped on **every** row so a page never has to test for a key that + may or may not be there: + + * `resolved` — the address list published for this `(name, port)`, or `null`. **Never `[]`**: + an entry with no addresses and no entry at all are one fact to a reader, and `is null` says + it in one test instead of a length check that has to know which empty it found. It is stamped + on the rows written as a **name** only: `address: true` already says the config wrote the + address, and copying the agent's `literal` echo of that same string into `resolved` would put + a `via: libc` lookup claim on a resolution that never happened. `endpoint_of` reads the half + that applies, exactly as the page's `address` test always has. + * `via` — the agent's own string (`"libc"`), copied verbatim, and `null` where nothing was + resolved. This module adds no vantage and invents no word for one. + * `resolved_ts` — that **entry's** timestamp, falling back to the section's (`ts`) when the + entry carries none: a name refreshed five minutes ago and one looked up on this poll are two + claims, and the page's tooltip should print the one that is true. + + Then the one behaviour change, the widened joinability. Whether a line gets a gauge is + `endpoint_of`'s answer — "does this row have at least one key the kernel can be compared with" — + so there is exactly one definition of joinable in this module and a malformed resolver address + (a name where an address should be) places nothing rather than naming a series no sample can ever + reach: + + ```python + # A line is countable when the kernel can name its far end: either the config wrote an + # address, or the gateway's own libc resolution — the one that opens this connection — + # published one. Phase 30 retired name-keyed gauges because they could only ever be + # empty; this is the fact that made that untrue. The NAME keeps the config's spelling so + # the stored series is the same one phase 29/30 would have written. + ``` + + A line the resolver could not place keeps `metric: null` and `connections: null`: no sample, no + zero, and the dash the footer explains. Nothing here counts a socket — that is `join_sockets`, + which runs after this and reads the keys this stamped onto `out.resolved`. + """ + for connection in connections: + out = connection.get("out") + if not isinstance(out, MutableMapping): + continue + host = _text(out.get("host")) + port = _port(out.get("port")) + if out.get("address") is True: + # nothing to resolve, so nothing was: the three fields stay null on an address row + out["resolved"] = out["via"] = out["resolved_ts"] = None + continue + entry = index.get((host, port)) if host is not None else None + addresses = _addresses(entry.get("addresses")) if entry is not None else [] + out["resolved"] = list(addresses) if addresses else None + out["via"] = via if addresses else None + stamped = _stamp(entry.get("ts")) if (entry is not None and addresses) else None + out["resolved_ts"] = stamped if stamped is not None else (ts if addresses else None) + # Joinable is `endpoint_of`'s answer, not a second opinion here: an entry whose addresses + # are not addresses (a malformed agent answer) places nothing, and a gauge name for a line + # that can never publish a sample would be a metric no history can ever answer. + if connection.get("metric") is None and endpoint_of(connection): + connection["metric"] = f"{SOURCE}.out_conns@{out.get('host')}:{port}" + + +def _address_key(value: Any) -> str | None: + """One address in the single spelling the **match** is made in, or ``None`` for a non-address. + + `ipaddress` parses text and never dials, which is why it answers this and a resolver may not + (`_is_ip_literal` uses it for the same reason). It canonicalises an IPv6 (`2001:0DB8::0001` → + `2001:db8::1`) and `ipv4_mapped` folds the `::ffff:10.1.2.64` form `inet_ntop` writes for an + IPv4 peer seen through the IPv6 table back onto the plain `10.1.2.64` a config or a resolver + wrote — so one address in two tables is one key. + + What is deliberately **not** folded is a spelling `ipaddress` refuses, a zero-padded IPv4 octet + (`010.1.2.64`) above all: `inet_aton` reads a leading `0` as *octal*, the two readings are two + addresses, and picking one here would book a socket on a route because of a guess. Such a target + is not an address to this module, so it is a **name**, and the agent's resolver section is what + turns it into one the kernel agrees with. + + Only the match key is normalised. `out.host` and `out.resolved` keep publishing exactly what the + config and the agent wrote, because the page prints the config's text, not this one. + """ + text = _text(value) + if text is None: + return None + try: + address = ipaddress.ip_address(text) + except ValueError: + return None + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped is not None: + return str(address.ipv4_mapped) + return str(address) + + +def endpoint_of(connection: Mapping[str, Any]) -> list[tuple[str, int]]: + """Every `(address, port)` key the kernel can name for this line's far end. + + Two sources and never a mixture: the address **the config wrote** (one key), or every address + the agent's resolver published for the name it wrote (one key each, because a multi-homed name + is dialled at any of them and the sockets are the same route either way). + + An empty list is what `None` was before task 03: nothing here can be compared with `/proc`, so + the line is drawn and never counted, publishes no gauge, and carries no `0`. A target that names + **no port** is in that arm too — the join needs the pair, and the port of a socket is not + something this module may supply. + + The keys are normalised (`_address_key`) and this function is the only place that normalisation + lives, for the endpoints map and for the socket lookup both. The payload above it stays verbatim. + """ + out = connection.get("out") + if not isinstance(out, Mapping): + return [] + port = _port(out.get("port")) + if port is None: + return [] + if out.get("address") is True: + key = _address_key(out.get("host")) + return [(key, port)] if key is not None else [] + keys: list[tuple[str, int]] = [] + for address in _addresses(out.get("resolved")): + key = _address_key(address) + if key is not None and (key, port) not in keys: + keys.append((key, port)) + return keys + + +def _endpoint_groups(key_sets: Sequence[Collection[tuple[str, int]]]) -> list[int]: + """One group number per connection: lines reaching **one** endpoint share one counter row. + + Two lines share an endpoint when their key sets *overlap*, which covers three shapes and not + merely the obvious one: one `(address, port)` written by two vhosts (phase 30's `shared`), two + **names** the resolver placed on one address, and a multi-homed name sharing one address with a + single-homed one. `-1` is "no endpoint": a name nobody resolved, or a target naming no port. + + Numbers are handed out in first-seen order so the grouping is deterministic, and the arithmetic + it protects is one line: a socket increments exactly one counter, so repeating a number down a + list of routes never repeats the traffic. + """ + parent: dict[tuple[str, int], tuple[str, int]] = {} + + def find(key: tuple[str, int]) -> tuple[str, int]: + root = key + while parent[root] != root: + root = parent[root] + while parent[key] != root: # path compression: the chains stay flat + parent[key], key = root, parent[key] + return root + + for keys in key_sets: + if not keys: + continue + anchor = min(keys) + parent.setdefault(anchor, anchor) + root = find(anchor) + for key in keys: + parent.setdefault(key, key) + other = find(key) + if other != root: + parent[other] = root + + numbers: dict[tuple[str, int], int] = {} + groups: list[int] = [] + for keys in key_sets: + if not keys: + groups.append(-1) + continue + root = find(min(keys)) + if root not in numbers: + numbers[root] = len(numbers) + groups.append(numbers[root]) + return groups + + +def distinct_endpoint_counts(connections: Any) -> list[Any]: + """One count per **distinct** endpoint, first-seen order — the only figures a total may add. + + `shared: true` says several lines carry one number because they reach one kernel endpoint, so a + total over the **lines** books the same sockets twice and prints a router busier than it is. + This is the reader `/api/summary`'s `live_out` goes through, and it lives here rather than there + because the answer depends on the same normalised `(address, port)` identity the join uses: since + task 03 two lines can reach one address while the two config files spell it as two different + **names**, and a total keyed on the config's text would count that endpoint once per name. + + A line with no endpoint contributes nothing at all: an unknown is not a zero, and the caller + turns "nothing measured" into `null` (`api._live_total`). + """ + rows = _mappings(connections) + groups = _endpoint_groups([set(endpoint_of(row)) for row in rows]) + values: dict[int, Any] = {} + for row, group in zip(rows, groups, strict=True): + if group < 0: + continue + values.setdefault(group, row.get("connections")) + return list(values.values()) + + # --------------------------------------------------------------------------- # # the join: two arms, and the ignore rule # # `outbound[]`, `by_port`, `unmatched`, `unmatched.groups`, `partial` and `MAX_GROUP_HOSTS` # were deleted here in phase 30 task 02. They existed to answer "which configured target does -# this socket belong to?" for a target the config wrote as a **name**, and the honest answer is -# that nothing in this stack can know (no DNS, ever). What was left over went into a card of -# `unmatched` counts that, on the live router, was mostly this app's own agent port and SSH — -# noise dressed as a signal. A socket the two configs do not describe is now ignored. +# this socket belong to?" for a target the config wrote as a **name** while nothing in this stack +# could know what that name meant, and the leftover went into a card of `unmatched` counts that, on +# the live router, was mostly this app's own agent port and SSH — noise dressed as a signal. Phase +# 32 did not bring the guessing back: it brought the document the **gateway** produced about its own +# connections, which `apply_resolution` reads. A socket the two configs do not describe is still +# ignored, and a name the gateway did not resolve is still counted nowhere. # --------------------------------------------------------------------------- # -def endpoint_of(connection: Mapping[str, Any]) -> tuple[str, int] | None: - """The joinable endpoint key `(address, port)` of a connection, or ``None``. - - ``None`` means "a name, or an address that names no port": there is nothing in `/proc` to - compare it with, so the connection is not an endpoint the join can count. - """ - out = connection.get("out") - if not isinstance(out, Mapping) or out.get("address") is not True: - return None - host = out.get("host") - port = out.get("port") - if not isinstance(host, str) or not isinstance(port, int) or isinstance(port, bool): - return None - return host, port - - def join_sockets(listeners: Mapping[int, dict[str, Any]], endpoints: MutableMapping[tuple[str, int], dict[str, Any]], established: Sequence[Mapping[str, Any]]) -> None: @@ -902,8 +1191,11 @@ def join_sockets(listeners: Mapping[int, dict[str, Any]], * a socket whose **local port** is a configured listener is inbound: that listener's `connections` goes up by one and its `clients` gains the remote address; - * otherwise a socket whose **remote `(address, port)`** is an endpoint the config wrote as - an address is outbound: that endpoint's `connections` goes up by one; + * otherwise a socket whose **remote `(address, port)`** is an endpoint the config wrote as an + address, or one the gateway's resolver published for the name it wrote, is outbound: that + endpoint's `connections` goes up by one. The lookup is `_address_key(remote)`, so the + `::ffff:`-mapped spelling `/proc` uses and the plain one a resolver returned are the same + key — string equality here is the bug class where a route resolves and still never counts; * anything else is **not counted at all**, by decision — see the module docstring. The identity that follows, and that `tests/test_gateway_source.py` pins: every socket is @@ -925,8 +1217,9 @@ def join_sockets(listeners: Mapping[int, dict[str, Any]], if remote_addr not in seen: seen.append(remote_addr) continue - if remote_addr is not None and isinstance(remote_port, int): - endpoint = endpoints.get((remote_addr, remote_port)) + key = _address_key(remote_addr) + if key is not None and remote_port is not None: + endpoint = endpoints.get((key, remote_port)) if endpoint is not None: endpoint["connections"] = int(endpoint["connections"]) + 1 for port, seen in clients.items(): @@ -954,6 +1247,11 @@ def _skeleton(host: str) -> dict[str, Any]: "connections": [], "warnings": [], "sockets": {"listens": None, "established": None}, + # Phase 32's fourth section, in the one place the key set is declared so `parse` and + # `stub` cannot drift. `available: null` is "this agent said nothing about resolution" + # — a 1.2.0 agent, `--no-resolve`, or a host that has not answered — and is deliberately + # not `false`, which is the agent's own word for "nothing to resolve". + "resolver": {"available": None, "via": None, "entries": []}, } @@ -1012,6 +1310,12 @@ def parse(snapshot: Mapping[str, Any], host: str) -> dict[str, Any]: version = _text(snapshot.get("agent_version")) if version: payload["agent_version"] = version + resolver = _section(snapshot, "resolver") + resolver_via = _text(resolver.get("via")) if resolver is not None else None + resolver_ts = _stamp(resolver.get("ts")) if resolver is not None else None + payload["resolver"] = {"available": _available(resolver), + "via": resolver_via, + "entries": resolver_entries(resolver)} if not supports: # The 1.1.0 / --no-proxy arm: every key the page indexes is there and empty, every # count is null, and `why` is the one sentence. Nothing here is red — an agent that @@ -1035,6 +1339,13 @@ def parse(snapshot: Mapping[str, Any], host: str) -> dict[str, Any]: # consulted and is the same in the `sockets`-silent arm. Its per-line `connections` is filled # in below, and stays `None` for a target the config wrote as a name. payload["connections"] = config_connections(haproxy, nginx) + # Phase 32's fourth section, applied between the graph and the join: the addresses the gateway + # resolved **on the box that dials them**. Nothing here asks a resolver — this reads the + # document the agent published — and a snapshot without one (1.2.0, `--no-resolve`) yields an + # empty index, which leaves every name-written line exactly as phase 30 drew it: no address, + # no count, no gauge, no sample. + apply_resolution(payload["connections"], resolution_index(snapshot), + resolver_via, resolver_ts) # Config integrity is config text too, for the same reason: a broken route is just as true when # the socket table never answered. Each half is judged only if its own section did answer, so # this is `[]` — not a guess — for a host with no haproxy, no nginx, or an unreadable file. @@ -1068,20 +1379,27 @@ def parse(snapshot: Mapping[str, Any], host: str) -> dict[str, Any]: payload["listeners"] = list(listeners.values()) # --- the endpoints the socket table can be joined to ----------------------- - # Only an endpoint the config wrote as an **address** is joinable: `/proc` spells addresses, - # this stack resolves nothing, and a name is therefore a route with no measurable end. One - # entry per `(address, port)` — several lines can reach one endpoint, and `routes` is what - # makes `shared` true on every one of them afterwards. - endpoints: dict[tuple[str, int], dict[str, Any]] = {} - for connection in payload["connections"]: - key = endpoint_of(connection) - if key is None: + # Joinable means the **kernel can name the far end**: an address the config wrote, or one the + # gateway's own resolver published for the name it wrote. A name nobody resolved has no key and + # therefore no count, no gauge and no sample. Several lines can reach one endpoint — two vhosts + # naming one address, or two names resolving to it — and `_endpoint_groups` is what makes them + # one counter row whose `routes` is what turns `shared` on afterwards. + key_sets: list[set[tuple[str, int]]] = [set(endpoint_of(connection)) + for connection in payload["connections"]] + groups = _endpoint_groups(key_sets) + rows: list[dict[str, Any]] = [] + for group in groups: + if group < 0: continue - seen = endpoints.get(key) - if seen is None: - endpoints[key] = {"connections": 0, "routes": 1} - else: - seen["routes"] = int(seen["routes"]) + 1 + while len(rows) <= group: + rows.append({"connections": 0, "routes": 0}) + rows[group]["routes"] = int(rows[group]["routes"]) + 1 + endpoints: dict[tuple[str, int], dict[str, Any]] = {} + for keys, group in zip(key_sets, groups, strict=True): + if group < 0: + continue + for key in keys: + endpoints[key] = rows[group] if not counting: # The configs are known and the live half is not: every line is drawn, every count stays @@ -1090,11 +1408,10 @@ def parse(snapshot: Mapping[str, Any], host: str) -> dict[str, Any]: # --- the join: inbound by local port, outbound by exact address, nothing else -- join_sockets(listeners, endpoints, established) - for connection in payload["connections"]: - key = endpoint_of(connection) - if key is None: - continue # a name, or an address with no port: never counted, so never a number - endpoint = endpoints[key] + for connection, group in zip(payload["connections"], groups, strict=True): + if group < 0: + continue # a name nobody resolved, or an address with no port: never counted + endpoint = rows[group] connection["connections"] = int(endpoint["connections"]) connection["shared"] = int(endpoint["routes"]) > 1 payload["sockets"] = {"listens": len(listening), "established": len(established)} @@ -1107,15 +1424,20 @@ def samples(payload: Mapping[str, Any], ts: float | None = None) -> list[Sample] Three refusals, and each is a different fact: * A `null` count publishes **nothing**. "Unknown" is not a zero, and a chart of zeros for a dead - socket table — or of confident zeros for every route whose target the config wrote as a - **name** — is the failure mode rule 2 names. + socket table — or of confident zeros for a route whose target the config wrote as a **name** + the gateway never resolved — is the failure mode rule 2 names. `0` is published, and only for + an endpoint whose address is known while no socket is open to it. * **One sample per metric name.** Several route lines can reach one endpoint and each of them carries that endpoint's whole count (`shared: true`); the gauge is keyed by the endpoint, so emitting per line would store the same number twice under one name for one reading. * A gauge never gets a bool. - That is also why the name-keyed `gateway.out_conns@:` series stopped being - written in phase 30 task 02: the figure they carried belonged to the port, not the domain. + Which lines have a metric at all is `apply_resolution`'s answer, not this function's: since + phase 32 task 03 that is an endpoint the config wrote as an address **or** a name the gateway + published addresses for, and the name-keyed `gateway.out_conns@:` series phase 30 + task 02 stopped writing are written again because there is now a figure that belongs to the + domain rather than to whatever else was listening on its port. + The timestamp is the poll's (when the dashboard read the count), carried on `ts`; nothing here is a rate, so nothing here needs a window. """ diff --git a/src/dashboards/web/pages/gateway.html b/src/dashboards/web/pages/gateway.html index 7648729..4f5e55e 100644 --- a/src/dashboards/web/pages/gateway.html +++ b/src/dashboards/web/pages/gateway.html @@ -133,7 +133,10 @@ hanging off it on the right. This is the structure of the file, not a summary of it. A leaf reads inbound first: line 1 is the config's word for who reaches the endpoint (`server_name`, `map` key, `default`) and line 2 is the counterpart it flows to, so do - not swap them back — the question this page answers is which domain goes where. --> + not swap them back — the question this page answers is which domain goes where. + `out.resolved` is the agent's own libc answer for a target the config wrote as a name, + and it is why a name line can carry a number at all: without it the line is drawn and + never counted. -->