services/dashboards
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase: 32_gateway_resolved_target_liveness

Browse Source 
Fixture and live stream.conf are byte-identical (modulo header comment). Let me test the parser directly:
This commit is contained in:
ducoterra
2026-10-26 14:03:42 -04:00
parent 25fd74b7c5
commit 51772ffef2
58 changed files with 5272 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/conftest.py
+19 -3
View File
@@ -257,7 +257,7 @@ GATEWAY_TREE = {
"proc/net/tcp6": "gateway_proc_net_tcp6.txt",
}
GATEWAY_MODES = ("full", "predates", "no_haproxy", "no_nginx", "sockets_silent")
GATEWAY_MODES = ("full", "resolved", "predates", "no_haproxy", "no_nginx", "sockets_silent")
@pytest.fixture
@@ -269,9 +269,19 @@ def gateway_snapshot(host_agent_module: Any, tmp_path: Path) -> Any:
`sources/gateway.py` are the bytes agent 1.2.0 serves. A field rename on either side of the
wire therefore fails a source test instead of drawing an empty pipe on the page (rule 7).
Modes are the degradation arms the page has to render:
One more axis is a **version**, not a degradation:
* `full` — all three sections `available: true`, live sockets from the capture
* `resolved` — the same tree as `full` plus agent 1.3.0's fourth section, the recorded
`gateway_resolver.json` attached verbatim (the document `capture-gateway-config.sh` reads
back with `getent ahostsv4/6` on the box that dials). This is what the deployed ingress
serves after phase 32 task 05, and it is the arm in which a name-written route becomes
countable.
Modes are otherwise the degradation arms the page has to render:
* `full` — all three sections `available: true`, live sockets from the capture, and **no**
`resolver` key at all: a 1.2.0 agent, or a 1.3.0 one run with `--no-resolve`. Phase 30's
payload, verbatim, which is what the phase-32 additivity pins compare against
* `predates` — none of the three keys at all (an agent older than 1.2.0, or `--no-proxy`)
* `no_haproxy` / `no_nginx` — that one config file is absent: `available: false` plus the
agent's own **not found** line, the other half of the page untouched
@@ -308,6 +318,12 @@ def gateway_snapshot(host_agent_module: Any, tmp_path: Path) -> Any:
snapshot["haproxy"] = collector.haproxy(errors)
snapshot["nginx"] = collector.nginx(errors)
snapshot["sockets"] = collector.sockets(errors)
if mode == "resolved":
# The recorded capture, not a re-derivation: the names are the ones `target_names()`
# derives from the config fixtures above and the answers are libc's on the router, so
# the join is tested against the document the agent will actually serve (rule 7). The
# dashboard never asks a resolver, and nothing here does either.
snapshot["resolver"] = json.loads((FIXTURES / "gateway_resolver.json").read_text())
return snapshot
return build
tests/fixtures/gateway_nginx_stream_d_stream.conf
Vendored
+1
View File
@@ -16,6 +16,7 @@ map $ssl_preread_server_name $targetBackend {
brain.reeseapps.com reeseapps-brain-of-reese.reeselink.com:443;
brain-of-code.reeseapps.com reeseapps-brain-of-code.reeselink.com:443;
immich.reeseapps.com reeseapps-immich.reeselink.com:443;
whiteboard.reeseapps.com reeseapps-whiteboard.reeselink.com:443;
stackexpected.com stackexpected.reeselink.com:443;
gitea.stackexpected.com stackexpected-gitea.reeselink.com:443;
tests/fixtures/gateway_proc_net_tcp.txt
Vendored
+72 -45
View File
@@ -11,48 +11,75 @@
# a diff here on re-capture is expected. The tests assert shape, the two states the agent
# keeps, and the ports the captured configs claim to bind — never a particular row.
sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode
0: 00000000:20FB 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9978 1 0000000000000000 100 0 0 10 0
1: 00000000:08AE 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 7797 1 0000000000000000 100 0 0 10 0
2: 00000000:0050 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9974 1 0000000000000000 100 0 0 10 0
3: 00000000:01BB 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9972 1 0000000000000000 100 0 0 10 0
4: 00000000:115B 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9976 1 0000000000000000 100 0 0 10 0
5: 1B01010A:23A0 00000000:0000 0A 00000000:00000000 00:00000000 00000000 994 0 9479 1 0000000000000000 100 0 0 10 0
6: 1B01010A:BE2A B001010A:01BB 06 00000000:00000000 03:000013F5 00000000 0 0 0 3 0000000000000000
7: 0100007F:115B 0100007F:818C 01 00000000:00000000 00:00000000 00000000 101 0 62860 1 0000000000000000 20 4 30 10 -1
8: 1B01010A:BE2E B001010A:01BB 06 00000000:00000000 03:00001407 00000000 0 0 0 3 0000000000000000
9: 1B01010A:23A0 0204A8C0:C626 06 00000000:00000000 03:00000820 00000000 0 0 0 3 0000000000000000
10: 0100007F:115B 0100007F:C096 01 00000000:00000000 00:00000000 00000000 101 0 49558 1 0000000000000000 20 4 26 10 -1
11: 0100007F:115B 0100007F:B502 06 00000000:00000000 03:00000841 00000000 0 0 0 3 0000000000000000
12: 1B01010A:8290 A401010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 11850 1 0000000000000000 20 4 28 10 -1
13: 1B01010A:01BB 017100CB:90FA 01 00000000:00000000 00:00000000 00000000 101 0 21476 1 0000000000000000 35 4 31 10 -1
14: 1B01010A:23A0 0204A8C0:992E 06 00000000:00000000 03:00000C0A 00000000 0 0 0 3 0000000000000000
15: 1B01010A:CB4C B001010A:01BB 06 00000000:00000000 03:00000ECB 00000000 0 0 0 3 0000000000000000
16: 1B01010A:9C42 2700010A:01BB 06 00000000:00000000 03:00001259 00000000 0 0 0 3 0000000000000000
17: 1B01010A:01BB 0100010A:B2AC 01 00000000:00000000 00:00000000 00000000 101 0 77631 1 0000000000000000 20 4 31 10 -1
18: 1B01010A:01BB E600020A:BBE8 06 00000000:00000000 03:00000992 00000000 0 0 0 3 0000000000000000
19: 0100007F:C99E 0100007F:115B 06 00000000:00000000 03:000016BE 00000000 0 0 0 3 0000000000000000
20: 1B01010A:C1FE 2700010A:01BB 06 00000000:00000000 03:00000662 00000000 0 0 0 3 0000000000000000
21: 1B01010A:01BB E600020A:B5C4 06 00000000:00000000 03:000005EE 00000000 0 0 0 3 0000000000000000
22: 1B01010A:840E B001010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 21477 1 0000000000000000 20 4 30 10 -1
23: 1B01010A:D22A 4E01010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 77632 1 0000000000000000 21 4 30 10 -1
24: 1B01010A:01BB 017100CB:956A 01 00000000:00000000 00:00000000 00000000 101 0 62858 1 0000000000000000 29 4 30 10 -1
25: 0100007F:115B 0100007F:B4F6 06 00000000:00000000 03:000005ED 00000000 0 0 0 3 0000000000000000
26: 1B01010A:23A0 0204A8C0:8EE6 06 00000000:00000000 03:00001408 00000000 0 0 0 3 0000000000000000
27: 1B01010A:01BB EC01010A:94DE 01 00000000:00000000 00:00000000 00000000 101 0 11849 1 0000000000000000 20 4 31 10 -1
28: 0100007F:818C 0100007F:115B 01 00000000:00000000 00:00000000 00000000 101 0 62859 1 0000000000000000 21 4 24 10 -1
29: 1B01010A:23A0 0204A8C0:D5DE 06 00000000:00000000 03:0000020F 00000000 0 0 0 3 0000000000000000
30: 0100007F:115B 0100007F:B4F8 06 00000000:00000000 03:000005F6 00000000 0 0 0 3 0000000000000000
31: 1B01010A:A828 7600010A:63DD 06 00000000:00000000 03:00000F7F 00000000 0 0 0 3 0000000000000000
32: 0100007F:C096 0100007F:115B 01 00000000:00000000 00:00000000 00000000 101 0 49557 1 0000000000000000 20 4 24 10 -1
33: 1B01010A:D444 FE00010A:2AF8 01 00000000:00000000 00:00000000 00000000 101 0 49559 1 0000000000000000 20 4 30 10 -1
34: 1B01010A:9032 B001010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 11954 1 0000000000000000 20 4 30 10 -1
35: 1B01010A:977E 2700010A:01BB 06 00000000:00000000 03:00000C72 00000000 0 0 0 3 0000000000000000
36: 0100007F:115B 0100007F:B4EA 06 00000000:00000000 03:000005F0 00000000 0 0 0 3 0000000000000000
37: 1B01010A:08AE 0204A8C0:8290 01 00000048:00000000 01:00000017 00000000 0 0 86478 4 0000000000000000 24 8 15 10 -1
38: 1B01010A:9ED0 2700010A:01BB 06 00000000:00000000 03:0000170F 00000000 0 0 0 3 0000000000000000
39: 1B01010A:0050 027100CB:843E 01 0000018C:00000000 01:00000E57 00000007 101 0 84306 2 0000000000000000 4864 4 31 1 7
40: 1B01010A:CB5C B001010A:01BB 06 00000000:00000000 03:00000ED6 00000000 0 0 0 3 0000000000000000
41: 1B01010A:9776 2700010A:01BB 06 00000000:00000000 03:00000875 00000000 0 0 0 3 0000000000000000
42: 1B01010A:C59C FE00010A:2AF8 01 00000000:00000000 00:00000000 00000000 101 0 62861 1 0000000000000000 21 4 30 10 -1
43: 1B01010A:9154 2700010A:01BB 06 00000000:00000000 03:00001572 00000000 0 0 0 3 0000000000000000
44: 1B01010A:23A0 0204A8C0:9C42 06 00000000:00000000 03:00000FF3 00000000 0 0 0 3 0000000000000000
0: 1B01010A:23A0 00000000:0000 0A 00000000:00000000 00:00000000 00000000 994 0 9291 1 0000000000000000 100 0 0 10 0
1: 00000000:20FB 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9804 1 0000000000000000 100 0 0 10 0
2: 00000000:08AE 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 8256 1 0000000000000000 100 0 0 10 0
3: 00000000:0050 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9800 1 0000000000000000 100 0 0 10 0
4: 00000000:01BB 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9798 1 0000000000000000 100 0 0 10 0
5: 00000000:115B 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9802 1 0000000000000000 100 0 0 10 0
6: 1B01010A:E27A 7600010A:63DD 06 00000000:00000000 03:00000D25 00000000 0 0 0 3 0000000000000000
7: 1B01010A:23A0 0204A8C0:A726 06 00000000:00000000 03:00000082 00000000 0 0 0 3 0000000000000000
8: 1B01010A:90A0 A401010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 788517 1 0000000000000000 21 4 31 10 -1
9: 1B01010A:C76A 2700010A:01BB 06 00000000:00000000 03:00001004 00000000 0 0 0 3 0000000000000000
10: 1B01010A:23A0 0204A8C0:A9EA 06 00000000:00000000 03:0000141C 00000000 0 0 0 3 0000000000000000
11: 1B01010A:AC88 2700010A:01BB 06 00000000:00000000 03:00001745 00000000 0 0 0 3 0000000000000000
12: 1B01010A:01BB 0204A8C0:D28C 01 00000000:00000000 00:00000000 00000000 101 0 470456 1 0000000000000000 23 4 31 2 2
13: 1B01010A:AFCE 2700010A:01BB 06 00000000:00000000 03:00000CA6 00000000 0 0 0 3 0000000000000000
14: 1B01010A:EA9C 6A01010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 894482 1 0000000000000000 20 4 30 10 -1
15: 1B01010A:01BB 0204A8C0:B050 01 00000000:00000000 00:00000000 00000000 101 0 901901 1 0000000000000000 20 4 31 10 -1
16: 1B01010A:C778 2700010A:01BB 06 00000000:00000000 03:00001077 00000000 0 0 0 3 0000000000000000
17: 1B01010A:01BB E600020A:C768 06 00000000:00000000 03:00000FBA 00000000 0 0 0 3 0000000000000000
18: 1B01010A:881A 2700010A:01BB 06 00000000:00000000 03:0000140B 00000000 0 0 0 3 0000000000000000
19: 1B01010A:C750 2700010A:01BB 06 00000000:00000000 03:00000E46 00000000 0 0 0 3 0000000000000000
20: 1B01010A:AFBE 2700010A:01BB 06 00000000:00000000 03:00000B6B 00000000 0 0 0 3 0000000000000000
21: 1B01010A:C74A 2700010A:01BB 06 00000000:00000000 03:00000D8B 00000000 0 0 0 3 0000000000000000
22: 1B01010A:E6AE B001010A:01BB 06 00000000:00000000 03:00000664 00000000 0 0 0 3 0000000000000000
23: 1B01010A:EDD0 2700010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 903407 1 0000000000000000 20 4 32 10 -1
24: 1B01010A:C916 FE00010A:2AF8 01 00000000:00000000 00:00000000 00000000 101 0 470459 1 0000000000000000 22 4 30 10 -1
25: 1B01010A:08AE 0204A8C0:9D44 01 00000000:00000000 02:000AFC6F 00000000 0 0 903449 4 0000000000000000 20 4 17 10 -1
26: 1B01010A:D958 4E01010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 890783 1 0000000000000000 21 4 30 10 -1
27: 1B01010A:87F6 2700010A:01BB 06 00000000:00000000 03:00001217 00000000 0 0 0 3 0000000000000000
28: 0100007F:BC28 0100007F:115B 01 00000000:00000000 00:00000000 00000000 101 0 879324 1 0000000000000000 20 4 24 10 -1
29: 1B01010A:EDC0 2700010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 901902 1 0000000000000000 20 4 33 10 -1
30: 1B01010A:01BB 0204A8C0:94D8 01 00000000:00000000 00:00000000 00000000 101 0 887225 1 0000000000000000 20 4 26 7 7
31: 1B01010A:880A 2700010A:01BB 06 00000000:00000000 03:000016C8 00000000 0 0 0 3 0000000000000000
32: 1B01010A:01BB 0204A8C0:B05E 01 00000000:00000000 00:00000000 00000000 101 0 902458 1 0000000000000000 20 4 14 10 -1
33: 1B01010A:CEC4 B001010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 684979 1 0000000000000000 20 4 30 10 -1
34: 1B01010A:CB90 DF00010A:1451 02 00000001:00000000 01:00000022 00000000 99 0 903275 2 0000000000000000 100 0 1 10 -1
35: 1B01010A:AC7A 2700010A:01BB 06 00000000:00000000 03:000016F1 00000000 0 0 0 3 0000000000000000
36: 1B01010A:23A0 0204A8C0:994E 06 00000000:00000000 03:0000046B 00000000 0 0 0 3 0000000000000000
37: 0100007F:C45A 0100007F:115B 06 00000000:00000000 03:00000C8A 00000000 0 0 0 3 0000000000000000
38: 0100007F:115B 0100007F:C454 06 00000000:00000000 03:00000D86 00000000 0 0 0 3 0000000000000000
39: 1B01010A:8140 6A01010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 887226 1 0000000000000000 20 4 32 10 -1
40: 1B01010A:01BB 0204A8C0:C044 06 00000000:00000000 03:00000839 00000000 0 0 0 3 0000000000000000
41: 1B01010A:9A6C 7600010A:63DE 06 00000000:00000000 03:00000CC8 00000000 0 0 0 3 0000000000000000
42: 1B01010A:8828 2700010A:01BB 06 00000000:00000000 03:00001552 00000000 0 0 0 3 0000000000000000
43: 1B01010A:C184 FE00010A:2AF8 01 00000000:00000000 00:00000000 00000000 101 0 880000 1 0000000000000000 20 4 30 10 -1
44: 1B01010A:01BB 0204A8C0:B052 01 00000000:00000000 00:00000000 00000000 101 0 903348 1 0000000000000000 21 4 31 10 -1
45: 1B01010A:EDD8 2700010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 902459 1 0000000000000000 20 4 33 43 -1
46: 1B01010A:01BB 0100010A:AD60 01 00000000:00000000 00:00000000 00000000 101 0 890782 1 0000000000000000 20 4 31 4 4
47: 1B01010A:01BB E600020A:C770 06 00000000:00000000 03:00000EC1 00000000 0 0 0 3 0000000000000000
48: 1B01010A:23A0 0204A8C0:B316 06 00000000:00000000 03:00001035 00000000 0 0 0 3 0000000000000000
49: 0100007F:115B 0100007F:BC28 01 00000000:00000000 00:00000000 00000000 101 0 879999 1 0000000000000000 20 4 26 10 -1
50: 1B01010A:01BB E600020A:A480 06 00000000:00000000 03:00000C14 00000000 0 0 0 3 0000000000000000
51: 1B01010A:8804 2700010A:01BB 06 00000000:00000000 03:0000126B 00000000 0 0 0 3 0000000000000000
52: 1B01010A:23A0 0204A8C0:B31E 06 00000000:00000000 03:00001036 00000000 0 0 0 3 0000000000000000
53: 0100007F:115B 0100007F:C436 06 00000000:00000000 03:00000C14 00000000 0 0 0 3 0000000000000000
54: 1B01010A:D16A B001010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 464535 1 0000000000000000 20 4 30 10 -1
55: 1B01010A:E490 6A01010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 853018 1 0000000000000000 20 4 32 10 -1
56: 1B01010A:C756 2700010A:01BB 06 00000000:00000000 03:00000E62 00000000 0 0 0 3 0000000000000000
57: 1B01010A:01BB 0204A8C0:9FA2 01 00000000:00000000 00:00000000 00000000 101 0 853017 1 0000000000000000 20 4 30 7 7
58: 0100007F:115B 0100007F:BA26 01 00000000:00000000 00:00000000 00000000 101 0 470458 1 0000000000000000 21 4 31 10 -1
59: 1B01010A:01BB 0303A8C0:8B88 01 00000000:00000000 00:00000000 00000000 101 0 684978 1 0000000000000000 30 4 30 3 2
60: 1B01010A:A132 2700010A:01BB 06 00000000:00000000 03:00000BF8 00000000 0 0 0 3 0000000000000000
61: 0100007F:115B 0100007F:C446 06 00000000:00000000 03:00000C1D 00000000 0 0 0 3 0000000000000000
62: 1B01010A:01BB 0204A8C0:BAFA 01 00000000:00000000 00:00000000 00000000 101 0 894481 1 0000000000000000 20 4 31 7 7
63: 0100007F:115B 0100007F:C456 06 00000000:00000000 03:00000DA6 00000000 0 0 0 3 0000000000000000
64: 1B01010A:B862 B001010A:01BB 01 00000000:00000000 00:00000000 00000000 101 0 705498 1 0000000000000000 20 4 31 10 -1
65: 1B01010A:23A0 0204A8C0:8618 06 00000000:00000000 03:00000C3A 00000000 0 0 0 3 0000000000000000
66: 0100007F:BA26 0100007F:115B 01 00000000:00000000 00:00000000 00000000 101 0 470457 1 0000000000000000 21 4 22 10 -1
67: 1B01010A:23A0 0204A8C0:AD66 06 00000000:00000000 03:00000852 00000000 0 0 0 3 0000000000000000
68: 1B01010A:23A0 0204A8C0:9952 06 00000000:00000000 03:0000046B 00000000 0 0 0 3 0000000000000000
69: 1B01010A:DC6A 2700010A:01BB 06 00000000:00000000 03:000009C9 00000000 0 0 0 3 0000000000000000
70: 1B01010A:01BB EC01010A:953E 01 00000000:00000000 00:00000000 00000000 101 0 788516 1 0000000000000000 20 4 31 10 -1
71: 1B01010A:E6AC B001010A:01BB 06 00000000:00000000 03:0000065D 00000000 0 0 0 3 0000000000000000
tests/fixtures/gateway_proc_net_tcp6.txt
Vendored
+14 -13
View File
@@ -11,16 +11,17 @@
# a diff here on re-capture is expected. The tests assert shape, the two states the agent
# keeps, and the ports the captured configs claim to bind — never a particular row.
sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode
0: 00000000000000000000000000000000:20FB 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9979 1 0000000000000000 100 0 0 10 0
1: 00000000000000000000000000000000:08AE 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 7799 1 0000000000000000 100 0 0 10 0
2: 00000000000000000000000000000000:1451 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9181 1 0000000000000000 100 0 0 10 0
3: 00000000000000000000000000000000:0050 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9975 1 0000000000000000 100 0 0 10 0
4: 00000000000000000000000000000000:0016 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9179 1 0000000000000000 100 0 0 10 0
5: 00000000000000000000000000000000:01BB 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9973 1 0000000000000000 100 0 0 10 0
6: 00000000000000000000000000000000:115B 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9977 1 0000000000000000 100 0 0 10 0
7: 00000000000000000000000000000000:63DE 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9184 1 0000000000000000 100 0 0 10 0
8: 00000000000000000000000000000000:63DD 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9183 1 0000000000000000 100 0 0 10 0
9: 00000000000000000000000000000000:07D0 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9180 1 0000000000000000 100 0 0 10 0
10: 00000000000000000000000000000000:1F90 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9182 1 0000000000000000 100 0 0 10 0
11: B80D0120000000000000000001000000:01BB 2377D3FDF349E0970000000003000000:C0AC 01 00000000:00000000 00:00000000 00000000 101 0 49556 1 0000000000000000 80 4 26 10 -1
12: B80D0120000000000000000001000000:01BB 2377D3FDF349E0970000000003000000:A120 01 00000000:00000000 00:00000000 00000000 101 0 11953 1 0000000000000000 59 4 30 7 7
0: 00000000000000000000000000000000:07D0 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9105 1 0000000000000000 100 0 0 10 0
1: 00000000000000000000000000000000:63DD 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9108 1 0000000000000000 100 0 0 10 0
2: 00000000000000000000000000000000:63DE 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9109 1 0000000000000000 100 0 0 10 0
3: 00000000000000000000000000000000:1F90 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9107 1 0000000000000000 100 0 0 10 0
4: 00000000000000000000000000000000:20FB 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9805 1 0000000000000000 100 0 0 10 0
5: 00000000000000000000000000000000:08AE 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 8264 1 0000000000000000 100 0 0 10 0
6: 00000000000000000000000000000000:0050 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9801 1 0000000000000000 100 0 0 10 0
7: 00000000000000000000000000000000:1451 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9106 1 0000000000000000 100 0 0 10 0
8: 00000000000000000000000000000000:0016 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 99 0 9104 1 0000000000000000 100 0 0 10 0
9: 00000000000000000000000000000000:01BB 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9799 1 0000000000000000 100 0 0 10 0
10: 00000000000000000000000000000000:115B 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 9803 1 0000000000000000 100 0 0 10 0
11: B80D0120000000000000000001000000:01BB B80D0120000000000000000002000000:D76A 01 00000000:00000000 00:00000000 00000000 101 0 705497 1 0000000000000000 20 4 30 10 -1
12: B80D0120000000000000000001000000:01BB B80D0120000000000000000003000000:9FBA 01 00000000:00000000 00:00000000 00000000 101 0 464534 1 0000000000000000 57 4 30 3 2
13: B80D0120000000000000000001000000:01BB B80D0120000000000000000003000000:E0C6 01 00000000:00000000 00:00000000 00000000 101 0 879323 1 0000000000000000 42 4 26 10 -1
tests/fixtures/gateway_resolver.json
Vendored
+297
View File
@@ -0,0 +1,297 @@
{
"available": true,
"via": "libc",
"config_directive": "10.1.0.1",
"ts": 1792886400.0,
"entries": [
{
"name": "reeseapps-gitea.reeselink.com",
"port": 2222,
"status": "ok",
"addresses": [
"10.1.1.164"
],
"ts": 1792886400.0
},
{
"name": "stackexpected-gitea.reeselink.com",
"port": 2222,
"status": "ok",
"addresses": [
"10.1.1.78"
],
"ts": 1792886400.0
},
{
"name": "web.reeselink.com",
"port": 5201,
"status": "ok",
"addresses": [
"10.1.0.223"
],
"ts": 1792886400.0
},
{
"name": "unifi-external.reeselink.com",
"port": 8080,
"status": "ok",
"addresses": [
"10.1.0.222"
],
"ts": 1792886400.0
},
{
"name": "minecraft.reeselink.com",
"port": 25565,
"status": "ok",
"addresses": [
"10.1.0.118"
],
"ts": 1792886400.0
},
{
"name": "minecraft.reeselink.com",
"port": 25566,
"status": "ok",
"addresses": [
"10.1.0.118"
],
"ts": 1792886400.0
},
{
"name": "nextcloud.reeselink.com",
"port": 11000,
"status": "ok",
"addresses": [
"10.1.0.254"
],
"ts": 1792886400.0
},
{
"name": "nextcloud.reeselink.com",
"port": 11001,
"status": "ok",
"addresses": [
"10.1.0.254"
],
"ts": 1792886400.0
},
{
"name": "gitea.reeselink.com",
"port": 3000,
"status": "ok",
"addresses": [
"10.1.0.137"
],
"ts": 1792886400.0
},
{
"name": "jellyfin.reeselink.com",
"port": 8096,
"status": "ok",
"addresses": [
"10.1.0.75"
],
"ts": 1792886400.0
},
{
"name": "127.0.0.1",
"port": 8081,
"status": "literal",
"addresses": [
"127.0.0.1"
],
"ts": 1792886400.0
},
{
"name": "bricktracker.reeselink.com",
"port": 3333,
"status": "ok",
"addresses": [
"10.1.0.66"
],
"ts": 1792886400.0
},
{
"name": "keycloak.reeselink.com",
"port": 8080,
"status": "ok",
"addresses": [
"10.1.1.75"
],
"ts": 1792886400.0
},
{
"name": "emmaleaf.reeselink.com",
"port": 80,
"status": "ok",
"addresses": [
"10.1.0.138"
],
"ts": 1792886400.0
},
{
"name": "127.0.0.1",
"port": 4443,
"status": "literal",
"addresses": [
"127.0.0.1"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-homepage.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.1.33"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-ntfy.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.1.176"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-gitea.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.1.164"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-litellm.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.0.39"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-openwebui.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.2.39"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-keycloak.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.3.148"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-forgejo.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.3.189"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-brain-of-reese.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.2.64"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-brain-of-code.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.0.106"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-immich.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.3.159"
],
"ts": 1792886400.0
},
{
"name": "reeseapps-whiteboard.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.2.79"
],
"ts": 1792886400.0
},
{
"name": "stackexpected.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.3.64"
],
"ts": 1792886400.0
},
{
"name": "stackexpected-gitea.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.1.78"
],
"ts": 1792886400.0
},
{
"name": "stackexpected-keycloak.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.0.42"
],
"ts": 1792886400.0
},
{
"name": "stackexpected-nextcloud.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.1.106"
],
"ts": 1792886400.0
},
{
"name": "stackexpected-hedgedoc.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.3.142"
],
"ts": 1792886400.0
},
{
"name": "wfc.reeselink.com",
"port": 443,
"status": "ok",
"addresses": [
"10.1.1.41"
],
"ts": 1792886400.0
}
],
"errors": []
}
tests/test_agent.py
+712 -15
View File
@@ -12,6 +12,7 @@ Four things are asserted here:
"""
from __future__ import annotations
import hashlib
import http.client
import importlib.util
import ipaddress
@@ -331,13 +332,14 @@ def write_fake_disk(root: Path, name: str, model: str | None = None) -> None:
write_file(root, f"sys/block/{name}/device/model", f"{model}\n")
#: The add-on sections report their own absence in `errors` (`smart:`, `haproxy:`), because
#: "this box has no smartmontools / no haproxy" is a fact the page that owns the add-on has to
#: be able to print. Tests that assert a snapshot is otherwise clean filter the same prefixes
#: the dashboard does — `collectors.ADDON_ERROR_PREFIXES`, pinned equal to this list by
#: `test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters`, so neither side can
#: The add-on sections report their own absence in `errors` (`smart:`, `haproxy:`, `resolver:`),
#: because
#: "this box has no smartmontools / no haproxy / nothing to resolve" is a fact the page that owns
#: the add-on has to be able to print. Tests that assert a snapshot is otherwise clean filter the
#: same prefixes the dashboard does — `collectors.ADDON_ERROR_PREFIXES`, pinned equal to this list
#: by `test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters`, so neither side can
#: quietly grow a fourth opinion.
ADDON_ERROR_PREFIXES = ("smart", "haproxy", "nginx", "sockets")
ADDON_ERROR_PREFIXES = ("smart", "haproxy", "nginx", "sockets", "resolver")
def test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters() -> None:
@@ -357,6 +359,36 @@ def core_errors(snapshot: dict[str, Any]) -> list[str]:
if not str(line).lstrip().lower().startswith(ADDON_ERROR_PREFIXES)]
@pytest.fixture(autouse=True)
def no_real_dns(monkeypatch: pytest.MonkeyPatch) -> None:
"""Not one test in this file resolves a name across the network (phase 32's hermetic mandate).
The patch is module-wide on purpose — `host_agent.socket` *is* the `socket` module — and the
stub answers a hostname with a deterministic pair of **documentation-space** addresses
(RFC 5737 / RFC 3849, the same substitution `scripts/capture-gateway-config.sh` makes to a
socket table, so nothing a resolver test prints could carry a real endpoint either). Every IP
literal and `localhost` goes to the real function: this file's HTTP and TLS tests connect to
numeric loopback, and a stub that answered those too would break them for a reason nobody is
testing. A test that wants its own answer — `nxdomain`, a slow resolver, a call ledger —
patches the same attribute in its own body, which runs after this fixture.
"""
real = socket.getaddrinfo
def stub(name: Any, port: Any = None, *args: Any, **kwargs: Any) -> Any:
text = str(name)
if host_agent.address_family(text) is not None or text in ("", "localhost"):
return real(name, port, *args, **kwargs)
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
number = int(digest[:4], 16)
host_port = port if isinstance(port, int) else 0
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "",
(f"203.0.113.{number % 256}", host_port)),
(socket.AF_INET6, socket.SOCK_STREAM, 0, "",
(f"2001:db8::{digest[4:8]}", host_port, 0, 0))]
monkeypatch.setattr(host_agent.socket, "getaddrinfo", stub)
def fake_calls(tmp_path: Path) -> list[str]:
called = tmp_path / "smart-out" / "_called"
return called.read_text().split() if called.exists() else []
@@ -1585,11 +1617,16 @@ def test_a_corrupt_nginx_config_leaves_the_rest_of_the_snapshot_intact(tmp_path:
def test_an_nginx_failure_inside_the_collector_still_yields_a_section(tmp_path: Path,
monkeypatch) -> None:
"""Whatever raises in here, the document keeps the key with `available: null` — a snapshot
that loses the key would read as a pre-1.2.0 agent, not as a failed add-on."""
that loses the key would read as a pre-1.2.0 agent, not as a failed add-on.
`nginx_document` is the seam `snapshot()` calls (it is the pair of the section and the
`resolver` directive); `Collector.nginx` is its one-line wrapper, so patching the document is
patching the real path a snapshot takes.
"""
def explode(*args: object) -> dict[str, Any]:
raise RuntimeError("boom")
monkeypatch.setattr(host_agent.Collector, "nginx", explode)
monkeypatch.setattr(host_agent.Collector, "nginx_document", explode)
snap = host_agent.Collector(str(tmp_path / "root")).snapshot()
assert snap["nginx"] == {"available": None, "config": host_agent.NGINX_CONFIG_PATH,
"files": [], "servers": [], "upstreams": [], "maps": []}
@@ -2767,15 +2804,13 @@ def test_the_tls_fields_are_documented_in_the_schema() -> None:
assert "--cert-bundle" in (host_agent.__doc__ or "")
def test_1_2_0_is_the_version_that_ships_the_three_proxy_sections() -> None:
"""The version bump is the fleet's way of saying "this file has the gateway sections".
def test_the_schema_names_the_three_proxy_sections() -> None:
"""Phase 28's three sections are all in the SCHEMA block, each with its three-way `available`.
Phase 28 added three sections and one flag to the payload, so an install that predates them
is distinguishable by one string: `--status`, `/health` and this test all read the same
constant. The SCHEMA block is `GET /schema`'s body, so it names all three, each with the
three-way `available` the dashboard reads, and `--no-proxy`, the flag that removes the family.
The SCHEMA block is `GET /schema`'s body, so a deployment can read what an agent publishes
from the agent itself. The version constant is pinned by the newest phase's test below — one
place names the number, so a bump moves one assertion and not a file's worth of history.
"""
assert host_agent.AGENT_VERSION == "1.2.0"
doc = host_agent.__doc__ or ""
for section in ("haproxy", "nginx", "sockets"):
assert schema_documents(section), f"{section} is not documented in the SCHEMA block"
@@ -2784,6 +2819,24 @@ def test_1_2_0_is_the_version_that_ships_the_three_proxy_sections() -> None:
assert "--no-proxy" in doc and "--haproxy-config" in doc and "--nginx-config" in doc
def test_the_agent_version_is_1_3_0_and_the_schema_names_the_resolver_section() -> None:
"""1.3.0 is the version that resolves the names the configs declare.
The bump is how the fleet tells an agent that publishes `resolver` from one that does not —
`--status`, `/health` and this test all read the same constant, and `sources/gateway.py`'s
"the agent is too old" arm keys on the sections the version belongs to. The SCHEMA block is
`GET /schema`'s body, so it names the section, its three-way `available`, and `--no-resolve`,
the flag that removes it in silence.
"""
assert host_agent.AGENT_VERSION == "1.3.0"
doc = host_agent.__doc__ or ""
assert schema_documents("resolver"), "resolver is not documented in the SCHEMA block"
block = doc.split(" resolver ", 1)[-1]
assert "available" in block, "the resolver block does not document its three-way answer"
assert "--no-resolve" in doc, "the flag that removes the section is undocumented"
assert "libc" in block, "how the addresses were obtained is not published in the schema"
# ---- the recorded gateway: the captures themselves (phase 28 task 04) -------------------- #
#: Every document `scripts/capture-gateway-config.sh` records, at the tree path the agent reads
@@ -2983,6 +3036,151 @@ def test_the_recorded_configs_and_the_recorded_sockets_agree(tmp_path: Path) ->
"the agent's own listener should be in the socket table and claimed by neither config"
#: Every top-level key a **1.2.0** snapshot published, recorded here so this phase's central claim
#: outlives the script it was measured with (`data/logs/verify-32-additive.py`, which imports the
#: released file out of git and compares the two documents value for value). `data/` is gitignored;
#: this table is not, and it is what makes "purely additive" a permanent pin.
SNAPSHOT_KEYS_BEFORE_1_3_0 = frozenset({
"agent_version", "cpu", "cpu_temperature_celsius", "disk_io", "duration_ms", "errors",
"filesystems", "gpus", "haproxy", "host", "memory", "network", "nginx", "processes",
"schema", "sockets", "ts", "temperatures",
})
def test_the_resolver_section_is_the_only_key_1_3_0_adds(tmp_path: Path) -> None:
"""1.3.0 = 1.2.0 + `resolver`, and `--no-resolve` is exactly the old document.
The phase's safety property: an agent upgrade must not move a number on a page that never
asked about name resolution, and a host that opts out must not be left with an empty shell of a
key. Compared as key **sets** against the recorded pre-1.3.0 shape rather than against another
module, so the pin is hermetic and permanent; the value-for-value half of the claim runs in the
script named above, which loads the released file from git and calls both against one `--root`.
"""
root = gateway_tree(tmp_path / "root")
with_resolver = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl").snapshot()
assert set(with_resolver) == SNAPSHOT_KEYS_BEFORE_1_3_0 | {"resolver"}, (
f"1.3.0 changed the document beyond adding one section: "
f"{sorted(set(with_resolver) ^ (SNAPSHOT_KEYS_BEFORE_1_3_0 | {'resolver'}))}")
assert with_resolver["resolver"]["available"] is True
opted_out = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl",
resolve=False).snapshot()
assert set(opted_out) == SNAPSHOT_KEYS_BEFORE_1_3_0, "--no-resolve leaves a key behind"
assert not [line for line in opted_out["errors"]
if str(line).lstrip().lower().startswith("resolver")], (
"opting out is silence, the `--no-smart` precedent")
no_proxy = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl",
proxy=False).snapshot()
assert set(no_proxy) == SNAPSHOT_KEYS_BEFORE_1_3_0 - {"haproxy", "nginx", "sockets"}, (
"the resolver belongs to the proxy family and is removed with it")
def test_the_recorded_resolver_document_is_the_agents_own_section(tmp_path: Path) -> None:
"""`gateway_resolver.json` is a section of the payload, not notes about one.
Nothing here is a hand-written expectation: the entry set and the `resolver` directive are
recomputed from the config fixtures in this directory with the agent's own `target_names()` and
`nginx_document()`, so the two captures cannot drift apart. A route added to the gateway shows
up in both on the next run; a fixture naming a target the parser no longer derives fails here
instead of teaching `sources/gateway.py` to join a route the config does not declare.
The per-entry assertions are the ones the page will rely on: a status out of the four the
section publishes, an `ok` with no address, an `nxdomain` with one, or a list the agent's own
ordering rule would reshuffle, all mean the fixture is not what a poll would serve.
"""
document = json.loads((FIXTURES / "gateway_resolver.json").read_text())
assert set(document) == {"available", "via", "config_directive", "ts", "entries", "errors"}
assert document["available"] is True, "the recorded gateway reads, so the section is available"
assert document["via"] == host_agent.RESOLVE_VIA
assert document["errors"] == []
collector = gateway_collector(gateway_tree(tmp_path / "root"))
errors: list[str] = []
haproxy, nginx = collector.haproxy(errors), collector.nginx(errors)
assert [(row["name"], row["port"]) for row in document["entries"]] == (
host_agent.target_names(haproxy, nginx)), (
"the recorded names are not the ones these configs declare — re-run "
"./scripts/capture-gateway-config.sh and re-tally in the same commit")
_section, directive = collector.nginx_document(errors)
assert document["config_directive"] == directive, (
"the fixture's `resolver` line is not what the agent reads out of the same text")
for row in document["entries"]:
assert set(row) == {"name", "port", "status", "addresses", "ts"}
assert row["status"] in {"ok", "nxdomain", "error", "literal"}, row
if row["status"] in {"ok", "literal"}:
assert row["addresses"], f"{row['name']} is {row['status']} with no address"
else:
assert row["addresses"] == [], f"{row['name']} publishes an address it did not resolve"
assert row["addresses"] == host_agent.sort_resolver_addresses(row["addresses"]), (
f"{row['name']} is not in the order the payload will publish it")
def test_the_lookup_the_agent_performs_is_one_real_libc_accepts(monkeypatch, tmp_path) -> None:
"""The one pin in this file with **no stub in sight**, because a stub cannot refuse a keyword.
`socket.getaddrinfo(name, port, proto=socket.SOCK_STREAM)` — the spelling that reads as "give
me the TCP addresses" — is not that. `proto` is a **protocol number** (`IPPROTO_TCP` is 6) and
`SOCK_STREAM` is 1, so libc is asked for a stream socket over protocol 1 and answers
`EAI_SERVICE` for every name. A fake `getaddrinfo` accepts any keyword, so every resolver test
in this file would have stayed green while the agent reported `error` on a healthy gateway and
left every route on `/gateway` uncounted.
So: the module-wide `no_real_dns` fixture is removed for this test, and the lookup runs against
real libc on a name libc answers from `/etc/hosts` — no resolver, no network, no skip. It
cannot be replaced with a numeric literal either: an address literal is the `literal` arm and
never reaches libc at all, which is the other half of the same pass.
"""
monkeypatch.setattr(host_agent.socket, "getaddrinfo", socket.getaddrinfo)
collector = host_agent.Collector(str(gateway_tree(tmp_path / "root")),
smartctl="/nonexistent/smartctl")
entry = collector._resolve_target("localhost", 9120)
assert entry["status"] == "ok", (
f"real libc refused the call the agent makes ({entry}); if it says `error` with "
"`EAI_SERVICE` in it, the keyword on that call has moved back to `proto=`")
assert entry["addresses"], "a name libc resolves must publish at least one address"
for value in entry["addresses"]:
assert ipaddress.ip_address(value) # type: ignore[call-overload]
def test_the_recorded_resolver_document_publishes_no_globally_routable_address() -> None:
"""Invariant 6 over the JSON, for the same reason and with one hazard of its own.
A resolver answer is an identifier the same way a socket row is: `getent` on this router answers
the WAN address of anything published through it, and a global IPv6 answer is EUI-64 — this
box's NIC, in hex. `scripts/capture-gateway-config.sh` runs the same substitution over the
resolver's answers as over the socket tables, so nothing routable is tracked.
The second assertion is specific to *this* fixture and is a warning, not a rule: on this fleet
every declared target answers a private LAN address, so nothing needs scrubbing. If a capture
ever moves an address into documentation space, that replacement is numbered per document in
first-seen order — it is therefore **not** guaranteed to be the same documentation address the
socket fixtures gave the same endpoint, and `tests/test_gateway_source.py`'s join of the two
would silently stop joining. Re-check that join before trusting the numbers.
"""
document = json.loads((FIXTURES / "gateway_resolver.json").read_text())
routable: list[str] = []
documented: list[str] = []
for row in document["entries"]:
for value in row["addresses"]:
address = ipaddress.ip_address(value)
subject = getattr(address, "ipv4_mapped", None) or address
if subject.is_global:
routable.append(f"{row['name']} -> {value}")
if not subject.is_private:
documented.append(f"{row['name']} -> {value}")
assert not routable, ("globally routable resolver answers are in a tracked fixture: "
+ ", ".join(routable)
+ " — scripts/capture-gateway-config.sh is the thing that stops them")
assert not documented, ("the recorded resolver document had to be scrubbed: "
+ ", ".join(documented)
+ " — reconcile the replacement with the socket fixtures before "
"trusting the join in tests/test_gateway_source.py")
def test_the_recorded_socket_tables_publish_no_globally_routable_endpoint() -> None:
"""AGENTS.md invariant 6, on the artifacts: identifiers never go in the repo.
@@ -3035,3 +3233,502 @@ def test_the_recorded_gateway_answers_all_three_sections_at_once(tmp_path: Path)
"temperatures", "processes", "errors"):
assert section in snap, f"the pre-1.2.0 payload lost {section}"
# ---- the resolver: the names the configs declare, resolved on this box (phase 32 task 01) -- #
def resolver_ledger(monkeypatch: pytest.MonkeyPatch, *,
nx: tuple[str, ...] = (), delay: float = 0.0,
fail: tuple[str, ...] = ()) -> list[tuple[str, int | None]]:
"""Stub `getaddrinfo` with a ledger of every `(name, port)` it was asked about.
The three knobs are the three things a resolver does that a page has to survive: it answers
(`203.0.113.x` + `2001:db8::…`, documentation space so a test payload can never carry a real
endpoint), it refuses a name (`nxdomain`), and it is slow or broken (`delay`, `fail`).
"""
calls: list[tuple[str, int | None]] = []
def stub(name: object, port: object = None, *args: Any, **kwargs: Any) -> Any:
text = str(name)
calls.append((text, port if isinstance(port, int) else None))
if text in fail:
raise socket.gaierror(socket.EAI_NONAME, "and the resolver refused to say")
if delay:
time.sleep(delay)
if text in nx:
raise socket.gaierror(socket.EAI_NONAME, "Name or service not known")
host_port = port if isinstance(port, int) else 0
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (f"203.0.113.{len(calls)}", host_port))]
monkeypatch.setattr(host_agent.socket, "getaddrinfo", stub)
return calls
def resolver_snapshot(root: Path, **kwargs: Any) -> dict[str, Any]:
"""One snapshot of a captured-gateway tree, the resolver's `getaddrinfo` stubbed by the caller."""
return host_agent.Collector(str(root), smartctl="/nonexistent/smartctl",
**kwargs).snapshot()
def test_only_the_targets_the_configs_declare_are_resolved(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""The allow-list is the config, in config order, once each — decision 3, pinned as a call set.
Everything the resolver dialed is a target the two parsed sections publish, in the order the
files write them, with no duplicate — and nothing that came from `/proc/net/tcp` is in there.
That last half is the point: a socket table is full of addresses this gateway talks to, and
resolving one of those would be a fact about whoever connected, not about the config.
"""
calls = resolver_ledger(monkeypatch)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
declared = host_agent.target_names(snap["haproxy"], snap["nginx"])
looked_up = [pair for pair in declared if host_agent.address_family(pair[0]) is None]
assert calls == looked_up, "the lookups are exactly the declared targets, in config order"
assert len(calls) == len(set(calls)), "one name on one port is one lookup, not two"
assert len(looked_up) < len(declared), "the tree writes literals too, and they are kept"
socket_addresses = {row["remote"]["addr"] for row in snap["sockets"]["established"]}
socket_addresses |= {row["local"]["addr"] for row in snap["sockets"]["established"]}
socket_addresses |= {row["addr"] for row in snap["sockets"]["listens"]}
assert not socket_addresses & {name for name, _ in calls}, (
"a name from the socket table was resolved: the config is the only allow-list")
# haproxy first (its six `server` lines), and this fleet's two Minecraft ports are two targets
assert calls[:6] == [("reeseapps-gitea.reeselink.com", 2222),
("stackexpected-gitea.reeselink.com", 2222),
("web.reeselink.com", 5201),
("unifi-external.reeselink.com", 8080),
("minecraft.reeselink.com", 25565),
("minecraft.reeselink.com", 25566)]
def test_an_address_literal_in_a_config_is_reported_and_never_dialled(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""`proxy_pass http://127.0.0.1:8081` is already an address: the entry says so, no lookup runs.
Looking a literal up would be a round trip to be told what the config wrote, and on a box with
a slow resolver it is time spent for nothing. The address still travels, because the gateway
page's join keys on it.
"""
calls = resolver_ledger(monkeypatch)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
entries = {host_agent.resolver_key(e["name"], e["port"]): e for e in snap["resolver"]["entries"]}
literal = entries[host_agent.resolver_key("127.0.0.1", 8081)]
assert literal["status"] == "literal"
assert literal["addresses"] == ["127.0.0.1"]
assert ("127.0.0.1", 8081) not in calls, "a literal costs no lookup"
assert not [call for call in calls if host_agent.address_family(call[0])], \
"no lookup was ever asked about an address"
def test_the_section_names_how_and_where_and_when(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""`via`, `config_directive` and the timestamps are the provenance of every address here.
`libc` is what `getaddrinfo` is — the resolution haproxy performs on this box — and the nginx
config's own `resolver` line travels beside it verbatim (read out of `stream.d/stream.conf`,
which is where this fleet writes it, so an include is as good as the main file). The
`nginx` section itself grows no key: that is what keeps the phase's additivity control a
top-level comparison.
"""
resolver_ledger(monkeypatch)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
section = snap["resolver"]
assert section["via"] == host_agent.RESOLVE_VIA == "libc"
assert section["config_directive"] == "10.1.0.1"
assert "resolver" not in snap["nginx"], "the nginx document stays byte-stable"
assert isinstance(section["ts"], float)
for entry in section["entries"]:
assert entry["ts"] <= section["ts"] + 1.0, "an entry is answered at or before the pass ends"
assert set(entry) == {"name", "port", "status", "addresses", "ts"}
def test_the_resolver_cache_means_one_lookup_per_target(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""Two snapshots, one set of lookups; an expiry refreshes in the background and never on the request.
The dashboard polls every 10 s. Without the cache that is 30 DNS lookups per host per minute
for names that move when the operator moves them; with it, the poll that finds an expired
cache is answered from the cache and carries the **original** per-entry `ts`, so it cannot
present a five-minute-old address as a fresh one.
"""
calls = resolver_ledger(monkeypatch)
root = gateway_tree(tmp_path / "root")
collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl")
first = collector.snapshot()["resolver"]
baseline = len(calls)
assert baseline and len(calls) == len(set(calls))
second = collector.snapshot()["resolver"]
assert len(calls) == baseline, "a fresh cache costs no lookup"
assert second["entries"] == first["entries"]
collector.resolve_cache_seconds = 0.0
third = collector.snapshot()["resolver"]
assert third["entries"] == first["entries"], "the expired cache is served as it was"
assert [e["ts"] for e in third["entries"]] == [e["ts"] for e in first["entries"]], \
"a cache-served entry keeps the time it was answered, not the time it was served"
wait_for_resolver_refresh(collector)
assert len(calls) > baseline, "the expiry did re-resolve, off the request path"
def test_a_slow_resolver_cannot_hold_the_snapshot_open(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""The pass stops at its budget and says so once — the SMART pass-budget lesson, applied to DNS.
`getaddrinfo` takes no timeout argument, so the only defence against a resolver that has
started black-holing is to stop asking. What the section must still be: `available: true`, the
targets it reached present, one line naming how many of them that was, and `/snapshot`
answering in a small fraction of the time the full set would have taken — because the *system*
page's timeout is what a slow pass would otherwise cost.
"""
calls = resolver_ledger(monkeypatch, delay=0.02)
root = gateway_tree(tmp_path / "root")
collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl")
collector.resolve_pass_budget_seconds = 0.1
started = time.monotonic()
snap = collector.snapshot()
elapsed = time.monotonic() - started
section = snap["resolver"]
budget_lines = [line for line in section["errors"] if "pass budget" in line]
assert section["available"] is True, "a truncated pass is still an answer"
assert len(budget_lines) == 1, snap["errors"]
reached = [e for e in section["entries"] if e["status"] != "literal"]
assert budget_lines[0] == (f"resolver: pass budget 0.1s exhausted, "
f"{len(reached)} of 30 targets looked up"), budget_lines[0]
assert len(reached) < 29, "the pass stopped rather than finished"
assert len(calls) == len(reached), "the ledger and the section agree on what was reached"
assert [e for e in section["entries"] if e["status"] == "literal"], \
"an address the config spells survives a pass that stopped early"
assert len(section["entries"]) == len(reached) + 2, "two literals, and nothing invented"
assert elapsed < 1.0, f"a bounded pass cost {elapsed:.2f}s"
assert snap["cpu"]["modes"]["total"] > 0 and snap["host"]["hostname"]
assert core_errors(snap) == [], snap["errors"]
def test_the_name_bound_reports_the_excess_in_one_line(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""More declared targets than the bound: the excess is one line, not 40, and `true` stands."""
calls = resolver_ledger(monkeypatch)
root = gateway_tree(tmp_path / "root")
collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl")
collector.resolve_max_names = 3
section = collector.snapshot()["resolver"]
bound_lines = [line for line in section["errors"] if "name bound" in line]
assert len(bound_lines) == 1, section["errors"]
assert bound_lines[0].startswith("resolver: 27 of 30 targets are beyond the 3-name bound"), \
bound_lines[0]
assert section["available"] is True
assert len(calls) == 3, "the bound is the cost of the section, not a display choice"
literals = [e for e in section["entries"] if e["status"] == "literal"]
assert len(section["entries"]) == 3 + len(literals)
def wait_for_resolver_refresh(collector: Any, attempts: int = 500) -> None:
"""Join the single-flight refresher, as far as a test can: it is a daemon thread on purpose.
The wait is on the flag rather than a `Thread.join()` because the agent keeps no handle to it
(a snapshot must never be able to wait on DNS), and 5 s of 10 ms polls is more than a stubbed
pass takes on any machine this suite runs on.
"""
for _ in range(attempts):
if not collector._resolver_refreshing:
return
time.sleep(0.01)
raise AssertionError("the background resolver refresh never finished")
def test_a_failed_background_refresh_keeps_the_last_addresses(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""A refresh that dies is one line, the old entries, and no hammering.
The thread is the only thing that ever re-reads DNS after the first pass, so a pass that
raises must leave the snapshot answering with the last truth it had (SMART's rule) and must
not turn every subsequent poll into a retry. One line, replaced rather than stacked: a
resolver that has been broken for a week is one fact the operator acts on once, not 2 000.
"""
resolver_ledger(monkeypatch)
collector = host_agent.Collector(str(gateway_tree(tmp_path / "root")),
smartctl="/nonexistent/smartctl")
first = collector.snapshot()["resolver"]
assert first["available"] is True and first["entries"]
def explode(*args: object) -> None:
raise RuntimeError("the resolver daemon is gone")
monkeypatch.setattr(host_agent.Collector, "resolver_pass", explode)
collector.resolve_cache_seconds = 0.0
for attempt in range(2):
served = collector.snapshot()["resolver"]
assert served["entries"] == first["entries"], "the last good answers stay published"
wait_for_resolver_refresh(collector)
# Read the collector's state, not this response: the snapshot was served *while* the
# refresh was in flight, so the line it could not have is not a defect in it.
failures = [line for line in collector._resolver_errors
if "background refresh failed" in line]
assert len(failures) == 1, (attempt, collector._resolver_errors)
assert failures[0].startswith("resolver: background refresh failed: RuntimeError")
assert collector._resolver_cache_ts is not None, "a failed pass still re-stamps the cache"
# …and the line reaches the page eventually, beside the entries it explains.
later = collector.snapshot()["resolver"]
assert any("background refresh failed" in line for line in later["errors"]), later["errors"]
assert later["entries"] == first["entries"]
def test_nxdomain_is_a_status_and_never_a_failure(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""One name that does not answer is one entry with no addresses — the snapshot is unbothered.
The route stays on the page and carries no count (task 03 does the joining); what must not
happen is a `0`, a dropped entry, or a poll that goes red because a domain left the zone file.
"""
resolver_ledger(monkeypatch, nx=("reeseapps-brain-of-reese.reeselink.com",))
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
section = snap["resolver"]
entries = {e["name"]: e for e in section["entries"]}
failed = entries["reeseapps-brain-of-reese.reeselink.com"]
assert failed["status"] == "nxdomain"
assert failed["addresses"] == [], "no address, never a 0 and never last month's"
answered = [e for e in section["entries"] if e["status"] == "ok"]
assert answered, "the others answered"
assert section["available"] is True
assert section["errors"] == [], "a name that does not resolve is not this section's failure"
assert snap["cpu"]["modes"]["total"] > 0
assert core_errors(snap) == [], snap["errors"]
def test_a_resolver_that_errors_is_an_error_and_not_an_nxdomain(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
""""The name has no record" and "the lookup could not be performed" are two fixes.
A temporary failure (`EAI_AGAIN`) is the one that most looks like `nxdomain` and must not be
reported as it: an operator reading `nxdomain` edits a zone file, an operator reading `error`
checks whether the resolver is up.
"""
def stub(name: object, port: object = None, *args: Any, **kwargs: Any) -> Any:
text = str(name)
if text == "gitea.reeselink.com":
raise socket.gaierror(socket.EAI_AGAIN, "Temporary failure in name resolution")
if text == "jellyfin.reeselink.com":
raise OSError("resolver socket refused to open")
if text == "bricktracker.reeselink.com":
raise ValueError("a platform-shaped answer nobody predicted")
host_port = port if isinstance(port, int) else 0
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("203.0.113.7", host_port))]
monkeypatch.setattr(host_agent.socket, "getaddrinfo", stub)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
entries = {e["name"]: e for e in snap["resolver"]["entries"]}
assert entries["gitea.reeselink.com"]["status"] == "error", "EAI_AGAIN is not nxdomain"
assert entries["jellyfin.reeselink.com"]["status"] == "error"
assert entries["bricktracker.reeselink.com"]["status"] == "error"
assert entries["bricktracker.reeselink.com"]["detail"].startswith("ValueError")
assert entries["gitea.reeselink.com"]["addresses"] == []
assert snap["resolver"]["available"] is True
assert core_errors(snap) == [], snap["errors"]
def test_the_addresses_are_unique_and_sorted_v4_then_v6(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""Two polls of one healthy gateway are byte-identical, whatever order libc answered in.
`getaddrinfo` returns whatever the resolver and `/etc/gai.conf` feel like today, and the entry
list is something a page renders: an unsorted set makes a healthy box look like its addresses
are shuffling.
"""
def stub(name: object, port: object = None, *args: Any, **kwargs: Any) -> Any:
host_port = port if isinstance(port, int) else 0
return [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", ("2001:db8::2", host_port, 0, 0)),
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("203.0.113.9", host_port)),
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("203.0.113.9", host_port)),
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("203.0.113.4", host_port))]
monkeypatch.setattr(host_agent.socket, "getaddrinfo", stub)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
entry = snap["resolver"]["entries"][0]
assert entry["status"] == "ok"
assert entry["addresses"] == ["203.0.113.4", "203.0.113.9", "2001:db8::2"]
def test_a_value_that_is_not_an_address_sorts_last_instead_of_raising() -> None:
"""The sort is defensive for the one input the section should never see.
`getaddrinfo` answers addresses, so this arm is a stub or a platform oddity — and the answer
to that is a stable order with the odd value at the end, not a `ValueError` on a snapshot path
and not the whole section lost because one entry could not be compared.
"""
assert host_agent.sort_resolver_addresses(["nope", "10.0.0.2", "::1", "also-bad", "10.0.0.2"]) \
== ["10.0.0.2", "::1", "also-bad", "nope"]
def test_no_resolve_removes_the_key_in_silence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""`--no-resolve` is `--no-smart`'s precedent: no key, no error line — "not collected".
An empty section would read as "collected, and nothing resolves", which is a different fact
and the wrong one to print for a host the operator told not to ask.
"""
calls = resolver_ledger(monkeypatch)
root = gateway_tree(tmp_path / "root")
snap = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl",
resolve=False).snapshot()
assert "resolver" not in snap
assert calls == [], "the flag is honoured before anything is dialled"
assert snap["nginx"]["available"] is True and snap["sockets"]["available"] is True, \
"the rest of the family is untouched"
assert [line for line in snap["errors"] if line.startswith("resolver:")] == []
def test_no_proxy_removes_the_resolver_with_the_rest_of_the_family(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""A box that collects no proxy has no declared names, so `--no-proxy` removes this too."""
calls = resolver_ledger(monkeypatch)
snap = host_agent.Collector(str(gateway_tree(tmp_path / "root")),
smartctl="/nonexistent/smartctl", proxy=False).snapshot()
for section in ("haproxy", "nginx", "sockets", "resolver"):
assert section not in snap
assert calls == []
def test_a_host_with_no_proxy_sections_says_so_once(tmp_path: Path) -> None:
"""`available: false` and exactly one `resolver:` line naming **not found** — invariant 3.
Two of this fleet's three hosts run no proxy at all. Their agents must report "nothing to
resolve" without putting a red banner, an overview alert, or fifty lines in `errors` anywhere.
"""
snap = host_agent.Collector(str(tmp_path / "root")).snapshot()
resolver_lines = [line for line in snap["errors"] if line.startswith("resolver:")]
assert snap["resolver"]["available"] is False
assert snap["resolver"]["entries"] == []
assert len(resolver_lines) == 1, snap["errors"]
assert "not found" in resolver_lines[0]
assert "--no-resolve" in resolver_lines[0]
assert snap["resolver"]["errors"] == resolver_lines
assert core_errors(snap) == [], "an unconfigured add-on is not a system degradation"
@pytest.mark.skipif(os.geteuid() == 0, reason="root reads a 0000 file")
def test_a_config_that_is_present_but_unreadable_says_that_instead(tmp_path: Path) -> None:
"""The `false` arm names the right reason: an unreadable file is not a missing one.
`not found` and `unreadable` are two operator actions — install the proxy, or fix the mode on
its config — and one line that says which is the difference between a `chmod` and a
`dnf install`. Still one line, still `available: false`, still no system degradation.
"""
root = tmp_path / "root"
nginx_tree(root, http=None, stream=None)
(root / "etc/nginx/nginx.conf").chmod(0o000)
snap = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl").snapshot()
line = next(line for line in snap["errors"] if line.startswith("resolver:"))
assert snap["resolver"]["available"] is False
assert "haproxy not found" in line
assert "nginx unreadable" in line
assert len([x for x in snap["errors"] if x.startswith("resolver:")]) == 1
def test_a_config_that_reads_cleanly_and_names_no_target_is_true_and_empty(
tmp_path: Path) -> None:
"""Configs read, no targets declared: `available: true` with `entries: []` and no line.
Absence of a target is not a fault, and the third arm (`null`) is reserved for a pass that
could not run at all — otherwise "no routes" and "no resolver" arrive as one value.
"""
root = tmp_path / "root"
nginx_tree(root, main="http { server { listen 8080; server_name quiet.test; } }\n",
http=None, stream=None)
snap = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl").snapshot()
assert snap["nginx"]["available"] is True
assert snap["resolver"]["available"] is True, snap["errors"]
assert snap["resolver"]["entries"] == []
assert snap["resolver"]["errors"] == []
assert core_errors(snap) == [], snap["errors"]
def test_the_resolver_lines_never_fail_the_snapshot(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""Every arm of the new section leaves `/snapshot` answering, with its key and its own line.
The family rule phase 28 pinned for `haproxy`, `nginx` and `sockets` extended to the fourth
section: whatever the resolver does — raises inside, cannot find a config, cannot reach a
name — the document keeps the key (so a reader can tell "the pass failed" from "the agent
predates it"), the other sections are intact, and the line is the add-on's own rather than a
system `degraded` (invariant 3).
"""
def explode(*args: object) -> dict[str, Any]:
raise RuntimeError("resolver exploded")
monkeypatch.setattr(host_agent.Collector, "resolver", explode)
snap = host_agent.Collector(str(gateway_tree(tmp_path / "root")),
smartctl="/nonexistent/smartctl").snapshot()
assert snap["resolver"] == {"available": None, "via": "libc", "config_directive": None,
"ts": None, "entries": [], "errors": []}
assert any(line.startswith("resolver:") for line in snap["errors"])
assert core_errors(snap) == [], snap["errors"]
assert snap["haproxy"]["available"] is True and snap["nginx"]["available"] is True
assert snap["cpu"]["modes"]["total"] > 0
def test_the_resolver_section_is_documented_in_the_schema() -> None:
"""`GET /schema` answers the docstring, so the section an operator can `curl` is described."""
assert schema_documents("resolver")
def test_the_resolver_reaches_the_command_line(tmp_path: Path, capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch) -> None:
"""`--once` prints the resolver block, and prints none with `--no-resolve` added.
This is the completion criterion an operator can run on a host with no dashboard: one command,
and the answer says which names this box turned into which addresses.
"""
resolver_ledger(monkeypatch)
root = gateway_tree(tmp_path / "root")
# The config paths are the ones the **config** writes: an absolute path resolves under
# `--root`, which is how a deployment reads /etc and a test reads a fake /etc.
nginx, haproxy = "/etc/nginx/nginx.conf", "/etc/haproxy/haproxy.cfg"
assert host_agent.main(["--once", "--no-smart", "--root", str(root),
"--haproxy-config", haproxy, "--nginx-config", nginx]) == 0
printed = json.loads(capsys.readouterr().out)
assert printed["resolver"]["available"] is True
assert printed["resolver"]["via"] == "libc"
assert printed["resolver"]["config_directive"] == "10.1.0.1"
assert printed["resolver"]["entries"]
assert host_agent.main(["--once", "--no-smart", "--no-resolve", "--root", str(root),
"--haproxy-config", haproxy, "--nginx-config", nginx]) == 0
without = json.loads(capsys.readouterr().out)
assert "resolver" not in without
assert without["nginx"]["available"] is True, "the rest of the family still prints"
def test_the_target_list_is_the_two_sections_and_survives_a_section_that_is_not_one() -> None:
"""`target_names` reads haproxy then nginx (upstreams, proxy_pass, maps) and nothing stranger.
A `proxy_pass $var` contributes nothing — the map decides, and the map's entries are already
in the list — and a `None`, a bare string, or a section of the wrong shape contributes an
empty list rather than a `TypeError` on a snapshot path.
"""
haproxy = {"backends": [{"servers": [{"name": "a", "target": "a.test", "port": 1},
{"name": "b", "target": "b.test", "port": None},
{"name": "c", "target": None, "port": 3},
{"name": "d", "target": "a.test", "port": 1}]}]}
nginx = {"upstreams": [{"servers": [{"host": "pool.test", "port": 80}]}],
"servers": [{"proxy_passes": [{"target": {"host": "vhost.test", "port": 443}},
{"target": None, "variable": "backend"}]}],
"maps": [{"entries": [{"key": "x", "target": {"host": "map.test", "port": 443}},
{"key": "default", "target": None}]}]}
assert host_agent.target_names(haproxy, nginx) == [
("a.test", 1), ("b.test", None), ("pool.test", 80),
("vhost.test", 443), ("map.test", 443)]
assert host_agent.target_names(None, None) == []
assert host_agent.target_names({"backends": "nonsense"}, {"maps": [7]}) == []
tests/test_alerts.py
+34
View File
@@ -24,6 +24,7 @@ import inspect
from typing import Any
from dashboards import alerts, severity
from dashboards.collectors import ADDON_ERROR_PREFIXES
from dashboards.config import Settings
from dashboards.sources import nut
from dashboards.sources.nut_sim import NutSimulator
@@ -351,6 +352,39 @@ def test_a_degraded_agent_warns_and_names_the_sections_that_failed(
assert "disk_io" in out[0]["message"]
def test_an_addon_absence_line_raises_no_row_and_a_section_that_failed_raises_one(
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
"""Invariant 3's last link, `resolver:` included: filtered out means no row, unfiltered means one.
`collectors.ADDON_ERROR_PREFIXES` is what keeps an agent's add-on lines out of `degraded`
(`tests/test_collectors.py` pins that), and `degraded` is what this module turns into the
overview row pinned just above. So: a host whose agent said it runs no haproxy, no nginx, no
smartmontools and had **nothing to resolve** is all-clear on every panel — the phase-32 pair of
lines (`nothing to resolve`, `pass budget … exhausted`) included, because a truncated lookup is
one route without a count and not a broken box. And a line that *did* reach `degraded` — which
is what dropping a prefix from that list would do — does raise exactly one row, naming it.
"""
addon_lines = (
"resolver: nothing to resolve — haproxy not found and nginx not found; --no-resolve opts out",
"resolver: pass budget 3.0s exhausted, 41 of 58 targets looked up",
"haproxy: /etc/haproxy/haproxy.cfg not found — haproxy is not installed here",
"smart: smartctl 7.5 not found",
)
assert all(line.lower().startswith(ADDON_ERROR_PREFIXES) for line in addon_lines), (
"a prefix dropped from collectors' list would put this fleet's non-gateway hosts on warn")
clean = host_scrape(agent_snapshot, cpu_busy=10.0)
clean["agent"]["errors"] = list(addon_lines)
assert "degraded" not in clean, "the collector filtered them, so there is nothing to grade"
assert alerts.build(SETTINGS, [state()], [clean]) == []
for line in addon_lines:
broken = host_scrape(agent_snapshot, cpu_busy=10.0)
broken["degraded"] = [line]
out = alerts.build(SETTINGS, [state()], [broken])
assert [row["id"] for row in out] == ["driveripper:agent.sections"], line
def test_a_degraded_list_from_an_older_agent_is_readable_too(
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
tests/test_api.py
+603 -65
View File
@@ -4907,9 +4907,17 @@ def test_the_gateway_job_is_handed_no_rate_estimator(tmp_path: Path) -> None:
def make_gateway_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
**overrides: Any) -> Any:
"""An app whose only host is the ingress, on clocks fast enough to poll inside a test."""
"""An app whose only host is the ingress, on clocks fast enough to poll inside a test.
The intervals are the poller's own floor, not 0.05: `Job` clamps its cadence at 0.5 s,
while the staleness threshold is three times the *configured* interval. At 0.05 s a healthy
job's gap (≥ 0.5 s) always outlived its own threshold (0.15 s), so its `:stale` row
appeared in the alert list 70 % of the time and any assertion on the list's length was
a test of when the event loop happened to tick. At 0.5 s the threshold (1.5 s) is
three times the gap a healthy job can make, and the premise jobs stay quiet whatever
the machine is doing."""
return build_client(tmp_path, monkeypatch, host_targets=(GATEWAY_TARGET,),
host_interval=0.05, disks_interval=0.05, gateway_interval=0.05,
host_interval=0.5, disks_interval=0.5, gateway_interval=0.5,
**overrides)
@@ -4975,25 +4983,56 @@ def test_the_summary_gateway_block_is_the_stub_of_a_source_that_never_answered(
assert block["href"] == GATEWAY_HREF
def endpoint_identity(out: dict[str, Any]) -> tuple[Any, ...] | None:
"""One endpoint, in the currency the kernel writes — derived here, not by the source.
An endpoint written as an address **is** that address; an endpoint written as a name is every
address the agent's resolver published for it (`out.resolved`), and `ipaddress` folds the
`::ffff:`-mapped and zero-padded spellings so one address in two tables is one key. `None`
means the join has nothing to name the far end with, which contributes nothing to a total.
This is deliberately a **third** reading of "what is one endpoint" (the others are
`gateway.endpoint_of` behind `api._endpoint_values`, and the join itself): since phase 32 two
route lines can reach one address while the configs spell it as two different names, and a
total keyed on the config's text would book it once per name.
"""
port = out.get("port")
if not isinstance(port, int):
return None
spellings = ([out["host"]] if out.get("address") is True
else list(out.get("resolved") or []))
keys: list[str] = []
for spelling in spellings:
try:
address = ipaddress.ip_address(str(spelling))
except ValueError:
continue
mapped = getattr(address, "ipv4_mapped", None)
keys.append(str(mapped if mapped is not None else address))
return (min(keys), port) if keys else None
def endpoint_sums(payload: dict[str, Any]) -> tuple[int | None, int | None]:
"""The two live totals, derived here so the block is checked against a **second** reading.
Inbound is the sum over trunks. Outbound is the sum over the **distinct** `(host, port)` the
route lines name: `sources/gateway.py` repeats an endpoint's whole count on every line that
reaches it (`shared: true`), so summing the lines would book the same sockets twice and print
a router busier than the kernel has connections for. Neither is a sum when nothing answered.
Inbound is the sum over trunks. Outbound is the sum over the **distinct endpoints** the route
lines reach: `sources/gateway.py` repeats an endpoint's whole count on every line that reaches
it (`shared: true`), so summing the lines would book the same sockets twice and print a router
busier than the kernel has connections for. Neither is a sum when nothing answered.
"""
live_in = [row["connections"] for row in payload["listeners"]
if isinstance(row["connections"], int)]
endpoints: dict[tuple[Any, Any], Any] = {}
endpoints: dict[tuple[Any, ...], Any] = {}
for row in payload["connections"]:
endpoints.setdefault((row["out"]["host"], row["out"]["port"]), row["connections"])
key = endpoint_identity(row["out"])
if key is not None:
endpoints.setdefault(key, row["connections"])
live_out = [value for value in endpoints.values() if isinstance(value, int)]
return (sum(live_in) if live_in else None,
sum(live_out) if live_out else None)
@pytest.mark.parametrize("mode", ["full", "predates", "sockets_silent"])
@pytest.mark.parametrize("mode", ["full", "resolved", "predates", "sockets_silent"])
def test_the_summary_gateway_block_is_the_payloads_sums_not_the_pages_arithmetic(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, gateway_snapshot: Any,
mode: str) -> None:
@@ -5003,12 +5042,18 @@ def test_the_summary_gateway_block_is_the_payloads_sums_not_the_pages_arithmetic
hand-written payload, so a rename in the source's shape fails here instead of printing a
plausible zero (rule 7).
The split that replaced `partial` is the point of the three arms. `listeners` / `connections`
The split that replaced `partial` is the point of the arms. `listeners` / `connections`
are **row counts** — a fact readable from a config file, so they are numbers even when the
socket table never answered — while `live_in` / `live_out` are measurements and go `null` with
it. The old block could only report one number per side and needed a `partial` flag to say it
was a subtotal; there is no residual to qualify now, because a socket outside the two files is
counted nowhere by decision.
The `resolved` arm is phase 32's, and it is here because the **block** must not change when the
join widens: twelve fields, the same names, and `live_out` still the sum over distinct
endpoints — now endpoints reached through a resolved name as well as an address, each booked
once however many lines reach it. A block that summed the lines instead would move from 15 to a
bigger, wronger number on the same capture.
"""
payload = gateway_source.parse(gateway_snapshot(mode), "gateway")
expected_in, expected_out = endpoint_sums(payload)
@@ -5054,22 +5099,40 @@ def test_the_summary_gateway_block_of_the_captured_ingress_reads_the_live_join(
"""The real capture, one figure at a time — the block an operator would actually get.
The parametrised test above proves the rule; this one pins the numbers, so a change to the join
that moves a total is visible somewhere rather than only in a browser. The outbound figure is
small **on purpose**: this capture's configs write almost every target as a domain, and a
domain is not something this stack can join to a socket, so only the two `127.0.0.1` endpoints
carry a count at all (`:4443` with 2 and `:8081` with 0).
that moves a total is visible somewhere rather than only in a browser. The three config-shaped
figures are literal, because only the operator moves them; the two live totals are derived from
the capture rather than quoted, because a re-capture of a busy router lawfully moves them.
Since phase 32 task 03 the outbound total is no longer pinned to the two `127.0.0.1` endpoints:
the recorded `gateway_resolver.json` places every name this ingress routes to, so the routes
that used to be dashes are counted through the address the gateway dialled. What is still pinned
is the **rule** the number comes from — one count per distinct endpoint, and never more than the
kernel has connections for.
"""
payload = gateway_source.parse(gateway_snapshot("resolved"), "gateway")
with make_gateway_client(tmp_path, monkeypatch) as client:
answer_as(client, "gateway", gateway_source.parse(gateway_snapshot("full"), "gateway"))
answer_as(client, "gateway", payload)
deadline = time.time() + 10
while time.time() < deadline and not client.app.state.ctx.hub.jobs["gateway"].state.ok:
time.sleep(0.05)
block = gateway_block(client)
assert block["listeners"] == 10, "6 haproxy binds + 4 nginx listen ports"
assert block["connections"] == 32, "every route the two captured configs declare"
assert block["live_in"] == 9, "9 of the 18 live sockets face a configured listen port"
assert block["live_out"] == 2, "the two loopback endpoints, counted once each"
assert block["map_targets"] == 17, "the stream map's routing entries are the fan-out"
assert block["connections"] == 33, "every route the two captured configs declare"
assert block["map_targets"] == 18, "the stream map's routing entries are the fan-out"
# The two live totals are the volatile half: a capture of a running router is a different set of
# connections every second, so they are compared with the payload the block was built from
# rather than with a number typed here (the doctrine this suite already keeps for the socket
# fixtures themselves — assert the shape and the rule, never a particular row). What stays
# literal is what only the two config files decide, above, and what stays asserted as a
# relationship is the direction of the error, below.
assert block["live_in"] == sum(row["connections"] or 0 for row in payload["listeners"])
_, expected_out = endpoint_sums(payload)
assert block["live_out"] == expected_out, "one count per distinct endpoint, never one per line"
assert sum(row["connections"] or 0 for row in payload["connections"]) >= (expected_out or 0), (
"the lines repeat a shared endpoint's count, which is exactly why the block adds the "
"endpoints up instead")
assert block["live_in"] + block["live_out"] <= payload["sockets"]["established"], (
"the reported figures may under-count the kernel's, never exceed it")
assert block["warnings"] == 0, (
"this fleet's two config files connect to themselves, and 0 is the truth rather than a "
"missing measurement — the overview line prints nothing for it either way")
@@ -5106,6 +5169,13 @@ def test_one_dangling_use_backend_is_one_warn_row_on_the_overview(
deadline = time.time() + 15
while time.time() < deadline and not client.app.state.ctx.hub.jobs["gateway"].state.ok:
time.sleep(0.05)
# The two host jobs are this test's premise, not its subject: on a slow machine the wait
# loops above outlive their 1.5 s staleness window, and the alert then gains a `:stale`
# row for the premise and the assertion below reads a three-row list. Stamping their
# `last_ok` pins the premise the docstring states ("the agent answers") and leaves the
# gateway row the only line.
for host_key in ("host:gateway", "disks:gateway"):
client.app.state.ctx.hub.jobs[host_key].state.last_ok = time.time()
rows = gateway_alerts(client)
block = client.get("/api/summary").json()["gateway"]
assert len(rows) == 1, rows
@@ -5123,6 +5193,8 @@ def test_one_dangling_use_backend_is_one_warn_row_on_the_overview(
deadline = time.time() + 15
while time.time() < deadline and client.get("/api/summary").json()["gateway"]["warnings"]:
time.sleep(0.05)
for host_key in ("host:gateway", "disks:gateway"):
client.app.state.ctx.hub.jobs[host_key].state.last_ok = time.time()
assert gateway_alerts(client) == [], "a clean ingress is an all clear, not a green line"
@@ -5164,6 +5236,13 @@ def test_a_gateway_job_that_stops_answering_keeps_one_row_not_two(
time.sleep(0.05)
else:
pytest.fail("the gateway job never went from answering to failing")
# The two host jobs are this test's premise, not its subject: on a slow machine the wait
# loops above outlive their 1.5 s staleness window, and the alert then gains a `:stale`
# row for the premise and the assertion below reads a three-row list. Stamping their
# `last_ok` pins the premise the docstring states ("the agent answers") and leaves the
# gateway row the only line.
for host_key in ("host:gateway", "disks:gateway"):
client.app.state.ctx.hub.jobs[host_key].state.last_ok = time.time()
alerts = gateway_alerts(client)
assert [row["id"] for row in alerts] == ["gateway:gateway:down"], (
"the reachability row is the gateway's only line while it is down")
@@ -5428,17 +5507,16 @@ def find_keys(node: Any, name: str) -> list[Any]:
return []
@pytest.fixture
def gateway_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
gateway_snapshot: Any) -> Iterator[TestClient]:
"""The ingress answering with the captured configs and sockets, its gauges already stored.
def _gateway_routes_client(tmp_path: Path, mode: str,
gateway_snapshot: Any) -> Iterator[TestClient]:
"""One app per capture mode, its gauges already stored, its payload already in the job state.
Both halves come from `sources/gateway` over the **captured** tree, so every path a page reads
and every series name a chart asks for is one the source really emits (rule 7). The target
points at a dead port: no poll can overwrite the payload, dial out, or add a second sample
between the store read and the route assertion below.
"""
db = str(tmp_path / "gateway-routes.sqlite3")
db = str(tmp_path / f"gateway-routes-{mode}.sqlite3")
store = SampleStore(db)
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
pbkdf2_iterations=FAST_PBKDF2,
@@ -5446,7 +5524,7 @@ def gateway_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
with TestClient(app) as test_client:
sign_in(test_client)
now = time.time()
snapshot = gateway_snapshot("full")
snapshot = gateway_snapshot(mode)
snapshot["ts"] = now # "the agent answered just now", so charts are in range
result = gateway_source.parse_gateway_snapshot("gateway", snapshot, now=now)
store.add_many(result.samples)
@@ -5455,6 +5533,31 @@ def gateway_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
store.close()
@pytest.fixture
def gateway_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
gateway_snapshot: Any) -> Iterator[TestClient]:
"""The ingress answering as agent **1.2.0** (or 1.3.0 with `--no-resolve`) does.
No `resolver` section, so the routes whose targets the two config files write as domains carry
no count, no gauge and no stored series — phase 30 verbatim, which is what the phase-32 pins
below compare against.
"""
yield from _gateway_routes_client(tmp_path, "full", gateway_snapshot)
@pytest.fixture
def resolved_gateway_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
gateway_snapshot: Any) -> Iterator[TestClient]:
"""The same ingress answering as agent **1.3.0**: the captured resolver section attached.
The document is the recorded `gateway_resolver.json`, so every series name this fixture's store
holds is one the deployed ingress will really write after task 05 — including the **name-keyed**
outbound gauges phase 30 stopped writing, which are writable again precisely because the
gateway published the address behind the name.
"""
yield from _gateway_routes_client(tmp_path, "resolved", gateway_snapshot)
def dial_spies(monkeypatch: pytest.MonkeyPatch, calls: list[str]) -> None:
"""Two spies that raise: a route that dials before its gate fails here, not in a timeout.
@@ -5568,11 +5671,19 @@ def test_gateway_snapshot_answers_the_pipes_the_projection_joined(
"the deleted join is back in the payload")
# the config half, verbatim from the capture the agent read (phase 28's six frontends, the
# stream map's 17 routing entries) — the echo the pipes were built from
# stream map's 18 routing entries) — the echo the pipes were built from
assert len(body["haproxy"]["frontends"]) == 6
assert len(body["nginx"]["map_targets"]) == 17
assert len(body["nginx"]["map_targets"]) == 18
# `/proc/net/tcp` is one row per live connection, so every count below is derived from the same
# capture the payload was built from instead of typed here: the rule is the pin, the number is a
# reading. The config-shaped figures above stay literal, because only the operator moves those.
established = gateway_snapshot("full")["sockets"]["established"]
stream = next(pipe for pipe in body["listeners"] if pipe["port"] == 443)
assert stream["connections"] == 6 and stream["metric"] == "gateway.in_conns@443", (
assert stream["connections"] == sum(1 for row in established
if row["local"]["port"] == 443), (
"the inbound pipe carries the sockets whose LOCAL port it is, which is the whole join")
assert stream["metric"] == "gateway.in_conns@443", (
"the pipe carries the series it is, so the page never rebuilds the name")
assert stream["clients"], "the inbound pipe names the addresses that are using it"
secure = next(pipe for pipe in body["listeners"] if pipe["port"] == 4443)
@@ -5581,14 +5692,19 @@ def test_gateway_snapshot_answers_the_pipes_the_projection_joined(
# The graph: every route the two captured configs declare, each line's inbound end a trunk this
# same payload draws, and a count on **only** the lines whose endpoint the config wrote as an
# address — this capture has exactly two (`127.0.0.1:4443` and `127.0.0.1:8081`).
assert len(body["connections"]) == 32
assert len(body["connections"]) == 33
listener_ports = {listener["port"] for listener in body["listeners"]}
assert listener_ports == {22, 2000, 5201, 8080, 25565, 25566, 80, 443, 4443, 8443}
assert all(row["in"]["port"] in listener_ports for row in body["connections"]), (
"a connection on a port no listener carries is an orphan the page cannot draw")
def sockets_to(host: str, port: int) -> int:
return sum(1 for row in established
if (row["remote"]["addr"], row["remote"]["port"]) == (host, port))
counted = {row["id"]: row["connections"] for row in body["connections"]
if row["connections"] is not None}
assert counted == {"nginx:443->127.0.0.1:4443": 2, "nginx:8443->127.0.0.1:8081": 0}, (
assert counted == {"nginx:443->127.0.0.1:4443": sockets_to("127.0.0.1", 4443),
"nginx:8443->127.0.0.1:8081": sockets_to("127.0.0.1", 8081)}, (
"a route written as a domain carries no count, and one written as an address carries the "
"endpoint's own — 0 included, because the socket table answered")
assert all(row["out"]["address"] is True for row in body["connections"]
@@ -5600,11 +5716,88 @@ def test_gateway_snapshot_answers_the_pipes_the_projection_joined(
if pipe["connections"] is not None)
endpoints = {(row["out"]["host"], row["out"]["port"]) for row in body["connections"]
if row["connections"] is not None}
assert known_in + sum(counted.values()) <= body["sockets"]["established"] == 18
assert known_in + sum(counted.values()) <= body["sockets"]["established"] == len(established)
assert len(endpoints) == 2
assert [row["key"] for row in body["states"]] == ["gateway"]
def test_the_served_snapshot_counts_a_resolved_name_and_says_which_address_it_used(
resolved_gateway_client) -> None:
"""`/api/gateway/snapshot` on the 1.3.0 document: the dash is gone, and the reason is in the row.
One route, both halves of the claim. `brain.reeseapps.com` leaves this ingress for
`reeseapps-brain-of-reese.reeselink.com:443`; the row now carries `address: false` (the config
wrote a name — verbatim, still), `resolved: ["10.1.2.64"]`, `via: "libc"` and that entry's own
`resolved_ts`, **and** a finite `connections`. A number with no provenance beside it would be
the invention this app refuses to print, which is why the two halves are one test.
"""
body = resolved_gateway_client.get("/api/gateway/snapshot").json()
assert body["resolver"]["available"] is True and body["resolver"]["via"] == "libc"
brain = next(row for row in body["connections"]
if row["out"]["host"] == "reeseapps-brain-of-reese.reeselink.com")
assert brain["out"]["selectors"] == ["brain.reeseapps.com"]
assert brain["out"]["address"] is False
assert brain["out"]["resolved"] == ["10.1.2.64"]
assert brain["out"]["via"] == "libc"
assert isinstance(brain["out"]["resolved_ts"], (int, float))
assert isinstance(brain["connections"], int) and not isinstance(brain["connections"], bool), (
"resolved and idle is a `0`, not the dash: the socket table answered and this endpoint has none")
assert brain["metric"] == "gateway.out_conns@reeseapps-brain-of-reese.reeselink.com:443"
ntfy = next(row for row in body["connections"]
if row["out"]["host"] == "reeseapps-ntfy.reeselink.com")
assert ntfy["connections"] > 0, "the capture has live ntfy sockets behind that name"
counted = [row for row in body["connections"] if row["connections"] is not None]
assert len(counted) == len(body["connections"]), (
"every route this ingress declares is placed by the recorded resolver, so no route on the "
"live page is a dash any more — the operator's complaint, in one assertion")
assert all(row["out"]["address"] is True or row["out"]["resolved"] for row in counted), (
"a count never arrives without the address it was counted through — one the config wrote, "
"or one the gateway published; `resolved` stays null on the address rows, because copying "
"the agent's `literal` echo of a string the config already wrote would claim a lookup that "
"never happened")
def test_an_injected_nxdomain_row_is_a_dash_in_the_served_document(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, gateway_snapshot: Any) -> None:
"""Both arms on one route list: placed is a number, unplaced is `null` / `null`.
The capture resolves everything, so the second arm is injected: one entry of the recorded
document emptied and marked `nxdomain`, exactly what agent 1.3.0 publishes when `getaddrinfo`
says a name has no record. Its line keeps its place in the graph with `resolved: null`,
`via: null`, `connections: null` and `metric: null` — no `0`, no series, no sample — while the
routes around it carry their counts, so a page cannot read one failed name as a dead ingress.
"""
broken = gateway_snapshot("resolved")
entries = [entry for entry in broken["resolver"]["entries"]
if entry["name"] == "reeseapps-immich.reeselink.com"]
assert entries, "the capture resolves immich, so the injection has something to un-resolve"
for entry in entries:
entry["addresses"] = []
entry["status"] = "nxdomain"
payload = gateway_source.parse(broken, "gateway")
with make_gateway_client(tmp_path, monkeypatch) as client:
answer_as(client, "gateway", payload)
deadline = time.time() + 10
while time.time() < deadline and not client.app.state.ctx.hub.jobs["gateway"].state.ok:
time.sleep(0.05)
body = client.get("/api/gateway/snapshot").json()
row = next(item for item in body["connections"]
if item["out"]["host"] == "reeseapps-immich.reeselink.com")
assert row["out"]["resolved"] is None, "never `[]`, so the page's dash arm is one `is null` test"
assert row["out"]["via"] is None and row["out"]["resolved_ts"] is None
assert row["connections"] is None and row["metric"] is None
assert row["in"] == {"port": 443}, "the route is still drawn: a name with no record is not no route"
placed = [item for item in body["connections"] if item["connections"] is not None]
assert len(placed) == len(body["connections"]) - 1, (
"one unresolved name dims one line and nothing else")
documented = {entry["name"]: entry["status"] for entry in body["resolver"]["entries"]}
assert documented["reeseapps-immich.reeselink.com"] == "nxdomain", (
"the entry the index refused to use is still the entry the page can quote")
def test_no_port_the_two_configs_do_not_name_appears_anywhere(gateway_client,
gateway_snapshot: Any) -> None:
"""The live router's own agent port is `9120`, and this payload never mentions it.
@@ -5700,10 +5893,12 @@ def test_gateway_history_serves_the_pipe_gauges_the_store_holds(
facts), and the `source` says the `gateway` family — a page must never ask the 10-second `host`
series for a connection count.
The outbound name asked for is an **address** (`127.0.0.1:4443`), and that is the whole point of
the phase-30 change to `samples()`: a route whose target the config wrote as a domain publishes
no series at all, so a name-keyed `gateway.out_conns@<domain>:<port>` is no longer stored and
the store's `gateway` family is exactly the endpoints the join could count.
The outbound name asked for is an **address** (`127.0.0.1:4443`) because this fixture answers as
a 1.2.0 agent. The rule `samples()` keeps is about the **address**, not the spelling: a route
publishes a gauge when the kernel can name its far end — an address the config wrote, or (since
phase 32 task 03, pinned beside this one on the `resolved` arm) a name the gateway itself
resolved. A name nobody resolved publishes nothing, which is what
`test_gateway_history_answers_empty_for_a_pipe_it_never_measured` still proves.
"""
store: SampleStore = gateway_client.app.state.ctx.store # type: ignore[attr-defined]
body = gateway_client.get("/api/gateway/history", params={
@@ -5743,6 +5938,65 @@ def test_gateway_history_answers_empty_for_a_pipe_it_never_measured(gateway_clie
assert series_source(body) == {"gateway"}
def test_a_resolved_name_publishes_its_gauge_and_history_answers_it(
resolved_gateway_client) -> None:
"""The name-keyed series phase 30 stopped writing, written again on an honest footing.
Phase 30 task 02 retired `gateway.out_conns@<domain>:<port>` because the only figure it could
have carried was the *port's* — the port-only join's count filed under whatever domain happened
to share it. Phase 32 task 03 gives the name a figure of its own: the sockets to the address the
**gateway** resolved that name to. So the name is stored, `/api/metrics` lists it, and
`/api/gateway/history` answers points for it — read through `store.query`, the third path, as
every other history pin is.
"""
store: SampleStore = resolved_gateway_client.app.state.ctx.store # type: ignore[attr-defined]
metric = "gateway.out_conns@reeseapps-ntfy.reeselink.com:443"
body = resolved_gateway_client.get("/api/gateway/history", params={
"hosts": "gateway", "metrics": metric, "range": "1h"}).json()
assert [series["metric"] for series in body["series"]] == [metric]
assert series_source(body) == {"gateway"}
rows = store.query("gateway", "gateway", metric, res=RES_RAW)
assert rows, "the resolved route's gauge was not stored"
assert [(point["ts"], point["value"]) for point in body["series"][0]["points"]] == rows
assert metric in resolved_gateway_client.get("/api/metrics",
params={"source": "gateway"}).json()["metrics"]
def test_one_shared_endpoint_costs_one_insert_per_poll_even_through_a_name(
tmp_path: Path, gateway_snapshot: Any) -> None:
"""Two lines, one gauge name, **one row per poll** — counted in the store, not asserted.
The captured ingress reaches `keycloak.reeselink.com:8080` from `:4443` and from `:8443`, and
since task 03 the endpoint is reachable through a **resolved name** rather than a written
address. `shared: true` says the two lines carry one number; `samples()` de-duplicating by
metric is what stops that number being inserted twice for one reading, and the row count is the
only proof worth having. Two polls, two rows: one per reading, never one per line.
"""
db = str(tmp_path / "gateway-shared.sqlite3")
store = SampleStore(db)
now = time.time()
metric = "gateway.out_conns@keycloak.reeselink.com:8080"
# both readings in the past: `store.query` bounds its window at now, so a simulated second poll
# that lands in the future is a row that never comes back, not a lost insert
for offset in (-60.0, -30.0):
snapshot = gateway_snapshot("resolved")
snapshot["ts"] = now + offset
result = gateway_source.parse_gateway_snapshot("gateway", snapshot, now=now + offset)
shared = [row for row in result.payload["connections"] if row["shared"]]
assert {row["metric"] for row in shared} == {metric}, (
"this capture's shared endpoint is the keycloak pair, and one poll writes it once")
assert len({row["connections"] for row in shared}) == 1, (
"both lines carry the same number, which is what makes one insert the honest count")
out = [row for row in result.samples if row.metric == metric]
assert len(out) == 1, "one sample per poll, however many lines reach the endpoint"
store.add_many(result.samples)
rows = store.query("gateway", "gateway", metric, res=RES_RAW)
assert len(rows) == 2, "two polls, two rows — not four (one per line) and not one (lost poll)"
store.close()
def test_gateway_history_names_the_ingress_and_no_other_host(gateway_client) -> None:
"""With no `hosts=` the answer is the one host this source reads — not the whole fleet.
@@ -5853,6 +6107,10 @@ GATEWAY_PAGE_FIELDS = (
"connections.id", "connections.source", "connections.in.port",
"connections.out.host", "connections.out.port", "connections.out.name",
"connections.out.kind", "connections.out.selectors", "connections.out.address",
# Phase 32 task 04: the provenance a counted line prints and the dash refuses. `resolved` is what
# the filter matches an IP against, `via` + `resolved_ts` are the tooltip's `address from libc,
# resolved 3m` — so a rename of any of the three fails by name here, not as a permanent silence.
"connections.out.resolved", "connections.out.via", "connections.out.resolved_ts",
"connections.connections", "connections.shared", "connections.metric",
"warnings.id", "warnings.kind", "warnings.subject", "warnings.message",
)
@@ -6415,11 +6673,16 @@ def test_the_trunk_pipe_is_an_accessible_image_of_its_port(client, gateway_clien
def _leaf_haystack(row: dict[str, Any]) -> str:
"""The one string the filter can see about one route, as `page-gateway.js` composes it."""
"""The one string the filter can see about one route, as `page-gateway.js` composes it.
Phase 32 task 04 adds the **sixth** field: `out.resolved`, the addresses the gateway published for
a name target. Typing an IP is how an operator asks "which route owns the socket I am looking at",
and a haystack of names alone cannot answer it.
"""
out = row.get("out") or {}
port = (row.get("in") or {}).get("port")
parts = ["" if port is None else str(port), out.get("host"), out.get("name"), out.get("kind"),
*(out.get("selectors") or [])]
*(out.get("selectors") or []), *(out.get("resolved") or [])]
return " ".join(p for p in parts if isinstance(p, str) and p).lower()
@@ -6434,7 +6697,8 @@ def filter_view(snapshot: dict[str, Any], query: str) -> dict[str, Any]:
There is no JS runner (a locked anchor), so this is `rendered_leaf`'s idiom: the rules are
re-applied here from the payload, and the served-byte pins in the test below are what keep the
mirror honest — `haystack` indexes `in.port`, `out.host`, `out.name`, `out.kind`, `out.selectors`;
mirror honest — `haystack` indexes `in.port`, `out.host`, `out.name`, `out.kind`,
`out.selectors`, `out.resolved`;
`trunkHaystack` indexes `:${port}`, `domains`, `clients`; a trunk that matches keeps **all** its
leaves; a group stands when it or one of its routes answers. A rename on either side therefore
fails a pin *and* changes what this mirror prints.
@@ -6476,7 +6740,8 @@ def filter_view(snapshot: dict[str, Any], query: str) -> dict[str, Any]:
return {"groups": groups, "standing": standing, "total": len(snapshot["connections"])}
def test_the_filter_narrows_the_map_without_asking_anything(client, gateway_client) -> None:
def test_the_filter_narrows_the_map_without_asking_anything(client, gateway_client,
gateway_snapshot: Any) -> None:
"""One field, four rules, and not one extra request.
The operator's complaint is a reading-speed one: *"I want to find a domain."* This ingress routes
@@ -6485,9 +6750,12 @@ def test_the_filter_narrows_the_map_without_asking_anything(client, gateway_clie
feature could go wrong:
* **It matches payload text and nothing else.** `in.port`, `out.host`, `out.name`, `out.kind`,
`out.selectors` for a route, `:${port}` + `domains` + `clients` for a trunk. No DNS (the module
still resolves no name), no re-ranking, no rule about `.reeselink.com` — a query can never
invent a route, because there is no other text to match.
`out.selectors` and `out.resolved` for a route, `:${port}` + `domains` + `clients` for a trunk.
The sixth field is phase 32 task 04's: an address the **gateway** published is text the payload
already carries, so typing the far end of a socket finds the route that owns it. No DNS (the
page still resolves no name — it matches a string, it never looks one up), no re-ranking, no
rule about `.reeselink.com` — a query can never invent a route, because there is no other text
to match.
* **It costs nothing.** The repaint runs on `state.snapshot`; a keystroke that re-fetched would be
a poll per character and a lie about what changed.
* **It hides rows, it does not remove them.** `li.hidden` in the patch pass, so a route the query
@@ -6498,18 +6766,27 @@ def test_the_filter_narrows_the_map_without_asking_anything(client, gateway_clie
"""
js = _code(client.get("/static/page-gateway.js").text)
# (1) the matcher's whole vocabulary: five payload fields for a route, three for a trunk.
# (1) the matcher's whole vocabulary: six payload fields for a route, three for a trunk.
hay = js[js.index("function haystack("):js.index("function trunkHaystack(")]
assert "String((conn.in || {}).port ?? '')" in hay, \
"the inbound port is no longer searchable — `4443` is the second thing an operator types"
for field in ("out.host", "out.name", "out.kind", "out.selectors"):
for field in ("out.host", "out.name", "out.kind", "out.selectors", "out.resolved"):
assert field in hay, f"the filter no longer sees {field}"
# Phase 32 task 04: an address the **gateway** resolved is payload text like any other, so typing
# the far end of a socket finds the route that owns it. Names alone could not answer that question.
assert "...(out.resolved || [])" in hay, \
"a name-written route stopped being findable by the address it resolves to"
assert "toLowerCase()" in hay, "the match became case-sensitive on a page of lowercase domains"
for absent in ("conn.connections", "severity", "metric", "getaddrinfo", "split(",
"endsWith", "startsWith", "slice("):
assert absent not in hay, (
f"the matcher reached for {absent} — it may read what the payload calls a route, never "
"what it reports about it, and never resolve a name")
# …and the three things that would stop it being an eye over the document in hand: a dial, a read
# of what is painted rather than of what the payload says, and a literal `null` a query could
# match against a row that published nothing.
for not_here in ("fetchJSON", "textContent", "innerText", '"null"', "'null'"):
assert not_here not in hay, f"the haystack reached for {not_here}"
trunk = js[js.index("function trunkHaystack("):js.index("function filtering(")]
for field in ("group.port", "listener.domains", "listener.clients"):
assert field in trunk, f"the trunk no longer answers for {field}"
@@ -6616,6 +6893,25 @@ def test_the_filter_narrows_the_map_without_asking_anything(client, gateway_clie
assert all(not leaf["hidden"] for leaf in home["leaves"]), \
"a trunk that matched lost leaves of its fan-out"
# (7) phase 32 task 04: an address the **gateway** resolved is searchable too, which is the field
# an operator reaches for when they are holding a socket rather than a domain. Over the 1.2.0
# document there is nothing to match (no resolver section, and the page resolves nothing), so
# this leg runs on the recorded 1.3.0 one.
placed = gateway_source.parse(gateway_snapshot("resolved"), "gateway")
assert not filter_view(payload, "10.1.2.64")["standing"], \
"the page found an address it was never given — it resolves nothing, and must keep not doing it"
brain_ip = filter_view(placed, "10.1.2.64")
found = [leaf["id"] for group in brain_ip["groups"] for leaf in group["leaves"]
if not leaf["hidden"]]
assert found and all("reeseapps-brain-of-reese" in route_id for route_id in found), \
f"typing the brain's address left {found}"
assert all(group["hidden"] or group["port"] == 443 for group in brain_ip["groups"]), \
"an address match let a trunk of another port stand"
shared_ip = filter_view(placed, "10.1.1.78")
expected = sum(1 for row in placed["connections"] if "10.1.1.78" in (row["out"]["resolved"] or []))
assert expected > 1 and shared_ip["standing"] == expected, (
f"one address reached by {expected} routes was found by {shared_ip['standing']} of them")
def test_the_filter_field_is_markup_the_sheet_and_nobody_else_uses(client) -> None:
"""The field is pinned markup, on one page, wearing the sheet's one field skin.
@@ -6972,19 +7268,39 @@ def test_the_page_says_where_a_line_comes_from_and_where_its_count_cannot(client
"""Four footnotes, one per claim the page is allowed to make, and not one more.
Why the page draws a line at all (the config declares the route) and what its motion means (a
socket is open this instant, so a resting line is not a dead one); why an endpoint written as a
**name** carries no number (nothing here resolves one, so a dash is not a fault); what the switch
does **not** do (toggling a route needs a config write path this app does not have); and — since
phase 31 task 04 — what the filter box does to the map and what it leaves alone (the tallies, the
warnings and the charts keep describing the whole ingress, because the field asks for nothing).
Each lives in the footer exactly once — the page does not repeat a why-line per panel, and it
does not need to: an uncounted line prints `—` and the footnote is the sentence that explains it
socket is open this instant, so a resting line is not a dead one); why a route carries no number
(phase 32 task 04: **the gateway could not resolve that name in this reading** — the gateway does
the resolving, so the sentence blames the reading and points at the hover that shows the address a
counted line was matched to, instead of blaming the config for having written a name or the page
for having no resolver); what the switch does **not** do (toggling a route needs a config write
path this app does not have); and — since phase 31 task 04 — what the filter box does to the map
and what it leaves alone (the tallies, the warnings and the charts keep describing the whole
ingress, because the field asks for nothing).
Each lives in the footer exactly once — the page does not repeat a why-line per panel, and it does
not need to: an uncounted line prints `—` and the footnote is the sentence that explains it
(rule 2).
"""
html = client.get(GATEWAY_PAGE).text
footer = html[html.index("<footer"):]
assert footer.count("resolves nothing") == 1, (
"the no-DNS rule is the page's single footnote about a missing count — printed once")
assert footer.count("a dash means") == 1, (
"the page's single footnote about a missing count, printed once, in the phase-32 reason")
assert "could not resolve that name" in footer, (
"a dash means the gateway's reading came back with nothing, not that the config wrote a name")
# The two sentences phase 32 made stale may not come back from either file: the blame on the page
# (`the page knows no`) and its no-DNS limit as a per-row excuse, which task 04 replaced with the
# thing to do about a dash rather than a reason to accept it.
assert "the page knows no" not in footer, (
"the dash stopped blaming the gateway's reading and went back to blaming the page")
# The served source wraps this paragraph, so the two halves are matched separately: what is
# pinned is that the sentence names a cause **and** the place to look.
flat = " ".join(footer.split())
assert "hover a counted line for the address it matched" in flat, (
"the footnote names a cause and no longer says where the address and its age are")
script = client.get("/static/page-gateway.js").text
assert "the gateway could not resolve this name in this reading" in script, (
"the per-row tooltip must say the same thing the footnote says")
assert "nothing here resolves names" not in script, (
"the old tooltip sentence blames the name, which is no longer why a line has no count")
assert "animates" in footer, "the animation rule lost its footnote"
band = footer[:footer.index("</footer>")]
# Four paragraphs and nothing else: one sentence per claim, in one ink band under the panels.
@@ -7054,6 +7370,21 @@ def test_the_gateway_page_renders_each_degradation_arm_in_the_payloads_own_words
assert sum(1 for field in ("listeners", "connections") if stub[field]) == 0, (
"the stub arm's rows are not empty, so the header's `n/a` would not be about the same thing")
# (5) a **resolver** that did not answer (agent 1.2.0, `--no-resolve`, or a pass that could not
# run) is the fifth arm and the quietest: the section is not one of the two config files, so
# it raises no banner and no colour, and every name-written line simply carries the dash with
# the resolution sentence. `full` is that document — no `resolver` key at all.
assert silent["resolver"]["available"] is None and silent["resolver"]["entries"] == [], (
"the `full` capture is supposed to be the no-resolver arm this case measures")
assert silent["why"] is None, "an absent resolver section became a page-wide complaint"
assert "severity" not in json.dumps(silent), "the fifth arm arrived with a verdict on it"
dashes = [row for row in silent["connections"] if row["connections"] is None]
assert dashes, "a document with no resolver counted a name-written route"
assert all(count_surfaces(row)["why"] == COUNT_WHY_UNRESOLVED for row in dashes), \
"a line nobody resolved is being explained by something other than resolution"
assert not any(count_surfaces(row)["glyph"] == "0" for row in dashes), \
"the uncounted became a claimed zero"
def test_the_gateway_page_keeps_its_last_drawing_when_a_poll_fails(client) -> None:
"""Invariant 4 on the fifth page: a failed read leaves the last picture on screen.
@@ -7072,6 +7403,43 @@ def test_the_gateway_page_keeps_its_last_drawing_when_a_poll_fails(client) -> No
"and the reason has to be printed where the operator is already looking"
COUNT_WHY_UNRESOLVED = "the gateway could not resolve this name in this reading"
COUNT_WHY_NO_SOCKETS = "the socket table did not answer this reading"
def count_surfaces(row: dict[str, Any]) -> dict[str, Any]:
"""The count column's two strings for one route, as `countWhy` / `countTitle` build them.
A **second** reading of the same decision, with the served-byte pins below as its other half:
* `glyph` — the em dash for a null, the number otherwise (`0` included, because a resolved
endpoint with no socket is a measurement and not a gap);
* `why` — for a null, which of the two unknowns the payload describes: an address is **known**
when the config wrote one (`out.address is True`) or the gateway published one
(`out.resolved` non-empty), and a row with neither is a name nothing could resolve;
* `title` — for a counted row, the number plus the provenance the tooltip owes it. The **age**
fragment is left out here and pinned as a prefix instead, because it is a clock: the test
asserts what comes before it and what the remainder looks like.
"""
out = row["out"]
shown = row["connections"]
if shown is None:
known = out["address"] is True or bool(out["resolved"])
why = COUNT_WHY_NO_SOCKETS if known else COUNT_WHY_UNRESOLVED
return {"glyph": "\u2014", "why": why, "title": f"no number: {why}",
"aria": f"connections not attributed: {why}", "age": False}
title = f"{shown} open connection(s) on this endpoint"
via = out["via"] if isinstance(out["via"], str) and out["via"] else None
if out["resolved"] is None or via is None:
return {"glyph": str(shown), "why": None, "title": title, "age": False,
"aria": f"{shown} live connections"}
title += f" \u00b7 address from {via}"
# `Dash.duration(Dash.ageSeconds(ts))`, whose text is a clock and therefore not this mirror's to
# reproduce — only the *decision* to print it is.
return {"glyph": str(shown), "why": None, "title": title,
"age": out["resolved_ts"] is not None, "aria": f"{shown} live connections"}
def rendered_leaf(row: dict[str, Any]) -> dict[str, str]:
"""The two lines and the two long strings one leaf row prints, derived as a **second** reading.
@@ -7082,6 +7450,10 @@ def rendered_leaf(row: dict[str, Any]) -> dict[str, str]:
`leafAria` compose those same two from the literals asserted verbatim below. A rename on either
side therefore fails a pin **and** changes what this mirror prints, which is the pair of
guarantees a page with no test runner can actually make.
The row's `aria-label` ends with the count's claim, and phase 32 task 04 made that claim carry its
cause: `count_surfaces()` is the one place that sentence is assembled, so the row and the count
element cannot disagree about which unknown a dash is.
"""
out = row["out"]
host = out["host"] or "?"
@@ -7099,8 +7471,7 @@ def rendered_leaf(row: dict[str, Any]) -> dict[str, str]:
if row["shared"] is True:
bits.append("this endpoint is shared with another route")
bits.append("in the config")
bits.append("connections not attributed" if row["connections"] is None
else f"{row['connections']} live connections")
bits.append(count_surfaces(row)["aria"])
return {"line1": inbound,
"line2": f"→ {endpoint}" + (f" · {kind}" if kind else ""),
"title": pair + shared,
@@ -7440,7 +7811,7 @@ def test_the_stream_map_is_rendered_inside_nginx_and_nowhere_else(
Pinned from both ends: the script ships exactly the two panel definitions and never indexes
`nginx.map_targets`, and the live document's map connections are all nginx's, all on one port,
and exactly as many as the map has routing entries (17 on this capture — 16 `*.reeselink.com`
and exactly as many as the map has routing entries (18 on this capture — 17 `*.reeselink.com`
plus the `default` the file writes for SNI nobody mapped).
"""
html = _code(client.get(GATEWAY_PAGE).text)
@@ -7570,22 +7941,31 @@ def test_warnings_render_in_the_overviews_alert_vocabulary_and_hide_when_clean(
def test_an_unattributed_endpoint_prints_a_dash_and_never_a_zero(client, gateway_client) -> None:
"""A leaf the join cannot count prints `—`, and the tooltip says why in one sentence.
"""A leaf the join cannot count prints `—`, and the tooltip says **which** unknown it is.
Sixteen of this fleet's 17 map entries name a **domain**, and `/proc` reports addresses, so no
count exists for them and none can ever exist (nothing here resolves a name). `0` would be a
claim about traffic (rule 2) and a column of `n/a` would be twelve identical words competing with
the page's one footnote — so the leaf prints an em dash whose `title` names the reason, while the
trunk keeps `n/a`, which is a different fact: the socket table itself went quiet.
`/proc` reports addresses, so a route whose far end has no address cannot be counted, and `0` would
be a claim about traffic (rule 2) while a column of `n/a` would be twelve identical words
competing with the page's one footnote — so the leaf prints an em dash whose `title` names the
reason, while the trunk keeps `n/a`, which is a different fact: the socket table itself went quiet.
Phase 32 task 04 sharpened the mechanism and not only the wording: the dash now has **two** causes
that `countCause` separates from `out.address` / `out.resolved` — a name the gateway could not
resolve, and an address that is known beside a table that answered nothing — and the sentence is
chosen from data, never guessed. A `0` is neither of them: it is a measurement.
"""
js = _code(client.get("/static/page-gateway.js").text)
head = js[:js.index("function countText(")]
assert "const NO_COUNT = '—'" in head, "the unattributed leaf count is no longer an em dash"
assert "nothing here resolves names" in head, "the dash stopped explaining itself"
assert "the gateway could not resolve this name in this reading" in head, \
"the dash stopped explaining itself"
assert "function countCause(" in head and "out.address === true" in head \
and "out.resolved" in head, \
"the dash's cause is no longer read from the two payload fields that decide it"
assert "nothing here resolves names" not in head, (
"the old sentence blamed the name, which phase 32 task 03 made a false reason")
leaf = js[js.index("function renderLeaf("):js.index("function groupElement(")]
assert "shown == null ? NO_COUNT : shown" in leaf, "a null count is not the dash any more"
assert "count.title = shown == null ? NO_COUNT_WHY" in leaf, \
"the dash lost the sentence that separates it from a zero"
assert "count.title = shown == null ? countWhy(conn) : countTitle(conn, shown)" in leaf, \
"the dash lost its sentence, or a counted line lost the provenance beside its number"
trunk = js[js.index("function renderTrunk("):js.index("function renderLeaves(")]
assert "live" in trunk and "n/a" not in trunk, (
"the trunk now prints the leaf's dash too — a quiet socket table is a different fact, and "
@@ -7600,6 +7980,164 @@ def test_an_unattributed_endpoint_prints_a_dash_and_never_a_zero(client, gateway
"table's arm, not the name arm, and it prints a different sentence")
def test_an_unresolved_name_and_an_idle_route_say_two_different_sentences(
client, gateway_snapshot: Any) -> None:
"""The em dash and the measured `0` are two facts, and every surface of the row says so.
Phase 32 task 03 made a name-written route countable, which left the page's one dash sentence
explaining two different things: a name the gateway could not resolve, and a route that **was**
resolved and simply has nothing open. Task 04 splits them, and the split has to be on every
surface a reader can meet — the glyph, the `title`, the `aria-label` — because a `0` read as a
missing measurement (or a dash read as an idle route) sends the operator to the wrong box.
Three arms, all from the recorded capture and one injection, all rendered through
`count_surfaces` (the mirror) with the served-byte pins below as its other half:
* **resolved and idle** — `brain.reeseapps.com` reaches
`reeseapps-brain-of-reese.reeselink.com:443`, the gateway resolved it to `10.1.2.64`, and the
kernel reports no socket: the glyph is `0`, and a `0` is a measurement.
* **resolved and busy** — the ntfy route, whose glyph is its own number.
* **not resolved** — one entry of the recorded document emptied and marked `nxdomain`, exactly
what agent 1.3.0 publishes for a name with no record: the glyph is the em dash and the sentence
is the resolution one, never the socket one.
And a fourth, because the third arm's sentence must not become a catch-all: with the kernel's
table unreadable, a line whose address the **config wrote** is uncounted too, and blaming
resolution there would be false — so it gets the socket-table sentence.
"""
js = _code(client.get("/static/page-gateway.js").text)
head = js[: js.index("function countText(")]
# (1) the two sentences exist, differ, name no `n/a`, and neither is the other's wording.
assert "const WHY_UNRESOLVED = 'the gateway could not resolve this name in this reading'" in head
assert "const WHY_NO_SOCKETS = 'the socket table did not answer this reading'" in head
assert COUNT_WHY_UNRESOLVED != COUNT_WHY_NO_SOCKETS
for sentence in (COUNT_WHY_UNRESOLVED, COUNT_WHY_NO_SOCKETS):
assert "n/a" not in sentence, "an unknown that has a cause stopped naming its cause"
# The choice is data: the same two fields `sources/gateway.py` joins on, and nothing else.
chooser = js[js.index("function countCause("):js.index("function countWhy(")]
assert "out.address === true" in chooser and "out.resolved" in chooser, \
"the dash's cause is no longer chosen from the address the config wrote or the gateway found"
assert "countCause(conn)" in js[js.index("function leafAria("):js.index("function leafTitle(")], \
"the `aria-label` no longer carries the cause the tooltip carries"
# (2) the three arms, rendered.
resolved = gateway_source.parse(gateway_snapshot("resolved"), "gateway")
def route(host: str, port: int) -> dict[str, Any]:
return next(row for row in resolved["connections"]
if row["out"]["host"] == host and row["in"]["port"] == port)
idle = count_surfaces(route("reeseapps-brain-of-reese.reeselink.com", 443))
busy = count_surfaces(route("reeseapps-ntfy.reeselink.com", 443))
assert idle["glyph"] == "0", "a resolved endpoint with nothing open is a measurement, not a gap"
assert busy["glyph"] == "3" and busy["why"] is None
assert idle["title"].startswith("0 open connection(s) on this endpoint") \
and busy["title"].startswith("3 open connection(s) on this endpoint")
broken = gateway_snapshot("resolved")
for entry in broken["resolver"]["entries"]:
if entry["name"] == "reeseapps-immich.reeselink.com":
entry["addresses"] = []
entry["status"] = "nxdomain"
unplaced = gateway_source.parse(broken, "gateway")
dash = count_surfaces(next(row for row in unplaced["connections"]
if row["out"]["host"] == "reeseapps-immich.reeselink.com"))
assert dash["glyph"] == "\u2014", "a name with no record grew someone else's number"
assert dash["why"] == COUNT_WHY_UNRESOLVED and dash["title"] == f"no number: {dash['why']}"
assert dash["aria"] == f"connections not attributed: {COUNT_WHY_UNRESOLVED}"
# (3) the fourth arm: an address the config wrote, beside a kernel that said nothing.
quiet = gateway_source.parse(gateway_snapshot("sockets_silent"), "gateway")
unwritten = count_surfaces(next(row for row in quiet["connections"]
if row["out"]["address"] is True))
assert unwritten["glyph"] == "\u2014" and unwritten["why"] == COUNT_WHY_NO_SOCKETS, (
"a line whose address the config wrote was told to go resolve something — the socket table "
"is what went quiet, and that is the sentence the trunk already prints as `n/a`")
# (4) five lines read aloud: distinct claims, no invented number, no `null` leaking as text.
spoken = [count_surfaces(row) for row in resolved["connections"][:5]]
claims = {entry["title"] for entry in spoken} | {entry["aria"] for entry in spoken}
assert len(claims) > 1, "five lines said one thing"
for entry in [*spoken, idle, busy, dash, unwritten]:
for text in (entry["glyph"], entry["title"] or "", entry["aria"]):
assert "undefined" not in text and "NaN" not in text and "null" not in text, \
f"a row printed {text!r}"
def test_a_counted_line_names_where_its_address_came_from(client, gateway_snapshot: Any,
resolved_gateway_client) -> None:
"""Every counted line says, on hover, which address it was counted to and when that was learned.
A number that arrived through the gateway's own `libc` resolution is a different claim from one
the config wrote as an address, and the tooltip is the one place both can say it without growing
a column of ink per row. Three arms, one rule: the provenance is the payload's (`out.via`,
`out.resolved_ts`) copied, never paraphrased, and a field that is not there prints **nothing**
rather than `n/a` or `NaN`.
* a **resolved name** — `… · address from libc`, and `, resolved <age>` while the entry has a
timestamp (the age is `Dash.duration(Dash.ageSeconds(…))`, so its text is a clock and only its
presence is pinned here);
* the same row with `resolved_ts` removed — the age fragment goes, the rest of the sentence
stays, and nothing prints the shared `n/a` where an age belongs;
* an **address the config wrote** — the bare sentence. It was never resolved, and stamping the
agent's `literal` echo of it as a lookup would be rule 2 with a tooltip's clothes on.
"""
js = _code(client.get("/static/page-gateway.js").text)
builder = js[js.index("function countTitle("):js.index("const NO_TARGET")]
for field in ("out.via", "out.resolved", "out.resolved_ts"):
assert field in builder, f"the counted line's tooltip stopped reading {field}"
for helper in ("duration(", "ageSeconds("):
assert helper in builder, f"the age stopped going through the shared helper {helper}"
assert "out.resolved_ts == null" in builder, \
"a missing timestamp now prints whatever duration() makes of Infinity"
assert "address from" in builder and "resolved " in builder
assert "n/a" not in builder and "\u2014" not in builder, \
"the counted tooltip grew a fallback glyph; a missing fragment is silence, not a mark"
resolved = gateway_source.parse(gateway_snapshot("resolved"), "gateway")
brain = next(row for row in resolved["connections"]
if row["out"]["host"] == "reeseapps-brain-of-reese.reeselink.com")
printed = count_surfaces(brain)
assert printed["title"] == "0 open connection(s) on this endpoint \u00b7 address from libc"
assert printed["age"] is True
served = client.get("/static/page-gateway.js").text
assert "\u00b7 address from ${via}" in served and ", resolved ${age}`" in served, \
"the two provenance fragments are no longer composed from the payload's own words"
aged = dict(brain, out=dict(brain["out"], resolved_ts=None))
lost = count_surfaces(aged)
assert lost["title"] == printed["title"], "losing the timestamp lost the address it came from"
assert lost["age"] is False, "an age was printed for a row that publishes no timestamp"
assert "NaN" not in lost["title"] and "n/a" not in lost["title"]
written = next(row for row in resolved["connections"] if row["out"]["address"] is True)
plain = count_surfaces(written)
assert plain["title"] == f"{written['connections']} open connection(s) on this endpoint", \
"an address the config wrote was given a resolution provenance it never had"
assert "address from" not in plain["title"]
# …and the field table, in **both** directions (the `SLOT_FIELDS` device): the three names the
# tooltip and the filter index are named there, and the 1.3.0 document answers each with real
# data, so a rename in `sources/gateway.py` fails by name here instead of printing a tooltip that
# quietly stopped carrying its provenance.
provenance = ("connections.out.resolved", "connections.out.via",
"connections.out.resolved_ts")
live = resolved_gateway_client.get("/api/gateway/snapshot").json()
for path in provenance:
assert path in GATEWAY_PAGE_FIELDS, (
f"the counted line reads {path.removeprefix('connections.')}, which the page's field "
"table does not name")
found, values = _payload_field(live, path)
assert found and any(value is not None for value in values), (
f"{path} is named but the resolved document never answers it, so the pin above would "
"pass over a payload that carries nothing")
assert any(value is None for path in provenance
for value in _payload_field(live, path)[1]), (
"every row published provenance, including the addresses the config wrote — the arm that "
"must print nothing is no longer distinguishable")
def test_the_docs_name_the_page_its_knob_and_its_gauges() -> None:
"""The phase's own completion criterion, pinned so it cannot rot quietly.
tests/test_collectors.py
+10 -2
View File
@@ -261,6 +261,11 @@ async def test_a_host_without_a_proxy_config_does_not_degrade_the_system_source(
`collectors.ADDON_ERROR_PREFIXES` is the one list of these prefixes; this asserts every
name by behaviour, so dropping any of them fails here, and `tests/test_agent.py` pins both
the wording of the lines the agent actually writes and the equality of the two lists.
The `resolver:` line is phase 32's and the same case with a DNS-shaped name: a host with no
proxy config has nothing to resolve, and a truncated lookup pass (`resolver: pass budget …`)
is one route without a count, not a broken box. Both arms belong in this list or every
upgraded host gets an overview WARN it has no way to clear.
"""
missing_haproxy = ("haproxy: /etc/haproxy/haproxy.cfg not found — haproxy is not installed "
"here, or its config lives elsewhere (--haproxy-config); --no-proxy "
@@ -269,9 +274,12 @@ async def test_a_host_without_a_proxy_config_does_not_degrade_the_system_source(
"its config lives elsewhere (--nginx-config); --no-proxy opts out")
no_sockets = ("sockets: /proc/net/tcp not found, /proc/net/tcp6 not found — this host "
"publishes no kernel socket table; --no-proxy opts out")
nothing_to_resolve = ("resolver: nothing to resolve — haproxy not found and nginx not found; "
"--no-resolve opts out")
truncated_lookups = ("resolver: pass budget 3.0s exhausted, 41 of 58 targets looked up")
body = agent_body()
body["errors"] = [missing_haproxy, missing_nginx, no_sockets,
"smart: smartctl 7.5 not found"]
"smart: smartctl 7.5 not found", nothing_to_resolve, truncated_lookups]
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=body)
@@ -280,7 +288,7 @@ async def test_a_host_without_a_proxy_config_does_not_degrade_the_system_source(
result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
HostCollector(RateEstimator()))
assert "degraded" not in result.payload, "unconfigured is not broken (invariant 3)"
for prefix in ("haproxy:", "nginx:", "sockets:"):
for prefix in ("haproxy:", "nginx:", "sockets:", "resolver:"):
assert any(prefix in str(e) for e in result.payload["agent"]["errors"]), \
"the sentence survives for the page that explains it"
tests/test_deploy_scripts.py
+265 -4
View File
@@ -27,6 +27,7 @@ from __future__ import annotations
import difflib
import importlib.util
import ipaddress
import json
import os
import re
@@ -2391,6 +2392,23 @@ case "$command" in
[ -f "$file" ] || { echo "cat: $path: No such file or directory" >&2; exit 1; }
cat "$file"
;;
"getent "*)
# libc resolution on the fake box. Answers come from `$FAKE_REMOTE/getent/<name>.<family>`;
# a file that is not there is `getent`'s own "no such record" (exit 2, no output), which the
# script records as a status rather than a failure. Two env switches stand in for the two
# reads that are *not* a fact about the name: `GETENT_MODE=transport` is ssh dying, and
# `GETENT_MODE=missing` is a gateway with no getent to ask.
family="${command#getent }"
family="${family%% *}"
name="${command##* }"
case "${GETENT_MODE:-ok}" in
transport) echo "ssh: connect to host $host port 22: Connection refused" >&2; exit 255 ;;
missing) echo "bash: line 1: getent: command not found" >&2; exit 127 ;;
esac
file="$FAKE_REMOTE/getent/$name.${family#ahosts}"
[ -f "$file" ] || exit 2
cat "$file"
;;
"ls "*)
spec="${command#ls }"
dir="${spec%/*}"
@@ -2444,7 +2462,10 @@ CAPTURE_FAKE_GATEWAY: dict[str, str] = {
"map $ssl_preread_server_name $targetBackend {\n"
" default 127.0.0.1:4443;\n"
" brain.example reeseapps-brain-of-reese.reeselink.com:443;\n}\n"
"server {\n listen 443;\n proxy_pass $targetBackend;\n}\n"
# `resolver` here, inside the stream server, is where this fleet writes it: the fixture's
# `config_directive` must come out of an *included* document, read from the bytes the run
# already captured, with no round trip of its own.
"server {\n listen 443;\n proxy_pass $targetBackend;\n resolver 10.9.8.7;\n}\n"
),
# 8.8.8.8 and 2603:6013::1 are globally routable, `::ffff:8.8.8.8` is a visitor behind a v6
# socket, and 203.0.113.7 is already documentation space. The tx/rx queue pair on the last row
@@ -2470,12 +2491,40 @@ CAPTURE_FAKE_GATEWAY: dict[str, str] = {
}
#: What the fake gateway's libc answers for the names its configs declare. The three arms are the
#: three the capture has to tell apart: an answer (private, so unscrubbed — this is what the real
#: fleet looks like), an answer that is **globally routable** (so it must become documentation
#: space), and no answer at all (a status, not a failure). `nextcloud` deliberately answers a v6
#: that is `::ffff:`-mapped, because `getent ahostsv6` synthesises that spelling for a name with no
#: AAAA record and the agent's own `getaddrinfo` never publishes it.
CAPTURE_FAKE_ANSWERS: dict[str, dict[str, str]] = {
# Private and answered twice over — the A record, and the `::ffff:` spelling `getent ahostsv6`
# synthesises for a name with no AAAA record. One address goes in; nothing moves.
"gitea.reeselink.com": {
"v4": "10.1.0.137 STREAM gitea.reeselink.com\n10.1.0.137 DGRAM \n",
"v6": "::ffff:10.1.0.137 STREAM \n",
},
# Two globally routable answers (one per family) beside one already in documentation space: the
# first two become `203.0.113.1` / `2001:db8::1` and the third stays exactly as it was, which is
# what makes a second run a no-op instead of a renumbering.
"nextcloud.reeselink.com": {
"v4": ("8.8.8.8 STREAM nextcloud.reeselink.com\n"
"203.0.113.9 STREAM nextcloud.reeselink.com\n"),
"v6": "2603:6013:3100:101:5054:ff:fe84:b2e0 STREAM nextcloud.reeselink.com\n",
},
# `reeseapps-brain-of-reese.reeselink.com` is answered by nothing on purpose: the absence of
# the file *is* the nxdomain arm, a status among successes rather than a failed run.
}
def _capture_sandbox(
tmp_path: Path,
gateway: dict[str, str] | None = None,
*,
recorded: dict[str, str] | None = None,
fail_ssh: bool = False,
fail_getent: str | None = None,
answers: dict[str, dict[str, str]] | None = None,
alias: str | None = None,
) -> tuple[Path, Path, Path, dict[str, str]]:
"""A copy of the capture script, a fake gateway, and a PATH whose `ssh` reaches only the fake.
@@ -2483,16 +2532,28 @@ def _capture_sandbox(
Returns (script, fixtures dir, stub call log, env). The script is copied into
`<tmp>/scripts/` because it derives `tests/fixtures/` from its own location, which is what
keeps the repository's fixtures out of reach of a test.
The parser step needs the repo, so `agent/host_agent.py` is copied beside the copied script:
the sandbox is a checkout shape, and the plan step imports `target_names` from it exactly as
it does when the real script runs. That copy is the pin — a script that re-implemented the
parser would not need it, and one that imported a different module would not find it.
"""
script = tmp_path / "scripts" / CAPTURE.name
script.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(CAPTURE, script)
(tmp_path / "agent").mkdir(parents=True, exist_ok=True)
shutil.copyfile(ROOT / "agent/host_agent.py", tmp_path / "agent/host_agent.py")
fake = tmp_path / "gateway"
for path, body in {**CAPTURE_FAKE_GATEWAY, **(gateway or {})}.items():
target = fake / path.lstrip("/")
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(body, encoding="utf-8")
for name, families in (CAPTURE_FAKE_ANSWERS if answers is None else answers).items():
for family, body in families.items():
answer = fake / "getent" / f"{name}.{family}"
answer.parent.mkdir(parents=True, exist_ok=True)
answer.write_text(body, encoding="utf-8")
fixtures = tmp_path / "tests" / "fixtures"
fixtures.mkdir(parents=True, exist_ok=True)
@@ -2511,7 +2572,11 @@ def _capture_sandbox(
"PATH": f"{stubs}{os.pathsep}{os.environ['PATH']}",
"STUB_LOG": str(call_log),
"FAKE_REMOTE": str(fake),
"FAIL_SSH": "1" if fail_ssh else "0"}
"FAIL_SSH": "1" if fail_ssh else "0",
"GETENT_MODE": fail_getent or "ok",
# The script prepends its own root to this, which is how `agent.host_agent` resolves
# when uv has no project to run in (this sandbox is not one).
"PYTHONPATH": str(tmp_path)}
if alias is not None:
env["GATEWAY_SSH"] = alias
return script, fixtures, call_log, env
@@ -2529,12 +2594,23 @@ def _fixture_body(path: Path) -> str:
def test_the_gateway_capture_exists_and_asks_for_no_privilege() -> None:
"""Six world-readable reads over an ssh key: nothing here needs root or a rewrite tool."""
"""Six world-readable reads plus `getent`, over an ssh key: no root, no rewrite tool.
`getent` belongs in this list because it is the seventh read and the only one that is not a
file: it is libc resolution on the box that dials, which is what `via: "libc"` claims, and it
is world-runnable for the same reason the six files are world-readable.
"""
assert CAPTURE.is_file(), "scripts/capture-gateway-config.sh is missing (phase 28 task 04)"
assert os.access(CAPTURE, os.X_OK)
text = CAPTURE.read_text()
for path in CAPTURE_TARGETS:
assert path in text, f"{path} is no longer captured"
for family in ("getent ahostsv4", "getent ahostsv6"):
assert family in text, f"{family} is no longer how the resolver document is recorded"
for tool in ("dig", "nslookup", "resolvectl", "drill"):
assert not re.search(rf"(?<![\w-]){tool}\s+[A-Za-z0-9.*_-]", text), (
f"{tool} assumes a DNS client the gateway may not ship; getent is libc, which is the "
"vantage the agent's own lookup performs")
code = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#"))
assert "sudo" not in code, (
"the capture must read exactly what dashboards-agent can read; a pass that needed root "
@@ -2586,7 +2662,9 @@ def test_the_gateway_capture_records_every_document_of_the_tree_it_read(tmp_path
expected = {"gateway_haproxy.cfg", "gateway_nginx_main.conf",
"gateway_nginx_conf_d_http.conf", "gateway_nginx_conf_d_tls.conf",
"gateway_nginx_stream_d_stream.conf",
"gateway_proc_net_tcp.txt", "gateway_proc_net_tcp6.txt"}
"gateway_proc_net_tcp.txt", "gateway_proc_net_tcp6.txt",
# The resolver document, derived from the configs in this same set.
"gateway_resolver.json"}
assert {p.name for p in fixtures.iterdir()} == expected, result.stdout
assert _fixture_body(fixtures / "gateway_haproxy.cfg") == \
CAPTURE_FAKE_GATEWAY["/etc/haproxy/haproxy.cfg"]
@@ -2703,6 +2781,189 @@ def test_the_socket_capture_publishes_no_globally_routable_endpoint(tmp_path: Pa
assert len(v6[2]) == len(raw_v6[2])
# --- the resolver document (phase 32 task 02) ------------------------------------------------
#: One entry per declared target of the fake gateway, in the plan's own order: haproxy's `server`
#: line first, then the nginx vhosts' `proxy_pass` targets, then the stream `map`'s entries — with
#: the map's `default` an address literal, which is why it costs no read on the remote.
CAPTURE_RESOLVER_TARGETS: tuple[tuple[str, int | None], ...] = (
("gitea.reeselink.com", 2222),
("nextcloud.reeselink.com", 11000),
("reeseapps-brain-of-reese.reeselink.com", 443),
("127.0.0.1", 4443),
)
RESOLVER_KEEP = {
"gateway_resolver.json": '{"available": true, "entries": ["keep me"]}\n',
}
def _resolver_fixture(path: Path) -> dict[str, Any]:
"""The recorded document, parsed — and a clear failure when the run refused to write it."""
assert path.is_file(), f"{path.name} was not written"
return json.loads(path.read_text())
def _resolver_run(tmp_path: Path, **kwargs: Any) -> tuple[subprocess.CompletedProcess[str], Path]:
"""One sandboxed capture run, returning its result and the fixture directory it wrote into."""
script, fixtures, _log, env = _capture_sandbox(tmp_path, **kwargs)
result = subprocess.run([BASH, str(script)], capture_output=True, text=True, env=env,
check=False)
return result, fixtures
#: The config documents of the captured tree, the same mapping `tests/conftest.py` lays down for
#: `sources/gateway.py`: an agent `--root` under which the agent can read this repo's captures.
def test_the_capture_asks_its_own_parser_and_not_its_memory() -> None:
"""The name list is `target_names()`, imported — never a second list somebody maintained.
This is the phase's rule 3 in the capture's own hands: the fixture can only ever name what the
agent will resolve, so `target_names()` changing moves the fixture with it instead of leaving a
document of names the agent stopped asking about.
"""
text = CAPTURE.read_text()
assert "from agent.host_agent import" in text, "the plan does not import the agent's parser"
assert "target_names(" in text, "the plan does not ask the agent what its configs declare"
assert "Collector(" in text, "the plan does not read the captured configs the agent reads them"
# And no domain is written into the script: the only domains in it are comments and the fake
# gateway's own fixtures live in the test, not here.
code = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#"))
assert ".reeselink.com" not in code and ".reeseapps.com" not in code, (
"a target name is hardcoded in the capture instead of derived from the configs")
def test_the_resolver_document_names_every_target_the_configs_declare(tmp_path: Path) -> None:
"""The happy path: one libc read per declared name, in the agent's order, in its own shape.
Four assertions the fixture's whole usefulness rests on: the entry set is the plan, `via` says
how the addresses were obtained, `config_directive` came out of the **captured** nginx text (so
it matches what the agent will read from the same bytes, with no extra round trip), and the
remote was asked exactly twice per name — and never about the literal the map writes.
"""
result, fixtures = _resolver_run(tmp_path)
assert result.returncode == 0, result.stdout + result.stderr
document = _resolver_fixture(fixtures / "gateway_resolver.json")
assert document["available"] is True
assert document["via"] == "libc"
assert document["config_directive"] == "10.9.8.7", (
"the `resolver` directive did not come out of the included stream config")
assert [(row["name"], row["port"]) for row in document["entries"]] == list(
CAPTURE_RESOLVER_TARGETS), "the entries are the plan, in the agent's order"
assert document["errors"] == []
for row in document["entries"]:
assert set(row) == {"name", "port", "status", "addresses", "ts"}
assert row["ts"] == document["ts"], "a fixed capture ts keeps the fixture byte-stable"
script, _fixtures, log, _env = _capture_sandbox(tmp_path / "ledger")
subprocess.run([BASH, str(script)], capture_output=True, text=True, env=_env, check=False)
asked = [line for line in log.read_text().splitlines() if "getent " in line]
for name, _port in CAPTURE_RESOLVER_TARGETS:
if ipaddress.ip_address(name) if name[0].isdigit() else False:
assert not [line for line in asked if name in line], (
"an address literal cost a lookup on the remote")
continue
families = [line for line in asked if "getent ahostsv" in line and line.endswith(name)]
assert len(families) == 2, f"{name} was not read once per family: {families}"
def test_the_resolver_document_scrubs_global_addresses_and_keeps_everything_else(
tmp_path: Path) -> None:
"""What the document edits, and what it must not, in one run.
Three arms at once, because they are one rule: `8.8.8.8` and a global EUI-64 become
documentation space (invariant 6 — a resolved address is as much an identifier as the socket
table's), `10.1.0.137` and an address already in documentation space stay exactly as they were
(which is what makes a re-run a no-op), and the `::ffff:` spelling `getent ahostsv6` synthesises
for a name with no AAAA record is dropped — the agent's one `getaddrinfo` never publishes it, so
recording it would claim an address the agent does not. A name nobody answers is `nxdomain`, the
status the page splits out from "resolved and idle".
"""
result, fixtures = _resolver_run(tmp_path)
assert result.returncode == 0, result.stdout + result.stderr
entries = {row["name"]: row for row in
_resolver_fixture(fixtures / "gateway_resolver.json")["entries"]}
private = entries["gitea.reeselink.com"]
assert private["status"] == "ok"
assert private["addresses"] == ["10.1.0.137"], private
assert not any(a.startswith("::ffff:") for a in private["addresses"]), (
"the synthesised v4-mapped spelling survived: the agent never publishes it")
moved = entries["nextcloud.reeselink.com"]
assert moved["status"] == "ok"
assert moved["addresses"] == ["203.0.113.1", "203.0.113.9", "2001:db8::1"], moved
assert "8.8.8.8" not in json.dumps(moved), "a globally routable answer is in a tracked file"
assert "2603:6013" not in json.dumps(moved), "an EUI-64 address is in a tracked file"
missing = entries["reeseapps-brain-of-reese.reeselink.com"]
assert missing["status"] == "nxdomain", "one name failing among successes is a fact, kept"
assert missing["addresses"] == [], "an unresolved name publishes no address, not 0 and not a guess"
literal = entries["127.0.0.1"]
assert literal["status"] == "literal" and literal["addresses"] == ["127.0.0.1"]
def test_the_resolver_document_is_idempotent_so_a_re_run_does_not_churn(tmp_path: Path) -> None:
"""Two runs over the same answers write the same bytes.
The `ts` is a constant and the scrub is first-seen order over a deterministic plan, so a
re-capture of a gateway nobody changed is a no-op — the property that makes `git diff` on this
directory a statement about the ingress rather than about the clock.
"""
once, first = _resolver_run(tmp_path)
assert once.returncode == 0, once.stdout + once.stderr
again, second = _resolver_run(tmp_path / "again")
assert again.returncode == 0, again.stdout + again.stderr
assert (second / "gateway_resolver.json").read_bytes() == (
first / "gateway_resolver.json").read_bytes()
@pytest.mark.parametrize("mode, needle", [
pytest.param("transport", "did not answer", id="ssh-dies"),
pytest.param("missing", "getent is not available", id="no-getent"),
])
def test_a_broken_resolver_vantage_is_not_a_fixture(tmp_path: Path, mode: str,
needle: str) -> None:
"""The configs capture fine and only the resolver is broken — and it still refuses.
This is the arm the socket rule cannot cover: `getent` answering *nothing* for every name is a
fact about the box's resolver, not about the names, and a document of `nxdomain` rows would
make every live route on the page a permanent dash and every join test a test of nothing.
"""
result, fixtures = _resolver_run(tmp_path, recorded=dict(RESOLVER_KEEP), fail_getent=mode)
assert result.returncode == 1, result.stdout
assert needle in result.stdout, result.stdout
assert "gateway_resolver.json" in result.stdout
assert (fixtures / "gateway_resolver.json").read_text() == RESOLVER_KEEP[
"gateway_resolver.json"], "a broken vantage overwrote the recorded document"
def test_not_one_name_answering_is_refused_the_way_a_dead_transport_is(tmp_path: Path) -> None:
"""Every name answered `nxdomain`: each individual read succeeded, and the run still refuses.
A resolver that returns "no records" for the whole config is a firewall, a dead upstream DNS,
or a box that has never been pointed at one — and the fixture it would produce is the same
nothing the other two arms produce.
"""
result, fixtures = _resolver_run(tmp_path, recorded=dict(RESOLVER_KEEP), answers={})
assert result.returncode == 1, result.stdout
assert "none of the" in result.stdout and "vantage is broken" in result.stdout, result.stdout
assert (fixtures / "gateway_resolver.json").read_text() == RESOLVER_KEEP[
"gateway_resolver.json"]
def test_a_dead_gateway_does_not_attempt_the_resolver_document(tmp_path: Path) -> None:
"""With no configs there is no name list, and a document of the wrong question is worse than none."""
result = _capture(tmp_path, recorded=dict(RESOLVER_KEEP), fail_ssh=True)
fixtures = tmp_path / "tests" / "fixtures"
assert result.returncode == 1, result.stdout
assert "not attempted" in result.stdout, result.stdout
assert (fixtures / "gateway_resolver.json").read_text() == RESOLVER_KEEP[
"gateway_resolver.json"]
assert sorted(p.name for p in fixtures.iterdir()) == ["gateway_resolver.json"], (
"a refused resolver capture left a temp file in tests/fixtures/")
def test_the_socket_capture_is_idempotent_so_a_re_run_does_not_churn(tmp_path: Path) -> None:
"""Re-recording the same moment must not diff, or every capture moves the fixture twice.
tests/test_gateway_source.py
+741 -26
View File
@@ -18,13 +18,18 @@ outside the two configs is *reported*, because it no longer is —
"""
from __future__ import annotations
import ipaddress
import json
import socket as socket_module
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from dashboards.sources import gateway
#: This repo's recorded captures, for the pins that read a fixture directly.
FIXTURES = Path(__file__).resolve().parent / "fixtures"
# --------------------------------------------------------------------------- #
# synthetic builders: a section exactly as the agent serves it, minus the file
# --------------------------------------------------------------------------- #
@@ -111,10 +116,31 @@ def conn(local: tuple[str, int], remote: tuple[str, int], *,
"remote": {"addr": remote[0], "port": remote[1]}}
def resolver_section(entries: Sequence[dict[str, Any]] = (), *, available: bool | None = True,
via: str | None = "libc", config_directive: str | None = "10.1.0.1",
ts: float = 1_760_000_000.0,
errors: Sequence[str] = ()) -> dict[str, Any]:
"""Agent 1.3.0's fourth section, in the shape `host_agent.Collector.resolver` publishes."""
return {"available": available, "via": via, "config_directive": config_directive, "ts": ts,
"entries": list(entries), "errors": list(errors)}
def resolved(name: str, port: int | None, addresses: Sequence[str], *,
status: str = "ok", ts: float | None = 1_760_000_000.0) -> dict[str, Any]:
"""One resolver entry: what libc answered for `(name, port)`, and when."""
return {"name": name, "port": port, "status": status, "addresses": list(addresses), "ts": ts}
def snapshot(*, haproxy: dict[str, Any] | None = None, nginx: dict[str, Any] | None = None,
sockets: dict[str, Any] | None = None, errors: Sequence[str] = (),
sockets: dict[str, Any] | None = None, resolver: dict[str, Any] | None = None,
errors: Sequence[str] = (),
agent_version: str | None = "1.2.0") -> dict[str, Any]:
"""The document one agent `/snapshot` would answer, with only what a test names."""
"""The document one agent `/snapshot` would answer, with only what a test names.
No `resolver` key is the default, because that is the arm most of this file predates: a 1.2.0
agent, or a 1.3.0 one run with `--no-resolve`. A test that wants a name to be countable passes
`resolver=resolver_section([...])` and says so.
"""
doc: dict[str, Any] = {"ok": True, "schema": 1, "ts": 1_760_000_000.0, "host": "gateway",
"errors": list(errors)}
if agent_version is not None:
@@ -125,6 +151,8 @@ def snapshot(*, haproxy: dict[str, Any] | None = None, nginx: dict[str, Any] | N
doc["nginx"] = nginx
if sockets is not None:
doc["sockets"] = sockets
if resolver is not None:
doc["resolver"] = resolver
return doc
@@ -162,6 +190,52 @@ def out_metrics(payload: Mapping[str, Any]) -> list[str]:
return [row["metric"] for row in payload["connections"] if row["metric"] is not None]
def endpoint_keys(row: Mapping[str, Any]) -> set[tuple[str, int]]:
"""The normalised `(address, port)` keys one route reaches, derived **in the test**.
Read straight off `out.address` / `out.resolved` with `ipaddress`, deliberately not by calling
`gateway.endpoint_of`: a total a test compares against must not come from the function under
test. The normalisation is the same one the join makes, because `::ffff:10.1.2.64` and
`10.1.2.64` are one endpoint and a ledger that counted them apart would reconcile to nothing.
"""
out = row["out"]
port = out.get("port")
if not isinstance(port, int):
return set()
spellings = [out["host"]] if out.get("address") is True else list(out.get("resolved") or [])
keys: set[tuple[str, int]] = set()
for spelling in spellings:
try:
address = ipaddress.ip_address(str(spelling))
except ValueError:
continue
mapped = getattr(address, "ipv4_mapped", None)
keys.add((str(mapped if mapped is not None else address), port))
return keys
def endpoint_groups(payload: Mapping[str, Any]) -> dict[tuple[str, int], list[Mapping[str, Any]]]:
"""The route lines grouped by the **distinct endpoint** they reach, keyed by min(key).
Lines whose key sets are identical share a group, which is every case a capture can produce
(one address per name, or one name written by two vhosts). `gateway._endpoint_groups` merges
*overlapping* sets as well, for a multi-homed name sharing one address with a single-homed one;
no capture has that shape, and if one ever does this helper disagrees by keeping two groups —
which fails the totals test below loudly rather than quietly.
"""
groups: dict[tuple[str, int], list[Mapping[str, Any]]] = {}
for row in payload["connections"]:
keys = endpoint_keys(row)
if keys:
groups.setdefault(min(keys), []).append(row)
return groups
def distinct_endpoint_totals(payload: Mapping[str, Any]) -> dict[tuple[str, int], Any]:
"""One count per distinct endpoint — the figure a total may add, never one per line."""
return {key: rows[0]["connections"] for key, rows in endpoint_groups(payload).items()}
def walk_keys(node: Any) -> set[str]:
"""Every key name anywhere in a payload — the device the "no severity / no window" pins use."""
if isinstance(node, Mapping):
@@ -200,7 +274,7 @@ def test_the_captured_configs_yield_every_configured_pipe(gateway_snapshot: Any)
"five vhosts share 4443 and their domains belong to the one pipe for that port")
assert payload["nginx"]["map_targets"], "the stream map is the fan-out the page draws"
assert len(payload["nginx"]["map_targets"]) == 17
assert len(payload["nginx"]["map_targets"]) == 18
homepage = next(row for row in payload["nginx"]["map_targets"]
if row["key"] == "homepage.reeseapps.com")
assert homepage == {"key": "homepage.reeseapps.com",
@@ -229,7 +303,12 @@ def test_the_source_reads_the_keys_the_agent_actually_publishes(gateway_snapshot
assert {row["port"] for row in payload["nginx"]["servers"]} == agent_listen_ports
# and the three section names themselves, as the snapshot publishes them
assert {"haproxy", "nginx", "sockets"} <= set(snap)
assert host_agent_module.AGENT_VERSION == "1.2.0", "the sections these keys belong to"
# At least 1.2.0, the version that shipped these three keys. It is a floor and not an
# equality on purpose: phase 32 added a fourth section and bumped the constant, and the
# coupling this test pins is that the three the source joins still exist in what an agent
# publishes, whatever the number on it has since grown.
version = tuple(int(part) for part in host_agent_module.AGENT_VERSION.split("."))
assert version >= (1, 2, 0), "the sections these keys belong to"
def test_the_live_counts_reconcile_with_the_socket_table(gateway_snapshot: Any) -> None:
@@ -247,29 +326,60 @@ def test_the_live_counts_reconcile_with_the_socket_table(gateway_snapshot: Any)
What stays pinned is the direction of the error and the absence of a residual: the reported
figures are always **under** the kernel's number, and there is no field anywhere that quietly
absorbs the difference to make the two agree.
Run against the **`resolved`** capture, because that is what the deployed ingress serves since
phase 32 task 05: the widened match set is the one the arithmetic has to hold for. Three
figures, read three ways:
* `in_total` — the sum over the listeners (each socket at most once, inbound);
* `out_total` — the sum over **distinct endpoints**, derived here from `out.address` /
`out.resolved` rather than from `gateway.endpoint_of`;
* `lines_total` — the sum over the **lines**, which is deliberately larger wherever a line
repeats a number (`shared`), and is pinned to be exactly `out_total` plus the repetitions.
The `127.0.0.1:4443` loopback hop is the boundary case: it is booked **once inbound** (the rows
whose local port is 4443, on the `:4443` trunk) and **once outbound** (the rows whose remote is
4443, on the `:443 → 127.0.0.1:4443` pipe) and no single `/proc` row is in both.
"""
snap = gateway_snapshot("full")
snap = gateway_snapshot("resolved")
payload = gateway.parse(snap, "gateway")
established = len(snap["sockets"]["established"])
established = snap["sockets"]["established"]
configured = {row["port"] for row in payload["listeners"]}
assert configured == {22, 2000, 5201, 8080, 25565, 25566, 80, 443, 4443, 8443}
assert all(row["port"] in configured for row in payload["listeners"]), (
"an inbound count on a port no config names cannot exist in this payload")
in_total = sum(row["connections"] for row in payload["listeners"])
endpoints = {(row["out"]["host"], row["out"]["port"]): row["connections"]
for row in payload["connections"] if row["metric"] is not None}
out_total = sum(value or 0 for value in endpoints.values())
totals = distinct_endpoint_totals(payload)
out_total = sum(value or 0 for value in totals.values())
lines_total = sum(row["connections"] or 0 for row in payload["connections"])
repeated = sum((len(rows) - 1) * (rows[0]["connections"] or 0)
for rows in endpoint_groups(payload).values() if len(rows) > 1)
rows_in_table = len(established)
assert payload["sockets"] == {"listens": len(snap["sockets"]["listens"]),
"established": established}
assert in_total + out_total <= established
assert in_total + out_total < established, (
"established": rows_in_table}
assert in_total + out_total <= rows_in_table
assert in_total + out_total < rows_in_table, (
"this capture has traffic outside both config files; if it ever stops having any, this "
"test is proving nothing and should be re-written against a capture that does")
assert "unmatched" not in payload and "partial" not in payload
# the loopback hop is the case that used to double count: `127.0.0.1:4443` is both a listener's
# remote end and an outbound endpoint, and it may be booked on only one of them
assert sum(row["connections"] or 0 for row in payload["connections"]) == out_total
assert lines_total == out_total + repeated, (
"the only difference between the lines and the endpoints is a repeated count — no socket "
"is booked twice and no endpoint is booked once per name that reaches it")
# the loopback hop, both sides, from the raw table rather than from either counter
accepted = [row for row in established if (row["local"]["port"]) == 4443]
dialled = [row for row in established
if row["local"]["port"] != 4443 and (row["remote"]["addr"], row["remote"]["port"])
== ("127.0.0.1", 4443)]
assert accepted and dialled, "this capture is the loop the stream map's `default` produces"
assert listener_at(payload, 4443)["connections"] == len(accepted)
assert endpoint_count(payload, "127.0.0.1", 4443) == len(dialled)
assert not [row for row in established
if row["local"]["port"] == 4443
and (row["remote"]["addr"], row["remote"]["port"]) == ("127.0.0.1", 4443)], (
"a row that was both would be counted on two pipes by two rules, and the ≤ above would "
"still hold while every figure in the payload was wrong")
def test_a_socket_neither_config_describes_is_counted_nowhere(gateway_snapshot: Any) -> None:
@@ -293,7 +403,7 @@ def test_every_connection_the_captured_configs_declare_is_drawn(gateway_snapshot
"""The graph as a **set**, so a missing or invented route fails by name.
The join this task rewrote stamps counts onto these rows, so the rows themselves are pinned
here too: 6 haproxy `server` lines, 17 stream-`map` entries (the 16 `*.reeselink.com:443` fan-
here too: 6 haproxy `server` lines, 18 stream-`map` entries (the 17 `*.reeselink.com:443` fan-
out plus the literal `default` → `127.0.0.1:4443`), and 9 vhost `proxy_pass` targets across
`:4443` and `:8443`. `:80` is the refusal that lives on this fleet: the catch-all vhost declares
no `proxy_pass`, so it is a trunk with no connection and no invented line beside it.
@@ -317,6 +427,7 @@ def test_every_connection_the_captured_configs_declare_is_drawn(gateway_snapshot
(443, "reeseapps-gitea.reeselink.com", 443),
(443, "reeseapps-homepage.reeselink.com", 443),
(443, "reeseapps-immich.reeselink.com", 443),
(443, "reeseapps-whiteboard.reeselink.com", 443),
(443, "reeseapps-keycloak.reeselink.com", 443),
(443, "reeseapps-litellm.reeselink.com", 443),
(443, "reeseapps-ntfy.reeselink.com", 443),
@@ -362,9 +473,9 @@ def test_the_map_fanout_is_routes_and_never_a_variable_host(gateway_snapshot: An
"a proxy_pass $var carries no address, so it must not become a line of its own")
brain = next(row for row in map_routes
if row["out"]["host"] == "reeseapps-brain-of-reese.reeselink.com")
assert brain["out"]["address"] is False, "the config wrote a name, and this stack resolves none"
assert brain["connections"] is None, "so the line carries no count"
assert brain["metric"] is None, "and publishes no series"
assert brain["out"]["address"] is False, "the config wrote a name, and the config is verbatim"
assert brain["connections"] is None, "and on this arm (no resolver section) nothing placed it"
assert brain["metric"] is None, "so it publishes no series"
# --------------------------------------------------------------------------- #
@@ -476,10 +587,12 @@ def test_a_name_target_is_never_credited_for_a_socket_on_its_port() -> None:
"""The rule that replaced `by_port`, stated as a fact about the payload and not a chip.
`proxy_pass https://nextcloud.example.test:11000` and the socket to `10.1.0.254:11000` are the
same connection in the operator's head and two unrelated strings in this stack. Attributing one
to the other needed a resolver, and the port-only guess the old join made in its place was a
guess about a **port**, not about a host — which is why its count was the port's and its chip
said so. Now the line simply has no number, no gauge, and no apology to render.
same connection in the operator's head and two unrelated strings **in this stack**. The bridge is
the gateway's own resolution (phase 32, tested below), and the port-only guess the old join made
in its place was a guess about a **port**, not about a host — which is why its count was the
port's and its chip said so. With nothing to bridge them the line has no number, no gauge, and no
apology to render; with the agent's document it is counted through the address, and this test
stays true because the snapshot it builds has no resolver section.
"""
snap = snapshot(
nginx=nginx_section([nginx_server([4443], ["cloud.example.test"],
@@ -503,7 +616,7 @@ def test_a_name_target_is_never_credited_for_a_socket_on_its_port() -> None:
def test_a_port_shared_by_several_names_still_resolves_none_of_them() -> None:
"""The real `stream.conf` shape: sixteen names on 443, none of them counted.
"""The real `stream.conf` shape: sixteen names on 443, none of them counted **without DNS**.
The answer is the same `null` the old join gave, minus the machinery that produced it and the
`unmatched.outbound` total that used to double-count the same sockets.
@@ -557,6 +670,68 @@ def test_no_join_ever_asks_a_resolver() -> None:
assert calls == []
def test_the_recorded_resolver_document_reaches_every_name_written_route(
gateway_snapshot: Any) -> None:
"""What phase 32 buys, priced against the captures — and spent, since task 03.
31 of this ingress's 33 routes carried an em dash because this module joins sockets by
**address** and the two config files write those endpoints as **names**. The recorded
`gateway_resolver.json` is the thing that removed that reason, so this pins the two properties
the join needs and no more:
* every name-written route has an entry in the document, for that name **and that port**, with
at least one address — so no route on `/gateway` is dashed forever because the resolver had
nothing to say about its endpoint;
* the addresses are the same currency the socket table is written in: the two captures overlap
on live remote endpoints, which is what a join has to have to join anything.
and, in the last few lines, that spending it is a fact about the **document** rather than a
coincidence of the port table: the name-written routes are counted exactly where the entry puts
an address and nowhere else. This module still resolves nothing — the spy pin above stays exactly
as it was, and is the reason the fixture has to exist.
"""
snap = gateway_snapshot("full")
payload = gateway.parse(snap, "gateway")
document = json.loads((FIXTURES / "gateway_resolver.json").read_text())
answered = {(row["name"], row["port"]) for row in document["entries"] if row["addresses"]}
written = {(row["out"]["host"], row["out"]["port"]) for row in payload["connections"]
if not row["out"]["address"]}
assert written, "this capture writes its targets as names, which is the gap the fixture fills"
unreached = written - answered
assert not unreached, ("name-written routes the resolver document cannot reach: "
f"{sorted(unreached)} — re-run ./scripts/capture-gateway-config.sh")
resolved = {address for row in document["entries"] for address in row["addresses"]}
remote = {row["remote"]["addr"] for row in snap["sockets"]["established"]}
assert resolved & remote, (
"the resolver document and the socket table share no address, so a join built on it could "
"attribute nothing — the two captures are from different boxes or one scrub moved a number")
assert all(row["connections"] is None for row in payload["connections"]
if (row["out"]["host"], row["out"]["port"]) in written), (
"this arm is the snapshot with no `resolver` key at all, so a name-written route still "
"carries no count here — and adding one address to that section is what changes it")
spent = gateway.parse(gateway_snapshot("resolved"), "gateway")
rows = {(row["out"]["host"], row["out"]["port"]): row for row in spent["connections"]}
trunks = {row["port"] for row in spent["listeners"]}
for name, port in sorted(written):
row = rows[(name, port)]
entry = next(item for item in document["entries"]
if (item["name"], item["port"]) == (name, port))
assert row["out"]["resolved"] == entry["addresses"], f"{name}:{port} rewrites the capture"
assert row["connections"] is not None, f"{name}:{port} is placed and still carries no count"
# the exact figure, re-counted from the raw table by this test: the rows that reach this
# endpoint, minus the ones the inbound arm claimed first (a socket is counted at most once)
keys = {(address, port) for address in entry["addresses"]}
live = [item for item in snap["sockets"]["established"]
if item["local"]["port"] not in trunks
and (item["remote"]["addr"], item["remote"]["port"]) in keys]
assert row["connections"] == len(live), (
f"{name}:{port} resolves to {sorted(keys)} and the table holds {len(live)} such "
f"sockets, but the line reads {row['connections']}")
def test_two_services_on_one_host_are_two_pipes_not_one_sum() -> None:
"""The reason the outbound series name carries its port (`minecraft` 25565 and 25566)."""
snap = snapshot(
@@ -750,6 +925,546 @@ def test_an_endpoint_reached_by_one_line_is_not_shared() -> None:
assert row["connections"] == 1
# --------------------------------------------------------------------------- #
# phase 32: the resolver section — a name counted through the address the gateway dialled
#
# Everything above this line runs on a snapshot with **no** `resolver` key, which is agent 1.2.0 (or
# 1.3.0 with `--no-resolve`) and therefore phase 30 verbatim. Everything below it spends the
# document `scripts/capture-gateway-config.sh` recorded with `getent` on the router: the resolution
# haproxy and nginx perform **on the box that opens the connection**, which is the only vantage that
# can join `reeseapps-brain-of-reese.reeselink.com` to the `10.1.2.64` `/proc` reports. The
# dashboard asking the same name through its own `resolv.conf` would be a different vantage
# answering a different question — and `test_the_dashboard_still_never_resolves` is the spy that
# keeps it that way.
# --------------------------------------------------------------------------- #
def test_a_name_route_counts_through_the_address_the_gateway_published(
gateway_snapshot: Any) -> None:
"""The whole point of phase 32, measured against the captures.
`stream.conf` routes `brain.reeseapps.com` to `reeseapps-brain-of-reese.reeselink.com:443`,
`/proc` would report a socket to `10.1.2.64:443`, and until this section arrived nothing could
join the two. Now the line carries the config's text and the agent's answer side by side, and
the count comes through the answer.
"""
snap = gateway_snapshot("resolved")
payload = gateway.parse(snap, "gateway")
brain = next(row for row in payload["connections"]
if row["out"]["host"] == "reeseapps-brain-of-reese.reeselink.com")
assert brain["in"] == {"port": 443}
assert brain["out"]["selectors"] == ["brain.reeseapps.com"], "the config's word, untouched"
assert brain["out"]["address"] is False, "the config wrote a name, and still says so"
assert brain["out"]["resolved"] == ["10.1.2.64"], "the agent's answer, verbatim"
assert brain["out"]["via"] == "libc", "the vantage is the agent's own string, copied"
assert brain["out"]["resolved_ts"] == 1_792_886_400.0, "that entry's own timestamp"
assert brain["metric"] == "gateway.out_conns@reeseapps-brain-of-reese.reeselink.com:443", (
"the gauge keeps the config's spelling, so history continues under one name")
assert brain["connections"] == 0, "resolved and idle is `0`, never the dash"
ntfy = next(row for row in payload["connections"]
if row["out"]["host"] == "reeseapps-ntfy.reeselink.com")
live = [row for row in snap["sockets"]["established"]
if (row["remote"]["addr"], row["remote"]["port"]) == ("10.1.1.176", 443)]
assert live, "the recorded table has live ntfy sockets for the join to find"
assert ntfy["connections"] == len(live), "every socket to the address the name became"
assert ntfy["shared"] is False, "one route reaches that endpoint, so nothing to repeat"
def test_an_endpoint_the_config_wrote_as_an_address_claims_no_lookup(
gateway_snapshot: Any) -> None:
"""`resolved` answers *what did this name become*, and an address was never a name.
The agent publishes a `literal` entry for a target that is already an address, and the resolver
block lists it verbatim. Stamping that echo onto the route as `resolved` beside `via: "libc"`
would report a `getaddrinfo` this gateway never ran, so the three provenance fields stay `null`
on an `address: true` row while the count and the gauge keep working exactly as phase 30 left
them. Two facts stay two sentences: `address` is what the config wrote, `resolved` is what the
gateway looked up.
"""
payload = gateway.parse(gateway_snapshot("resolved"), "gateway")
written = [row for row in payload["connections"] if row["out"]["address"] is True]
assert written, "this capture writes two targets as addresses, which is phase 30's joinable pair"
for row in written:
assert row["out"]["host"] in {entry["name"]
for entry in payload["resolver"]["entries"]}, (
"the agent did publish this literal, and the block still lists it")
assert row["out"]["resolved"] is None and row["out"]["via"] is None \
and row["out"]["resolved_ts"] is None, (
f"{row['id']}: an address row must not borrow a lookup claim")
assert row["metric"] == f"gateway.out_conns@{row['out']['host']}:{row['out']['port']}"
assert row["connections"] is not None, "phase 30's address join is untouched by this"
def test_a_socket_lands_on_the_name_route_that_resolved_to_its_address() -> None:
"""The synthetic version, so the claim does not depend on what a capture happened to contain."""
snap = snapshot(
nginx=nginx_section([nginx_server([443], [], [proxy_pass(None, variable="targetBackend")])],
maps=[mapping_block("$targetBackend",
[("brain.example.test",
("brain-backend.example.test", 443))])]),
sockets=sockets_section(
listens=[listen(443)],
established=[conn(("::", 51000), ("10.1.2.64", 443), family="v6"),
conn(("10.1.1.27", 443), ("203.0.113.7", 900))]),
resolver=resolver_section([resolved("brain-backend.example.test", 443, ["10.1.2.64"])]))
payload = gateway.parse(snap, "gateway")
row = routes_to(payload, "brain-backend.example.test")[0]
assert row["out"]["address"] is False and row["out"]["resolved"] == ["10.1.2.64"]
assert row["connections"] == 1
assert row["metric"] == "gateway.out_conns@brain-backend.example.test:443"
assert listener_at(payload, 443)["connections"] == 1, "the inbound half is untouched"
gauge = [g for g in gateway.samples(payload, 5.0)
if g.metric == "gateway.out_conns@brain-backend.example.test:443"]
assert [(g.metric, g.value) for g in gauge] == [
("gateway.out_conns@brain-backend.example.test:443", 1.0)]
def test_the_match_is_an_address_not_a_string() -> None:
"""One address, two spellings, in either direction: the join still lands.
`/proc`'s `inet_ntop` writes an IPv4 peer seen through the IPv6 table as `::ffff:10.1.2.64`
while a resolver answers `10.1.2.64`, and an IPv6 in one table can carry leading zeros and
capitals the other does not. `ipaddress` canonicalises both, so the **match key** is one address
while the **payload** keeps publishing whatever the config and the agent wrote — a page that
printed the normalised form would be quoting a string nobody configured.
A join that compared strings instead would report a resolved, dialled route as idle forever:
the exact failure this phase exists to remove, arriving back through a spelling nobody thought
about. The two directions are asserted separately because each half is a different table.
"""
def build(written: str, socketed: str) -> dict[str, Any]:
snap = snapshot(
haproxy=haproxy_section([frontend("peer", [8001], default="peer-be")],
[backend("peer-be", [server("p", "peer.example.test", 443)])]),
sockets=sockets_section(
established=[conn(("::", 40000), (socketed, 443), family="v6")]),
resolver=resolver_section([resolved("peer.example.test", 443, [written])]))
return gateway.parse(snap, "gateway")
for written, socketed in (("10.1.2.64", "::ffff:10.1.2.64"),
("::ffff:10.1.2.64", "10.1.2.64"),
("2001:db8::1", "2001:0DB8::0001"),
("2001:0DB8::0001", "2001:db8::1")):
payload = build(written, socketed)
row = routes_to(payload, "peer.example.test")[0]
assert row["connections"] == 1, f"{written} vs {socketed} is one address and did not match"
assert row["out"]["resolved"] == [written], "the payload keeps the agent's own spelling"
def test_a_zero_padded_ipv4_is_no_address_to_anyone_and_matches_nothing() -> None:
"""The one spelling this module refuses to fold, and why refusing is the safe half.
`ipaddress` rejects `010.1.2.64` because `inet_aton` reads a leading `0` as **octal**: the
decimal and octal readings are two different addresses, and picking one here would book a
socket on a route by coin flip. So a padded target is a **name** to this module — and the
resolver section, which is libc's own answer, is what turns it into an address the kernel agrees
with. Nothing is guessed in between.
"""
snap = snapshot(
haproxy=haproxy_section([frontend("be", [8080], default="be")],
[backend("be", [server("odd", "010.1.2.64", 443)])]),
sockets=sockets_section(
established=[conn(("10.1.1.27", 40000), ("10.1.2.64", 443)),
conn(("10.1.1.27", 40002), ("8.1.2.64", 443))]))
row = routes_to(gateway.parse(snap, "gateway"), "010.1.2.64")[0]
assert row["out"]["address"] is False, "not an address here, so it is a name"
assert row["connections"] is None and row["metric"] is None, (
"and with no resolver entry it is counted nowhere — neither reading is chosen for it")
# …and the fix is the resolver's, not this module's: libc's own answer places the same route,
# the padded spelling travels in the payload untouched, and neither socket is read as an octet.
placed = snapshot(
haproxy=haproxy_section([frontend("be", [8080], default="be")],
[backend("be", [server("odd", "010.1.2.64", 443)])]),
sockets=sockets_section(
established=[conn(("10.1.1.27", 40000), ("10.1.2.64", 443)),
conn(("10.1.1.27", 40002), ("8.1.2.64", 443))]),
resolver=resolver_section([resolved("010.1.2.64", 443, ["10.1.2.64"])]))
fixed = routes_to(gateway.parse(placed, "gateway"), "010.1.2.64")[0]
assert fixed["out"]["resolved"] == ["10.1.2.64"]
assert fixed["connections"] == 1, (
"the one socket to the address libc returned, not the octal-reading 8.1.2.64 and not both")
def test_two_names_one_endpoint_is_one_count_shared() -> None:
"""Two **names** the gateway placed on one address are one endpoint and one number.
Phase 30 knew this shape only for one address written twice; resolving makes it ordinary, and
the rule did not change: the endpoint's whole count is repeated on both lines with
`shared: true`, never halved and never attributed to one of them.
Two readers then differ, and both are right. **Every total** books the endpoint once
(`distinct_endpoint_counts`, which is what `/api/summary`'s `live_out` reads), while the gauge
is per line-family — one series per name the config wrote, each carrying that endpoint's number,
because a route's own history is asked for by the name on the route. Two names therefore store
two series of one figure; lines that share a **name** (`:4443` and `:8443` reaching one target)
still store exactly one.
"""
snap = snapshot(
nginx=nginx_section([nginx_server([4443], ["a.example.test"],
[proxy_pass(("one.example.test", 8080))]),
nginx_server([8443], ["b.example.test"],
[proxy_pass(("two.example.test", 8080))])]),
sockets=sockets_section(
established=[conn(("10.1.1.27", 40000), ("10.5.5.5", 8080)),
conn(("10.1.1.27", 40002), ("10.5.5.5", 8080))]),
resolver=resolver_section([resolved("one.example.test", 8080, ["10.5.5.5"]),
resolved("two.example.test", 8080, ["10.5.5.5"])]))
payload = gateway.parse(snap, "gateway")
rows = [row for row in payload["connections"] if row["in"]["port"] in (4443, 8443)]
assert [row["connections"] for row in rows] == [2, 2], "the endpoint's total, twice"
assert [row["shared"] for row in rows] == [True, True]
assert gateway.distinct_endpoint_counts(payload["connections"]) == [2], (
"and one figure for anything that adds the outbound side up")
assert out_metrics(payload) == ["gateway.out_conns@one.example.test:8080",
"gateway.out_conns@two.example.test:8080"]
def test_two_spellings_of_one_multi_homed_name_are_one_endpoint() -> None:
"""A name with three addresses is **one** route, not three, and its sockets are one count.
Every key the resolver published points at the same counter row, so a client round-robin across
`10.5.5.5`/`10.5.5.6` reads as three connections on one pipe. The case union-find exists for is
two names whose address sets *overlap* — a multi-homed host and a single-homed alias for one of
its addresses — which would otherwise count the same endpoint twice in one total.
"""
snap = snapshot(
nginx=nginx_section([nginx_server([4443], ["a.example.test"],
[proxy_pass(("multi.example.test", 443))]),
nginx_server([8443], ["b.example.test"],
[proxy_pass(("alias.example.test", 443))])]),
sockets=sockets_section(
established=[conn(("10.1.1.27", 40000), ("10.5.5.5", 443)),
conn(("10.1.1.27", 40002), ("10.5.5.6", 443))]),
resolver=resolver_section([resolved("multi.example.test", 443,
["10.5.5.5", "10.5.5.6"]),
resolved("alias.example.test", 443, ["10.5.5.6"])]))
payload = gateway.parse(snap, "gateway")
rows = [row for row in payload["connections"] if row["in"]["port"] in (4443, 8443)]
assert [row["connections"] for row in rows] == [2, 2], (
"the shared endpoint's two sockets, on both lines — not one socket each")
assert [row["shared"] for row in rows] == [True, True]
assert gateway.distinct_endpoint_counts(payload["connections"]) == [2], (
"overlapping address sets are one endpoint, so a total books them once")
def test_a_name_that_bridges_two_others_merges_all_three_lines_into_one_endpoint() -> None:
"""A→{x}, B→{x,y}, C→{y}: three routes, one endpoint, one figure — transitivity included.
The bridging name is what makes this a *merge* rather than three pairwise comparisons: the first
and third share no address, and yet the same kernel endpoint sits behind both, so counting them
separately would book two endpoints where `/proc` has one. **The config order is load-bearing**
beyond the transitivity: the bridge arrives second and its lowest address (`10.5.5.5`) is lower
than the root the first route already built, so that whole group is re-parented, and the third
route's lookup then has to walk through two links to find it — the path the union-find compresses
so a fan-out of this size stays cheap on the next poll.
"""
snap = snapshot(
nginx=nginx_section([nginx_server([4443], ["a.example.test"],
[proxy_pass(("multi.example.test", 443))]),
nginx_server([8443], ["b.example.test"],
[proxy_pass(("bridge.example.test", 443))]),
nginx_server([8444], ["c.example.test"],
[proxy_pass(("alias.example.test", 443))])]),
sockets=sockets_section(
established=[conn(("10.1.1.27", 40000), ("10.5.5.6", 443)),
conn(("10.1.1.27", 40002), ("10.5.5.7", 443))]),
resolver=resolver_section([resolved("multi.example.test", 443,
["10.5.5.6", "10.5.5.7"]),
resolved("bridge.example.test", 443,
["10.5.5.5", "10.5.5.6"]),
resolved("alias.example.test", 443, ["10.5.5.7"])]))
payload = gateway.parse(snap, "gateway")
rows = [row for row in payload["connections"] if row["in"]["port"] in (4443, 8443, 8444)]
assert len(rows) == 3
assert [row["connections"] for row in rows] == [2, 2, 2], (
"one endpoint's two sockets, repeated on all three lines")
assert [row["shared"] for row in rows] == [True, True, True]
assert gateway.distinct_endpoint_counts(payload["connections"]) == [2], (
"and one figure for any total, however many names bridged their way to it")
def test_a_resolver_entry_that_publishes_nothing_address_like_places_nothing() -> None:
"""A malformed answer is a name with no address, not an address that matches by luck.
The agent publishes strings it got from `getaddrinfo`, and this module has no reason to believe
them: an entry whose `addresses` are not addresses yields no endpoint key, so the route keeps its
dash and its missing gauge exactly as an `nxdomain` does. The alternative — comparing the junk
text against socket addresses — is how a typo in one file could credit a socket to a route.
"""
snap = snapshot(
nginx=nginx_section([nginx_server([4443], ["odd.example.test"],
[proxy_pass(("odd.example.test", 443))])]),
sockets=sockets_section(established=[conn(("10.1.1.27", 40000), ("10.5.5.5", 443))]),
resolver=resolver_section([
resolved("odd.example.test", 443, ["localhost", "10.5.5.5.6", ""])]))
row = routes_to(gateway.parse(snap, "gateway"), "odd.example.test")[0]
assert row["out"]["resolved"] == ["localhost", "10.5.5.5.6"], (
"the agent's bytes verbatim, minus the empty string — the payload is not where this is fixed")
assert row["connections"] is None and row["metric"] is None, (
"and not one of them is an address to match a socket with")
def test_a_line_with_no_endpoint_contributes_nothing_to_a_total() -> None:
"""`distinct_endpoint_counts` skips the unplaced line instead of offering it as a zero.
The caller turns "no figures at all" into `null` (`api._live_total`), which is the arm an
ingress of nothing-but-unresolved-names lands in. A `null` inside the list instead would make
every total on the page `null` because one route has a name nobody could place — and rule 2's
complaint about that shape is precisely that it invents a fault in the healthy routes beside it.
"""
snap = snapshot(
nginx=nginx_section([nginx_server([4443], ["up.example.test"],
[proxy_pass(("up.example.test", 443))]),
nginx_server([8443], ["gone.example.test"],
[proxy_pass(("gone.example.test", 443))])]),
sockets=sockets_section(established=[conn(("10.1.1.27", 40000), ("10.5.5.5", 443))]),
resolver=resolver_section([resolved("up.example.test", 443, ["10.5.5.5"]),
resolved("gone.example.test", 443, [], status="nxdomain")]))
payload = gateway.parse(snap, "gateway")
placed, unplaced = (routes_to(payload, "up.example.test")[0],
routes_to(payload, "gone.example.test")[0])
assert (placed["connections"], unplaced["connections"]) == (1, None)
assert gateway.distinct_endpoint_counts(payload["connections"]) == [1], (
"the placed endpoint alone: no second entry, no None, no zero")
assert gateway.distinct_endpoint_counts([unplaced]) == [], (
"and a payload of nothing-but-dashes is no figures at all, which the API renders as null")
def test_an_unresolved_name_publishes_no_metric_and_no_sample() -> None:
"""`nxdomain`, an error, and no entry at all are one answer here: drawn, never counted.
Three arms in one snapshot — a name the resolver placed, a name it looked up and got nothing
for, and a name it never mentioned — so the difference between "no record" and "nobody asked"
stays visible in the resolver block while the **route rows** behave identically: `connections`
`null`, `metric` `null`, no sample, and never a `0` that reads as an idle route.
"""
snap = snapshot(
nginx=nginx_section([
nginx_server([4443], ["up.example.test"], [proxy_pass(("up.example.test", 443))]),
nginx_server([8443], ["gone.example.test"], [proxy_pass(("gone.example.test", 443))]),
nginx_server([9443], ["silent.example.test"],
[proxy_pass(("silent.example.test", 443))])]),
sockets=sockets_section(
established=[conn(("10.1.1.27", 40000), ("10.5.5.5", 443)),
conn(("10.1.1.27", 40002), ("10.5.5.9", 443))]),
resolver=resolver_section([
resolved("up.example.test", 443, ["10.5.5.5"]),
resolved("gone.example.test", 443, [], status="nxdomain")]))
payload = gateway.parse(snap, "gateway")
assert routes_to(payload, "up.example.test")[0]["connections"] == 1
for name in ("gone.example.test", "silent.example.test"):
row = routes_to(payload, name)[0]
assert row["out"]["resolved"] is None, "never `[]`: unknown is one test, not a length"
assert row["out"]["via"] is None and row["out"]["resolved_ts"] is None
assert row["connections"] is None, "no count, and no 0 standing in for one"
assert row["metric"] is None, "no series for a number that cannot exist"
stored = [g.metric for g in gateway.samples(payload, 5.0)
if g.metric.startswith("gateway.out_conns@")]
assert stored == ["gateway.out_conns@up.example.test:443"]
assert sum(row["connections"] or 0 for row in payload["connections"]) == 1, (
"the socket to the unresolved target is credited to nothing at all")
def test_a_resolver_section_that_did_not_answer_places_nothing(gateway_snapshot: Any) -> None:
"""`available: false` and `available: null` are the agent's answers, and neither joins.
`false` is "there was nothing to resolve" (no proxy config was read) and `null` is "a pass that
could not run"; both arrive with no usable entries, and the payload's `resolver.available` says
which one it was rather than leaving a page to infer it from an empty list.
"""
for available in (False, None):
snap = snapshot(
nginx=nginx_section([nginx_server([4443], ["a.example.test"],
[proxy_pass(("x.example.test", 443))])]),
sockets=sockets_section(
established=[conn(("10.1.1.27", 40000), ("10.5.5.5", 443))]),
resolver=resolver_section([resolved("x.example.test", 443, ["10.5.5.5"])],
available=available, via=None))
payload = gateway.parse(snap, "gateway")
assert payload["resolver"]["available"] is available
assert payload["resolver"]["entries"] == [], "nothing was indexed from a dead pass"
assert routes_to(payload, "x.example.test")[0]["connections"] is None, (
"an entry beside `available: null` is a stale answer, not a joinable address")
def test_the_resolver_block_names_the_entries_the_index_was_built_from(
gateway_snapshot: Any) -> None:
"""The payload carries the document it was drawn from, so provenance is one read.
`entries` is every entry the section published — the index filters **within** it, so a page can
still say *this name has no record* — and each is projected to the five keys the agent owns.
`via` is the section's string. The `stub` arm keeps the same key set with nulls, which is what
the parse/stub parity pin in `tests/test_api.py` compares.
"""
payload = gateway.parse(gateway_snapshot("resolved"), "gateway")
block = payload["resolver"]
assert set(block) == {"available", "via", "entries"}
assert block["available"] is True and block["via"] == "libc"
document = json.loads((FIXTURES / "gateway_resolver.json").read_text())
assert [row["name"] for row in block["entries"]] == [row["name"] for row in document["entries"]]
assert [row["port"] for row in block["entries"]] == [row["port"] for row in document["entries"]]
assert all(set(row) == {"name", "port", "status", "addresses", "ts"}
for row in block["entries"]), "no agent field reaches a page unreviewed"
assert {row["status"] for row in block["entries"]} == {"ok", "literal"}, (
"this capture resolves every name it was asked about, plus the two `127.0.0.1` targets it "
"recognised as addresses without asking at all")
empty = gateway.stub("gateway")
assert empty["resolver"] == {"available": None, "via": None, "entries": []}, (
"a host that has not answered has not said resolution is unavailable")
def test_the_dashboard_still_never_resolves(gateway_snapshot: Any) -> None:
"""The spy pin, restated for phase 32 rather than relaxed by it.
`parse()` may **read** a resolver document and must not produce one, so the spy now runs over
three paths: a synthetic snapshot whose every target is a name, the same with a resolver section
full of answers, and the captured `resolved` ingress — then `samples()` over the result, since
a gauge name is built from a host string and is the other place a resolver could creep in.
Why the split is the design and not a limitation: the agent resolves with `getaddrinfo` on the
host that **opens** these connections (haproxy's `init-addr last,libc,none`), so its answer is
the one that decides what `/proc` then says. The dashboard asking the same name through its own
`resolv.conf` would answer a different question — a different server, a different view, and on
this fleet a name that resolves back through this same router — and would then attribute a
socket to a route on the strength of a lookup the router never made.
"""
calls: list[tuple[Any, ...]] = []
def spy(*args: Any, **kwargs: Any) -> Any: # pragma: no cover - must never run
calls.append((args, kwargs))
raise AssertionError("the gateway join must never resolve a name")
named = snapshot(
haproxy=haproxy_section([frontend("mc", [25565])],
[backend("mc-be", [server("mc1", "minecraft.example.test",
25565)])]),
nginx=nginx_section([nginx_server([443], ["a.example.test"],
[proxy_pass(("app.example.test", 8080))])]),
sockets=sockets_section(listens=[listen(25565), listen(443)],
established=[conn(("10.1.1.27", 443), ("203.0.113.5", 900)),
conn(("10.1.1.27", 1), ("10.2.2.2", 8080))]))
answered = snapshot(
haproxy=named["haproxy"], nginx=named["nginx"], sockets=named["sockets"],
resolver=resolver_section([resolved("minecraft.example.test", 25565, ["10.9.9.9"]),
resolved("app.example.test", 8080, ["10.2.2.2"])]))
original = socket_module.getaddrinfo
socket_module.getaddrinfo = spy
try:
for document in (named, answered, gateway_snapshot("resolved")):
payload = gateway.parse(document, "gateway")
gateway.samples(payload, 5.0)
gateway.warnings(gateway._section(document, "haproxy"),
gateway._section(document, "nginx"))
finally:
socket_module.getaddrinfo = original
assert calls == []
def test_a_snapshot_without_a_resolver_section_is_phase_30_verbatim(
gateway_snapshot: Any) -> None:
"""A 1.2.0 agent, or a 1.3.0 one run with `--no-resolve`, is **not** a changed page.
The topology, the sections, the warnings and the two loopback counts are byte-for-byte what
phase 30 produced; what differs is three `null`s on every `out` and an empty `resolver` block,
plus the routes phase 32 makes countable. `supported` stays `true` — a missing resolver is a
missing measurement, not an unsupported agent, and a fleet running two versions (which this one
did until task 05) must not read half-red.
"""
without = gateway.parse(gateway_snapshot("full"), "gateway")
with_resolver = gateway.parse(gateway_snapshot("resolved"), "gateway")
assert without["supported"] is True and with_resolver["supported"] is True
assert without["resolver"] == {"available": None, "via": None, "entries": []}
assert all(row["out"]["resolved"] is None and row["out"]["via"] is None
and row["out"]["resolved_ts"] is None for row in without["connections"])
def shape(payload: Mapping[str, Any]) -> Any:
"""Everything except the counts, which is the one thing the section may move."""
return {
"routes": [row["id"] for row in payload["connections"]],
"out": [(row["out"]["host"], row["out"]["port"], row["out"]["name"],
row["out"]["kind"], tuple(row["out"]["selectors"]), row["out"]["address"])
for row in payload["connections"]],
"trunks": [(row["port"], row["source"], tuple(row["domains"]))
for row in payload["listeners"]],
"sections": payload["sections"],
"why": payload["why"],
"warnings": payload["warnings"],
"sockets": payload["sockets"],
}
assert shape(without) == shape(with_resolver), (
"the resolver section counts through an address; it moves no route, trunk or warning")
counted = {row["id"] for row in without["connections"] if row["connections"] is not None}
assert counted == {"nginx:443->127.0.0.1:4443", "nginx:8443->127.0.0.1:8081"}, (
"phase 30 counted exactly the two endpoints the config wrote as addresses")
def test_resolution_raises_the_counted_route_count_from_two_to_every_name(
gateway_snapshot: Any) -> None:
"""The change, as one printed number rather than as an implied one.
Phase 30 could count the two routes whose target the configs wrote as an address. The recorded
resolver document places the other thirty-one, so against the captures every route on the
ingress now carries a figure — which is what the operator's "the only animated line is `:443 ->
127.0.0.1:4443`" was really asking for.
"""
before = gateway.parse(gateway_snapshot("full"), "gateway")
after = gateway.parse(gateway_snapshot("resolved"), "gateway")
counted_before = [row for row in before["connections"] if row["connections"] is not None]
counted_after = [row for row in after["connections"] if row["connections"] is not None]
written_as_name = [row for row in after["connections"] if row["out"]["address"] is not True]
unplaced = sorted(row["out"]["host"] for row in written_as_name
if row["connections"] is None)
report = (f"/gateway counted routes: {len(counted_before)} "
f"(phase 30, addresses only) → {len(counted_after)} of {len(after['connections'])}; "
f"the resolver placed {len(written_as_name)} name-written routes")
print(report)
assert len(counted_before) == 2, "phase 30's figure, so the delta is measurable"
assert not unplaced, f"{report} — unplaced: {unplaced}"
assert len(counted_after) == len(after["connections"]), (
f"{report} — a route the document places must carry a count")
def test_two_ports_on_one_host_are_two_gauges_even_when_both_names_resolve() -> None:
"""`minecraft:25565` and `minecraft:25566` stay two series — the port earns its place in the name.
This is why phase 29 put the port in `gateway.out_conns@<host>:<port>` in the first place, and
resolution does not remove the reason: this fleet's two haproxy Minecraft backends name the *same*
domain on two ports, libc answers one address for both, and a gauge keyed on the name alone would
have the second poll overwrite the first in silence. Two endpoints, two counters, two series — and
the fact that they share an address is invisible in the names and visible in the traffic.
"""
snap = snapshot(
haproxy=haproxy_section(
[frontend("mc", [25565], default="mc-a"), frontend("mc2", [25566], default="mc-b")],
[backend("mc-a", [server("mc1", "minecraft.example.test", 25565)]),
backend("mc-b", [server("mc2", "minecraft.example.test", 25566)])]),
sockets=sockets_section(established=[
conn(("10.1.1.27", 40000), ("10.9.9.9", 25565)),
conn(("10.1.1.27", 40002), ("10.9.9.9", 25565)),
conn(("10.1.1.27", 40004), ("10.9.9.9", 25566))]),
resolver=resolver_section([resolved("minecraft.example.test", 25565, ["10.9.9.9"]),
resolved("minecraft.example.test", 25566, ["10.9.9.9"])]))
payload = gateway.parse(snap, "gateway")
by_port = {row["in"]["port"]: row for row in payload["connections"]}
assert by_port[25565]["metric"] == "gateway.out_conns@minecraft.example.test:25565"
assert by_port[25566]["metric"] == "gateway.out_conns@minecraft.example.test:25566"
assert [by_port[25565]["connections"], by_port[25566]["connections"]] == [2, 1], (
"two endpoints, two counts — one address does not merge two ports")
assert by_port[25565]["shared"] is False and by_port[25566]["shared"] is False, (
"sharing an *address* is not sharing an *endpoint*, which is what `shared` means")
stored = gateway.samples(payload)
assert [(s.metric, s.value) for s in stored if "out_conns" in s.metric] == [
("gateway.out_conns@minecraft.example.test:25565", 2),
("gateway.out_conns@minecraft.example.test:25566", 1)], (
"neither gauge overwrites the other on the way to the store")
assert gateway.distinct_endpoint_counts(payload["connections"]) == [2, 1], (
"and the total books both, because the kernel has both")
# --------------------------------------------------------------------------- #
# the map: which maps route, and which are just header words
# --------------------------------------------------------------------------- #
@@ -1336,9 +2051,9 @@ def test_the_captured_configs_produce_no_warnings(gateway_snapshot: Any) -> None
assert used == {"targetBackend"} and used <= defined, (
"the one variable a `proxy_pass` passes IS defined by a map")
referenced = [block for block in nginx["maps"] if block["variable"].lstrip("$") in used]
assert len(referenced) == 1 and len(referenced[0]["entries"]) == 17
assert len(referenced) == 1 and len(referenced[0]["entries"]) == 18
assert all(entry["target"] for entry in referenced[0]["entries"]), (
"all 17 stream routes name an address, `default` included")
"all 18 stream routes name an address, `default` included")
unreferenced = [block for block in nginx["maps"] if block not in referenced]
assert any(entry["target"] is None for block in unreferenced for entry in block["entries"]), (
"the capture must still hold null-target entries on the UNREFERENCED map, or this pin "
tests/test_live_fleet.py
+93 -6
View File
@@ -21,6 +21,7 @@ a unit test.
from __future__ import annotations
import asyncio
import ipaddress
import json
import os
import sqlite3
@@ -44,6 +45,31 @@ from ups_contract import assert_ups_hero_contract
pytestmark = pytest.mark.live
def endpoint_identity(out: Any) -> tuple[Any, ...] | None:
"""One endpoint in the kernel's currency, derived **here** rather than by the source.
An address endpoint is its own address; a name endpoint is the addresses the agent's resolver
published for it (`out.resolved`), normalised so `::ffff:10.1.2.64` and `10.1.2.64` are one
key. `None` is "nothing can name this far end", which is counted nowhere. Phase 32 made this
helper necessary: a live router can have two route lines reaching one address through two
different names, and a total keyed on the config's text would book it twice.
"""
port = out.get("port")
if not isinstance(port, int):
return None
spellings = ([out["host"]] if out.get("address") is True
else list(out.get("resolved") or []))
keys: list[str] = []
for spelling in spellings:
try:
address = ipaddress.ip_address(str(spelling))
except ValueError:
continue
mapped = getattr(address, "ipv4_mapped", None)
keys.append(str(mapped if mapped is not None else address))
return (min(keys), port) if keys else None
@pytest.fixture(scope="module")
def settings() -> Settings:
"""Live env, not the import-time SETTINGS singleton."""
@@ -270,6 +296,11 @@ async def test_the_gateway_source_joins_the_live_ingress(settings: Settings,
against the kernel rather than against a fixture. The totals are an inequality by design
(`sum(listeners) + sum(endpoints) <= established`): a socket neither config file describes
is counted nowhere, and on this router that is most of the table.
Phase 32 widened **which** routes can be counted — an endpoint the config wrote as an address,
or one the gateway's own resolver published for the name it wrote — and the two assertions near
the end are stated in those terms, with the endpoint identity derived here so the outbound total
is not simply the source's own arithmetic re-added.
"""
target = next((item for item in settings.host_targets if item.name == settings.gateway_host),
None)
@@ -307,11 +338,14 @@ async def test_the_gateway_source_joins_the_live_ingress(settings: Settings,
for route in payload["connections"]:
assert route["in"]["port"] in configured, "a route hangs off no trunk the page draws"
out = route["out"]
assert (route["connections"] is None) is (out["address"] is not True), (
f"{route['id']}: only an endpoint the config wrote as an address can be counted")
if out["address"] is True and isinstance(out["port"], int):
placed = out["address"] is True or bool(out.get("resolved"))
assert (route["connections"] is None) is (not placed), (
f"{route['id']}: only an endpoint whose address is known — written, or resolved by the "
"gateway itself — can carry a count")
key = endpoint_identity(out)
if key is not None:
joinable.add(str(route["metric"]))
endpoints.setdefault((out["host"], out["port"]), route["connections"])
endpoints.setdefault(key, route["connections"])
else:
assert route["metric"] is None, "a route nobody can count publishes no series"
live_out = sum(value for value in endpoints.values() if isinstance(value, int))
@@ -325,8 +359,61 @@ async def test_the_gateway_source_joins_the_live_ingress(settings: Settings,
stored = gateway.samples(payload, 0.0)
assert all(sample.source == gateway.SOURCE for sample in stored)
assert {sample.metric for sample in stored if "out_conns" in sample.metric} <= joinable, (
"only an address endpoint publishes an outbound gauge — a domain-keyed series is the one "
"phase 30 stopped writing, because the figure it carried belonged to the port")
"only an endpoint whose address the join knows publishes an outbound gauge — written as an "
"address, or resolved by the gateway itself (phase 32); an unplaced name publishes nothing")
async def test_a_resolved_target_on_the_live_ingress_is_never_a_dash(
settings: Settings, agent_tls: dict[str, collectors.AgentTls]) -> None:
"""Phase 32 task 03 against the real ingress: `ok` in the resolver means a number on the page.
The operator's complaint was that one line out of thirty-three moved, and the acceptance line for
removing the dash is that it goes because the gateway published the address — not because a
column was hidden. So every route the resolver placed carries a **finite** `connections` (`0`
allowed: resolved and idle is a reading), a non-empty `out.resolved`, the agent's own `via`, the
timestamp of the lookup, and the gauge that goes with it. Skips when the ingress runs an agent
with no `resolver` section yet — which is every host until task 05 re-stages it.
"""
target = next((item for item in settings.host_targets if item.name == settings.gateway_host),
None)
if target is None:
pytest.skip(f"GATEWAY_HOST={settings.gateway_host!r} is not in HOST_TARGETS")
async with httpx.AsyncClient(timeout=settings.gateway_timeout) as client:
try:
snapshot = await collectors.fetch_snapshot(client, target,
timeout=settings.gateway_timeout,
tls=agent_tls.get(target.name))
except collectors.UpstreamError as exc:
pytest.skip(f"{target.name} did not answer: {exc}")
section = snapshot.get("resolver")
if not isinstance(section, dict):
pytest.skip(f"{target.name} publishes no resolver section — agent 1.3.0 is task 05's "
"re-stage; until then every name-written route is a dash by design")
payload = gateway.parse(snapshot, target.name)
if section.get("available") is not True:
pytest.skip(f"{target.name}'s resolver did not answer "
f"(available={section.get('available')!r}): nothing is placed, so nothing "
"is expected to be counted")
placed = [route for route in payload["connections"] if route["out"].get("resolved")]
assert placed, (f"{target.name} published {len(section.get('entries') or [])} resolver "
"entries and the join placed none of them")
for route in placed:
out = route["out"]
# The count lives on the **route** (`connections[]`), not on its `out` endpoint — the same
# shape `test_the_gateway_source_joins_the_live_ingress` reads. This path only ever ran once
# agent 1.3.0 was installed (before that the whole test skipped for want of a `resolver`
# section), which is how a wrong key survived task 03: a KeyError is not a dash.
assert isinstance(route["connections"], int) and not isinstance(route["connections"], bool), (
f"{route['id']}: resolved to {out['resolved']} and the line is still a dash")
assert out["via"] == section.get("via") == "libc", (
"the vantage is the agent's string, copied, never one this module invented")
assert isinstance(out["resolved_ts"], int | float), (
f"{route['id']}: a count obtained through a lookup must say when the lookup happened")
assert route["metric"] is not None, "a counted route always has its gauge"
stored = {sample.metric for sample in gateway.samples(payload, 0.0)}
assert {route["metric"] for route in placed} <= stored, (
"a resolved route's count must reach the store, or its chart stays empty forever")
@pytest.mark.parametrize("engine", ["gufo", "vllm"])