services/dashboards
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase: 32_gateway_resolved_target_liveness

Browse Source 
Fixture and live stream.conf are byte-identical (modulo header comment). Let me test the parser directly:
This commit is contained in:
ducoterra
2026-10-26 14:03:42 -04:00
parent 25fd74b7c5
commit 51772ffef2
58 changed files with 5272 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_agent.py
+712 -15
View File
@@ -12,6 +12,7 @@ Four things are asserted here:
"""
from __future__ import annotations
import hashlib
import http.client
import importlib.util
import ipaddress
@@ -331,13 +332,14 @@ def write_fake_disk(root: Path, name: str, model: str | None = None) -> None:
write_file(root, f"sys/block/{name}/device/model", f"{model}\n")
#: The add-on sections report their own absence in `errors` (`smart:`, `haproxy:`), because
#: "this box has no smartmontools / no haproxy" is a fact the page that owns the add-on has to
#: be able to print. Tests that assert a snapshot is otherwise clean filter the same prefixes
#: the dashboard does — `collectors.ADDON_ERROR_PREFIXES`, pinned equal to this list by
#: `test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters`, so neither side can
#: The add-on sections report their own absence in `errors` (`smart:`, `haproxy:`, `resolver:`),
#: because
#: "this box has no smartmontools / no haproxy / nothing to resolve" is a fact the page that owns
#: the add-on has to be able to print. Tests that assert a snapshot is otherwise clean filter the
#: same prefixes the dashboard does — `collectors.ADDON_ERROR_PREFIXES`, pinned equal to this list
#: by `test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters`, so neither side can
#: quietly grow a fourth opinion.
ADDON_ERROR_PREFIXES = ("smart", "haproxy", "nginx", "sockets")
ADDON_ERROR_PREFIXES = ("smart", "haproxy", "nginx", "sockets", "resolver")
def test_the_addon_error_prefixes_are_the_same_list_the_dashboard_filters() -> None:
@@ -357,6 +359,36 @@ def core_errors(snapshot: dict[str, Any]) -> list[str]:
if not str(line).lstrip().lower().startswith(ADDON_ERROR_PREFIXES)]
@pytest.fixture(autouse=True)
def no_real_dns(monkeypatch: pytest.MonkeyPatch) -> None:
"""Not one test in this file resolves a name across the network (phase 32's hermetic mandate).
The patch is module-wide on purpose — `host_agent.socket` *is* the `socket` module — and the
stub answers a hostname with a deterministic pair of **documentation-space** addresses
(RFC 5737 / RFC 3849, the same substitution `scripts/capture-gateway-config.sh` makes to a
socket table, so nothing a resolver test prints could carry a real endpoint either). Every IP
literal and `localhost` goes to the real function: this file's HTTP and TLS tests connect to
numeric loopback, and a stub that answered those too would break them for a reason nobody is
testing. A test that wants its own answer — `nxdomain`, a slow resolver, a call ledger —
patches the same attribute in its own body, which runs after this fixture.
"""
real = socket.getaddrinfo
def stub(name: Any, port: Any = None, *args: Any, **kwargs: Any) -> Any:
text = str(name)
if host_agent.address_family(text) is not None or text in ("", "localhost"):
return real(name, port, *args, **kwargs)
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
number = int(digest[:4], 16)
host_port = port if isinstance(port, int) else 0
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "",
(f"203.0.113.{number % 256}", host_port)),
(socket.AF_INET6, socket.SOCK_STREAM, 0, "",
(f"2001:db8::{digest[4:8]}", host_port, 0, 0))]
monkeypatch.setattr(host_agent.socket, "getaddrinfo", stub)
def fake_calls(tmp_path: Path) -> list[str]:
called = tmp_path / "smart-out" / "_called"
return called.read_text().split() if called.exists() else []
@@ -1585,11 +1617,16 @@ def test_a_corrupt_nginx_config_leaves_the_rest_of_the_snapshot_intact(tmp_path:
def test_an_nginx_failure_inside_the_collector_still_yields_a_section(tmp_path: Path,
monkeypatch) -> None:
"""Whatever raises in here, the document keeps the key with `available: null` — a snapshot
that loses the key would read as a pre-1.2.0 agent, not as a failed add-on."""
that loses the key would read as a pre-1.2.0 agent, not as a failed add-on.
`nginx_document` is the seam `snapshot()` calls (it is the pair of the section and the
`resolver` directive); `Collector.nginx` is its one-line wrapper, so patching the document is
patching the real path a snapshot takes.
"""
def explode(*args: object) -> dict[str, Any]:
raise RuntimeError("boom")
monkeypatch.setattr(host_agent.Collector, "nginx", explode)
monkeypatch.setattr(host_agent.Collector, "nginx_document", explode)
snap = host_agent.Collector(str(tmp_path / "root")).snapshot()
assert snap["nginx"] == {"available": None, "config": host_agent.NGINX_CONFIG_PATH,
"files": [], "servers": [], "upstreams": [], "maps": []}
@@ -2767,15 +2804,13 @@ def test_the_tls_fields_are_documented_in_the_schema() -> None:
assert "--cert-bundle" in (host_agent.__doc__ or "")
def test_1_2_0_is_the_version_that_ships_the_three_proxy_sections() -> None:
"""The version bump is the fleet's way of saying "this file has the gateway sections".
def test_the_schema_names_the_three_proxy_sections() -> None:
"""Phase 28's three sections are all in the SCHEMA block, each with its three-way `available`.
Phase 28 added three sections and one flag to the payload, so an install that predates them
is distinguishable by one string: `--status`, `/health` and this test all read the same
constant. The SCHEMA block is `GET /schema`'s body, so it names all three, each with the
three-way `available` the dashboard reads, and `--no-proxy`, the flag that removes the family.
The SCHEMA block is `GET /schema`'s body, so a deployment can read what an agent publishes
from the agent itself. The version constant is pinned by the newest phase's test below — one
place names the number, so a bump moves one assertion and not a file's worth of history.
"""
assert host_agent.AGENT_VERSION == "1.2.0"
doc = host_agent.__doc__ or ""
for section in ("haproxy", "nginx", "sockets"):
assert schema_documents(section), f"{section} is not documented in the SCHEMA block"
@@ -2784,6 +2819,24 @@ def test_1_2_0_is_the_version_that_ships_the_three_proxy_sections() -> None:
assert "--no-proxy" in doc and "--haproxy-config" in doc and "--nginx-config" in doc
def test_the_agent_version_is_1_3_0_and_the_schema_names_the_resolver_section() -> None:
"""1.3.0 is the version that resolves the names the configs declare.
The bump is how the fleet tells an agent that publishes `resolver` from one that does not —
`--status`, `/health` and this test all read the same constant, and `sources/gateway.py`'s
"the agent is too old" arm keys on the sections the version belongs to. The SCHEMA block is
`GET /schema`'s body, so it names the section, its three-way `available`, and `--no-resolve`,
the flag that removes it in silence.
"""
assert host_agent.AGENT_VERSION == "1.3.0"
doc = host_agent.__doc__ or ""
assert schema_documents("resolver"), "resolver is not documented in the SCHEMA block"
block = doc.split(" resolver ", 1)[-1]
assert "available" in block, "the resolver block does not document its three-way answer"
assert "--no-resolve" in doc, "the flag that removes the section is undocumented"
assert "libc" in block, "how the addresses were obtained is not published in the schema"
# ---- the recorded gateway: the captures themselves (phase 28 task 04) -------------------- #
#: Every document `scripts/capture-gateway-config.sh` records, at the tree path the agent reads
@@ -2983,6 +3036,151 @@ def test_the_recorded_configs_and_the_recorded_sockets_agree(tmp_path: Path) ->
"the agent's own listener should be in the socket table and claimed by neither config"
#: Every top-level key a **1.2.0** snapshot published, recorded here so this phase's central claim
#: outlives the script it was measured with (`data/logs/verify-32-additive.py`, which imports the
#: released file out of git and compares the two documents value for value). `data/` is gitignored;
#: this table is not, and it is what makes "purely additive" a permanent pin.
SNAPSHOT_KEYS_BEFORE_1_3_0 = frozenset({
"agent_version", "cpu", "cpu_temperature_celsius", "disk_io", "duration_ms", "errors",
"filesystems", "gpus", "haproxy", "host", "memory", "network", "nginx", "processes",
"schema", "sockets", "ts", "temperatures",
})
def test_the_resolver_section_is_the_only_key_1_3_0_adds(tmp_path: Path) -> None:
"""1.3.0 = 1.2.0 + `resolver`, and `--no-resolve` is exactly the old document.
The phase's safety property: an agent upgrade must not move a number on a page that never
asked about name resolution, and a host that opts out must not be left with an empty shell of a
key. Compared as key **sets** against the recorded pre-1.3.0 shape rather than against another
module, so the pin is hermetic and permanent; the value-for-value half of the claim runs in the
script named above, which loads the released file from git and calls both against one `--root`.
"""
root = gateway_tree(tmp_path / "root")
with_resolver = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl").snapshot()
assert set(with_resolver) == SNAPSHOT_KEYS_BEFORE_1_3_0 | {"resolver"}, (
f"1.3.0 changed the document beyond adding one section: "
f"{sorted(set(with_resolver) ^ (SNAPSHOT_KEYS_BEFORE_1_3_0 | {'resolver'}))}")
assert with_resolver["resolver"]["available"] is True
opted_out = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl",
resolve=False).snapshot()
assert set(opted_out) == SNAPSHOT_KEYS_BEFORE_1_3_0, "--no-resolve leaves a key behind"
assert not [line for line in opted_out["errors"]
if str(line).lstrip().lower().startswith("resolver")], (
"opting out is silence, the `--no-smart` precedent")
no_proxy = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl",
proxy=False).snapshot()
assert set(no_proxy) == SNAPSHOT_KEYS_BEFORE_1_3_0 - {"haproxy", "nginx", "sockets"}, (
"the resolver belongs to the proxy family and is removed with it")
def test_the_recorded_resolver_document_is_the_agents_own_section(tmp_path: Path) -> None:
"""`gateway_resolver.json` is a section of the payload, not notes about one.
Nothing here is a hand-written expectation: the entry set and the `resolver` directive are
recomputed from the config fixtures in this directory with the agent's own `target_names()` and
`nginx_document()`, so the two captures cannot drift apart. A route added to the gateway shows
up in both on the next run; a fixture naming a target the parser no longer derives fails here
instead of teaching `sources/gateway.py` to join a route the config does not declare.
The per-entry assertions are the ones the page will rely on: a status out of the four the
section publishes, an `ok` with no address, an `nxdomain` with one, or a list the agent's own
ordering rule would reshuffle, all mean the fixture is not what a poll would serve.
"""
document = json.loads((FIXTURES / "gateway_resolver.json").read_text())
assert set(document) == {"available", "via", "config_directive", "ts", "entries", "errors"}
assert document["available"] is True, "the recorded gateway reads, so the section is available"
assert document["via"] == host_agent.RESOLVE_VIA
assert document["errors"] == []
collector = gateway_collector(gateway_tree(tmp_path / "root"))
errors: list[str] = []
haproxy, nginx = collector.haproxy(errors), collector.nginx(errors)
assert [(row["name"], row["port"]) for row in document["entries"]] == (
host_agent.target_names(haproxy, nginx)), (
"the recorded names are not the ones these configs declare — re-run "
"./scripts/capture-gateway-config.sh and re-tally in the same commit")
_section, directive = collector.nginx_document(errors)
assert document["config_directive"] == directive, (
"the fixture's `resolver` line is not what the agent reads out of the same text")
for row in document["entries"]:
assert set(row) == {"name", "port", "status", "addresses", "ts"}
assert row["status"] in {"ok", "nxdomain", "error", "literal"}, row
if row["status"] in {"ok", "literal"}:
assert row["addresses"], f"{row['name']} is {row['status']} with no address"
else:
assert row["addresses"] == [], f"{row['name']} publishes an address it did not resolve"
assert row["addresses"] == host_agent.sort_resolver_addresses(row["addresses"]), (
f"{row['name']} is not in the order the payload will publish it")
def test_the_lookup_the_agent_performs_is_one_real_libc_accepts(monkeypatch, tmp_path) -> None:
"""The one pin in this file with **no stub in sight**, because a stub cannot refuse a keyword.
`socket.getaddrinfo(name, port, proto=socket.SOCK_STREAM)` — the spelling that reads as "give
me the TCP addresses" — is not that. `proto` is a **protocol number** (`IPPROTO_TCP` is 6) and
`SOCK_STREAM` is 1, so libc is asked for a stream socket over protocol 1 and answers
`EAI_SERVICE` for every name. A fake `getaddrinfo` accepts any keyword, so every resolver test
in this file would have stayed green while the agent reported `error` on a healthy gateway and
left every route on `/gateway` uncounted.
So: the module-wide `no_real_dns` fixture is removed for this test, and the lookup runs against
real libc on a name libc answers from `/etc/hosts` — no resolver, no network, no skip. It
cannot be replaced with a numeric literal either: an address literal is the `literal` arm and
never reaches libc at all, which is the other half of the same pass.
"""
monkeypatch.setattr(host_agent.socket, "getaddrinfo", socket.getaddrinfo)
collector = host_agent.Collector(str(gateway_tree(tmp_path / "root")),
smartctl="/nonexistent/smartctl")
entry = collector._resolve_target("localhost", 9120)
assert entry["status"] == "ok", (
f"real libc refused the call the agent makes ({entry}); if it says `error` with "
"`EAI_SERVICE` in it, the keyword on that call has moved back to `proto=`")
assert entry["addresses"], "a name libc resolves must publish at least one address"
for value in entry["addresses"]:
assert ipaddress.ip_address(value) # type: ignore[call-overload]
def test_the_recorded_resolver_document_publishes_no_globally_routable_address() -> None:
"""Invariant 6 over the JSON, for the same reason and with one hazard of its own.
A resolver answer is an identifier the same way a socket row is: `getent` on this router answers
the WAN address of anything published through it, and a global IPv6 answer is EUI-64 — this
box's NIC, in hex. `scripts/capture-gateway-config.sh` runs the same substitution over the
resolver's answers as over the socket tables, so nothing routable is tracked.
The second assertion is specific to *this* fixture and is a warning, not a rule: on this fleet
every declared target answers a private LAN address, so nothing needs scrubbing. If a capture
ever moves an address into documentation space, that replacement is numbered per document in
first-seen order — it is therefore **not** guaranteed to be the same documentation address the
socket fixtures gave the same endpoint, and `tests/test_gateway_source.py`'s join of the two
would silently stop joining. Re-check that join before trusting the numbers.
"""
document = json.loads((FIXTURES / "gateway_resolver.json").read_text())
routable: list[str] = []
documented: list[str] = []
for row in document["entries"]:
for value in row["addresses"]:
address = ipaddress.ip_address(value)
subject = getattr(address, "ipv4_mapped", None) or address
if subject.is_global:
routable.append(f"{row['name']} -> {value}")
if not subject.is_private:
documented.append(f"{row['name']} -> {value}")
assert not routable, ("globally routable resolver answers are in a tracked fixture: "
+ ", ".join(routable)
+ " — scripts/capture-gateway-config.sh is the thing that stops them")
assert not documented, ("the recorded resolver document had to be scrubbed: "
+ ", ".join(documented)
+ " — reconcile the replacement with the socket fixtures before "
"trusting the join in tests/test_gateway_source.py")
def test_the_recorded_socket_tables_publish_no_globally_routable_endpoint() -> None:
"""AGENTS.md invariant 6, on the artifacts: identifiers never go in the repo.
@@ -3035,3 +3233,502 @@ def test_the_recorded_gateway_answers_all_three_sections_at_once(tmp_path: Path)
"temperatures", "processes", "errors"):
assert section in snap, f"the pre-1.2.0 payload lost {section}"
# ---- the resolver: the names the configs declare, resolved on this box (phase 32 task 01) -- #
def resolver_ledger(monkeypatch: pytest.MonkeyPatch, *,
nx: tuple[str, ...] = (), delay: float = 0.0,
fail: tuple[str, ...] = ()) -> list[tuple[str, int | None]]:
"""Stub `getaddrinfo` with a ledger of every `(name, port)` it was asked about.
The three knobs are the three things a resolver does that a page has to survive: it answers
(`203.0.113.x` + `2001:db8::…`, documentation space so a test payload can never carry a real
endpoint), it refuses a name (`nxdomain`), and it is slow or broken (`delay`, `fail`).
"""
calls: list[tuple[str, int | None]] = []
def stub(name: object, port: object = None, *args: Any, **kwargs: Any) -> Any:
text = str(name)
calls.append((text, port if isinstance(port, int) else None))
if text in fail:
raise socket.gaierror(socket.EAI_NONAME, "and the resolver refused to say")
if delay:
time.sleep(delay)
if text in nx:
raise socket.gaierror(socket.EAI_NONAME, "Name or service not known")
host_port = port if isinstance(port, int) else 0
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (f"203.0.113.{len(calls)}", host_port))]
monkeypatch.setattr(host_agent.socket, "getaddrinfo", stub)
return calls
def resolver_snapshot(root: Path, **kwargs: Any) -> dict[str, Any]:
"""One snapshot of a captured-gateway tree, the resolver's `getaddrinfo` stubbed by the caller."""
return host_agent.Collector(str(root), smartctl="/nonexistent/smartctl",
**kwargs).snapshot()
def test_only_the_targets_the_configs_declare_are_resolved(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""The allow-list is the config, in config order, once each — decision 3, pinned as a call set.
Everything the resolver dialed is a target the two parsed sections publish, in the order the
files write them, with no duplicate — and nothing that came from `/proc/net/tcp` is in there.
That last half is the point: a socket table is full of addresses this gateway talks to, and
resolving one of those would be a fact about whoever connected, not about the config.
"""
calls = resolver_ledger(monkeypatch)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
declared = host_agent.target_names(snap["haproxy"], snap["nginx"])
looked_up = [pair for pair in declared if host_agent.address_family(pair[0]) is None]
assert calls == looked_up, "the lookups are exactly the declared targets, in config order"
assert len(calls) == len(set(calls)), "one name on one port is one lookup, not two"
assert len(looked_up) < len(declared), "the tree writes literals too, and they are kept"
socket_addresses = {row["remote"]["addr"] for row in snap["sockets"]["established"]}
socket_addresses |= {row["local"]["addr"] for row in snap["sockets"]["established"]}
socket_addresses |= {row["addr"] for row in snap["sockets"]["listens"]}
assert not socket_addresses & {name for name, _ in calls}, (
"a name from the socket table was resolved: the config is the only allow-list")
# haproxy first (its six `server` lines), and this fleet's two Minecraft ports are two targets
assert calls[:6] == [("reeseapps-gitea.reeselink.com", 2222),
("stackexpected-gitea.reeselink.com", 2222),
("web.reeselink.com", 5201),
("unifi-external.reeselink.com", 8080),
("minecraft.reeselink.com", 25565),
("minecraft.reeselink.com", 25566)]
def test_an_address_literal_in_a_config_is_reported_and_never_dialled(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""`proxy_pass http://127.0.0.1:8081` is already an address: the entry says so, no lookup runs.
Looking a literal up would be a round trip to be told what the config wrote, and on a box with
a slow resolver it is time spent for nothing. The address still travels, because the gateway
page's join keys on it.
"""
calls = resolver_ledger(monkeypatch)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
entries = {host_agent.resolver_key(e["name"], e["port"]): e for e in snap["resolver"]["entries"]}
literal = entries[host_agent.resolver_key("127.0.0.1", 8081)]
assert literal["status"] == "literal"
assert literal["addresses"] == ["127.0.0.1"]
assert ("127.0.0.1", 8081) not in calls, "a literal costs no lookup"
assert not [call for call in calls if host_agent.address_family(call[0])], \
"no lookup was ever asked about an address"
def test_the_section_names_how_and_where_and_when(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""`via`, `config_directive` and the timestamps are the provenance of every address here.
`libc` is what `getaddrinfo` is — the resolution haproxy performs on this box — and the nginx
config's own `resolver` line travels beside it verbatim (read out of `stream.d/stream.conf`,
which is where this fleet writes it, so an include is as good as the main file). The
`nginx` section itself grows no key: that is what keeps the phase's additivity control a
top-level comparison.
"""
resolver_ledger(monkeypatch)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
section = snap["resolver"]
assert section["via"] == host_agent.RESOLVE_VIA == "libc"
assert section["config_directive"] == "10.1.0.1"
assert "resolver" not in snap["nginx"], "the nginx document stays byte-stable"
assert isinstance(section["ts"], float)
for entry in section["entries"]:
assert entry["ts"] <= section["ts"] + 1.0, "an entry is answered at or before the pass ends"
assert set(entry) == {"name", "port", "status", "addresses", "ts"}
def test_the_resolver_cache_means_one_lookup_per_target(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""Two snapshots, one set of lookups; an expiry refreshes in the background and never on the request.
The dashboard polls every 10 s. Without the cache that is 30 DNS lookups per host per minute
for names that move when the operator moves them; with it, the poll that finds an expired
cache is answered from the cache and carries the **original** per-entry `ts`, so it cannot
present a five-minute-old address as a fresh one.
"""
calls = resolver_ledger(monkeypatch)
root = gateway_tree(tmp_path / "root")
collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl")
first = collector.snapshot()["resolver"]
baseline = len(calls)
assert baseline and len(calls) == len(set(calls))
second = collector.snapshot()["resolver"]
assert len(calls) == baseline, "a fresh cache costs no lookup"
assert second["entries"] == first["entries"]
collector.resolve_cache_seconds = 0.0
third = collector.snapshot()["resolver"]
assert third["entries"] == first["entries"], "the expired cache is served as it was"
assert [e["ts"] for e in third["entries"]] == [e["ts"] for e in first["entries"]], \
"a cache-served entry keeps the time it was answered, not the time it was served"
wait_for_resolver_refresh(collector)
assert len(calls) > baseline, "the expiry did re-resolve, off the request path"
def test_a_slow_resolver_cannot_hold_the_snapshot_open(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""The pass stops at its budget and says so once — the SMART pass-budget lesson, applied to DNS.
`getaddrinfo` takes no timeout argument, so the only defence against a resolver that has
started black-holing is to stop asking. What the section must still be: `available: true`, the
targets it reached present, one line naming how many of them that was, and `/snapshot`
answering in a small fraction of the time the full set would have taken — because the *system*
page's timeout is what a slow pass would otherwise cost.
"""
calls = resolver_ledger(monkeypatch, delay=0.02)
root = gateway_tree(tmp_path / "root")
collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl")
collector.resolve_pass_budget_seconds = 0.1
started = time.monotonic()
snap = collector.snapshot()
elapsed = time.monotonic() - started
section = snap["resolver"]
budget_lines = [line for line in section["errors"] if "pass budget" in line]
assert section["available"] is True, "a truncated pass is still an answer"
assert len(budget_lines) == 1, snap["errors"]
reached = [e for e in section["entries"] if e["status"] != "literal"]
assert budget_lines[0] == (f"resolver: pass budget 0.1s exhausted, "
f"{len(reached)} of 30 targets looked up"), budget_lines[0]
assert len(reached) < 29, "the pass stopped rather than finished"
assert len(calls) == len(reached), "the ledger and the section agree on what was reached"
assert [e for e in section["entries"] if e["status"] == "literal"], \
"an address the config spells survives a pass that stopped early"
assert len(section["entries"]) == len(reached) + 2, "two literals, and nothing invented"
assert elapsed < 1.0, f"a bounded pass cost {elapsed:.2f}s"
assert snap["cpu"]["modes"]["total"] > 0 and snap["host"]["hostname"]
assert core_errors(snap) == [], snap["errors"]
def test_the_name_bound_reports_the_excess_in_one_line(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""More declared targets than the bound: the excess is one line, not 40, and `true` stands."""
calls = resolver_ledger(monkeypatch)
root = gateway_tree(tmp_path / "root")
collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl")
collector.resolve_max_names = 3
section = collector.snapshot()["resolver"]
bound_lines = [line for line in section["errors"] if "name bound" in line]
assert len(bound_lines) == 1, section["errors"]
assert bound_lines[0].startswith("resolver: 27 of 30 targets are beyond the 3-name bound"), \
bound_lines[0]
assert section["available"] is True
assert len(calls) == 3, "the bound is the cost of the section, not a display choice"
literals = [e for e in section["entries"] if e["status"] == "literal"]
assert len(section["entries"]) == 3 + len(literals)
def wait_for_resolver_refresh(collector: Any, attempts: int = 500) -> None:
"""Join the single-flight refresher, as far as a test can: it is a daemon thread on purpose.
The wait is on the flag rather than a `Thread.join()` because the agent keeps no handle to it
(a snapshot must never be able to wait on DNS), and 5 s of 10 ms polls is more than a stubbed
pass takes on any machine this suite runs on.
"""
for _ in range(attempts):
if not collector._resolver_refreshing:
return
time.sleep(0.01)
raise AssertionError("the background resolver refresh never finished")
def test_a_failed_background_refresh_keeps_the_last_addresses(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""A refresh that dies is one line, the old entries, and no hammering.
The thread is the only thing that ever re-reads DNS after the first pass, so a pass that
raises must leave the snapshot answering with the last truth it had (SMART's rule) and must
not turn every subsequent poll into a retry. One line, replaced rather than stacked: a
resolver that has been broken for a week is one fact the operator acts on once, not 2 000.
"""
resolver_ledger(monkeypatch)
collector = host_agent.Collector(str(gateway_tree(tmp_path / "root")),
smartctl="/nonexistent/smartctl")
first = collector.snapshot()["resolver"]
assert first["available"] is True and first["entries"]
def explode(*args: object) -> None:
raise RuntimeError("the resolver daemon is gone")
monkeypatch.setattr(host_agent.Collector, "resolver_pass", explode)
collector.resolve_cache_seconds = 0.0
for attempt in range(2):
served = collector.snapshot()["resolver"]
assert served["entries"] == first["entries"], "the last good answers stay published"
wait_for_resolver_refresh(collector)
# Read the collector's state, not this response: the snapshot was served *while* the
# refresh was in flight, so the line it could not have is not a defect in it.
failures = [line for line in collector._resolver_errors
if "background refresh failed" in line]
assert len(failures) == 1, (attempt, collector._resolver_errors)
assert failures[0].startswith("resolver: background refresh failed: RuntimeError")
assert collector._resolver_cache_ts is not None, "a failed pass still re-stamps the cache"
# …and the line reaches the page eventually, beside the entries it explains.
later = collector.snapshot()["resolver"]
assert any("background refresh failed" in line for line in later["errors"]), later["errors"]
assert later["entries"] == first["entries"]
def test_nxdomain_is_a_status_and_never_a_failure(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""One name that does not answer is one entry with no addresses — the snapshot is unbothered.
The route stays on the page and carries no count (task 03 does the joining); what must not
happen is a `0`, a dropped entry, or a poll that goes red because a domain left the zone file.
"""
resolver_ledger(monkeypatch, nx=("reeseapps-brain-of-reese.reeselink.com",))
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
section = snap["resolver"]
entries = {e["name"]: e for e in section["entries"]}
failed = entries["reeseapps-brain-of-reese.reeselink.com"]
assert failed["status"] == "nxdomain"
assert failed["addresses"] == [], "no address, never a 0 and never last month's"
answered = [e for e in section["entries"] if e["status"] == "ok"]
assert answered, "the others answered"
assert section["available"] is True
assert section["errors"] == [], "a name that does not resolve is not this section's failure"
assert snap["cpu"]["modes"]["total"] > 0
assert core_errors(snap) == [], snap["errors"]
def test_a_resolver_that_errors_is_an_error_and_not_an_nxdomain(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
""""The name has no record" and "the lookup could not be performed" are two fixes.
A temporary failure (`EAI_AGAIN`) is the one that most looks like `nxdomain` and must not be
reported as it: an operator reading `nxdomain` edits a zone file, an operator reading `error`
checks whether the resolver is up.
"""
def stub(name: object, port: object = None, *args: Any, **kwargs: Any) -> Any:
text = str(name)
if text == "gitea.reeselink.com":
raise socket.gaierror(socket.EAI_AGAIN, "Temporary failure in name resolution")
if text == "jellyfin.reeselink.com":
raise OSError("resolver socket refused to open")
if text == "bricktracker.reeselink.com":
raise ValueError("a platform-shaped answer nobody predicted")
host_port = port if isinstance(port, int) else 0
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("203.0.113.7", host_port))]
monkeypatch.setattr(host_agent.socket, "getaddrinfo", stub)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
entries = {e["name"]: e for e in snap["resolver"]["entries"]}
assert entries["gitea.reeselink.com"]["status"] == "error", "EAI_AGAIN is not nxdomain"
assert entries["jellyfin.reeselink.com"]["status"] == "error"
assert entries["bricktracker.reeselink.com"]["status"] == "error"
assert entries["bricktracker.reeselink.com"]["detail"].startswith("ValueError")
assert entries["gitea.reeselink.com"]["addresses"] == []
assert snap["resolver"]["available"] is True
assert core_errors(snap) == [], snap["errors"]
def test_the_addresses_are_unique_and_sorted_v4_then_v6(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""Two polls of one healthy gateway are byte-identical, whatever order libc answered in.
`getaddrinfo` returns whatever the resolver and `/etc/gai.conf` feel like today, and the entry
list is something a page renders: an unsorted set makes a healthy box look like its addresses
are shuffling.
"""
def stub(name: object, port: object = None, *args: Any, **kwargs: Any) -> Any:
host_port = port if isinstance(port, int) else 0
return [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", ("2001:db8::2", host_port, 0, 0)),
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("203.0.113.9", host_port)),
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("203.0.113.9", host_port)),
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("203.0.113.4", host_port))]
monkeypatch.setattr(host_agent.socket, "getaddrinfo", stub)
snap = resolver_snapshot(gateway_tree(tmp_path / "root"))
entry = snap["resolver"]["entries"][0]
assert entry["status"] == "ok"
assert entry["addresses"] == ["203.0.113.4", "203.0.113.9", "2001:db8::2"]
def test_a_value_that_is_not_an_address_sorts_last_instead_of_raising() -> None:
"""The sort is defensive for the one input the section should never see.
`getaddrinfo` answers addresses, so this arm is a stub or a platform oddity — and the answer
to that is a stable order with the odd value at the end, not a `ValueError` on a snapshot path
and not the whole section lost because one entry could not be compared.
"""
assert host_agent.sort_resolver_addresses(["nope", "10.0.0.2", "::1", "also-bad", "10.0.0.2"]) \
== ["10.0.0.2", "::1", "also-bad", "nope"]
def test_no_resolve_removes_the_key_in_silence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""`--no-resolve` is `--no-smart`'s precedent: no key, no error line — "not collected".
An empty section would read as "collected, and nothing resolves", which is a different fact
and the wrong one to print for a host the operator told not to ask.
"""
calls = resolver_ledger(monkeypatch)
root = gateway_tree(tmp_path / "root")
snap = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl",
resolve=False).snapshot()
assert "resolver" not in snap
assert calls == [], "the flag is honoured before anything is dialled"
assert snap["nginx"]["available"] is True and snap["sockets"]["available"] is True, \
"the rest of the family is untouched"
assert [line for line in snap["errors"] if line.startswith("resolver:")] == []
def test_no_proxy_removes_the_resolver_with_the_rest_of_the_family(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""A box that collects no proxy has no declared names, so `--no-proxy` removes this too."""
calls = resolver_ledger(monkeypatch)
snap = host_agent.Collector(str(gateway_tree(tmp_path / "root")),
smartctl="/nonexistent/smartctl", proxy=False).snapshot()
for section in ("haproxy", "nginx", "sockets", "resolver"):
assert section not in snap
assert calls == []
def test_a_host_with_no_proxy_sections_says_so_once(tmp_path: Path) -> None:
"""`available: false` and exactly one `resolver:` line naming **not found** — invariant 3.
Two of this fleet's three hosts run no proxy at all. Their agents must report "nothing to
resolve" without putting a red banner, an overview alert, or fifty lines in `errors` anywhere.
"""
snap = host_agent.Collector(str(tmp_path / "root")).snapshot()
resolver_lines = [line for line in snap["errors"] if line.startswith("resolver:")]
assert snap["resolver"]["available"] is False
assert snap["resolver"]["entries"] == []
assert len(resolver_lines) == 1, snap["errors"]
assert "not found" in resolver_lines[0]
assert "--no-resolve" in resolver_lines[0]
assert snap["resolver"]["errors"] == resolver_lines
assert core_errors(snap) == [], "an unconfigured add-on is not a system degradation"
@pytest.mark.skipif(os.geteuid() == 0, reason="root reads a 0000 file")
def test_a_config_that_is_present_but_unreadable_says_that_instead(tmp_path: Path) -> None:
"""The `false` arm names the right reason: an unreadable file is not a missing one.
`not found` and `unreadable` are two operator actions — install the proxy, or fix the mode on
its config — and one line that says which is the difference between a `chmod` and a
`dnf install`. Still one line, still `available: false`, still no system degradation.
"""
root = tmp_path / "root"
nginx_tree(root, http=None, stream=None)
(root / "etc/nginx/nginx.conf").chmod(0o000)
snap = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl").snapshot()
line = next(line for line in snap["errors"] if line.startswith("resolver:"))
assert snap["resolver"]["available"] is False
assert "haproxy not found" in line
assert "nginx unreadable" in line
assert len([x for x in snap["errors"] if x.startswith("resolver:")]) == 1
def test_a_config_that_reads_cleanly_and_names_no_target_is_true_and_empty(
tmp_path: Path) -> None:
"""Configs read, no targets declared: `available: true` with `entries: []` and no line.
Absence of a target is not a fault, and the third arm (`null`) is reserved for a pass that
could not run at all — otherwise "no routes" and "no resolver" arrive as one value.
"""
root = tmp_path / "root"
nginx_tree(root, main="http { server { listen 8080; server_name quiet.test; } }\n",
http=None, stream=None)
snap = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl").snapshot()
assert snap["nginx"]["available"] is True
assert snap["resolver"]["available"] is True, snap["errors"]
assert snap["resolver"]["entries"] == []
assert snap["resolver"]["errors"] == []
assert core_errors(snap) == [], snap["errors"]
def test_the_resolver_lines_never_fail_the_snapshot(tmp_path: Path,
monkeypatch: pytest.MonkeyPatch) -> None:
"""Every arm of the new section leaves `/snapshot` answering, with its key and its own line.
The family rule phase 28 pinned for `haproxy`, `nginx` and `sockets` extended to the fourth
section: whatever the resolver does — raises inside, cannot find a config, cannot reach a
name — the document keeps the key (so a reader can tell "the pass failed" from "the agent
predates it"), the other sections are intact, and the line is the add-on's own rather than a
system `degraded` (invariant 3).
"""
def explode(*args: object) -> dict[str, Any]:
raise RuntimeError("resolver exploded")
monkeypatch.setattr(host_agent.Collector, "resolver", explode)
snap = host_agent.Collector(str(gateway_tree(tmp_path / "root")),
smartctl="/nonexistent/smartctl").snapshot()
assert snap["resolver"] == {"available": None, "via": "libc", "config_directive": None,
"ts": None, "entries": [], "errors": []}
assert any(line.startswith("resolver:") for line in snap["errors"])
assert core_errors(snap) == [], snap["errors"]
assert snap["haproxy"]["available"] is True and snap["nginx"]["available"] is True
assert snap["cpu"]["modes"]["total"] > 0
def test_the_resolver_section_is_documented_in_the_schema() -> None:
"""`GET /schema` answers the docstring, so the section an operator can `curl` is described."""
assert schema_documents("resolver")
def test_the_resolver_reaches_the_command_line(tmp_path: Path, capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch) -> None:
"""`--once` prints the resolver block, and prints none with `--no-resolve` added.
This is the completion criterion an operator can run on a host with no dashboard: one command,
and the answer says which names this box turned into which addresses.
"""
resolver_ledger(monkeypatch)
root = gateway_tree(tmp_path / "root")
# The config paths are the ones the **config** writes: an absolute path resolves under
# `--root`, which is how a deployment reads /etc and a test reads a fake /etc.
nginx, haproxy = "/etc/nginx/nginx.conf", "/etc/haproxy/haproxy.cfg"
assert host_agent.main(["--once", "--no-smart", "--root", str(root),
"--haproxy-config", haproxy, "--nginx-config", nginx]) == 0
printed = json.loads(capsys.readouterr().out)
assert printed["resolver"]["available"] is True
assert printed["resolver"]["via"] == "libc"
assert printed["resolver"]["config_directive"] == "10.1.0.1"
assert printed["resolver"]["entries"]
assert host_agent.main(["--once", "--no-smart", "--no-resolve", "--root", str(root),
"--haproxy-config", haproxy, "--nginx-config", nginx]) == 0
without = json.loads(capsys.readouterr().out)
assert "resolver" not in without
assert without["nginx"]["available"] is True, "the rest of the family still prints"
def test_the_target_list_is_the_two_sections_and_survives_a_section_that_is_not_one() -> None:
"""`target_names` reads haproxy then nginx (upstreams, proxy_pass, maps) and nothing stranger.
A `proxy_pass $var` contributes nothing — the map decides, and the map's entries are already
in the list — and a `None`, a bare string, or a section of the wrong shape contributes an
empty list rather than a `TypeError` on a snapshot path.
"""
haproxy = {"backends": [{"servers": [{"name": "a", "target": "a.test", "port": 1},
{"name": "b", "target": "b.test", "port": None},
{"name": "c", "target": None, "port": 3},
{"name": "d", "target": "a.test", "port": 1}]}]}
nginx = {"upstreams": [{"servers": [{"host": "pool.test", "port": 80}]}],
"servers": [{"proxy_passes": [{"target": {"host": "vhost.test", "port": 443}},
{"target": None, "variable": "backend"}]}],
"maps": [{"entries": [{"key": "x", "target": {"host": "map.test", "port": 443}},
{"key": "default", "target": None}]}]}
assert host_agent.target_names(haproxy, nginx) == [
("a.test", 1), ("b.test", None), ("pool.test", 80),
("vhost.test", 443), ("map.test", 443)]
assert host_agent.target_names(None, None) == []
assert host_agent.target_names({"backends": "nonsense"}, {"maps": [7]}) == []