diff --git a/infra/ansible/roles/node_health/README.md b/infra/ansible/roles/node_health/README.md index db3f269..95f7dd6 100644 --- a/infra/ansible/roles/node_health/README.md +++ b/infra/ansible/roles/node_health/README.md @@ -47,7 +47,7 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0 | `storage.io_errors` | kernel SCSI/ATA `ioerr_cnt` / `iotmo_cnt` per disk (no root needed) | | `storage.capacity`, `storage.inodes` | worst mounted filesystem, space and inodes | | `storage.layout` | physical disks, and whether any device is genuinely free for `node_baseline_longhorn_device` / `node_baseline_etcd_device` | -| `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite, EDAC correctable/uncorrectable counters | +| `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite **and whether it survives a reboot** (see [Swap off is not the same as swap disabled](#swap-off-is-not-the-same-as-swap-disabled)), EDAC correctable/uncorrectable counters | | `memory.physical_total` | **is this machine using the RAM it has?** `/proc/zoneinfo` present-vs-managed pages (no root, metal or VM) and `MemTotal` vs the DIMM capacity SMBIOS reports (`dmidecode -t 17`, root, bare metal). Cause-independent by design — see [The EFI memattr check](#the-efi-memattr-check) | | `cpu.throttle`, `cpu.inventory`, `cpu.virtualisation` | thermal throttle events since boot, topology + frequency scaling, VT-x/AMD-V + `/dev/kvm` (KubeVirt, R-PLAT-03) | | `thermal.max` | hottest sensor across thermal zones and hwmon chips | @@ -59,6 +59,37 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0 Thresholds live in `defaults/main.yml` (`node_health_thresholds`); no verdict is hardcoded in task logic. +### Swap off is not the same as swap disabled + +`SwapTotal == 0` only says nothing has been swapped on *yet this boot*. It is not evidence +that swap is disabled, and on these nodes it was not: measured 2026-09-07, all three prod +nodes came back from a reboot with 977 MiB of swap active (`k3s-prod-1` `/dev/sda4`, +`k3s-prod-3` `/dev/sda4`, `k3s-prod-2` `/dev/nvme0n1p4`) even though `roles/node_baseline` +had run `swapoff -a` and commented their `/etc/fstab` swap line. + +The cause is `systemd-gpt-auto-generator`. It claims any GPT partition whose partition +**type GUID** is `0657fd6d-a4ab-43c4-84e5-0933c84b4f4f` ("Linux swap") straight from the +partition table, synthesises a `.swap` unit into `/run/systemd/generator.late/` and links +it into `swap.target.wants/`. `/etc/fstab` is never consulted, so neutralising fstab +cannot stop it. `roles/node_baseline` masks `swap.target` — the one unit every generated +`.swap` unit is `WantedBy` — and masks the generated units as a second barrier. + +So `memory.swap` reports on both halves, from `memory.swap_persistence` in the collector +(all world-readable, no root): + +| fact | meaning | +|---|---| +| `swap_target_masked` | is `/etc/systemd/system/swap.target` a symlink to `/dev/null`? | +| `generated_units` | `.swap` units systemd's generators built for this boot | +| `masked_units` | `.swap` units masked in `/etc/systemd/system` | +| `unmasked_generated_units` | `generated_units` minus `masked_units` — what a reboot would bring back | +| `fstab_active_swap_lines` | uncommented `swap` lines left in `/etc/fstab` | + +A node with no swap active but an unmasked generated unit and an unmasked `swap.target` +**WARNs**: it is clean now and will be swapping again after the next reboot. The check +used to blame that state on "this node has not been baselined yet", which sent the reader +back to re-run the role that had already run. + ## The EFI memattr check diff --git a/infra/ansible/roles/node_health/files/node_health_collect.py b/infra/ansible/roles/node_health/files/node_health_collect.py index 3bbc537..8eb79e7 100644 --- a/infra/ansible/roles/node_health/files/node_health_collect.py +++ b/infra/ansible/roles/node_health/files/node_health_collect.py @@ -301,9 +301,94 @@ def collect_memory(): # Firmware's own inventory of what is physically installed. Root-only; attempted # here anyway so the unprivileged pass records the exact reason it could not read. mem["devices"] = collect_memory_devices() + # "Is swap off right now?" and "will it still be off after a reboot?" are DIFFERENT + # questions, and only the second one is kubelet's actual prerequisite. See below. + mem["swap_persistence"] = collect_swap_persistence() return mem +# --- Will swap still be off after the next reboot? ----------------------------------- +# SwapTotal=0 does not mean swap is DISABLED — it means nothing has been swapped on yet +# this boot. Measured 2026-09-07 on all three prod nodes: swap was active again after a +# reboot, 977 MiB each, with /etc/fstab's swap line commented out by node_baseline. The +# cause is systemd-gpt-auto-generator, which claims any GPT partition typed +# 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") straight from the PARTITION TABLE, +# synthesises a .swap unit into /run/systemd/generator.late/ and links it into +# swap.target.wants/. It never reads fstab, so neutralising fstab cannot stop it. +# +# roles/node_baseline masks swap.target — the single unit every generated .swap unit is +# `WantedBy` — and masks the generated units themselves as a second barrier. This +# collects the facts that let the report tell "off, and staying off" apart from "off +# until the next reboot", which a /proc/meminfo reading alone can never do. +# +# All of it is world-readable (the generator dirs are 0755 root:root), so it works in the +# UNPRIVILEGED pass: no root, no systemctl, no parsing of systemctl output. +def _is_systemd_mask(path): + """True when `path` is a systemd mask, i.e. a symlink to /dev/null.""" + try: + return os.path.islink(path) and os.path.realpath(path) == "/dev/null" + except OSError: + return False + + +def collect_swap_persistence(): + out = { + "readable": False, + "swap_target_masked": None, + # .swap units systemd's generators built for THIS boot (gpt-auto, fstab, + # cryptsetup). These are the ones that would be activated again next boot. + "generated_units": [], + # .swap units masked in /etc/systemd/system — what node_baseline wrote. + "masked_units": [], + # generated MINUS masked: the units that would still come back. Empty is good. + "unmasked_generated_units": [], + "fstab_active_swap_lines": None, + } + + generated = set() + for directory in ( + "/run/systemd/generator.early", + "/run/systemd/generator", + "/run/systemd/generator.late", + ): + try: + names = os.listdir(directory) + except FileNotFoundError: + # A generator produced nothing. That is a real answer, not a read failure: + # dev's VMs have no swap partition, so gpt-auto emits no unit at all. + continue + except OSError as exc: + UNREADABLE.append({"path": directory, "reason": "read error: %s" % exc}) + return out + for name in names: + if name.endswith(".swap") and os.path.isfile(os.path.join(directory, name)): + generated.add(name) + + masked = set() + try: + for name in os.listdir("/etc/systemd/system"): + if name.endswith(".swap") and _is_systemd_mask("/etc/systemd/system/" + name): + masked.add(name) + except OSError as exc: + UNREADABLE.append({"path": "/etc/systemd/system", "reason": "read error: %s" % exc}) + return out + + out["swap_target_masked"] = _is_systemd_mask("/etc/systemd/system/swap.target") + out["generated_units"] = sorted(generated) + out["masked_units"] = sorted(masked) + out["unmasked_generated_units"] = sorted(generated - masked) + out["readable"] = True + + fstab = read_text("/etc/fstab") + if fstab is not None: + out["fstab_active_swap_lines"] = sum( + 1 + for line in fstab.splitlines() + if not line.lstrip().startswith("#") and re.search(r"\sswap\s", line) + ) + return out + + # --- Memory the kernel has but never hands out -------------------------------------- # /proc/zoneinfo is WORLD-READABLE on every Linux, VM or metal — no root, no dmidecode, # no DMI. Per zone it reports: diff --git a/infra/ansible/roles/node_health/tasks/assess.yml b/infra/ansible/roles/node_health/tasks/assess.yml index 9efe251..b2a1a80 100644 --- a/infra/ansible/roles/node_health/tasks/assess.yml +++ b/infra/ansible/roles/node_health/tasks/assess.yml @@ -499,25 +499,76 @@ if phys_status in ['WARN', 'FAIL'] else '') ~ (phys_cause if phys_status in ['WARN', 'FAIL'] else '') }} -- name: Assess — swap (kubelet requires it off) +# WHY THIS CHECK IS NOT JUST `SwapTotal == 0`. It used to be, and it read the answer +# wrong in both directions. It called active swap "this node has not been baselined yet", +# which was a MIS-DIAGNOSIS — measured 2026-09-07, all three prod nodes had been +# baselined, had their fstab swap line commented out by node_baseline, and were STILL +# swapping 977 MiB each after a reboot. Whoever read that WARN would have re-run the very +# role that had already run, and watched swap come back at the next boot again. +# +# The real mechanism is systemd-gpt-auto-generator: it claims any GPT partition typed +# 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") from the PARTITION TABLE, builds a +# .swap unit into /run/systemd/generator.late/ and links it into swap.target.wants/. It +# never reads /etc/fstab, so `swapoff -a` + a commented fstab line only holds until the +# next boot. roles/node_baseline now masks swap.target (the single unit those generated +# units are `WantedBy`) plus the generated units themselves. +# +# The other direction matters just as much: SwapTotal == 0 is NOT proof that swap is +# disabled — a node freshly `swapoff`-ed reads zero and swaps again on reboot. So this +# check now asks both questions, and WARNs on a node that is clean right now but has an +# unmasked generated swap unit waiting for the next boot. +- name: Assess — swap (kubelet requires it off, and off ACROSS REBOOTS) ansible.builtin.set_fact: node_health_checks: "{{ node_health_checks + [{ 'id': 'memory.swap', - 'title': 'Swap disabled (kubelet prerequisite)', + 'title': 'Swap disabled, and disabled across reboots (kubelet prerequisite)', 'status': swap_status, 'detail': swap_detail }] }}" vars: ansible_become: false m: "{{ node_health_facts.memory }}" + p: "{{ node_health_facts.memory.swap_persistence | default({}, true) }}" + # Durability was actually measurable on this node (the collector could read + # /etc/systemd/system and the generator dirs). When false, say so rather than + # implying a clean bill from a failed read. + swap_durable_known: "{{ (p.readable | default(false)) | bool }}" + # Units a reboot would bring back: produced by a generator, not masked, and with + # swap.target itself unmasked so something can still pull them in. + swap_will_return: >- + {{ (swap_durable_known | bool) + and not (p.swap_target_masked | default(false) | bool) + and (p.unmasked_generated_units | default([], true) | length) > 0 }} + swap_units_returning: "{{ p.unmasked_generated_units | default([], true) | join(', ') }}" + swap_active_gib: "{{ (m.swap_total_kb | int / 1048576) | round(2) | string }}" swap_status: >- {{ 'SKIP' if m.swap_total_kb is not number - else ('PASS' if m.swap_total_kb | int == 0 else 'WARN') }} + else ('WARN' if (m.swap_total_kb | int) > 0 + else ('WARN' if (swap_will_return | bool) else 'PASS')) }} + swap_cause: >- + systemd-gpt-auto-generator re-creates a .swap unit for the GPT partition typed + 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") at EVERY boot, straight from + the partition table and without reading /etc/fstab. This is NOT "the node has not + been baselined" — `swapoff -a` and a commented fstab line simply do not survive a + reboot. The fix is roles/node_baseline masking swap.target and the generated units: + re-run `node-baseline.yml --tags system` against this node. swap_detail: >- {{ 'swap state unreadable' if m.swap_total_kb is not number - else ('no swap active' if m.swap_total_kb | int == 0 - else ((m.swap_total_kb | int / 1048576) | round(2) | string - ~ ' GiB swap active — roles/node_baseline runs swapoff and neutralises - the fstab entry; this node has not been baselined yet')) }} + else ((swap_active_gib ~ ' GiB swap active' + ~ (' (' ~ swap_units_returning ~ ')' if swap_units_returning else '') + ~ ' — ' ~ swap_cause) + if (m.swap_total_kb | int) > 0 + else ('no swap active, but it would come back at the next boot via ' + ~ swap_units_returning ~ ' — ' ~ swap_cause + if (swap_will_return | bool) + else ('no swap active' + ~ (' and none can be activated: swap.target is masked' + if (p.swap_target_masked | default(false) | bool) + else (' and no swap unit exists on this node' + if (p.generated_units | default([], true) | length) == 0 + else ' and every generated swap unit is masked')) + if (swap_durable_known | bool) + else 'no swap active (durability NOT verified — the collector + could not read /etc/systemd/system or /run/systemd)'))) }} - name: Assess — ECC / EDAC correctable errors ansible.builtin.set_fact: