merge: fix/swap-off-persists-across-reboot

infra/ansible/roles/node_baseline/handlers/main.yml

@@ -11,3 +11,13 @@
   ansible.builtin.systemd:
     name: systemd-resolved
     state: restarted
+
+# Masking a unit writes a symlink to /dev/null into /etc/systemd/system; the RUNNING
+# systemd keeps its old view of the unit until it re-reads unit files. The mask is read
+# fresh at the next boot either way, so this only exists so the running manager does not
+# disagree with on-disk state in between. `daemon_reload` re-parses unit files and
+# re-runs generators — it starts and stops nothing, so it cannot disturb k3s or Longhorn.
+# Fires only when a mask actually changed, so a re-run of the role is silent.
+- name: Reload systemd unit files
+  ansible.builtin.systemd_service:
+    daemon_reload: true

infra/ansible/roles/node_baseline/tasks/disks.yml

@@ -331,6 +331,15 @@
 # The devices this role must never, under any circumstances, hand to e2label. Built from
 # gathered facts (`gather_facts: true` in the play) plus the live swap list, so it is the
 # NODE's account of what is load-bearing, not a hardcoded list of partition names.
+#
+# EXPECT THIS LIST TO BE EMPTY. Since the swap tasks in tasks/main.yml mask swap.target,
+# a baselined node has no active swap for `swapon` to report, so the swap half of
+# node_baseline_lh_protected_devices below contributes nothing. That is not a weakened
+# guard: the claim CANDIDATE is derived from whatever is mounted at
+# node_baseline_longhorn_claim_mountpoint, and a swap partition is neither mounted nor an
+# ext4 filesystem, so it can never be the candidate in the first place. The task stays
+# because the list must be the node's own account of what is load-bearing — including on
+# a node reached before baseline has run, or one an operator has deliberately swapped on.
 - name: "Longhorn (claim): list the devices currently backing active swap"
   ansible.builtin.command: swapon --noheadings --show=NAME
   register: node_baseline_lh_swapdevs

infra/ansible/roles/node_baseline/tasks/main.yml

@@ -646,9 +646,109 @@
       tags: [disks]
   tags: [disks]
 
-- name: Disable swap now
-  ansible.builtin.command: swapoff -a
+# =====================================================================================
+# SWAP — off NOW, and still off AFTER A REBOOT — tag `system`
+# =====================================================================================
+# `swapoff -a` plus commenting the fstab line is NOT durable on these nodes, and never
+# was. Measured 2026-09-07: all three prod nodes were running with 977 MiB of swap
+# ACTIVE (k3s-prod-1 /dev/sda4, k3s-prod-3 /dev/sda4, k3s-prod-2 /dev/nvme0n1p4) while
+# their fstab swap line was commented out by this very role. The role reported success
+# and the setting silently expired at the next boot, on every node.
+#
+# CAUSE. systemd-gpt-auto-generator (systemd 257, Debian 13) claims any GPT partition
+# whose partition TYPE GUID is 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap"),
+# synthesises a .swap unit for it into /run/systemd/generator.late/ and drops a symlink
+# to it into swap.target.wants/. It decides that from the PARTITION TABLE and never
+# consults /etc/fstab, so neutralising fstab cannot stop it. Re-running the generator by
+# hand on k3s-prod-1 says so in as many words:
+#     Dissecting swap partition with label n/a and UUID acd3876c-fd8e-4389-b856-...
+#     Probed fstype 'swap' on partition /dev/disk/by-diskseq/1-part4.
+#     Adding swap: /dev/disk/by-diskseq/1-part4
+#
+# MECHANISM — mask swap.target.
+# `systemctl show` on the generated unit says its ONLY inbound dependency is
+# `WantedBy=swap.target`: Requires/RequiredBy/BoundBy are empty, and the matching
+# .device unit has an empty `Wants=`, so a device appearing does NOT pull its swap unit
+# in by itself. swap.target is therefore the single choke point that every swap
+# activation path goes through — the gpt-auto units, the units systemd-fstab-generator
+# builds from /etc/fstab, and the ones systemd-cryptsetup-generator builds for encrypted
+# swap. Masking it is:
+#   * DEVICE-SHAPE INDEPENDENT. No device name appears anywhere in the change, so one
+#     task is correct on prod's sda4 AND nvme0n1p4 and on dev's vda VMs, and there is
+#     nothing to keep in sync when a disk is replaced. (The generated unit is named after
+#     the kernel's DISKSEQ, not the disk — it is `dev-disk-by\x2ddiskseq-1\x2dpart4.swap`
+#     on all three prod nodes, NVMe and SATA alike — but diskseq is assigned per boot in
+#     device-probe order, so it is not a name worth depending on either.)
+#   * BOOT-SAFE. swap.target is reached by `Wants=`/`After=` from sysinit.target and by
+#     `After=` from tmp.mount, and `RequiredBy=` is EMPTY (verified on all three prod
+#     nodes and all three dev nodes). A masked unit reached only through Wants= is
+#     skipped with a log line; it cannot fail the boot transaction. Ordering deps on a
+#     unit that never starts are no-ops.
+#   * INERT WHEN THE DEVICE IS ABSENT OR RENAMED. The mask names no device, so there is
+#     nothing to go stale: a disk swap, a new partition layout or a restore onto
+#     different hardware all stay swap-free without touching this role.
+# The per-unit masks that follow are a SECOND, independent barrier over the units that
+# exist right now, so `systemctl status <unit>` says "masked" to whoever looks, and an
+# operator who unmasks swap.target does not silently get swap back. Their failure mode is
+# benign in the other direction too: a mask whose unit no longer exists is an unreferenced
+# symlink to /dev/null, and the only thing it could ever shadow is a .swap unit.
+#
+# ALTERNATIVES REJECTED.
+#   * `systemd.gpt_auto=0` on the kernel cmdline — disables GPT auto-discovery WHOLESALE,
+#     not just swap, and needs a GRUB edit + update-grub + a reboot before it is even
+#     true. On prod that happens to be swap-only today (root= is already on the cmdline
+#     so root auto-detection is off, and the /boot/efi fstab entry already suppresses the
+#     ESP/XBOOTLDR mounts — the generator says so in its debug output), but on DEV the
+#     root partition carries the discoverable GUID 4f68bce3-e8cd-4db1-96e7-fbcaf984b709
+#     ("Linux root (x86-64)"), so the same flag would switch off root auto-discovery
+#     there. Editing a bootloader to fix a swap unit is the wrong blast radius, and the
+#     change would not be verifiable without the reboot it depends on.
+#   * Clearing or retyping the swap partition's GPT type GUID — a partition-table write
+#     to the live boot disk of a node currently serving etcd and Longhorn replicas, to
+#     fix something a symlink fixes. It would also decide, irreversibly and without
+#     asking, that those 977 MiB are never swap again and are simply unused; that is the
+#     operator's call to make, not this role's, and it needs a maintenance window.
+#   * `systemd.swap=0` — read by systemd-fstab-generator ONLY. The swap here does not
+#     come from fstab, so it would change nothing at all.
+#
+# DEV. Dev's nodes have no swap partition, no swap unit and no fstab swap line at all
+# (checked on k3s-dev-1/2/3), so the mask is inert there: it flips to masked once and
+# every later run is changed=0. It is correct rather than merely harmless — if a future
+# dev image ever ships a swap partition, the same mask already covers it.
+
+- name: Mask swap.target so no generator can pull a swap unit in at boot
+  ansible.builtin.systemd_service:
+    name: swap.target
+    masked: true
+  notify: Reload systemd unit files
+  tags: [system]
+
+- name: Find the .swap units systemd's generators produced for this boot
+  ansible.builtin.find:
+    # gpt-auto writes to generator.late; systemd-fstab-generator and
+    # systemd-cryptsetup-generator (the other two that can emit a .swap unit) write to
+    # generator. /run/systemd/generator.early is deliberately NOT listed: nothing that
+    # makes swap units runs at early priority, and `find` warns on a path that does not
+    # exist. Anything a future early generator invented is still caught by the
+    # swap.target mask above, which is the mechanism this task only reinforces.
+    paths:
+      - /run/systemd/generator
+      - /run/systemd/generator.late
+    patterns: "*.swap"
+    file_type: file
+  register: node_baseline_generated_swap
   changed_when: false
+  check_mode: false
+  tags: [system]
+
+- name: Mask each generated .swap unit (second barrier, independent of swap.target)
+  ansible.builtin.systemd_service:
+    name: "{{ item }}"
+    masked: true
+  loop: "{{ node_baseline_generated_swap.files | map(attribute='path') | map('basename') | unique | sort }}"
+  loop_control:
+    label: "{{ item }}"
+  notify: Reload systemd unit files
   tags: [system]
 
 - name: Neutralize swap entries in fstab (kubelet requires swap off)
@@ -658,6 +758,23 @@
     replace: '# \1  # disabled for k3s/kubelet'
   tags: [system]
 
+# The masks above only decide what the NEXT boot does. Anything swapped on already stays
+# on until it is told otherwise, so read the live state and act only when there is
+# something to act on — that is what makes a second run honestly changed=0 instead of
+# `changed_when: false` hiding a command that ran anyway.
+- name: Read the swap devices that are active right now
+  ansible.builtin.command: swapon --noheadings --show=NAME
+  register: node_baseline_swap_active
+  changed_when: false
+  check_mode: false
+  tags: [system]
+
+- name: Disable swap now
+  ansible.builtin.command: swapoff -a
+  when: node_baseline_swap_active.stdout | trim | length > 0
+  changed_when: true
+  tags: [system]
+
 # =====================================================================================
 # DNS RESILIENCE — tag `dns` (R-DNS-01 / R-DNS-03)
 # =====================================================================================

infra/ansible/roles/node_health/README.md

@@ -47,7 +47,7 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0
 | `storage.io_errors` | kernel SCSI/ATA `ioerr_cnt` / `iotmo_cnt` per disk (no root needed) |
 | `storage.capacity`, `storage.inodes` | worst mounted filesystem, space and inodes |
 | `storage.layout` | physical disks, and whether any device is genuinely free for `node_baseline_longhorn_device` / `node_baseline_etcd_device` |
-| `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite, EDAC correctable/uncorrectable counters |
+| `memory.available`, `memory.swap`, `memory.ecc` | headroom, kubelet's swap-off prerequisite **and whether it survives a reboot** (see [Swap off is not the same as swap disabled](#swap-off-is-not-the-same-as-swap-disabled)), EDAC correctable/uncorrectable counters |
 | `memory.physical_total` | **is this machine using the RAM it has?** `/proc/zoneinfo` present-vs-managed pages (no root, metal or VM) and `MemTotal` vs the DIMM capacity SMBIOS reports (`dmidecode -t 17`, root, bare metal). Cause-independent by design — see [The EFI memattr check](#the-efi-memattr-check) |
 | `cpu.throttle`, `cpu.inventory`, `cpu.virtualisation` | thermal throttle events since boot, topology + frequency scaling, VT-x/AMD-V + `/dev/kvm` (KubeVirt, R-PLAT-03) |
 | `thermal.max` | hottest sensor across thermal zones and hwmon chips |
@@ -59,6 +59,37 @@ Serves **R-ENV-01** (you cannot certify a node you have not measured), **R-ENV-0
 Thresholds live in `defaults/main.yml` (`node_health_thresholds`); no verdict is hardcoded
 in task logic.
 
+### Swap off is not the same as swap disabled
+
+`SwapTotal == 0` only says nothing has been swapped on *yet this boot*. It is not evidence
+that swap is disabled, and on these nodes it was not: measured 2026-09-07, all three prod
+nodes came back from a reboot with 977 MiB of swap active (`k3s-prod-1` `/dev/sda4`,
+`k3s-prod-3` `/dev/sda4`, `k3s-prod-2` `/dev/nvme0n1p4`) even though `roles/node_baseline`
+had run `swapoff -a` and commented their `/etc/fstab` swap line.
+
+The cause is `systemd-gpt-auto-generator`. It claims any GPT partition whose partition
+**type GUID** is `0657fd6d-a4ab-43c4-84e5-0933c84b4f4f` ("Linux swap") straight from the
+partition table, synthesises a `.swap` unit into `/run/systemd/generator.late/` and links
+it into `swap.target.wants/`. `/etc/fstab` is never consulted, so neutralising fstab
+cannot stop it. `roles/node_baseline` masks `swap.target` — the one unit every generated
+`.swap` unit is `WantedBy` — and masks the generated units as a second barrier.
+
+So `memory.swap` reports on both halves, from `memory.swap_persistence` in the collector
+(all world-readable, no root):
+
+| fact | meaning |
+|---|---|
+| `swap_target_masked` | is `/etc/systemd/system/swap.target` a symlink to `/dev/null`? |
+| `generated_units` | `.swap` units systemd's generators built for this boot |
+| `masked_units` | `.swap` units masked in `/etc/systemd/system` |
+| `unmasked_generated_units` | `generated_units` minus `masked_units` — what a reboot would bring back |
+| `fstab_active_swap_lines` | uncommented `swap` lines left in `/etc/fstab` |
+
+A node with no swap active but an unmasked generated unit and an unmasked `swap.target`
+**WARNs**: it is clean now and will be swapping again after the next reboot. The check
+used to blame that state on "this node has not been baselined yet", which sent the reader
+back to re-run the role that had already run.
+
 
 ## The EFI memattr check
 

infra/ansible/roles/node_health/files/node_health_collect.py

@@ -301,9 +301,94 @@ def collect_memory():
     # Firmware's own inventory of what is physically installed. Root-only; attempted
     # here anyway so the unprivileged pass records the exact reason it could not read.
     mem["devices"] = collect_memory_devices()
+    # "Is swap off right now?" and "will it still be off after a reboot?" are DIFFERENT
+    # questions, and only the second one is kubelet's actual prerequisite. See below.
+    mem["swap_persistence"] = collect_swap_persistence()
     return mem
 
 
+# --- Will swap still be off after the next reboot? -----------------------------------
+# SwapTotal=0 does not mean swap is DISABLED — it means nothing has been swapped on yet
+# this boot. Measured 2026-09-07 on all three prod nodes: swap was active again after a
+# reboot, 977 MiB each, with /etc/fstab's swap line commented out by node_baseline. The
+# cause is systemd-gpt-auto-generator, which claims any GPT partition typed
+# 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") straight from the PARTITION TABLE,
+# synthesises a .swap unit into /run/systemd/generator.late/ and links it into
+# swap.target.wants/. It never reads fstab, so neutralising fstab cannot stop it.
+#
+# roles/node_baseline masks swap.target — the single unit every generated .swap unit is
+# `WantedBy` — and masks the generated units themselves as a second barrier. This
+# collects the facts that let the report tell "off, and staying off" apart from "off
+# until the next reboot", which a /proc/meminfo reading alone can never do.
+#
+# All of it is world-readable (the generator dirs are 0755 root:root), so it works in the
+# UNPRIVILEGED pass: no root, no systemctl, no parsing of systemctl output.
+def _is_systemd_mask(path):
+    """True when `path` is a systemd mask, i.e. a symlink to /dev/null."""
+    try:
+        return os.path.islink(path) and os.path.realpath(path) == "/dev/null"
+    except OSError:
+        return False
+
+
+def collect_swap_persistence():
+    out = {
+        "readable": False,
+        "swap_target_masked": None,
+        # .swap units systemd's generators built for THIS boot (gpt-auto, fstab,
+        # cryptsetup). These are the ones that would be activated again next boot.
+        "generated_units": [],
+        # .swap units masked in /etc/systemd/system — what node_baseline wrote.
+        "masked_units": [],
+        # generated MINUS masked: the units that would still come back. Empty is good.
+        "unmasked_generated_units": [],
+        "fstab_active_swap_lines": None,
+    }
+
+    generated = set()
+    for directory in (
+        "/run/systemd/generator.early",
+        "/run/systemd/generator",
+        "/run/systemd/generator.late",
+    ):
+        try:
+            names = os.listdir(directory)
+        except FileNotFoundError:
+            # A generator produced nothing. That is a real answer, not a read failure:
+            # dev's VMs have no swap partition, so gpt-auto emits no unit at all.
+            continue
+        except OSError as exc:
+            UNREADABLE.append({"path": directory, "reason": "read error: %s" % exc})
+            return out
+        for name in names:
+            if name.endswith(".swap") and os.path.isfile(os.path.join(directory, name)):
+                generated.add(name)
+
+    masked = set()
+    try:
+        for name in os.listdir("/etc/systemd/system"):
+            if name.endswith(".swap") and _is_systemd_mask("/etc/systemd/system/" + name):
+                masked.add(name)
+    except OSError as exc:
+        UNREADABLE.append({"path": "/etc/systemd/system", "reason": "read error: %s" % exc})
+        return out
+
+    out["swap_target_masked"] = _is_systemd_mask("/etc/systemd/system/swap.target")
+    out["generated_units"] = sorted(generated)
+    out["masked_units"] = sorted(masked)
+    out["unmasked_generated_units"] = sorted(generated - masked)
+    out["readable"] = True
+
+    fstab = read_text("/etc/fstab")
+    if fstab is not None:
+        out["fstab_active_swap_lines"] = sum(
+            1
+            for line in fstab.splitlines()
+            if not line.lstrip().startswith("#") and re.search(r"\sswap\s", line)
+        )
+    return out
+
+
 # --- Memory the kernel has but never hands out --------------------------------------
 # /proc/zoneinfo is WORLD-READABLE on every Linux, VM or metal — no root, no dmidecode,
 # no DMI. Per zone it reports:

infra/ansible/roles/node_health/tasks/assess.yml

@@ -499,25 +499,76 @@
             if phys_status in ['WARN', 'FAIL'] else '')
          ~ (phys_cause if phys_status in ['WARN', 'FAIL'] else '') }}
 
-- name: Assess — swap (kubelet requires it off)
+# WHY THIS CHECK IS NOT JUST `SwapTotal == 0`. It used to be, and it read the answer
+# wrong in both directions. It called active swap "this node has not been baselined yet",
+# which was a MIS-DIAGNOSIS — measured 2026-09-07, all three prod nodes had been
+# baselined, had their fstab swap line commented out by node_baseline, and were STILL
+# swapping 977 MiB each after a reboot. Whoever read that WARN would have re-run the very
+# role that had already run, and watched swap come back at the next boot again.
+#
+# The real mechanism is systemd-gpt-auto-generator: it claims any GPT partition typed
+# 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") from the PARTITION TABLE, builds a
+# .swap unit into /run/systemd/generator.late/ and links it into swap.target.wants/. It
+# never reads /etc/fstab, so `swapoff -a` + a commented fstab line only holds until the
+# next boot. roles/node_baseline now masks swap.target (the single unit those generated
+# units are `WantedBy`) plus the generated units themselves.
+#
+# The other direction matters just as much: SwapTotal == 0 is NOT proof that swap is
+# disabled — a node freshly `swapoff`-ed reads zero and swaps again on reboot. So this
+# check now asks both questions, and WARNs on a node that is clean right now but has an
+# unmasked generated swap unit waiting for the next boot.
+- name: Assess — swap (kubelet requires it off, and off ACROSS REBOOTS)
   ansible.builtin.set_fact:
     node_health_checks: "{{ node_health_checks + [{
       'id': 'memory.swap',
-      'title': 'Swap disabled (kubelet prerequisite)',
+      'title': 'Swap disabled, and disabled across reboots (kubelet prerequisite)',
       'status': swap_status,
       'detail': swap_detail }] }}"
   vars:
     ansible_become: false
     m: "{{ node_health_facts.memory }}"
+    p: "{{ node_health_facts.memory.swap_persistence | default({}, true) }}"
+    # Durability was actually measurable on this node (the collector could read
+    # /etc/systemd/system and the generator dirs). When false, say so rather than
+    # implying a clean bill from a failed read.
+    swap_durable_known: "{{ (p.readable | default(false)) | bool }}"
+    # Units a reboot would bring back: produced by a generator, not masked, and with
+    # swap.target itself unmasked so something can still pull them in.
+    swap_will_return: >-
+      {{ (swap_durable_known | bool)
+         and not (p.swap_target_masked | default(false) | bool)
+         and (p.unmasked_generated_units | default([], true) | length) > 0 }}
+    swap_units_returning: "{{ p.unmasked_generated_units | default([], true) | join(', ') }}"
+    swap_active_gib: "{{ (m.swap_total_kb | int / 1048576) | round(2) | string }}"
     swap_status: >-
       {{ 'SKIP' if m.swap_total_kb is not number
-         else ('PASS' if m.swap_total_kb | int == 0 else 'WARN') }}
+         else ('WARN' if (m.swap_total_kb | int) > 0
+               else ('WARN' if (swap_will_return | bool) else 'PASS')) }}
+    swap_cause: >-
+      systemd-gpt-auto-generator re-creates a .swap unit for the GPT partition typed
+      0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap") at EVERY boot, straight from
+      the partition table and without reading /etc/fstab. This is NOT "the node has not
+      been baselined" — `swapoff -a` and a commented fstab line simply do not survive a
+      reboot. The fix is roles/node_baseline masking swap.target and the generated units:
+      re-run `node-baseline.yml --tags system` against this node.
     swap_detail: >-
       {{ 'swap state unreadable' if m.swap_total_kb is not number
-         else ('no swap active' if m.swap_total_kb | int == 0
-               else ((m.swap_total_kb | int / 1048576) | round(2) | string
-                     ~ ' GiB swap active — roles/node_baseline runs swapoff and neutralises
-                       the fstab entry; this node has not been baselined yet')) }}
+         else ((swap_active_gib ~ ' GiB swap active'
+                ~ (' via ' ~ swap_units_returning if swap_units_returning else '')
+                ~ ' — ' ~ swap_cause)
+               if (m.swap_total_kb | int) > 0
+               else ('no swap active, but it would come back at the next boot via '
+                     ~ swap_units_returning ~ ' — ' ~ swap_cause
+                     if (swap_will_return | bool)
+                     else ('no swap active'
+                           ~ (' and none can be activated: swap.target is masked'
+                              if (p.swap_target_masked | default(false) | bool)
+                              else (' and no swap unit exists on this node'
+                                    if (p.generated_units | default([], true) | length) == 0
+                                    else ' and every generated swap unit is masked'))
+                           if (swap_durable_known | bool)
+                           else 'no swap active (durability NOT verified — the collector
+                                 could not read /etc/systemd/system or /run/systemd)'))) }}
 
 - name: Assess — ECC / EDAC correctable errors
   ansible.builtin.set_fact: