diff --git a/infra/ansible/roles/node_baseline/handlers/main.yml b/infra/ansible/roles/node_baseline/handlers/main.yml index 5bd7788..f360b26 100644 --- a/infra/ansible/roles/node_baseline/handlers/main.yml +++ b/infra/ansible/roles/node_baseline/handlers/main.yml @@ -11,3 +11,13 @@ ansible.builtin.systemd: name: systemd-resolved state: restarted + +# Masking a unit writes a symlink to /dev/null into /etc/systemd/system; the RUNNING +# systemd keeps its old view of the unit until it re-reads unit files. The mask is read +# fresh at the next boot either way, so this only exists so the running manager does not +# disagree with on-disk state in between. `daemon_reload` re-parses unit files and +# re-runs generators — it starts and stops nothing, so it cannot disturb k3s or Longhorn. +# Fires only when a mask actually changed, so a re-run of the role is silent. +- name: Reload systemd unit files + ansible.builtin.systemd_service: + daemon_reload: true diff --git a/infra/ansible/roles/node_baseline/tasks/main.yml b/infra/ansible/roles/node_baseline/tasks/main.yml index b2fb6a7..11c914f 100644 --- a/infra/ansible/roles/node_baseline/tasks/main.yml +++ b/infra/ansible/roles/node_baseline/tasks/main.yml @@ -646,9 +646,109 @@ tags: [disks] tags: [disks] -- name: Disable swap now - ansible.builtin.command: swapoff -a +# ===================================================================================== +# SWAP — off NOW, and still off AFTER A REBOOT — tag `system` +# ===================================================================================== +# `swapoff -a` plus commenting the fstab line is NOT durable on these nodes, and never +# was. Measured 2026-09-07: all three prod nodes were running with 977 MiB of swap +# ACTIVE (k3s-prod-1 /dev/sda4, k3s-prod-3 /dev/sda4, k3s-prod-2 /dev/nvme0n1p4) while +# their fstab swap line was commented out by this very role. The role reported success +# and the setting silently expired at the next boot, on every node. +# +# CAUSE. systemd-gpt-auto-generator (systemd 257, Debian 13) claims any GPT partition +# whose partition TYPE GUID is 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap"), +# synthesises a .swap unit for it into /run/systemd/generator.late/ and drops a symlink +# to it into swap.target.wants/. It decides that from the PARTITION TABLE and never +# consults /etc/fstab, so neutralising fstab cannot stop it. Re-running the generator by +# hand on k3s-prod-1 says so in as many words: +# Dissecting swap partition with label n/a and UUID acd3876c-fd8e-4389-b856-... +# Probed fstype 'swap' on partition /dev/disk/by-diskseq/1-part4. +# Adding swap: /dev/disk/by-diskseq/1-part4 +# +# MECHANISM — mask swap.target. +# `systemctl show` on the generated unit says its ONLY inbound dependency is +# `WantedBy=swap.target`: Requires/RequiredBy/BoundBy are empty, and the matching +# .device unit has an empty `Wants=`, so a device appearing does NOT pull its swap unit +# in by itself. swap.target is therefore the single choke point that every swap +# activation path goes through — the gpt-auto units, the units systemd-fstab-generator +# builds from /etc/fstab, and the ones systemd-cryptsetup-generator builds for encrypted +# swap. Masking it is: +# * DEVICE-SHAPE INDEPENDENT. No device name appears anywhere in the change, so one +# task is correct on prod's sda4 AND nvme0n1p4 and on dev's vda VMs, and there is +# nothing to keep in sync when a disk is replaced. (The generated unit is named after +# the kernel's DISKSEQ, not the disk — it is `dev-disk-by\x2ddiskseq-1\x2dpart4.swap` +# on all three prod nodes, NVMe and SATA alike — but diskseq is assigned per boot in +# device-probe order, so it is not a name worth depending on either.) +# * BOOT-SAFE. swap.target is reached by `Wants=`/`After=` from sysinit.target and by +# `After=` from tmp.mount, and `RequiredBy=` is EMPTY (verified on all three prod +# nodes and all three dev nodes). A masked unit reached only through Wants= is +# skipped with a log line; it cannot fail the boot transaction. Ordering deps on a +# unit that never starts are no-ops. +# * INERT WHEN THE DEVICE IS ABSENT OR RENAMED. The mask names no device, so there is +# nothing to go stale: a disk swap, a new partition layout or a restore onto +# different hardware all stay swap-free without touching this role. +# The per-unit masks that follow are a SECOND, independent barrier over the units that +# exist right now, so `systemctl status ` says "masked" to whoever looks, and an +# operator who unmasks swap.target does not silently get swap back. Their failure mode is +# benign in the other direction too: a mask whose unit no longer exists is an unreferenced +# symlink to /dev/null, and the only thing it could ever shadow is a .swap unit. +# +# ALTERNATIVES REJECTED. +# * `systemd.gpt_auto=0` on the kernel cmdline — disables GPT auto-discovery WHOLESALE, +# not just swap, and needs a GRUB edit + update-grub + a reboot before it is even +# true. On prod that happens to be swap-only today (root= is already on the cmdline +# so root auto-detection is off, and the /boot/efi fstab entry already suppresses the +# ESP/XBOOTLDR mounts — the generator says so in its debug output), but on DEV the +# root partition carries the discoverable GUID 4f68bce3-e8cd-4db1-96e7-fbcaf984b709 +# ("Linux root (x86-64)"), so the same flag would switch off root auto-discovery +# there. Editing a bootloader to fix a swap unit is the wrong blast radius, and the +# change would not be verifiable without the reboot it depends on. +# * Clearing or retyping the swap partition's GPT type GUID — a partition-table write +# to the live boot disk of a node currently serving etcd and Longhorn replicas, to +# fix something a symlink fixes. It would also decide, irreversibly and without +# asking, that those 977 MiB are never swap again and are simply unused; that is the +# operator's call to make, not this role's, and it needs a maintenance window. +# * `systemd.swap=0` — read by systemd-fstab-generator ONLY. The swap here does not +# come from fstab, so it would change nothing at all. +# +# DEV. Dev's nodes have no swap partition, no swap unit and no fstab swap line at all +# (checked on k3s-dev-1/2/3), so the mask is inert there: it flips to masked once and +# every later run is changed=0. It is correct rather than merely harmless — if a future +# dev image ever ships a swap partition, the same mask already covers it. + +- name: Mask swap.target so no generator can pull a swap unit in at boot + ansible.builtin.systemd_service: + name: swap.target + masked: true + notify: Reload systemd unit files + tags: [system] + +- name: Find the .swap units systemd's generators produced for this boot + ansible.builtin.find: + # gpt-auto writes to generator.late; systemd-fstab-generator and + # systemd-cryptsetup-generator (the other two that can emit a .swap unit) write to + # generator. /run/systemd/generator.early is deliberately NOT listed: nothing that + # makes swap units runs at early priority, and `find` warns on a path that does not + # exist. Anything a future early generator invented is still caught by the + # swap.target mask above, which is the mechanism this task only reinforces. + paths: + - /run/systemd/generator + - /run/systemd/generator.late + patterns: "*.swap" + file_type: file + register: node_baseline_generated_swap changed_when: false + check_mode: false + tags: [system] + +- name: Mask each generated .swap unit (second barrier, independent of swap.target) + ansible.builtin.systemd_service: + name: "{{ item }}" + masked: true + loop: "{{ node_baseline_generated_swap.files | map(attribute='path') | map('basename') | unique | sort }}" + loop_control: + label: "{{ item }}" + notify: Reload systemd unit files tags: [system] - name: Neutralize swap entries in fstab (kubelet requires swap off) @@ -658,6 +758,23 @@ replace: '# \1 # disabled for k3s/kubelet' tags: [system] +# The masks above only decide what the NEXT boot does. Anything swapped on already stays +# on until it is told otherwise, so read the live state and act only when there is +# something to act on — that is what makes a second run honestly changed=0 instead of +# `changed_when: false` hiding a command that ran anyway. +- name: Read the swap devices that are active right now + ansible.builtin.command: swapon --noheadings --show=NAME + register: node_baseline_swap_active + changed_when: false + check_mode: false + tags: [system] + +- name: Disable swap now + ansible.builtin.command: swapoff -a + when: node_baseline_swap_active.stdout | trim | length > 0 + changed_when: true + tags: [system] + # ===================================================================================== # DNS RESILIENCE — tag `dns` (R-DNS-01 / R-DNS-03) # =====================================================================================