--- # TAGS — every task here carries one of `packages`, `kernel`, `system`, `wireless`, `disks` or # `dns`. An untagged run is unchanged (all six run, in this order). The split exists because the PACKAGE floor # and the DATA-DISK work have different readiness: packages are safe on any node from # first contact, while the disk tasks need block devices that a new environment may not # have decided yet (prod: infra/PROD-PROVISIONING.md §2, whose decision is itself blocked # on a SMART read that needs `smartmontools` — which ships in node_baseline_packages). # `--tags packages` breaks that circle: it installs the package floor and NOTHING else. # ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags packages # Skipping is the guard: with `--tags packages` no `disks`-tagged task is selected, so a # BLOCKED-pending-disk-decision device path is never even templated, let alone handed to # mkfs. A FULL run against such a node stops at the assert below instead. `dns` (the block # at the bottom of this file) works the same way and guards its own BLOCKED placeholder: # ansible-playbook playbooks/node-baseline.yml --tags dns # `kernel` (the backports-kernel block below) is separated for the same reason `packages` # is: it is the ONE thing that must land on a prod node BEFORE k3s is installed (§0b), and # it must be runnable without touching a disk decision that is still open: # ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags kernel # `wireless` (a cluster node gets exactly ONE path onto the network) is separated for a third # reason: it must be runnable BEFORE `dns` and without touching packages, the kernel or a disk # decision, because a node with a second DHCP-managed interface reverts the `dns` write. It is # placed before the disk work for the same reason, and the pair is the whole remedy for the # k3s-prod-3 incident: # ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags wireless,dns - name: Install baseline packages (guest agent, Longhorn/iscsi/nfs prereqs) ansible.builtin.apt: name: "{{ node_baseline_packages }}" state: present update_cache: true cache_valid_time: 3600 tags: [packages] # ===================================================================================== # BACKPORTS KERNEL — tag `kernel` (EFI memattr; infra/PROD-PROVISIONING.md §0b and §3) # ===================================================================================== # A STANDING DEVIATION from the Debian 13 baseline, encoded here so every current AND # FUTURE prod node gets it from provisioning instead of from someone remembering. # # WHAT IT FIXES. HP R21 firmware on the prod EliteDesk 800 G5 minis publishes a CORRUPT EFI # Memory Attributes Table. `efi_memattr_init()` in Linux < 6.13 computes # `tbl_size = sizeof(*tbl) + num_entries * desc_size` from two UNVALIDATED u32 fields into a # signed int and `memblock_reserve()`s the product; when bit 31 lands the value sign-extends # through phys_addr_t, `memblock_cap_size()` clamps it to PHYS_ADDR_MAX, and EVERY physical # address above the table is reserved. k3s-prod-2 booted with 2.6 GiB of its 16 GB, twice. # Upstream commit 8fbe4c49c0cc "efi/memattr: Ignore table if the size is clearly bogus" # (first released in v6.13) validates the size and rejects the table instead. It is absent # from v6.12 and from linux-6.12.y, has no `Cc: stable` and no `Fixes:` tag, so no automatic # stable feed will ever pick it up. There is NO kernel cmdline workaround — # `efi_memattr_init()` runs from `efi_config_parse_tables()` before any `efi=` bail-out could # take effect. Booting a >= 6.13 kernel is the only fix, and ALL THREE prod nodes are on the # same lottery — node 1 and node 3 have merely drawn safe tickets so far. # # WHY IT IS SCOPED OFF FOR DEV. `node_baseline_backports_kernel` defaults to FALSE in # defaults/main.yml and is set true in inventory/group_vars/k3s_prod/vars.yml only. Dev is # Proxmox VMs whose firmware publishes no MAT at all — genuinely unaffected — and pulling a # LIVE cluster onto a backports kernel as a side effect of sharing this role would be a real # change for no benefit. See defaults/main.yml for why this is a variable rather than a # `k3s-prod-*` hostname test. # # IT DOES NOT REBOOT. A new kernel is inert until the node reboots, and this role # deliberately will not take one: k3s-prod-2 hung on a warm reboot on 2026-09-03 — powered, # off the LAN for ~38 minutes, power button required (§0b "SECOND FAULT"). Rebooting is # therefore a SERIALIZED, SUPERVISED operator step — one node at a time, verify each returns # before touching the next, never node 2 first — and a role cannot enforce that. The last # task here reports the pending reboot loudly instead of letting a green play imply the node # is already fixed. - name: Backports kernel for the EFI memattr fix (>= 6.13, commit 8fbe4c49c0cc) when: node_baseline_backports_kernel | bool tags: [kernel] block: # The suite name and the running release must agree, or apt resolves no candidate and # `state: latest` reports "ok" while leaving the node on the vulnerable 6.12 kernel — # a green play that fixed nothing. Fail on the mismatch instead (e.g. a node # dist-upgraded to forky while group_vars still says trixie-backports). - name: Assert the configured backports suite matches this node's Debian release ansible.builtin.assert: that: - ansible_facts['distribution'] == 'Debian' - node_baseline_backports_suite == (ansible_facts['distribution_release'] ~ '-backports') fail_msg: >- node_baseline_backports_kernel is enabled but node_baseline_backports_suite ({{ node_baseline_backports_suite }}) is not the backports suite for this node's release ({{ ansible_facts['distribution'] }} {{ ansible_facts['distribution_release'] | default('?') }}). apt would resolve no candidate and this play would report success while leaving the node on the vulnerable kernel. Fix the suite in group_vars — do not remove this assert. quiet: true # deb822 `.sources`, not a legacy one-line `deb` entry: it is the format Debian 13 # prefers, it carries `Signed-By` explicitly (binding this source to the archive keyring # rather than to whatever is in /etc/apt/trusted.gpg.d), and a `copy` of exact content is # idempotent in a way `apt_repository` — which rewrites and reformats — is not. # # THERE IS DELIBERATELY NO /etc/apt/preferences.d PIN, and that is not an oversight. The # backports Release file carries `NotAutomatic: yes` + `ButAutomaticUpgrades: yes`, which # apt already reads as: priority 100 so NOTHING is pulled from backports by accident, but # 500 for a package that was explicitly installed from there so it keeps getting updates. # Writing an explicit `Pin-Priority: 100` looks like belt-and-braces and is actively # HARMFUL — it overrides the ButAutomaticUpgrades half, and the kernel would then never # be upgraded again by `apt upgrade`. Enabling the source IS the whole configuration. - name: Enable the Debian backports apt source (deb822) ansible.builtin.copy: dest: "/etc/apt/sources.list.d/{{ node_baseline_backports_suite }}.sources" owner: root group: root mode: "0644" content: | # Managed by the Ansible node_baseline role (tags: kernel) — do not edit by hand. # A DELIBERATE, STANDING deviation from the Debian 13 baseline, enabled ONLY so this # machine can boot a kernel >= 6.13 that validates the corrupt EFI Memory Attributes # Table its firmware publishes (upstream 8fbe4c49c0cc). See # infra/PROD-PROVISIONING.md §0b/§3, and {{ node_baseline_backports_kernel_stamp }} # on this node. # # Backports is NotAutomatic, so nothing is taken from here unless asked for by name # with `-t {{ node_baseline_backports_suite }}`. Do NOT add an apt pin for it: # ButAutomaticUpgrades is what keeps the installed kernel patched, and an explicit # Pin-Priority would switch that off. Types: deb URIs: {{ node_baseline_backports_mirror }} Suites: {{ node_baseline_backports_suite }} Components: {{ node_baseline_backports_components | join(' ') }} Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg register: node_baseline_backports_source - name: Refresh the apt cache after enabling backports ansible.builtin.apt: update_cache: true when: node_baseline_backports_source is changed # `state: latest`, NOT `state: present`. linux-image-amd64 is ALREADY installed on these # nodes from trixie main (6.12.107-1), so `present` would find the name satisfied and do # nothing at all — the single most likely way to write this task and have it silently fix # nothing forever. `latest` + `default_release` resolves the candidate inside backports # and upgrades the metapackage to it. Once the node is current a re-run is a genuine # no-op: no change, no churn, no spurious reboot flag. Scope is exactly # node_baseline_backports_kernel_packages — one metapackage — so nothing else on the node # is dragged out of the main archive. - name: Install the fixed kernel from backports (>= 6.13) ansible.builtin.apt: name: "{{ node_baseline_backports_kernel_packages }}" state: latest default_release: "{{ node_baseline_backports_suite }}" update_cache: true cache_valid_time: 3600 register: node_baseline_backports_kernel_apt # WHICH VERSION DID THIS NODE ACTUALLY LAND ON. Read back rather than assumed: the # metapackage tracks a moving target, so "we installed linux-image-amd64" is not an # answer anybody can roll back from six months later. - name: Read back the installed backports kernel metapackage version ansible.builtin.command: dpkg-query -W -f '${Version}' {{ node_baseline_backports_kernel_packages | first }} register: node_baseline_backports_kernel_version changed_when: false check_mode: false # `linux-version` ships in linux-base and knows Debian's kernel version ordering, which # neither `sort -V` nor a Jinja sort gets right across +deb13 / ABI suffixes. The full # list is the ROLLBACK INVENTORY: every entry is still installed and selectable from # GRUB's "Advanced options" without uninstalling anything. - name: List every kernel image installed on this node, newest first ansible.builtin.shell: cmd: set -o pipefail; linux-version list | linux-version sort --reverse executable: /bin/bash register: node_baseline_installed_kernels changed_when: false check_mode: false # The node must be able to explain itself to an operator standing in front of it with no # repo checkout. Content is deliberately time-INVARIANT (no timestamp) so re-running the # role is a real no-op rather than a permanent daily "changed". - name: Record the deviation, the exact version, and the rollback path on the node ansible.builtin.copy: dest: "{{ node_baseline_backports_kernel_stamp }}" owner: root group: root mode: "0644" content: | # Managed by the Ansible node_baseline role (tags: kernel) — do not edit by hand. # # THIS NODE DELIBERATELY DOES NOT RUN THE DEBIAN 13 STOCK KERNEL. # # WHY # This machine's HP R21 firmware publishes a CORRUPT EFI Memory Attributes Table. # Linux < 6.13 reserves memory from it without validating its size, and an unlucky # boot reserves everything above the table: 2.6 GiB of 16 GB, observed twice on # k3s-prod-2. Upstream commit 8fbe4c49c0cc "efi/memattr: Ignore table if the size # is clearly bogus" (first released in v6.13) adds the validation; it is NOT in # v6.12 and NOT in linux-6.12.y. There is NO kernel cmdline workaround. # Full write-up: infra/PROD-PROVISIONING.md §0b in the home-network-k3s-impl repo. # # PROOF THE FIX IS ACTIVE (not merely a lucky boot) # BOOT-INDEPENDENT, and this is the one to trust — the string only exists in # >= 6.13, so its presence in the RUNNING image is proof regardless of the boot: # zstd -dc < <(tail -c +$(...)) ... | grep 'Corrupted EFI Memory Attributes' # (roles/node_health does the version check for you; see the health report's # firmware.efi_memattr finding.) # # FROM dmesg, WHICH IS A LOTTERY. efi_memattr_init() on >= 6.13 has THREE gates # and each prints a DIFFERENT line — all three are rejections: # tbl->version > 2 'Unexpected EFI Memory Attributes table version %d' # (NOTE: printed with %d, so garbage may read NEGATIVE) # bad desc_size 'Unexpected EFI Memory Attributes descriptor size ...' # num_entries > 64K '[Firmware Bug]: Corrupted EFI Memory Attributes # Table detected! (version == ..., num_entries == ...)' # Which one appears depends on WHICH field this boot's garbage corrupted, so do # NOT treat the absence of the FW_BUG line as the fix being absent — all three # prod nodes printed the *version* line on 2026-09-04. # The one line that means the fix is NOT active: # 'Failed to map EFI Memory Attributes table @ 0x...' — the kernel trusted the # corrupt table. On a >= 6.13 kernel this should be unreachable; if it appears, # escalate, do not shrug. # # INSTALLED # source /etc/apt/sources.list.d/{{ node_baseline_backports_suite }}.sources # package {{ node_baseline_backports_kernel_packages | join(' ') }} # version {{ node_baseline_backports_kernel_version.stdout | default('(not read)') }} # kernels installed on this node, newest first: {% for kv in (node_baseline_installed_kernels.stdout_lines | default([])) %} # {{ kv }} {% endfor %} # # SECURITY COST # Packages from *-backports are provided AS IS. They are NOT covered by the Debian # Security Team's guarantees for the main archive: no security-tracker commitment, # no DSAs. Accepted deliberately, in exchange for the node keeping its RAM. # # ROLLBACK — no repo checkout needed # 1. NON-DESTRUCTIVE, try this first. Every kernel listed above is still installed # and bootable: pick the 6.12 one from GRUB -> "Advanced options for Debian # GNU/Linux". That answers "is the new kernel the problem?" without # uninstalling anything. Note it reinstates the memattr bug for that boot. # 2. PERMANENT. Set node_baseline_backports_kernel: false in # inventory/group_vars/k3s_prod/vars.yml FIRST — otherwise the next Ansible run # puts the backports kernel straight back — then, on this node: # apt-get install --reinstall linux-image-amd64/{{ ansible_facts['distribution_release'] }} # apt-get purge # rm /etc/apt/sources.list.d/{{ node_baseline_backports_suite }}.sources # apt-get update && update-grub && reboot # # EXIT CONDITION # If 8fbe4c49c0cc is ever picked up into linux-6.12.y (and so into Debian's 6.12 # rebase), this deviation can be removed entirely: flip the variable to false, # roll back as above, and delete the `kernel` block from roles/node_baseline. # Say the pending reboot out loud. A green play here does NOT mean the node is fixed — # it means the fixed kernel is on disk. See the block header for why the role will not # reboot on its own. - name: Report whether this node is running the kernel it now has installed ansible.builtin.debug: msg: >- {{ ('REBOOT REQUIRED — running ' ~ ansible_facts['kernel'] ~ ', newest installed is ' ~ newest ~ '. The EFI memattr fix is NOT active until this node reboots. Reboot prod nodes ONE AT A TIME, verify each returns before touching the next, and NEVER node 2 first (infra/PROD-PROVISIONING.md §0b "SECOND FAULT"). Afterwards confirm with: dmesg | grep -i "Corrupted EFI Memory Attributes Table"') if newest != ansible_facts['kernel'] else ('Running the newest installed kernel (' ~ ansible_facts['kernel'] ~ '); no reboot pending for the memattr fix.') }} vars: newest: "{{ (node_baseline_installed_kernels.stdout_lines | default([]) | first) | default('', true) }}" - name: Persist kernel modules for k3s/Cilium/Longhorn ansible.builtin.copy: dest: /etc/modules-load.d/k3s-cilium-longhorn.conf owner: root group: root mode: "0644" content: "{{ node_baseline_kernel_modules | join('\n') }}\n" tags: [system] - name: Load kernel modules now community.general.modprobe: name: "{{ item }}" state: present loop: "{{ node_baseline_kernel_modules }}" tags: [system] - name: Install sysctl tuning for k3s/Cilium ansible.builtin.copy: dest: /etc/sysctl.d/99-k3s-cilium.conf owner: root group: root mode: "0644" content: | net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.ipv4.ip_forward = 1 net.ipv6.conf.all.forwarding = 1 fs.inotify.max_user_instances = 8192 fs.inotify.max_user_watches = 524288 notify: Reload sysctl tags: [system] - name: Enable and start iscsid (Longhorn prerequisite) ansible.builtin.systemd: name: iscsid enabled: true state: started tags: [system] # qemu-guest-agent binds to the virtio-serial channel /dev/virtio-ports/org.qemu.guest_agent.0, # which Proxmox attaches only when the VM's `agent` option is enabled. Our tofu keeps agent # DISABLED on purpose (enabling it makes `tofu apply` block up to 15m waiting for an agent the # stock Debian cloud image hasn't installed yet — bpg gotcha). So we ENABLE the unit for the day # a channel exists, but only START it when the channel is actually present — never fail the # baseline over a missing hypervisor channel. - name: Check whether the qemu guest-agent virtio channel is attached ansible.builtin.stat: path: /dev/virtio-ports/org.qemu.guest_agent.0 register: node_baseline_qga_channel tags: [system] - name: Enable qemu-guest-agent (start only when its virtio channel is present) ansible.builtin.systemd: name: qemu-guest-agent enabled: true state: "{{ 'started' if node_baseline_qga_channel.stat.exists else 'stopped' }}" tags: [system] # ===================================================================================== # WIRELESS — tag `wireless`: a cluster node gets exactly ONE path onto the network # ===================================================================================== # WHAT WENT WRONG (k3s-prod-3, reported by the operator 2026-09-04). The node was up on BOTH # eno1 (192.168.2.12 — the inventory address) and wlp1s0 (192.168.2.195, DHCP, the SAME /24). # The wifi lease carried a dhcpcd run with it, and dhcpcd's `20-resolv.conf` hook REWROTE # /etc/resolv.conf, undoing the resolvers this role had just written: the file read # "# Generated by dhcpcd from wlp1s0.dhcp, wlp1s0.dhcp6, wlp1s0.ra" with nameserver # 192.168.2.1 plus two Google IPv6 servers, timestamped THREE MINUTES AFTER the role's own # write (01:32 -> 01:35). The role had reported `changed` and green. That is exactly the # "successful run that fixed nothing" the `contended` branch of the DNS block below exists to # prevent, and the detection blind spot that let it through is closed there. # # WHY THE INTERFACE IS REMOVED RATHER THAN JUST REPORTED. Closing the detection hole alone # would have turned a silent revert into a loud failure — an improvement, but it leaves the # node with a second, DHCP-managed path onto the LAN, and on a k3s server that is three # distinct problems beyond DNS: # * a SECOND DEFAULT ROUTE, whose metric decides which source address the node uses; # * a SECOND ADDRESS ON THE SAME L2 — the segment where kube-vip floats 192.168.2.9 and # Cilium L2-announces 192.168.2.20-.30 (§4). One host answering ARP on one subnet from # two NICs is the exact shape neither mechanism can reason about; # * an UNMANAGED address: nothing in this repo knows about .195, and the DHCP server is # free to hand out a different one after any lease expiry. # None of that is a resolver bug, and none of it is fixed by writing resolv.conf harder. # # WHY THE DRIVER LAYER — the four alternatives, and why each is weaker. What has to hold is: # survives a REBOOT, survives a LEASE RENEWAL, and cannot be undone by a routine `ifup`. # Measured on the nodes themselves — all three are HP EliteDesk 800 G5 minis, eno1 is # `e1000e`, the wifi NIC is an Intel AX200 on `iwlwifi`, and the OS is Debian 13 trixie with # ifupdown + `networking.service`, NO NetworkManager, NO systemd-networkd, NO systemd-resolved: # `ip link set down` — NOT persistent. Any `ifup`, any hotplug event, any reboot # undoes it. It is what had already been done by hand on node 3, and the stale dhcpcd # /etc/resolv.conf outlived it — the link was down and the wrong resolvers were still # in the file, across a reboot. # mask the service that manages it — THERE IS NO SUCH SERVICE HERE. dhcpcd on these nodes # is `dhcpcd-base`: no dhcpcd.service exists at all (`systemctl is-enabled dhcpcd` = # not-found), it is spawned per-interface by ifupdown. The only unit that can bring # wireless up is `networking.service` — which ALSO owns eno1's static address, so # masking it would cut the one SSH path to a machine whose operator is not on site. # `rfkill` soft-block — weaker on every axis. The netdev still EXISTS, so ifupdown and # dhcpcd can still be pointed at it; persistence depends on systemd-rfkill restoring # saved state (the kernel's own `persistent` flag reads 0 on these devices, and the # `rfkill` CLI is not even installed); and one `rfkill unblock wlan` undoes it. # the BIOS `M.2 WLAN/BT` toggle — genuinely the deepest layer, and it reads `Enable` on all # three nodes (hp-bioscfg survey, §1b). NOT taken here: it needs a REBOOT to take # effect, and a prod reboot is a serialized, supervised operator step (§0b "SECOND # FAULT" — k3s-prod-2 hung on a warm reboot and needed the power button). It also # disables Bluetooth on the same M.2 card. Left as an operator recommendation in §4. # THE DRIVER MODULE — chosen. `blacklist` plus `install /bin/false` in # /etc/modprobe.d means kmod refuses to load the wifi driver on this boot and every # future one, so THE NETDEV NEVER EXISTS. No interface means no DHCP lease, no # resolv.conf hook, no second route, no second address — and every re-activation path # above (udev hotplug, a hand-run `ifup`, wpa_supplicant, a hand-run dhcpcd) needs a # device that is not there. `blacklist` alone would still allow a dependency- or # hand-triggered `modprobe`; the `install ... /bin/false` line is what makes it refuse # outright. It is ONE declarative file, readable on the node, and undone by deleting # it. Paired with the runtime unload below, it needs NO reboot to take effect. # # SAFE AND IDEMPOTENT ON ALL THREE PROD NODES *AND* ON THE LIVE DEV CLUSTER. Nothing here is # hostname- or hardware-specific: the block acts only on interfaces the node itself reports as # real 802.11 devices (a `phy80211` link in sysfs) and on the driver modules behind them. # * dev (Proxmox VMs, virtio NICs) has no such interface, so the discovery comes back empty, # every task below is skipped, and NOT EVEN THE modprobe.d FILE IS WRITTEN. The block # cannot touch dev's networking, which is why it is safe to have on by default. # * k3s-prod-1 (wlp1s0) and k3s-prod-2 (wlp2s0) already have the link DOWN, so the link-down # task is skipped by its own `when` and only the modprobe.d file and the module unload # remain — and both are no-ops on a re-run. # * k3s-prod-3 (wlp1s0) is the reported node. # A re-run changes nothing on any of them: the file is content-identical, `ip link set down` # is gated on the link not already being down, and the unload is gated on the module still # being loaded. # # AND IT REFUSES TO CUT ITS OWN PATH IN. The assert below stops the play if the address # Ansible is connected to is bound to a wireless interface, or if the default route leaves # over one, or if the wireless driver module is also the driver of the default-route # interface. That is the one case where doing this would lock everybody out of a machine # nobody can walk up to, so it is checked per node, on every run, from that node's own facts. - name: Survey this node's 802.11 devices, their drivers and any live DHCP client ansible.builtin.shell: executable: /bin/sh cmd: | set -u # Flat KEY|value lines so the Ansible side needs one single-group regex per fact and # no parsing cleverness. A wireless interface is one with a phy80211 link in sysfs — # that is the kernel's own answer to "is this 802.11", not a name pattern like wlp*. wifi='' for d in /sys/class/net/*; do [ -e "$d/phy80211" ] || continue i=$(basename "$d") wifi="$wifi $i" m=$(basename "$(readlink -f "$d/device/driver" 2>/dev/null || echo unknown)") state=$(cat "$d/operstate" 2>/dev/null || echo unknown) printf 'IFACE|%s\n' "$i" printf 'DETAIL|%s driver=%s operstate=%s\n' "$i" "$m" "$state" # LIVE means "not already down" — the link-down task keys off this, so a node whose # radio is already down reports no change. [ "$state" = down ] || printf 'IFACE_LIVE|%s\n' "$i" ip -o addr show dev "$i" 2>/dev/null | awk '{print "IFACE_ADDR|" $4}' [ "$m" = unknown ] || printf 'MODULE|%s\n' "$m" done for m in $(printf '%s\n' $wifi | while read -r i; do [ -n "$i" ] && basename "$(readlink -f "/sys/class/net/$i/device/driver" 2>/dev/null || echo unknown)" done | sort -u); do [ "$m" = unknown ] && continue [ -d "/sys/module/$m" ] || continue printf 'MODULE_LOADED|%s\n' "$m" # Holders must be unloaded FIRST: `modprobe -r iwlwifi` fails while iwlmvm still # holds it. Read from sysfs rather than assumed, so a different wifi driver stack # (ath*, rtw*, mt76*) works without editing this role. for h in $(ls "/sys/module/$m/holders" 2>/dev/null); do printf 'MODULE_HOLDER|%s\n' "$h"; done done dr=$(ip -o route show default 2>/dev/null | awk '{for (i = 1; i < NF; i++) if ($i == "dev") print $(i + 1)}' | head -1) printf 'DEFROUTE_IFACE|%s\n' "${dr:-none}" if [ -n "${dr:-}" ] && [ -e "/sys/class/net/$dr" ]; then printf 'DEFROUTE_MODULE|%s\n' "$(basename "$(readlink -f "/sys/class/net/$dr/device/driver" 2>/dev/null || echo unknown)")" fi # Any DHCP client process, with the interfaces it was started for. Used only to stop a # client that holds a WIRELESS interface and nothing else. for p in $(pgrep -x dhcpcd 2>/dev/null; pgrep -x dhclient 2>/dev/null); do printf 'DHCPCLIENT|%s\n' "$(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null)" done exit 0 register: node_baseline_wifi_scan changed_when: false check_mode: false when: node_baseline_disable_wireless | bool tags: [wireless] - name: Interpret the wireless survey ansible.builtin.set_fact: node_baseline_wifi_ifaces: "{{ scan | regex_findall('(?m)^IFACE\\|(.+)$') }}" node_baseline_wifi_live: "{{ scan | regex_findall('(?m)^IFACE_LIVE\\|(.+)$') }}" node_baseline_wifi_addrs: "{{ scan | regex_findall('(?m)^IFACE_ADDR\\|(\\S+)') | map('regex_replace', '/.*$', '') | list }}" node_baseline_wifi_modules: "{{ scan | regex_findall('(?m)^MODULE\\|(.+)$') | unique }}" node_baseline_wifi_modules_loaded: "{{ scan | regex_findall('(?m)^MODULE_LOADED\\|(.+)$') | unique }}" node_baseline_wifi_module_holders: "{{ scan | regex_findall('(?m)^MODULE_HOLDER\\|(.+)$') | unique }}" node_baseline_wifi_detail: "{{ scan | regex_findall('(?m)^DETAIL\\|(.+)$') }}" node_baseline_wifi_dhcp_clients: "{{ scan | regex_findall('(?m)^DHCPCLIENT\\|(.+)$') }}" node_baseline_defroute_iface: "{{ (scan | regex_findall('(?m)^DEFROUTE_IFACE\\|(.+)$') | first) | default('none', true) }}" node_baseline_defroute_module: "{{ (scan | regex_findall('(?m)^DEFROUTE_MODULE\\|(.+)$') | first) | default('unknown', true) }}" vars: scan: "{{ node_baseline_wifi_scan.stdout | default('') }}" when: node_baseline_disable_wireless | bool tags: [wireless] - name: Report this node's wireless hardware ansible.builtin.debug: msg: >- wireless devices = {{ node_baseline_wifi_detail | default([]) | join('; ') | default('(none)', true) }} | driver modules = {{ node_baseline_wifi_modules | default([]) | join(', ') | default('(none)', true) }} (loaded: {{ node_baseline_wifi_modules_loaded | default([]) | join(', ') | default('(none)', true) }}, held by: {{ node_baseline_wifi_module_holders | default([]) | join(', ') | default('(none)', true) }}) | default route via {{ node_baseline_defroute_iface | default('?') }} (driver {{ node_baseline_defroute_module | default('?') }}) when: node_baseline_disable_wireless | bool tags: [wireless] # The one way this block could do real harm is by removing the interface it is being run # over. Checked from the node's own facts, per node, every run — not assumed from the fact # that prod "should" be wired. - name: Refuse to disable wireless if the path to this node runs OVER the wireless interface ansible.builtin.assert: that: - node_baseline_defroute_iface not in node_baseline_wifi_ifaces - node_baseline_defroute_module not in node_baseline_wifi_modules - (ansible_host | default(inventory_hostname)) not in node_baseline_wifi_addrs fail_msg: >- REFUSING to disable wireless on this node: the connection to it appears to BE the wireless path. ansible_host={{ ansible_host | default(inventory_hostname) }}, wireless interfaces={{ node_baseline_wifi_ifaces | join(', ') }} carrying {{ node_baseline_wifi_addrs | join(', ') | default('no addresses', true) }}, default route via {{ node_baseline_defroute_iface }} (driver {{ node_baseline_defroute_module }}). Disabling the driver here would drop this node off the network with no remote way back. Give the node a wired address, point the inventory at it, and re-run. quiet: true when: - node_baseline_disable_wireless | bool - node_baseline_wifi_ifaces | length > 0 tags: [wireless] # THE PERSISTENT HALF. This file is the whole reboot story: kmod consults /etc/modprobe.d on # every load attempt, so from here on the driver cannot be loaded by udev at boot, as a # dependency, or by hand. Written ONLY when the node actually has a wireless driver, so a # node without wifi hardware (every dev VM) gets no file and no change at all. # `update-initramfs` is deliberately NOT run: no wifi module is loaded from the initramfs on # these nodes (root is a local SATA/NVMe device), the driver is autoloaded by udev after # switch-root, and re-generating an initramfs on a prod node is a bigger risk than the one it # would cover. - name: Block the wireless driver module(s) from ever loading again ansible.builtin.copy: dest: "{{ node_baseline_wireless_modprobe_file }}" owner: root group: root mode: "0644" content: | # Managed by the Ansible node_baseline role (tags: wireless) — do not edit by hand. # # WHY THIS NODE HAS NO WIRELESS. A k3s server gets exactly ONE path onto the network. # A second, DHCP-managed interface on the same LAN gives the node a second default # route, a second address nothing in the cluster's config knows about, and a DHCP # client whose resolv.conf hook silently overwrites the resolvers Ansible manages # (that is what happened on k3s-prod-3, 2026-09-04). The link is also brought down and # the module unloaded at the same time, so no reboot is needed for this to take effect. # # `blacklist` stops udev's alias-driven autoload; the `install ... /bin/false` line is # what also refuses a dependency-triggered or hand-run `modprobe`. Both are needed. # # TO RE-ENABLE WIRELESS ON THIS NODE: delete this file and `modprobe` the module(s) # below (or reboot). To stop Ansible putting it back, set # node_baseline_disable_wireless: false for this host first — otherwise the next # node-baseline run recreates it. Full rationale: roles/node_baseline/tasks/main.yml, # the `wireless` block, and infra/PROD-PROVISIONING.md §4. {% for module in node_baseline_wifi_modules %} blacklist {{ module }} install {{ module }} /bin/false {% endfor %} when: - node_baseline_disable_wireless | bool - node_baseline_wifi_modules | length > 0 tags: [wireless] # THE RUNTIME HALF — everything below makes the change true NOW, so the node does not have to # be rebooted to stop carrying a second address (this role never reboots: §0b "SECOND FAULT"). # # Stop a DHCP client that is holding a wireless interface first, so it releases its lease # instead of being left holding a device that is about to disappear. HEAVILY GUARDED: it runs # only for clients whose command line names a wireless interface and names NO non-wireless # interface, because `dhcpcd -x` against a shared master process would also drop a wired # lease. On these prod nodes eno1 is a static ifupdown address with no DHCP client at all, so # this task has nothing to do; it exists so the ordering is correct on a node where wifi is # still actively leasing. - name: Stop the dhcpcd instance that holds a wireless interface (and nothing else) ansible.builtin.command: argv: [dhcpcd, --exit, "{{ item }}"] loop: "{{ node_baseline_wifi_ifaces }}" register: node_baseline_wifi_dhcpcd_exit changed_when: rc == 0 failed_when: false when: - node_baseline_disable_wireless | bool - node_baseline_wifi_dhcp_wifi_only | length > 0 - node_baseline_wifi_dhcp_wifi_only | select('search', 'dhcpcd') | list | length > 0 vars: # A client is "wireless-only" when its argv mentions a wireless interface and mentions no # other interface this node has. Empty interface sets are turned into a pattern that # cannot match, so an odd node fails towards doing nothing rather than towards guessing. node_baseline_non_wifi_ifaces: "{{ ansible_facts['interfaces'] | default([]) | difference(node_baseline_wifi_ifaces) | difference(['lo']) }}" node_baseline_wifi_dhcp_wifi_only: >- {{ node_baseline_wifi_dhcp_clients | select('search', '\b(' ~ ((node_baseline_wifi_ifaces | join('|')) | default('__no_such_interface__', true)) ~ ')\b') | reject('search', '\b(' ~ ((node_baseline_non_wifi_ifaces | join('|')) | default('__no_such_interface__', true)) ~ ')\b') | list }} tags: [wireless] - name: Take each wireless link down now ansible.builtin.command: argv: [ip, link, set, dev, "{{ item }}", down] loop: "{{ node_baseline_wifi_live }}" changed_when: true when: node_baseline_disable_wireless | bool tags: [wireless] # Unload the driver so the netdev is gone immediately — holders (iwlmvm on these nodes) first, # then the driver itself, because `modprobe -r` refuses a module that is still held. # NOT fatal on failure: the persistent block above is already in place, so a module that # cannot be unloaded right now is gone at the next boot regardless. The report below says so # out loud rather than letting a green play imply the radio is already off. - name: Unload the wireless driver module(s) now (no reboot needed) community.general.modprobe: name: "{{ item }}" state: absent loop: "{{ node_baseline_wifi_module_holders + node_baseline_wifi_modules_loaded }}" register: node_baseline_wifi_unload failed_when: false when: node_baseline_disable_wireless | bool tags: [wireless] - name: Report the wireless state this node is left in ansible.builtin.debug: msg: >- {{ 'No 802.11 device on this node — nothing to disable (no modprobe.d file written).' if (node_baseline_wifi_ifaces | length) == 0 else ((('WIRELESS DISABLED: ' ~ (node_baseline_wifi_modules | join(', ')) ~ ' blocked in ' ~ node_baseline_wireless_modprobe_file ~ ' (survives reboot) and unloaded now; ' ~ (node_baseline_wifi_ifaces | join(', ')) ~ ' is down and the netdev is gone.') if (failed_unloads | length) == 0 else ('WIRELESS BLOCKED FOR THE NEXT BOOT BUT STILL LOADED NOW: could not unload ' ~ (failed_unloads | join(', ')) ~ '. ' ~ node_baseline_wireless_modprobe_file ~ ' is in place, so the driver will not load again after a reboot, but the ' ~ 'radio is live until then — check `ip link` and whether something still ' ~ 'holds the device.'))) }} vars: failed_unloads: >- {{ (node_baseline_wifi_unload.results | default([])) | selectattr('failed', 'defined') | selectattr('failed') | map(attribute='item') | list }} when: node_baseline_disable_wireless | bool tags: [wireless] # ===================================================================================== # DATA DISKS — tag `disks`, in tasks/disks.yml # ===================================================================================== # Split into its own file for two reasons. It is the only DESTRUCTIVE part of the role # (prod re-purposes the partition already mounted at /srv as the Longhorn store), so it # should be readable end-to-end on its own; and both environments' storage shapes — dev's # two dedicated virtio disks, prod's single partitioned physical disk with etcd left on # the OS filesystem — are declared there as an explicit enum rather than inferred from # which variables happen to be set. See tasks/disks.yml and infra/PROD-PROVISIONING.md §2. # # The `disks` tag is applied ONCE here, in BOTH places it is needed: # * `tags:` on the include itself, so `--tags packages` never even reaches the include # and a placeholder device path is not so much as templated; and # * `apply: tags:`, so the tag reaches the tasks INSIDE the file. This half is not # optional and not decoration — a DYNAMIC include (`include_tasks`) does NOT propagate # its own tags to its children the way `import_tasks` does, so without `apply` the # included tasks are untagged and `--tags disks` selects the include and then runs # NOTHING inside it. That failure is silent and reports success, which is the worst # shape a guard can have. # A FULL run against a node whose layout is undeclared stops on the assert at the top of # that file instead of failing obscurely at mkfs time. - name: Provision the data disks (Longhorn store, etcd data dir) ansible.builtin.include_tasks: file: disks.yml apply: tags: [disks] tags: [disks] # ===================================================================================== # SWAP — off NOW, and still off AFTER A REBOOT — tag `system` # ===================================================================================== # `swapoff -a` plus commenting the fstab line is NOT durable on these nodes, and never # was. Measured 2026-09-07: all three prod nodes were running with 977 MiB of swap # ACTIVE (k3s-prod-1 /dev/sda4, k3s-prod-3 /dev/sda4, k3s-prod-2 /dev/nvme0n1p4) while # their fstab swap line was commented out by this very role. The role reported success # and the setting silently expired at the next boot, on every node. # # CAUSE. systemd-gpt-auto-generator (systemd 257, Debian 13) claims any GPT partition # whose partition TYPE GUID is 0657fd6d-a4ab-43c4-84e5-0933c84b4f4f ("Linux swap"), # synthesises a .swap unit for it into /run/systemd/generator.late/ and drops a symlink # to it into swap.target.wants/. It decides that from the PARTITION TABLE and never # consults /etc/fstab, so neutralising fstab cannot stop it. Re-running the generator by # hand on k3s-prod-1 says so in as many words: # Dissecting swap partition with label n/a and UUID acd3876c-fd8e-4389-b856-... # Probed fstype 'swap' on partition /dev/disk/by-diskseq/1-part4. # Adding swap: /dev/disk/by-diskseq/1-part4 # # MECHANISM — mask swap.target. # `systemctl show` on the generated unit says its ONLY inbound dependency is # `WantedBy=swap.target`: Requires/RequiredBy/BoundBy are empty, and the matching # .device unit has an empty `Wants=`, so a device appearing does NOT pull its swap unit # in by itself. swap.target is therefore the single choke point that every swap # activation path goes through — the gpt-auto units, the units systemd-fstab-generator # builds from /etc/fstab, and the ones systemd-cryptsetup-generator builds for encrypted # swap. Masking it is: # * DEVICE-SHAPE INDEPENDENT. No device name appears anywhere in the change, so one # task is correct on prod's sda4 AND nvme0n1p4 and on dev's vda VMs, and there is # nothing to keep in sync when a disk is replaced. (The generated unit is named after # the kernel's DISKSEQ, not the disk — it is `dev-disk-by\x2ddiskseq-1\x2dpart4.swap` # on all three prod nodes, NVMe and SATA alike — but diskseq is assigned per boot in # device-probe order, so it is not a name worth depending on either.) # * BOOT-SAFE. swap.target is reached by `Wants=`/`After=` from sysinit.target and by # `After=` from tmp.mount, and `RequiredBy=` is EMPTY (verified on all three prod # nodes and all three dev nodes). A masked unit reached only through Wants= is # skipped with a log line; it cannot fail the boot transaction. Ordering deps on a # unit that never starts are no-ops. # * INERT WHEN THE DEVICE IS ABSENT OR RENAMED. The mask names no device, so there is # nothing to go stale: a disk swap, a new partition layout or a restore onto # different hardware all stay swap-free without touching this role. # The per-unit masks that follow are a SECOND, independent barrier over the units that # exist right now, so `systemctl status ` says "masked" to whoever looks, and an # operator who unmasks swap.target does not silently get swap back. Their failure mode is # benign in the other direction too: a mask whose unit no longer exists is an unreferenced # symlink to /dev/null, and the only thing it could ever shadow is a .swap unit. # # ALTERNATIVES REJECTED. # * `systemd.gpt_auto=0` on the kernel cmdline — disables GPT auto-discovery WHOLESALE, # not just swap, and needs a GRUB edit + update-grub + a reboot before it is even # true. On prod that happens to be swap-only today (root= is already on the cmdline # so root auto-detection is off, and the /boot/efi fstab entry already suppresses the # ESP/XBOOTLDR mounts — the generator says so in its debug output), but on DEV the # root partition carries the discoverable GUID 4f68bce3-e8cd-4db1-96e7-fbcaf984b709 # ("Linux root (x86-64)"), so the same flag would switch off root auto-discovery # there. Editing a bootloader to fix a swap unit is the wrong blast radius, and the # change would not be verifiable without the reboot it depends on. # * Clearing or retyping the swap partition's GPT type GUID — a partition-table write # to the live boot disk of a node currently serving etcd and Longhorn replicas, to # fix something a symlink fixes. It would also decide, irreversibly and without # asking, that those 977 MiB are never swap again and are simply unused; that is the # operator's call to make, not this role's, and it needs a maintenance window. # * `systemd.swap=0` — read by systemd-fstab-generator ONLY. The swap here does not # come from fstab, so it would change nothing at all. # # DEV. Dev's nodes have no swap partition, no swap unit and no fstab swap line at all # (checked on k3s-dev-1/2/3), so the mask is inert there: it flips to masked once and # every later run is changed=0. It is correct rather than merely harmless — if a future # dev image ever ships a swap partition, the same mask already covers it. - name: Mask swap.target so no generator can pull a swap unit in at boot ansible.builtin.systemd_service: name: swap.target masked: true notify: Reload systemd unit files tags: [system] - name: Find the .swap units systemd's generators produced for this boot ansible.builtin.find: # gpt-auto writes to generator.late; systemd-fstab-generator and # systemd-cryptsetup-generator (the other two that can emit a .swap unit) write to # generator. /run/systemd/generator.early is deliberately NOT listed: nothing that # makes swap units runs at early priority, and `find` warns on a path that does not # exist. Anything a future early generator invented is still caught by the # swap.target mask above, which is the mechanism this task only reinforces. paths: - /run/systemd/generator - /run/systemd/generator.late patterns: "*.swap" file_type: file register: node_baseline_generated_swap changed_when: false check_mode: false tags: [system] - name: Mask each generated .swap unit (second barrier, independent of swap.target) ansible.builtin.systemd_service: name: "{{ item }}" masked: true loop: "{{ node_baseline_generated_swap.files | map(attribute='path') | map('basename') | unique | sort }}" loop_control: label: "{{ item }}" notify: Reload systemd unit files tags: [system] - name: Neutralize swap entries in fstab (kubelet requires swap off) ansible.builtin.replace: path: /etc/fstab regexp: '^([^#].*\sswap\s.*)$' replace: '# \1 # disabled for k3s/kubelet' tags: [system] # The masks above only decide what the NEXT boot does. Anything swapped on already stays # on until it is told otherwise, so read the live state and act only when there is # something to act on — that is what makes a second run honestly changed=0 instead of # `changed_when: false` hiding a command that ran anyway. - name: Read the swap devices that are active right now ansible.builtin.command: swapon --noheadings --show=NAME register: node_baseline_swap_active changed_when: false check_mode: false tags: [system] - name: Disable swap now ansible.builtin.command: swapoff -a when: node_baseline_swap_active.stdout | trim | length > 0 changed_when: true tags: [system] # ===================================================================================== # DNS RESILIENCE — tag `dns` (R-DNS-01 / R-DNS-03) # ===================================================================================== # Root cause (verified on dev): the nodes carried a SINGLE upstream resolver. PiHole # (192.168.1.18) answers `*.jackson.lan` instantly but its external recursion flaps — # 0/25 for registry-1.docker.io / quay.io / ghcr.io while gitea.jackson.lan resolved 5/5. # One upstream is a hard SPOF: containerd image pulls died with "lookup # registry-1.docker.io: Try again", and CoreDNS (`forward . /etc/resolv.conf`) flooded its # log with "i/o timeout" to 192.168.1.18. The fix is ADDITIVE — PiHole stays first so the # local zone and the ad-blocking policy are untouched; a second upstream is only consulted # when the first fails. See defaults/main.yml for the ordering rule. # # WHY THIS DETECTS INSTEAD OF ASSUMING. An earlier attempt (fix/node-dns-resilient-upstream, # dropped) wrote /etc/netplan/99-k3s-dns.yaml and ran `netplan apply`. netplan is NOT # installed on these nodes — they run the Debian 13 trixie generic cloud image # (infra/tofu/variables.tf) and netplan is not in node_baseline_packages — so that handler # would have failed command-not-found and broken node-baseline.yml on EVERY node. Worse, # the two environments do not even share a resolver stack: dev's /etc/resolv.conf is the # systemd-resolved stub (127.0.0.53) while prod's is a directly-managed file listing # 192.168.2.1 (it has no cloud-init at all — bare metal). So the mechanism is DETECTED per # node and the matching seam is written. Adding a third mechanism means adding a branch # here, not rewriting the role. # # AND DETECTION IS ONLY AS GOOD AS THE SET OF OWNERS IT KNOWS ABOUT. It originally looked for # resolvconf/openresolv and NetworkManager, missed dhcpcd, and therefore reported a green, # `changed` run on k3s-prod-3 whose resolvers a wifi lease reverted three minutes later. dhcpcd # is now part of that set — see the detection task below and the `contended` failure. # # All of it is `dns`-tagged, so `--tags packages` (the prod package-floor escape hatch # documented at the top of this file) still installs nothing but packages. - name: Determine whether systemd-resolved is running on this node ansible.builtin.command: systemctl is-active systemd-resolved register: node_baseline_resolved_active changed_when: false failed_when: false check_mode: false tags: [dns] - name: Determine whether NetworkManager is running (it rewrites /etc/resolv.conf) ansible.builtin.command: systemctl is-active NetworkManager register: node_baseline_nm_active changed_when: false failed_when: false check_mode: false tags: [dns] # THE dhcpcd BLIND SPOT, CLOSED (k3s-prod-3, 2026-09-04). dhcpcd's `20-resolv.conf` hook # rewrites /etc/resolv.conf on every lease, renewal and carrier change, and it is a THIRD # way to own that file — neither resolvconf nor NetworkManager. On node 3 it silently reverted # this role's write three minutes later, and because nothing below looked for it the role had # already reported `changed` and green: the precise failure the `contended` branch exists to # make impossible. # # WHY THIS LOOKS FOR A LIVE CLIENT AND NOT FOR AN INSTALLED PACKAGE. `dhcpcd-base` is # installed on all three prod nodes and always will be — it is the DHCP client ifupdown # reaches for — and there is NO dhcpcd.service on them at all (`systemctl is-active dhcpcd` = # not-found; it is spawned per-interface by ifupdown). Keying off the package, or off a unit # that does not exist, would put every prod node permanently in `contended` and leave the DNS # block unable to configure a node it is perfectly safe to configure. What actually matters is # whether a DHCP client CAN still write the file, so all three real signals are checked: # CLIENT| a dhcpcd/dhclient process running right now (with the argv that started it); # UNIT| a dhcpcd/dhclient unit that IS active or IS enabled, on a distro that has one; # STANZA| an UNCOMMENTED `iface inet[6] dhcp` in /etc/network/interfaces{,.d} — the # configuration that makes ifupdown spawn a client at the next boot or `ifup`, even # when none is running at this instant. # Any one of them means "another manager is in play" and the play stops below. None of them # means no client can run, and the file is genuinely ours to own. - name: Determine whether a DHCP client (dhcpcd/dhclient) can rewrite /etc/resolv.conf ansible.builtin.shell: executable: /bin/sh cmd: | set -u for p in $(pgrep -x dhcpcd 2>/dev/null; pgrep -x dhclient 2>/dev/null); do printf 'CLIENT|%s\n' "$(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null)" done for u in dhcpcd dhcpcd5 dhclient isc-dhcp-client; do a=$(systemctl is-active "$u" 2>/dev/null || true) e=$(systemctl is-enabled "$u" 2>/dev/null || true) # Compared exactly, never with a substring match: `is-active` answers "inactive", # which contains the word "active". if [ "$a" = active ] || [ "$e" = enabled ] || [ "$e" = enabled-runtime ]; then printf 'UNIT|%s active=%s enabled=%s\n' "$u" "$a" "$e" fi done # `^[[:space:]]*iface` cannot match a commented-out stanza, which is how the wireless # stanza is left on these nodes. grep -rhE '^[[:space:]]*iface[[:space:]]+[^[:space:]]+[[:space:]]+inet6?[[:space:]]+dhcp' \ /etc/network/interfaces /etc/network/interfaces.d/ 2>/dev/null \ | sed -e 's/^[[:space:]]*/STANZA|/' exit 0 register: node_baseline_dhcp_client_scan changed_when: false check_mode: false tags: [dns] # follow: false — we need to know whether /etc/resolv.conf IS a symlink and where it # points, which is the strongest single signal of who owns the file. - name: Inspect /etc/resolv.conf itself (symlink or real file, and its target) ansible.builtin.stat: path: /etc/resolv.conf follow: false register: node_baseline_resolv_stat tags: [dns] - name: Read the current /etc/resolv.conf contents ansible.builtin.slurp: src: /etc/resolv.conf register: node_baseline_resolv_slurp failed_when: false tags: [dns] # resolvconf/openresolv keeps the authoritative copy under here and regenerates # /etc/resolv.conf from it, so writing /etc/resolv.conf directly would be silently undone. - name: Detect a resolvconf/openresolv installation that would clobber /etc/resolv.conf ansible.builtin.stat: path: /etc/resolvconf/resolv.conf.d register: node_baseline_resolvconf_dir tags: [dns] # THE DETECTION. Three outcomes, in priority order: # systemd-resolved — resolved is running AND it owns /etc/resolv.conf, proven either by # the symlink pointing into /run/systemd/resolve/ or by the file containing only # resolved's loopback stub (127.0.0.53 / 127.0.0.54). Writing a resolved drop-in is # then the ONLY thing that changes real behaviour; writing resolv.conf would be # overwritten by resolved (or, worse, land in /run and vanish at reboot). # contended — nobody's stub, but resolvconf, NetworkManager or a DHCP client (dhcpcd / # dhclient) is present/running/configured and would regenerate the file underneath us. # We do NOT write. The role fails loudly below rather than leaving an operator # believing a change stuck when it did not. # resolv-conf-file — a plain, directly-managed /etc/resolv.conf (prod's shape today). # Safe to own outright. # resolved running but NOT owning resolv.conf correctly falls through to the file branch: # in that state the drop-in would change nothing glibc can see. - name: Detect which resolver mechanism this node actually uses ansible.builtin.set_fact: node_baseline_dns_mechanism: >- {{ 'systemd-resolved' if (node_baseline_resolved_running and node_baseline_resolved_owns_resolv) else ('contended' if (node_baseline_nm_running or node_baseline_resolvconf_present or node_baseline_dhcp_client_live) else 'resolv-conf-file') }} # Kept as facts (not just task vars) so the failure message, the report and the # reclaim note below all quote the SAME evidence this decision was made on. node_baseline_dhcp_client_evidence: "{{ (node_baseline_dhcp_client_scan.stdout | default('')) | regex_findall('(?m)^(?:CLIENT|UNIT|STANZA)\\|(.+)$') }}" # A file dhcpcd WROTE, versus a dhcpcd that can still write it — two different facts, and # only the second one is contention. See the reclaim note below. node_baseline_resolv_dhcpcd_written: "{{ 'Generated by dhcpcd' in (node_baseline_resolv_slurp.content | default('') | b64decode) }}" vars: node_baseline_resolved_running: "{{ (node_baseline_resolved_active.stdout | default('') | trim) == 'active' }}" node_baseline_nm_running: "{{ (node_baseline_nm_active.stdout | default('') | trim) == 'active' }}" node_baseline_resolvconf_present: "{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}" node_baseline_dhcp_client_live: "{{ (node_baseline_dhcp_client_scan.stdout | default('')) is search('(?m)^(?:CLIENT|UNIT|STANZA)\\|') }}" node_baseline_resolv_text: "{{ (node_baseline_resolv_slurp.content | default('') | b64decode) }}" node_baseline_resolv_ns: "{{ node_baseline_resolv_text | regex_findall('(?m)^\\s*nameserver\\s+(\\S+)') }}" node_baseline_resolved_owns_resolv: >- {{ ('/run/systemd/resolve/' in (node_baseline_resolv_stat.stat.lnk_source | default(''))) or ((node_baseline_resolv_ns | length) > 0 and (node_baseline_resolv_ns | reject('in', ['127.0.0.53', '127.0.0.54']) | list | length) == 0) }} tags: [dns] - name: Report the detected resolver mechanism ansible.builtin.debug: msg: >- resolver mechanism = {{ node_baseline_dns_mechanism }} (systemd-resolved is-active={{ node_baseline_resolved_active.stdout | default('') | trim }}, NetworkManager is-active={{ node_baseline_nm_active.stdout | default('') | trim }}, /etc/resolv.conf islnk={{ node_baseline_resolv_stat.stat.islnk | default(false) }} target={{ node_baseline_resolv_stat.stat.lnk_source | default('(not a symlink)') }}, resolvconf dir={{ node_baseline_resolvconf_dir.stat.exists | default(false) }}, DHCP client evidence={{ node_baseline_dhcp_client_evidence | default([]) | join(' ; ') | default('(none)', true) }}, resolv.conf written by dhcpcd={{ node_baseline_resolv_dhcpcd_written | default(false) }}) tags: [dns] # Same shape, and the same reason, as the data-disk assert above: this environment's # upstream list is an OPERATOR DECISION, and a placeholder that fails LOUDLY beats both a # guess and a silent inheritance of dev's PiHole. group_vars/k3s_prod/vars.yml deliberately # carries a BLOCKED second entry (see its comments and PROD-PROVISIONING.md §4): prod's # 192.168.2.1 is currently gateway AND only upstream, and choosing what sits behind it is a # network decision this role must not make. Running against prod stops HERE, before # anything is written. - name: Assert this environment's DNS upstream list has actually been decided ansible.builtin.assert: that: - node_baseline_dns_servers | length >= 2 - "'BLOCKED' not in (node_baseline_dns_servers | join(' '))" fail_msg: >- node_baseline_dns_servers is not a decided, redundant upstream list for this host's environment (got: {{ node_baseline_dns_servers | join(', ') }}). A single upstream is the exact SPOF this task exists to remove — it already broke containerd image pulls and CoreDNS forwarding on dev — so the role will not configure one, and it will NOT guess a second server: a public resolver would bypass the local zone and the ad-blocking policy, which is a decision, not a default. Decide the second upstream, record it in infra/PROD-PROVISIONING.md §4, and set node_baseline_dns_servers in inventory/group_vars/k3s_prod/vars.yml in the SAME change. To install only the package floor meanwhile, re-run with `--tags packages`, which skips every DNS task. quiet: true tags: [dns] # The `contended` outcome. Not a warning: an operator who is told "changed" while another # daemon quietly reverts the file is worse off than one who is told nothing happened. - name: Refuse to manage /etc/resolv.conf while another resolver manager owns it ansible.builtin.fail: msg: >- /etc/resolv.conf on this node is not the systemd-resolved stub, but something else would regenerate the file underneath this role: resolvconf/openresolv (/etc/resolvconf/resolv.conf.d present: {{ node_baseline_resolvconf_dir.stat.exists | default(false) }}), NetworkManager (is-active: {{ node_baseline_nm_active.stdout | default('') | trim }}), or a DHCP client that can still write it — dhcpcd/dhclient evidence: {{ node_baseline_dhcp_client_evidence | default([]) | join(' ; ') | default('(none)', true) }}. Writing it here would look like it worked and silently revert — which is exactly what happened on k3s-prod-3 on 2026-09-04, where a wifi lease's dhcpcd hook reverted this role's resolvers three minutes after a green run. If the evidence above is a DHCP client on a WIRELESS interface, the fix is the `wireless` block earlier in this role (a cluster node gets one network path) — run it, then re-run this. If it is a DHCP client on the node's real interface, or another manager entirely, add the matching branch to roles/node_baseline for the manager this node really uses — do not force it. when: node_baseline_dns_mechanism == 'contended' tags: [dns] # THE CORPSE CASE, and why it is NOT a hole in the fail-closed rule above. After the # `wireless` block disables the radio, k3s-prod-3 is left with a /etc/resolv.conf that dhcpcd # WROTE (its "# Generated by dhcpcd from wlp1s0.dhcp..." header, the gateway as sole IPv4 # resolver, two Google IPv6 servers) and NO dhcpcd that can ever run again. `contended` means # "another manager is live and will undo this write"; a file left behind by a client that no # longer exists is not a manager, it is the damage — and refusing to write here would leave the # node permanently mis-resolved, which is the very outcome the operator reported. So the file # branch below reclaims it, and says so. - name: Note that a dhcpcd-written /etc/resolv.conf is being reclaimed ansible.builtin.debug: msg: >- /etc/resolv.conf on this node was written by dhcpcd, but no DHCP client can write it any more (no process, no active/enabled unit, no `inet dhcp` stanza). Taking ownership and replacing it with the declared upstreams ({{ node_baseline_dns_servers | join(', ') }}). If a DHCP client ever comes back on this node, the `contended` check above will stop the play instead of letting it revert silently. when: - node_baseline_dns_mechanism == 'resolv-conf-file' - node_baseline_resolv_dhcpcd_written | default(false) tags: [dns] # --- Mechanism A: systemd-resolved (dev's shape today) ------------------------------- # A drop-in, never an edit of /etc/systemd/resolved.conf, so a package upgrade of that file # never conflicts. `DNS=` is one ORDERED list inside a single resolved scope: resolved uses # the first server and only rotates to the next when it fails, which is exactly the # PiHole-primary/fallback behaviour wanted — it is NOT a race between two servers, so a # gateway NXDOMAIN can never beat PiHole to a jackson.lan answer while PiHole is healthy. - name: Ensure the systemd-resolved drop-in directory exists ansible.builtin.file: path: /etc/systemd/resolved.conf.d state: directory owner: root group: root mode: "0755" when: node_baseline_dns_mechanism == 'systemd-resolved' tags: [dns] - name: Configure resilient DNS upstreams via a systemd-resolved drop-in ansible.builtin.copy: dest: /etc/systemd/resolved.conf.d/99-k3s-dns.conf owner: root group: root mode: "0644" content: | # Managed by the Ansible node_baseline role (tags: dns) — do not edit by hand. # Ordered upstream list: the FIRST server is the local-zone/ad-blocking resolver and # stays authoritative; the rest exist only so external resolution (containerd image # pulls, CoreDNS `forward . /etc/resolv.conf`) survives its recursion flapping. [Resolve] DNS={{ node_baseline_dns_servers | join(' ') }} Domains={{ node_baseline_dns_search | join(' ') }} # Reset to empty deliberately. systemd ships compiled-in public fallback resolvers # (Cloudflare/Google) that resolved would reach for if the list above ever came up # empty — which would silently bypass BOTH the local zone and the network's # ad-blocking. This fix adds redundancy; it never adds a bypass. FallbackDNS= notify: Restart systemd-resolved when: node_baseline_dns_mechanism == 'systemd-resolved' tags: [dns] # --- Mechanism B: a directly-managed /etc/resolv.conf (prod's shape today) ----------- # Only reached when nothing else owns the file (the `contended` branch above already # stopped the play otherwise). glibc walks `nameserver` lines in order and falls through # on timeout, so ordering carries the same policy as the resolved list. # NOTE this REPLACES the file: any nameserver currently there and NOT in # node_baseline_dns_servers is dropped. That is the point — the list in group_vars is the # whole declared truth — so an environment that wants its existing IPv6 upstreams kept # must list them there too. `backup: true` leaves the pre-role copy beside it the first # time it changes, because this file is load-bearing for reaching the node at all. - name: Configure resilient DNS upstreams directly in /etc/resolv.conf ansible.builtin.copy: dest: /etc/resolv.conf owner: root group: root mode: "0644" backup: true content: | # Managed by the Ansible node_baseline role (tags: dns) — do not edit by hand. # Ordered upstream list: the FIRST server is the local-zone resolver and stays # authoritative; the rest exist only so external resolution (containerd image pulls, # CoreDNS `forward . /etc/resolv.conf`) survives its recursion flapping. {% if node_baseline_dns_search | length > 0 %} search {{ node_baseline_dns_search | join(' ') }} {% endif %} {% for server in node_baseline_dns_servers %} nameserver {{ server }} {% endfor %} {% if node_baseline_dns_options | length > 0 %} options {{ node_baseline_dns_options | join(' ') }} {% endif %} when: node_baseline_dns_mechanism == 'resolv-conf-file' tags: [dns] # /etc/hosts pins — tag `dns` (and `hosts` on its own). Some node processes must reach a # public hostname at the in-cluster Gateway, not at its public answer. On prod, the # kube-apiserver's OIDC issuer resolves publicly to the WAN IP, which the router does not # hairpin. glibc reads /etc/hosts before any nameserver (nsswitch `files dns`), so a pin wins # regardless of which resolver mechanism above is in effect. One marked block: entries # dropped from node_baseline_hosts_pins disappear from the node, and an empty list # removes the block. - name: Pin node-process hostnames in /etc/hosts ansible.builtin.blockinfile: path: /etc/hosts marker: "# {mark} node_baseline hosts pins (Ansible, tags: hosts)" block: | {% for pin in node_baseline_hosts_pins %} {{ pin.ip }} {{ pin.names | join(' ') }} {% endfor %} state: "{{ 'present' if node_baseline_hosts_pins | length > 0 else 'absent' }}" backup: true tags: [dns, hosts]