--- # TAGS — every task here carries one of `packages`, `system`, `disks` or `dns`. An untagged # run is unchanged (all four run, in this order). The split exists because the PACKAGE floor # and the DATA-DISK work have different readiness: packages are safe on any node from # first contact, while the disk tasks need block devices that a new environment may not # have decided yet (prod: infra/PROD-PROVISIONING.md §2, whose decision is itself blocked # on a SMART read that needs `smartmontools` — which ships in node_baseline_packages). # `--tags packages` breaks that circle: it installs the package floor and NOTHING else. # ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags packages # Skipping is the guard: with `--tags packages` no `disks`-tagged task is selected, so a # BLOCKED-pending-disk-decision device path is never even templated, let alone handed to # mkfs. A FULL run against such a node stops at the assert below instead. `dns` (the block # at the bottom of this file) works the same way and guards its own BLOCKED placeholder: # ansible-playbook playbooks/node-baseline.yml --tags dns - name: Install baseline packages (guest agent, Longhorn/iscsi/nfs prereqs) ansible.builtin.apt: name: "{{ node_baseline_packages }}" state: present update_cache: true cache_valid_time: 3600 tags: [packages] - name: Persist kernel modules for k3s/Cilium/Longhorn ansible.builtin.copy: dest: /etc/modules-load.d/k3s-cilium-longhorn.conf owner: root group: root mode: "0644" content: "{{ node_baseline_kernel_modules | join('\n') }}\n" tags: [system] - name: Load kernel modules now community.general.modprobe: name: "{{ item }}" state: present loop: "{{ node_baseline_kernel_modules }}" tags: [system] - name: Install sysctl tuning for k3s/Cilium ansible.builtin.copy: dest: /etc/sysctl.d/99-k3s-cilium.conf owner: root group: root mode: "0644" content: | net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.ipv4.ip_forward = 1 net.ipv6.conf.all.forwarding = 1 fs.inotify.max_user_instances = 8192 fs.inotify.max_user_watches = 524288 notify: Reload sysctl tags: [system] - name: Enable and start iscsid (Longhorn prerequisite) ansible.builtin.systemd: name: iscsid enabled: true state: started tags: [system] # qemu-guest-agent binds to the virtio-serial channel /dev/virtio-ports/org.qemu.guest_agent.0, # which Proxmox attaches only when the VM's `agent` option is enabled. Our tofu keeps agent # DISABLED on purpose (enabling it makes `tofu apply` block up to 15m waiting for an agent the # stock Debian cloud image hasn't installed yet — bpg gotcha). So we ENABLE the unit for the day # a channel exists, but only START it when the channel is actually present — never fail the # baseline over a missing hypervisor channel. - name: Check whether the qemu guest-agent virtio channel is attached ansible.builtin.stat: path: /dev/virtio-ports/org.qemu.guest_agent.0 register: node_baseline_qga_channel tags: [system] - name: Enable qemu-guest-agent (start only when its virtio channel is present) ansible.builtin.systemd: name: qemu-guest-agent enabled: true state: "{{ 'started' if node_baseline_qga_channel.stat.exists else 'stopped' }}" tags: [system] # Fail LOUD and EARLY rather than at mkfs time when an environment has not yet decided # its data-disk layout. group_vars/k3s_prod/vars.yml deliberately carries # `/dev/BLOCKED-pending-disk-decision-see-PROD-PROVISIONING-md-2` for both devices (see # its comments): leaving them unset would fall back to dev's virtio /dev/vdb|/dev/vdc, # which do not exist on bare metal either — a confusing failure that looks like a wiring # bug. This assert names the real blocker instead. It is a no-op on dev (/dev/vdb, # /dev/vdc) and is `disks`-tagged, so `--tags packages` skips it along with the disk work. - name: Assert this environment's data-disk devices have actually been decided ansible.builtin.assert: that: - "'BLOCKED' not in node_baseline_longhorn_device" - "'BLOCKED' not in node_baseline_etcd_device" fail_msg: >- node_baseline_longhorn_device / node_baseline_etcd_device are still the BLOCKED placeholders for this host's environment (longhorn={{ node_baseline_longhorn_device }}, etcd={{ node_baseline_etcd_device }}). The disk layout is an OPEN operator decision — see infra/PROD-PROVISIONING.md §2 — and must NOT be guessed. To install ONLY the package floor (e.g. smartmontools, so the SMART read that decision waits on can be taken) re-run this play with `--tags packages`, which skips every disk task including this assert. quiet: true tags: [disks] - name: Create ext4 filesystem on the Longhorn data disk (data-safe; skips if present) community.general.filesystem: fstype: ext4 dev: "{{ node_baseline_longhorn_device }}" opts: "-L {{ node_baseline_longhorn_fs_label }}" tags: [disks] - name: Mount the Longhorn data disk ansible.posix.mount: path: "{{ node_baseline_longhorn_mount }}" src: "LABEL={{ node_baseline_longhorn_fs_label }}" fstype: ext4 opts: "defaults,nofail,x-systemd.device-timeout=5s" state: mounted tags: [disks] # Dedicated SSD etcd disk (docs/plans/ssd-etcd-isolation.md). Formats /dev/vdc and # mounts it at the k3s etcd data dir BEFORE k3s starts, so a rebuilt node comes up with # etcd already isolated on SSD. Data-safe: community.general.filesystem never reformats # a disk that already carries a filesystem, so re-running against an already-migrated # node leaves the etcd data intact. On EXISTING nodes the one-time data migration # (stop k3s -> rsync db -> mount) is done manually per the plan's §4.3 before this # role would mount over a populated directory. - name: Create ext4 filesystem on the dedicated etcd SSD disk (data-safe; skips if present) community.general.filesystem: fstype: ext4 dev: "{{ node_baseline_etcd_device }}" opts: "-L {{ node_baseline_etcd_fs_label }}" tags: [disks] - name: Mount the dedicated etcd SSD disk at the k3s etcd data dir ansible.posix.mount: path: "{{ node_baseline_etcd_mount }}" src: "LABEL={{ node_baseline_etcd_fs_label }}" fstype: ext4 # noatime: fewer metadata writes under etcd's fsync-heavy load. No `nofail`: a # missing etcd disk MUST fail loudly rather than let k3s start etcd on the root HDD. opts: "defaults,noatime" dump: "0" passno: "2" state: mounted tags: [disks] - name: Disable swap now ansible.builtin.command: swapoff -a changed_when: false tags: [system] - name: Neutralize swap entries in fstab (kubelet requires swap off) ansible.builtin.replace: path: /etc/fstab regexp: '^([^#].*\sswap\s.*)$' replace: '# \1 # disabled for k3s/kubelet' tags: [system] # ===================================================================================== # DNS RESILIENCE — tag `dns` (R-DNS-01 / R-DNS-03) # ===================================================================================== # Root cause (verified on dev): the nodes carried a SINGLE upstream resolver. PiHole # (192.168.1.18) answers `*.jackson.lan` instantly but its external recursion flaps — # 0/25 for registry-1.docker.io / quay.io / ghcr.io while gitea.jackson.lan resolved 5/5. # One upstream is a hard SPOF: containerd image pulls died with "lookup # registry-1.docker.io: Try again", and CoreDNS (`forward . /etc/resolv.conf`) flooded its # log with "i/o timeout" to 192.168.1.18. The fix is ADDITIVE — PiHole stays first so the # local zone and the ad-blocking policy are untouched; a second upstream is only consulted # when the first fails. See defaults/main.yml for the ordering rule. # # WHY THIS DETECTS INSTEAD OF ASSUMING. An earlier attempt (fix/node-dns-resilient-upstream, # dropped) wrote /etc/netplan/99-k3s-dns.yaml and ran `netplan apply`. netplan is NOT # installed on these nodes — they run the Debian 13 trixie generic cloud image # (infra/tofu/variables.tf) and netplan is not in node_baseline_packages — so that handler # would have failed command-not-found and broken node-baseline.yml on EVERY node. Worse, # the two environments do not even share a resolver stack: dev's /etc/resolv.conf is the # systemd-resolved stub (127.0.0.53) while prod's is a directly-managed file listing # 192.168.2.1 (it has no cloud-init at all — bare metal). So the mechanism is DETECTED per # node and the matching seam is written. Adding a third mechanism means adding a branch # here, not rewriting the role. # # All of it is `dns`-tagged, so `--tags packages` (the prod package-floor escape hatch # documented at the top of this file) still installs nothing but packages. - name: Determine whether systemd-resolved is running on this node ansible.builtin.command: systemctl is-active systemd-resolved register: node_baseline_resolved_active changed_when: false failed_when: false check_mode: false tags: [dns] - name: Determine whether NetworkManager is running (it rewrites /etc/resolv.conf) ansible.builtin.command: systemctl is-active NetworkManager register: node_baseline_nm_active changed_when: false failed_when: false check_mode: false tags: [dns] # follow: false — we need to know whether /etc/resolv.conf IS a symlink and where it # points, which is the strongest single signal of who owns the file. - name: Inspect /etc/resolv.conf itself (symlink or real file, and its target) ansible.builtin.stat: path: /etc/resolv.conf follow: false register: node_baseline_resolv_stat tags: [dns] - name: Read the current /etc/resolv.conf contents ansible.builtin.slurp: src: /etc/resolv.conf register: node_baseline_resolv_slurp failed_when: false tags: [dns] # resolvconf/openresolv keeps the authoritative copy under here and regenerates # /etc/resolv.conf from it, so writing /etc/resolv.conf directly would be silently undone. - name: Detect a resolvconf/openresolv installation that would clobber /etc/resolv.conf ansible.builtin.stat: path: /etc/resolvconf/resolv.conf.d register: node_baseline_resolvconf_dir tags: [dns] # THE DETECTION. Three outcomes, in priority order: # systemd-resolved — resolved is running AND it owns /etc/resolv.conf, proven either by # the symlink pointing into /run/systemd/resolve/ or by the file containing only # resolved's loopback stub (127.0.0.53 / 127.0.0.54). Writing a resolved drop-in is # then the ONLY thing that changes real behaviour; writing resolv.conf would be # overwritten by resolved (or, worse, land in /run and vanish at reboot). # contended — nobody's stub, but resolvconf or NetworkManager is present/running and # would regenerate the file underneath us. We do NOT write. The role fails loudly # below rather than leaving an operator believing a change stuck when it did not. # resolv-conf-file — a plain, directly-managed /etc/resolv.conf (prod's shape today). # Safe to own outright. # resolved running but NOT owning resolv.conf correctly falls through to the file branch: # in that state the drop-in would change nothing glibc can see. - name: Detect which resolver mechanism this node actually uses ansible.builtin.set_fact: node_baseline_dns_mechanism: >- {{ 'systemd-resolved' if (node_baseline_resolved_running and node_baseline_resolved_owns_resolv) else ('contended' if (node_baseline_nm_running or node_baseline_resolvconf_present) else 'resolv-conf-file') }} vars: node_baseline_resolved_running: "{{ (node_baseline_resolved_active.stdout | default('') | trim) == 'active' }}" node_baseline_nm_running: "{{ (node_baseline_nm_active.stdout | default('') | trim) == 'active' }}" node_baseline_resolvconf_present: "{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}" node_baseline_resolv_text: "{{ (node_baseline_resolv_slurp.content | default('') | b64decode) }}" node_baseline_resolv_ns: "{{ node_baseline_resolv_text | regex_findall('(?m)^\\s*nameserver\\s+(\\S+)') }}" node_baseline_resolved_owns_resolv: >- {{ ('/run/systemd/resolve/' in (node_baseline_resolv_stat.stat.lnk_source | default(''))) or ((node_baseline_resolv_ns | length) > 0 and (node_baseline_resolv_ns | reject('in', ['127.0.0.53', '127.0.0.54']) | list | length) == 0) }} tags: [dns] - name: Report the detected resolver mechanism ansible.builtin.debug: msg: >- resolver mechanism = {{ node_baseline_dns_mechanism }} (systemd-resolved is-active={{ node_baseline_resolved_active.stdout | default('') | trim }}, NetworkManager is-active={{ node_baseline_nm_active.stdout | default('') | trim }}, /etc/resolv.conf islnk={{ node_baseline_resolv_stat.stat.islnk | default(false) }} target={{ node_baseline_resolv_stat.stat.lnk_source | default('(not a symlink)') }}, resolvconf dir={{ node_baseline_resolvconf_dir.stat.exists | default(false) }}) tags: [dns] # Same shape, and the same reason, as the data-disk assert above: this environment's # upstream list is an OPERATOR DECISION, and a placeholder that fails LOUDLY beats both a # guess and a silent inheritance of dev's PiHole. group_vars/k3s_prod/vars.yml deliberately # carries a BLOCKED second entry (see its comments and PROD-PROVISIONING.md §4): prod's # 192.168.2.1 is currently gateway AND only upstream, and choosing what sits behind it is a # network decision this role must not make. Running against prod stops HERE, before # anything is written. - name: Assert this environment's DNS upstream list has actually been decided ansible.builtin.assert: that: - node_baseline_dns_servers | length >= 2 - "'BLOCKED' not in (node_baseline_dns_servers | join(' '))" fail_msg: >- node_baseline_dns_servers is not a decided, redundant upstream list for this host's environment (got: {{ node_baseline_dns_servers | join(', ') }}). A single upstream is the exact SPOF this task exists to remove — it already broke containerd image pulls and CoreDNS forwarding on dev — so the role will not configure one, and it will NOT guess a second server: a public resolver would bypass the local zone and the ad-blocking policy, which is a decision, not a default. Decide the second upstream, record it in infra/PROD-PROVISIONING.md §4, and set node_baseline_dns_servers in inventory/group_vars/k3s_prod/vars.yml in the SAME change. To install only the package floor meanwhile, re-run with `--tags packages`, which skips every DNS task. quiet: true tags: [dns] # The `contended` outcome. Not a warning: an operator who is told "changed" while another # daemon quietly reverts the file is worse off than one who is told nothing happened. - name: Refuse to manage /etc/resolv.conf while another resolver manager owns it ansible.builtin.fail: msg: >- /etc/resolv.conf on this node is not the systemd-resolved stub, but resolvconf/openresolv (/etc/resolvconf/resolv.conf.d present: {{ node_baseline_resolvconf_dir.stat.exists | default(false) }}) or NetworkManager (is-active: {{ node_baseline_nm_active.stdout | default('') | trim }}) is in play and would regenerate the file underneath this role. Writing it here would look like it worked and silently revert. Add the matching branch to roles/node_baseline (the `dns`-tagged block) for whichever manager this node really uses — do not force it. when: node_baseline_dns_mechanism == 'contended' tags: [dns] # --- Mechanism A: systemd-resolved (dev's shape today) ------------------------------- # A drop-in, never an edit of /etc/systemd/resolved.conf, so a package upgrade of that file # never conflicts. `DNS=` is one ORDERED list inside a single resolved scope: resolved uses # the first server and only rotates to the next when it fails, which is exactly the # PiHole-primary/fallback behaviour wanted — it is NOT a race between two servers, so a # gateway NXDOMAIN can never beat PiHole to a jackson.lan answer while PiHole is healthy. - name: Ensure the systemd-resolved drop-in directory exists ansible.builtin.file: path: /etc/systemd/resolved.conf.d state: directory owner: root group: root mode: "0755" when: node_baseline_dns_mechanism == 'systemd-resolved' tags: [dns] - name: Configure resilient DNS upstreams via a systemd-resolved drop-in ansible.builtin.copy: dest: /etc/systemd/resolved.conf.d/99-k3s-dns.conf owner: root group: root mode: "0644" content: | # Managed by the Ansible node_baseline role (tags: dns) — do not edit by hand. # Ordered upstream list: the FIRST server is the local-zone/ad-blocking resolver and # stays authoritative; the rest exist only so external resolution (containerd image # pulls, CoreDNS `forward . /etc/resolv.conf`) survives its recursion flapping. [Resolve] DNS={{ node_baseline_dns_servers | join(' ') }} Domains={{ node_baseline_dns_search | join(' ') }} # Reset to empty deliberately. systemd ships compiled-in public fallback resolvers # (Cloudflare/Google) that resolved would reach for if the list above ever came up # empty — which would silently bypass BOTH the local zone and the network's # ad-blocking. This fix adds redundancy; it never adds a bypass. FallbackDNS= notify: Restart systemd-resolved when: node_baseline_dns_mechanism == 'systemd-resolved' tags: [dns] # --- Mechanism B: a directly-managed /etc/resolv.conf (prod's shape today) ----------- # Only reached when nothing else owns the file (the `contended` branch above already # stopped the play otherwise). glibc walks `nameserver` lines in order and falls through # on timeout, so ordering carries the same policy as the resolved list. # NOTE this REPLACES the file: any nameserver currently there and NOT in # node_baseline_dns_servers is dropped. That is the point — the list in group_vars is the # whole declared truth — so an environment that wants its existing IPv6 upstreams kept # must list them there too. `backup: true` leaves the pre-role copy beside it the first # time it changes, because this file is load-bearing for reaching the node at all. - name: Configure resilient DNS upstreams directly in /etc/resolv.conf ansible.builtin.copy: dest: /etc/resolv.conf owner: root group: root mode: "0644" backup: true content: | # Managed by the Ansible node_baseline role (tags: dns) — do not edit by hand. # Ordered upstream list: the FIRST server is the local-zone resolver and stays # authoritative; the rest exist only so external resolution (containerd image pulls, # CoreDNS `forward . /etc/resolv.conf`) survives its recursion flapping. {% if node_baseline_dns_search | length > 0 %} search {{ node_baseline_dns_search | join(' ') }} {% endif %} {% for server in node_baseline_dns_servers %} nameserver {{ server }} {% endfor %} {% if node_baseline_dns_options | length > 0 %} options {{ node_baseline_dns_options | join(' ') }} {% endif %} when: node_baseline_dns_mechanism == 'resolv-conf-file' tags: [dns]