diff --git a/infra/ansible/roles/node_baseline/defaults/main.yml b/infra/ansible/roles/node_baseline/defaults/main.yml index 37ce87c..83728d5 100644 --- a/infra/ansible/roles/node_baseline/defaults/main.yml +++ b/infra/ansible/roles/node_baseline/defaults/main.yml @@ -15,3 +15,12 @@ node_baseline_kernel_modules: node_baseline_longhorn_device: /dev/vdb node_baseline_longhorn_mount: /var/lib/longhorn node_baseline_longhorn_fs_label: longhorn + +# DNS resilience (see tasks/main.yml "DNS resilience"). PiHole stays PRIMARY (local +# jackson.lan zone + network ad-blocking); the pfSense gateway is an additive FALLBACK so +# external resolution (containerd image pulls, CoreDNS→pods) survives PiHole recursion +# outages. Keep these aligned with tofu variables dns_server / network_gateway. +node_baseline_primary_iface: eth0 +node_baseline_dns_primary: 192.168.1.18 # PiHole — local zone + ad-block (PRIMARY) +node_baseline_dns_fallback: 192.168.1.1 # pfSense gateway — reliable external recursion +node_baseline_dns_search: jackson.lan diff --git a/infra/ansible/roles/node_baseline/handlers/main.yml b/infra/ansible/roles/node_baseline/handlers/main.yml index 15d80f0..4ac51a0 100644 --- a/infra/ansible/roles/node_baseline/handlers/main.yml +++ b/infra/ansible/roles/node_baseline/handlers/main.yml @@ -2,3 +2,9 @@ - name: Reload sysctl ansible.builtin.command: sysctl --system changed_when: true + +- name: Apply netplan + # `netplan apply` reconfigures systemd-resolved with the new nameserver list; it does not + # bounce the primary link's IP, so the SSH/NAT session survives (verified live on dev-1). + ansible.builtin.command: netplan apply + changed_when: true diff --git a/infra/ansible/roles/node_baseline/tasks/main.yml b/infra/ansible/roles/node_baseline/tasks/main.yml index bf84e9d..adef49c 100644 --- a/infra/ansible/roles/node_baseline/tasks/main.yml +++ b/infra/ansible/roles/node_baseline/tasks/main.yml @@ -81,3 +81,43 @@ path: /etc/fstab regexp: '^([^#].*\sswap\s.*)$' replace: '# \1 # disabled for k3s/kubelet' + +# --- DNS resilience (ROOT-CAUSE FIX) ----------------------------------------- +# The nodes shipped with a SINGLE upstream resolver — PiHole (node_baseline_dns_primary, +# 192.168.1.18) — set by cloud-init/netplan. PiHole answers the local `jackson.lan` zone +# instantly but its EXTERNAL/recursive resolution flaps (verified: 0/25 for public names +# like registry-1.docker.io/api.cloudflare.com while `gitea.jackson.lan` resolved 5/5). +# With one upstream that is a hard SPOF: BOTH containerd image pulls (`lookup +# registry-1.docker.io: Try again`) AND CoreDNS (`forward . /etc/resolv.conf`, so pods' +# external DNS — external-dns→Cloudflare, oauth, etc.) fail whenever PiHole's recursion is +# down. cert-manager already had to bypass it with public resolvers (DECISIONS-LOG Fix 5). +# +# FIX (additive, no policy bypass): keep PiHole PRIMARY (preserves `jackson.lan` + network +# ad-blocking) and ADD the pfSense gateway (node_baseline_dns_fallback, 192.168.1.1) as a +# FALLBACK. pfSense resolves the public zones reliably (verified). The resolver (glibc for +# containerd; CoreDNS `forward` health-checking for pods) fails over to the fallback when +# PiHole times out. A higher-priority netplan drop-in carries the change; cloud-init's +# 50-cloud-init.yaml is regenerated on rebuild, so tofu's `dns.servers` list is updated to +# match for provision-time parity (both layers converge on the same two servers). +# NOTE: the *underlying* PiHole external-recursion outage is a separate LAN-wide problem to +# investigate; this only makes the k3s nodes resilient to it. +- name: Deploy resilient DNS (PiHole primary + pfSense fallback) via netplan drop-in + ansible.builtin.copy: + dest: /etc/netplan/99-k3s-dns.yaml + owner: root + group: root + mode: "0600" + content: | + # Managed by the Ansible node_baseline role — do not edit by hand. + # Additive DNS redundancy for the k3s nodes (see roles/node_baseline/tasks/main.yml): + # PiHole PRIMARY (local zone + ad-block), pfSense gateway FALLBACK (reliable external + # recursion). Overrides the single nameserver from 50-cloud-init.yaml. + network: + version: 2 + ethernets: + {{ node_baseline_primary_iface }}: + nameservers: + addresses: [{{ node_baseline_dns_primary }}, {{ node_baseline_dns_fallback }}] + search: [{{ node_baseline_dns_search }}] + notify: Apply netplan + tags: [dns] diff --git a/infra/tofu/main.tf b/infra/tofu/main.tf index cfa2f5d..4d01d90 100644 --- a/infra/tofu/main.tf +++ b/infra/tofu/main.tf @@ -148,8 +148,14 @@ resource "proxmox_virtual_environment_vm" "node" { } } + # DNS resilience: PiHole (var.dns_server) stays PRIMARY for the local jackson.lan zone + + # network ad-blocking; the pfSense gateway (var.network_gateway) is an additive FALLBACK so + # a fresh node's containerd pulls + CoreDNS survive PiHole external-recursion outages (a + # single upstream was a hard SPOF — see the node_baseline "DNS resilience" task and + # DECISIONS-LOG). Existing nodes get the same two servers from the node_baseline netplan + # drop-in; this keeps cloud-init/netplan on rebuild in parity. dns { - servers = [var.dns_server] + servers = [var.dns_server, var.network_gateway] } user_account {