From 7a34590b19d192397bc6d85333e9f4d4e24b30e9 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Fri, 28 Aug 2026 23:30:59 -0500 Subject: [PATCH] fix(dns): add resilient fallback resolver to k3s nodes (PiHole primary + pfSense fallback) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: the k3s nodes shipped with a SINGLE upstream resolver, PiHole (192.168.1.18), set by cloud-init/netplan. PiHole answers the local jackson.lan zone instantly but its external/recursive resolution flaps — verified 0/25 for public names (registry-1.docker.io, api.cloudflare.com, quay.io, ghcr.io, google.com) while gitea.jackson.lan resolved 5/5. A single upstream is a hard SPOF: both containerd image pulls ("lookup registry-1.docker.io: Try again") and CoreDNS ("forward . /etc/resolv.conf" -> pods' external DNS, e.g. external-dns -> Cloudflare) fail whenever PiHole recursion is down. CoreDNS logs were flooded with "read udp ...->192.168.1.18:53: i/o timeout". Fix (additive, no policy bypass): keep PiHole PRIMARY (preserves jackson.lan + network ad-blocking) and ADD the pfSense gateway (192.168.1.1) as a FALLBACK, which resolves the public zones reliably (verified). The resolver fails over to the fallback when PiHole times out. - node_baseline: netplan drop-in (99-k3s-dns.yaml) with both nameservers + "Apply netplan" handler; new node_baseline_dns_* defaults; tagged `dns`. Applied to all 3 nodes via playbooks/node-baseline.yml --tags dns. - tofu: initialization.dns.servers now lists [dns_server, network_gateway] so a fresh cloud-init/netplan build stays in parity with the drop-in. Verified: all external names resolve cluster-wide (10/10, zero timeouts) from multiple pods after a CoreDNS restart; CoreDNS i/o-timeout floods stopped; split-horizon (keycloak.home.keithjackson.dev -> Gateway VIP) intact. NOTE: a SEPARATE, pre-existing WAN-egress outage (ICMP + TCP 443 to the internet time out from every node while the LAN/pfSense resolver work) still blocks image pulls, external-dns->Cloudflare, and Flux remote bases. That is a pfSense NAT/firewall/WAN problem outside this repo, flagged for PM follow-up. Co-Authored-By: Claude Opus 4.8 --- .../roles/node_baseline/defaults/main.yml | 9 +++++ .../roles/node_baseline/handlers/main.yml | 6 +++ .../roles/node_baseline/tasks/main.yml | 40 +++++++++++++++++++ infra/tofu/main.tf | 8 +++- 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/infra/ansible/roles/node_baseline/defaults/main.yml b/infra/ansible/roles/node_baseline/defaults/main.yml index 37ce87c..83728d5 100644 --- a/infra/ansible/roles/node_baseline/defaults/main.yml +++ b/infra/ansible/roles/node_baseline/defaults/main.yml @@ -15,3 +15,12 @@ node_baseline_kernel_modules: node_baseline_longhorn_device: /dev/vdb node_baseline_longhorn_mount: /var/lib/longhorn node_baseline_longhorn_fs_label: longhorn + +# DNS resilience (see tasks/main.yml "DNS resilience"). PiHole stays PRIMARY (local +# jackson.lan zone + network ad-blocking); the pfSense gateway is an additive FALLBACK so +# external resolution (containerd image pulls, CoreDNS→pods) survives PiHole recursion +# outages. Keep these aligned with tofu variables dns_server / network_gateway. +node_baseline_primary_iface: eth0 +node_baseline_dns_primary: 192.168.1.18 # PiHole — local zone + ad-block (PRIMARY) +node_baseline_dns_fallback: 192.168.1.1 # pfSense gateway — reliable external recursion +node_baseline_dns_search: jackson.lan diff --git a/infra/ansible/roles/node_baseline/handlers/main.yml b/infra/ansible/roles/node_baseline/handlers/main.yml index 15d80f0..4ac51a0 100644 --- a/infra/ansible/roles/node_baseline/handlers/main.yml +++ b/infra/ansible/roles/node_baseline/handlers/main.yml @@ -2,3 +2,9 @@ - name: Reload sysctl ansible.builtin.command: sysctl --system changed_when: true + +- name: Apply netplan + # `netplan apply` reconfigures systemd-resolved with the new nameserver list; it does not + # bounce the primary link's IP, so the SSH/NAT session survives (verified live on dev-1). + ansible.builtin.command: netplan apply + changed_when: true diff --git a/infra/ansible/roles/node_baseline/tasks/main.yml b/infra/ansible/roles/node_baseline/tasks/main.yml index bf84e9d..adef49c 100644 --- a/infra/ansible/roles/node_baseline/tasks/main.yml +++ b/infra/ansible/roles/node_baseline/tasks/main.yml @@ -81,3 +81,43 @@ path: /etc/fstab regexp: '^([^#].*\sswap\s.*)$' replace: '# \1 # disabled for k3s/kubelet' + +# --- DNS resilience (ROOT-CAUSE FIX) ----------------------------------------- +# The nodes shipped with a SINGLE upstream resolver — PiHole (node_baseline_dns_primary, +# 192.168.1.18) — set by cloud-init/netplan. PiHole answers the local `jackson.lan` zone +# instantly but its EXTERNAL/recursive resolution flaps (verified: 0/25 for public names +# like registry-1.docker.io/api.cloudflare.com while `gitea.jackson.lan` resolved 5/5). +# With one upstream that is a hard SPOF: BOTH containerd image pulls (`lookup +# registry-1.docker.io: Try again`) AND CoreDNS (`forward . /etc/resolv.conf`, so pods' +# external DNS — external-dns→Cloudflare, oauth, etc.) fail whenever PiHole's recursion is +# down. cert-manager already had to bypass it with public resolvers (DECISIONS-LOG Fix 5). +# +# FIX (additive, no policy bypass): keep PiHole PRIMARY (preserves `jackson.lan` + network +# ad-blocking) and ADD the pfSense gateway (node_baseline_dns_fallback, 192.168.1.1) as a +# FALLBACK. pfSense resolves the public zones reliably (verified). The resolver (glibc for +# containerd; CoreDNS `forward` health-checking for pods) fails over to the fallback when +# PiHole times out. A higher-priority netplan drop-in carries the change; cloud-init's +# 50-cloud-init.yaml is regenerated on rebuild, so tofu's `dns.servers` list is updated to +# match for provision-time parity (both layers converge on the same two servers). +# NOTE: the *underlying* PiHole external-recursion outage is a separate LAN-wide problem to +# investigate; this only makes the k3s nodes resilient to it. +- name: Deploy resilient DNS (PiHole primary + pfSense fallback) via netplan drop-in + ansible.builtin.copy: + dest: /etc/netplan/99-k3s-dns.yaml + owner: root + group: root + mode: "0600" + content: | + # Managed by the Ansible node_baseline role — do not edit by hand. + # Additive DNS redundancy for the k3s nodes (see roles/node_baseline/tasks/main.yml): + # PiHole PRIMARY (local zone + ad-block), pfSense gateway FALLBACK (reliable external + # recursion). Overrides the single nameserver from 50-cloud-init.yaml. + network: + version: 2 + ethernets: + {{ node_baseline_primary_iface }}: + nameservers: + addresses: [{{ node_baseline_dns_primary }}, {{ node_baseline_dns_fallback }}] + search: [{{ node_baseline_dns_search }}] + notify: Apply netplan + tags: [dns] diff --git a/infra/tofu/main.tf b/infra/tofu/main.tf index cfa2f5d..4d01d90 100644 --- a/infra/tofu/main.tf +++ b/infra/tofu/main.tf @@ -148,8 +148,14 @@ resource "proxmox_virtual_environment_vm" "node" { } } + # DNS resilience: PiHole (var.dns_server) stays PRIMARY for the local jackson.lan zone + + # network ad-blocking; the pfSense gateway (var.network_gateway) is an additive FALLBACK so + # a fresh node's containerd pulls + CoreDNS survive PiHole external-recursion outages (a + # single upstream was a hard SPOF — see the node_baseline "DNS resilience" task and + # DECISIONS-LOG). Existing nodes get the same two servers from the node_baseline netplan + # drop-in; this keeps cloud-init/netplan on rebuild in parity. dns { - servers = [var.dns_server] + servers = [var.dns_server, var.network_gateway] } user_account {