diff --git a/bootstrap/flux/bootstrap.sh b/bootstrap/flux/bootstrap.sh index 33e35c7..7617361 100755 --- a/bootstrap/flux/bootstrap.sh +++ b/bootstrap/flux/bootstrap.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash # -# bootstrap.sh — the SINGLE Flux + SOPS bootstrap entrypoint for the dev cluster. +# bootstrap.sh — the SINGLE Flux + SOPS bootstrap entrypoint. ENVIRONMENT-AGNOSTIC: which overlay +# it applies is FLUX_ENV_PATH (default clusters/dev). roles/flux_bootstrap exports that from +# `flux_bootstrap_env_path`, overridden to clusters/prod in group_vars/k3s_prod/vars.yml — so the +# same script bootstraps dev and prod with no fork (R-ENV-02). # # Phase 1, T-1.4 · ADR-0004 (Flux) · ADR-0007 (SOPS+age) · R-GITOPS-01 · R-SEC-01. # ============================================================================= @@ -33,7 +36,7 @@ # 2. sops-age Secret — the operator installs the age PRIVATE key out-of-band BEFORE reconcile so # decryption works on first pass (../sops/age-key-secret.md). This script # only VERIFIES it exists; it never handles key material. -# 3. `kubectl apply -k clusters/dev` — materialises the authored reconciliation graph: +# 3. `kubectl apply -k ${FLUX_ENV_PATH}` — materialises the authored reconciliation graph: # GitRepository `flux-system` (source) -> Kustomization `platform` -> # `platform-config` -> the app-wave Kustomizations `apps-wave-0/1/2` # (chained dependsOn) + `apps-vms`, with cluster-vars overlay @@ -52,7 +55,10 @@ set -euo pipefail # --- config (env-overridable) ------------------------------------------------ -FLUX_ENV_PATH="${FLUX_ENV_PATH:-clusters/dev}" # dev is the ONLY Phase-1 build target +# Which environment overlay to apply. Dev is the DEFAULT (it was the only Phase-1 build target); +# prod passes clusters/prod via roles/flux_bootstrap. Changing the default would silently retarget +# every existing caller — override it, never edit it. +FLUX_ENV_PATH="${FLUX_ENV_PATH:-clusters/dev}" FLUX_NAMESPACE="${FLUX_NAMESPACE:-flux-system}" SOPS_SECRET_NAME="${SOPS_SECRET_NAME:-sops-age}" # The two Phase-4 image-automation controllers are installed alongside the default set so their CRDs @@ -62,7 +68,7 @@ FLUX_COMPONENTS_EXTRA="${FLUX_COMPONENTS_EXTRA:-image-reflector-controller,image script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/../.." && pwd))" -echo "==> Flux + SOPS bootstrap (dev) — path=${FLUX_ENV_PATH}" +echo "==> Flux + SOPS bootstrap — path=${FLUX_ENV_PATH}" # --- 0. tooling present (from infra/devcontainer; versions pinned there) ------ for bin in flux kubectl; do @@ -134,7 +140,7 @@ if ! kubectl -n "${FLUX_NAMESPACE}" get secret "${SOPS_SECRET_NAME}" >/dev/null exit 1 fi -# --- 3. apply the authored dev entrypoint ------------------------------------ +# --- 3. apply the authored environment entrypoint ---------------------------- echo "==> [3/3] kubectl apply -k ${FLUX_ENV_PATH}" kubectl apply -k "${repo_root}/${FLUX_ENV_PATH}" diff --git a/bootstrap/gitea/bootstrap-gitea.sh b/bootstrap/gitea/bootstrap-gitea.sh index a3b923d..074444d 100755 --- a/bootstrap/gitea/bootstrap-gitea.sh +++ b/bootstrap/gitea/bootstrap-gitea.sh @@ -1,11 +1,30 @@ #!/usr/bin/env bash # bootstrap/gitea/bootstrap-gitea.sh — run a SINGLE Gitea container on the HOST (outside the -# cluster) as the Phase-1 GitOps bootstrap source. While the cluster is built, Flux (in-cluster) -# reconciles this repo from the host Gitea over the k3s-dev NAT gateway. This is the "local -# bootstrap source" that migrates to in-cluster Gitea in Phase 6 (docs/05 "Source-of-truth", -# CONTRIBUTING.md). +# cluster) as the GitOps bootstrap source. While a cluster is built, Flux (in-cluster) reconciles +# this repo from the host Gitea. This is the "local bootstrap source" that migrates to in-cluster +# Gitea at cutover (docs/05 "Source-of-truth", CONTRIBUTING.md). Dev has already cut over; PROD is +# using it now. # -# Run by the OPERATOR at bring-up — NOT during author-only, NOT by any agent. +# HOW THE CLUSTER REACHES THE HOST — read this before setting HOST_NAT_IP. +# It is an ADDRESS OF THIS HOST as seen from the cluster nodes, and it differs per environment: +# +# PROD (bare metal, current use): HOST_NAT_IP=192.168.2.132 +# The prod nodes (192.168.2.10/.11/.12) and this workstation are on the SAME LAN. There is +# NO NAT hop of any kind, and NO firewall change was needed — verified 2026-09-04 from all +# three nodes. The firewalld note further down is a DEV-ERA libvirt concern; it does not +# apply to prod, so do not "fix" prod by opening zones. +# +# DEV (libvirt VMs, HISTORICAL): HOST_NAT_IP=10.73.0.1 was the k3s-dev libvirt NAT gateway. +# *** THAT NETWORK IS RETIRED. 10.73.0.1 is stale and must not be reused. *** Dev's own +# bootstrap Gitea was later reached on 192.168.2.103:3000 and is retained STOPPED as +# recovery (see the comment on the GitRepository patch in clusters/dev/kustomization.yaml). +# +# WHO RUNS THIS: the OPERATOR, or an agent the operator has EXPLICITLY DIRECTED to stand up a +# bootstrap source for a specific bring-up (as happened 2026-09-04 for prod). It is NOT part of +# author-only work and must never be fired off casually or "to see if it works": it starts a +# long-lived container, generates and writes admin credentials to $HOME, creates an org and repo, +# and PUSHES THE CURRENT HEAD to that repo's `main` — which is the revision a cluster pointed at +# it will then reconcile. Know which HEAD you are publishing before you run it. set -euo pipefail ENGINE="${ENGINE:-podman}" GITEA_IMAGE="${GITEA_IMAGE:-docker.io/gitea/gitea:1.22}" # pinned; bump if needed @@ -16,11 +35,24 @@ SSH_PORT="${GITEA_SSH_PORT:-2222}" # subuid that can't write a host-owned dir, so Gitea fails to create /data/gitea/conf/app.ini. DATA_VOLUME="${GITEA_DATA_VOLUME:-gitea-bootstrap-data}" DATA_DIR="podman volume ${DATA_VOLUME}" # display label only (data lives in the named volume) -# The cluster VMs reach the host on the k3s-dev NAT gateway (10.73.0.1). ROOT_URL must match so +# The address the CLUSTER uses to reach this host (see the header). ROOT_URL must match it so # clone/redirect URLs are correct from inside the cluster. -# NOTE: firewalld's `libvirt` zone (owns virbrN) blocks VM->host:3000 by default. Open it once: +# +# The DEFAULT IS DELIBERATELY UNSET — there is no correct default any more. The old default +# (10.73.0.1) was the retired k3s-dev libvirt NAT gateway; leaving it in place would silently +# publish a ROOT_URL no cluster can resolve, and Flux would report what looks like a network fault. +# Pass it explicitly, e.g. for prod: HOST_NAT_IP=192.168.2.132 bash bootstrap-gitea.sh +# +# NOTE (DEV/libvirt ONLY, historical): firewalld's `libvirt` zone (owns virbrN) blocked +# VM->host:3000 by default and had to be opened once with # sudo firewall-cmd --zone=libvirt --add-port=3000/tcp --permanent && sudo firewall-cmd --reload -HOST_NAT_IP="${HOST_NAT_IP:-10.73.0.1}" +# PROD needs NO firewall change: same-LAN, no libvirt zone involved (verified 2026-09-04). +if [ -z "${HOST_NAT_IP:-}" ]; then + echo "ERROR: HOST_NAT_IP is not set — it is this host's address AS SEEN FROM THE CLUSTER." >&2 + echo " prod: HOST_NAT_IP=192.168.2.132 (same LAN as the nodes, no NAT, no firewall change)" >&2 + echo " There is no safe default: the old one (10.73.0.1) is the RETIRED libvirt NAT." >&2 + exit 1 +fi ORG="${GITEA_ORG:-Home-Network}" REPO="${GITEA_REPO:-home-network-k3s-impl}" @@ -75,8 +107,17 @@ cat <host:${HTTP_PORT} — +Reachability: prod nodes reach ${HOST_NAT_IP}:${HTTP_PORT} directly on the LAN — no NAT, no +firewall change (verified 2026-09-04). For libvirt-based clusters ONLY, firewalld's libvirt zone +must allow VM->host:${HTTP_PORT} — ansible-playbook infra/ansible/playbooks/host-prereqs.yml --ask-become-pass +REMINDER: the push above published the CURRENT HEAD as this repo's \`main\`. Whatever cluster points +at this source will reconcile THAT revision. +TEMPORARY BY DESIGN: this container is outside every cluster and is a single point of failure for +any cluster reconciling from it. Cut over to in-cluster Gitea (docs/05); for prod the cutover +procedure is written on the GitRepository patch in clusters/prod/kustomization.yaml. EOF diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index cb7e50e..aaaeb3d 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -115,8 +115,20 @@ patches: name: sops-age # --- GitOps source (operator decision 2026-09-01: prod tracks the SAME BRANCH as dev) --- # - # *** BOOTSTRAP BLOCKER — this URL CANNOT work on a cold prod cluster. READ BEFORE BRING-UP. *** + # ############################################################################################ + # ### ### + # ### *** T E M P O R A R Y B O O T S T R A P S O U R C E — NOT PERMANENT *** ### + # ### ### + # ### The url below is a Gitea container running ON THE OPERATOR'S WORKSTATION, outside ### + # ### any cluster. It exists ONLY to get prod's first reconcile off the ground. It MUST ### + # ### be cut over to the in-cluster URL once Gitea runs on prod. If you are reading this ### + # ### and prod has been serving traffic for months, THE CUTOVER WAS NEVER DONE — that is ### + # ### a live single point of failure, not a settled design. See "CUTOVER" below. ### + # ### ### + # ############################################################################################ # + # WHY THE IN-CLUSTER URL CANNOT BE USED YET (blocker found 2026-09-01, still true; the + # unusable value was `https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git`). # Two independent reasons, both consequences of the split-horizon design: # 1. SELF-HOSTING CIRCULARITY. Gitea runs ON THE DEV CLUSTER. Prod syncing from it means prod's # GitOps source lives inside the very cluster slated for decommission. Gitea must move to @@ -126,13 +138,46 @@ patches: # 192.168.2.20 — PROD's own gateway, where no Gitea is listening. Prod's Flux would resolve # gitea.home.keithjackson.dev to itself and never reach a repository. # - # Bring-up therefore needs a TEMPORARY source (the bootstrap host-Gitea, or an off-cluster - # mirror), with a cutover to the in-cluster URL once Gitea actually runs on prod — exactly the - # sequence docs/05 describes for dev's own Phase-6 cutover. Tracked in PROD-PROVISIONING.md §7. + # WHAT THE TEMPORARY SOURCE IS (stood up 2026-09-04 on explicit operator direction): + # bootstrap/gitea/bootstrap-gitea.sh with HOST_NAT_IP=192.168.2.132 — a podman container on the + # operator's workstation, Gitea 1.22.6, listening on 192.168.2.132:3000. All three prod nodes + # reach it DIRECTLY on the LAN: no libvirt NAT hop (that network is retired) and no firewall + # change was needed. This is the same "local bootstrap source" pattern dev itself used before + # its Phase-6 cutover (docs/05 "Source-of-truth"). + # + # PLAIN HTTP, AND WHY THAT IS ACCEPTED HERE: the bootstrap Gitea has no TLS, so there is no cert + # for Flux to trust and no caCertSecretRef to wire. Traffic is LAN-local between prod nodes and + # the workstation, the repo carries NO plaintext secrets (everything sensitive is SOPS-encrypted, + # .sops.yaml), and the window is the bring-up only. Do NOT extend this posture past cutover. + # + # NO secretRef, DELIBERATELY: the repo is PUBLIC on the bootstrap Gitea, so anonymous clone + # works and Flux needs no git credentials. VERIFIED 2026-09-04 by unauthenticated request: + # curl http://192.168.2.132:3000/api/v1/repos/Home-Network/home-network-k3s-impl -> "private":false + # git ls-remote http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git -> refs/heads/main + # If the GitRepository ever reports an auth failure that looks like a network problem, RE-CHECK + # THAT FIRST: a private repo here fails exactly that way. + # + # *** CUTOVER — the exit condition, mirroring dev's Phase-6 cutover (docs/05) *** + # WHEN: Gitea runs on prod (reconciled in-cluster) and answers gitea.home.keithjackson.dev + # through prod's own Gateway VIP with a publicly-trusted cert. + # DO: 1. push/mirror this repo into the prod in-cluster Gitea; + # 2. change the url below to + # https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git + # and DELETE this whole TEMPORARY block (dev's equivalent stanza, ~15 lines above in + # clusters/dev/kustomization.yaml, is what "done" looks like); + # 3. confirm `flux get sources git -n flux-system` reconciles the new revision; + # 4. only THEN stop the workstation container. Keep it stopped-not-deleted as recovery, + # exactly as dev retained its bootstrap Gitea. + # UNTIL THEN: prod's reconciliation depends on a container on a workstation that may be + # powered off. Flux keeps the last-reconciled state running, but NO new revision lands while + # it is down. Tracked as an open blocker in infra/PROD-PROVISIONING.md §7. # # The BRANCH below is settled and correct: prod tracks `main`, the same branch as dev. # CONSEQUENCE, recorded deliberately: once prod is live, EVERY MERGE TO main RECONCILES BOTH # CLUSTERS. There is no promotion step and no soak window — dev and prod move together. + # NOTE for bring-up: bootstrap-gitea.sh pushes the operator's CURRENT HEAD to the bootstrap + # Gitea's `main`, whatever branch that is. Before bootstrapping prod, confirm that remote `main` + # is the revision you intend prod to run (`git ls-remote` above, against local `main`). - patch: |- apiVersion: source.toolkit.fluxcd.io/v1 kind: GitRepository @@ -140,7 +185,9 @@ patches: name: flux-system namespace: flux-system spec: - url: https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git + # TEMPORARY bootstrap source — workstation container, NOT in any cluster. See the block + # above for the cutover to https://gitea.home.keithjackson.dev/... . Do not treat as final. + url: http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git ref: branch: main - target: diff --git a/infra/PROD-PROVISIONING.md b/infra/PROD-PROVISIONING.md index 407d489..99bbbd0 100644 --- a/infra/PROD-PROVISIONING.md +++ b/infra/PROD-PROVISIONING.md @@ -1193,12 +1193,15 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be as dev**. Eight `decryption: {provider: sops, secretRef: sops-age}` patches mirror dev exactly. The `sops-age` Secret is still created OUT OF BAND on the prod cluster (`bootstrap/sops/install-age-key.sh`) — the overlay references it by name only. - - [~] **GitRepository source** — BRANCH settled, URL still blocked. Operator decision - 2026-09-01: **prod tracks `main`, the same branch as dev**, because dev is temporary and - will be decommissioned once prod is stable. Recorded consequence: once prod is live, - **every merge to `main` reconciles BOTH clusters** — no promotion step, no soak window. - The URL is patched in but **cannot work on a cold prod cluster**, for two independent - reasons found 2026-09-01: + - [~] **GitRepository source** — **MITIGATED 2026-09-04, NOT SOLVED.** Branch settled; the + in-cluster URL is still unusable, so prod now points at a **temporary bootstrap source**. + Operator decision 2026-09-01: **prod tracks `main`, the same branch as dev**, because dev + is temporary and will be decommissioned once prod is stable. Recorded consequence: once + prod is live, **every merge to `main` reconciles BOTH clusters** — no promotion step, no + soak window. + + **Why the in-cluster URL (`https://gitea.home.keithjackson.dev/...`) cannot work on a + cold prod cluster** — two independent reasons, found 2026-09-01, both still true: 1. **Self-hosting circularity** — Gitea runs on the DEV cluster, the one slated for decommission. Gitea must move to prod (or off-cluster) BEFORE dev is retired, or prod loses its source of truth. **This is the decommission plan's first task, not its last.** @@ -1206,9 +1209,56 @@ no extra vars is byte-for-byte the dev behaviour it always had (verified by a be `home.keithjackson.dev` zone with `${GATEWAY_VIP}`, which on prod is `192.168.2.20`: prod's OWN gateway, where no Gitea listens. Prod's Flux would resolve `gitea.home.keithjackson.dev` to itself and never reach a repository. - Bring-up needs a TEMPORARY source (bootstrap host-Gitea, or an off-cluster mirror) and a - cutover once Gitea runs on prod — the same sequence `docs/05` describes for dev's own - Phase-6 cutover. + + **The mitigation now in `clusters/prod/kustomization.yaml`** (operator directed this to be + stood up, 2026-09-04): + + | | | + |---|---| + | GitRepository url | `http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git` | + | Branch | `main` (unchanged — same branch as dev) | + | What it is | Gitea 1.22.6 in a **podman container on the operator's workstation**, started by `bootstrap/gitea/bootstrap-gitea.sh` with `HOST_NAT_IP=192.168.2.132` | + | Reachability | All three prod nodes reach it **directly on the LAN** — no NAT hop (the libvirt network is retired), **no firewall change needed**. Verified from all three nodes. | + | Auth | **None, deliberately.** The repo is PUBLIC on that Gitea, so Flux needs no `secretRef`. Verified anonymously: `curl .../api/v1/repos/Home-Network/home-network-k3s-impl` → `"private":false`, and `git ls-remote` returns `refs/heads/main`. | + | Transport | **Plain HTTP, no TLS.** Accepted for the bring-up window only: LAN-local, and the repo holds no plaintext secrets (all SOPS-encrypted). | + | Admin creds | `~/.config/home-k3s-bootstrap-gitea-admin.txt`, mode 0600, host-only, never committed. | + + **THIS IS A SINGLE POINT OF FAILURE FOR PROD'S RECONCILIATION, and it is not even in a + cluster.** It is one container on one workstation, with no HA, no backup of its own, and + no restart guarantee beyond `--restart unless-stopped`. If that workstation is powered + off, suspended, or the podman volume `gitea-bootstrap-data` is lost: + - already-reconciled state **keeps running** — Flux does not tear anything down; + - but **no new revision can land on prod**, and every `GitRepository` reconcile fails. + So prod is *not* self-sufficient while this stands. Do not treat prod as production-ready + on this footing, and do not let it become the steady state by inattention. + + **One bring-up footgun:** `bootstrap-gitea.sh` pushes the operator's **current `HEAD`** to + the bootstrap Gitea's `main`, whatever branch that HEAD is on. Before bootstrapping prod, + confirm that remote `main` is the revision prod should run + (`git ls-remote http://192.168.2.132:3000/Home-Network/home-network-k3s-impl.git` vs local + `main`). As of 2026-09-04 the remote `main` is the tip of the working branch, **not** local + `main`. + + **EXIT CONDITION (the cutover) — mirrors dev's own Phase-6 cutover, `docs/05`.** + This item is not closable until all of it is done: + - [ ] Gitea runs **on prod**, reconciled in-cluster, answering + `gitea.home.keithjackson.dev` through prod's Gateway VIP with a publicly-trusted cert. + - [ ] This repo is pushed/mirrored into that in-cluster Gitea. + - [ ] The `clusters/prod` GitRepository url is changed to + `https://gitea.home.keithjackson.dev/Home-Network/home-network-k3s-impl.git` + and the TEMPORARY comment block is **deleted** (dev's stanza is what "done" looks like). + - [ ] `flux get sources git -n flux-system` on prod reconciles a revision from the new URL. + - [ ] **Only then** stop the workstation container — kept stopped-not-deleted as recovery, + exactly as dev retained its own bootstrap Gitea. + - [x] **`roles/flux_bootstrap` could only bootstrap dev** — FIXED 2026-09-04. It invoked + `bootstrap/flux/bootstrap.sh` without ever exporting `FLUX_ENV_PATH`, so the script fell + through to its `clusters/dev` default and §8 step 4 (`--path=clusters/prod`) was + unreachable without editing the role. Now parameterised on the standard group_vars seam: + role default `flux_bootstrap_env_path: clusters/dev` (defaults/main.yml) is exported as + `FLUX_ENV_PATH` by the role's task, and + `inventory/group_vars/k3s_prod/vars.yml` overrides it to `clusters/prod` — the same + mechanism as `kube_vip_address` / `k3s_data_dir` / `cluster_kubeconfig_name`. Dev has no + override anywhere, so dev resolves `clusters/dev` exactly as before. - [ ] Confirm prod's backup target: dev's S3 (`versitygw` on the Proxmox host, `192.168.1.9`) is on the dev side of the network. Off-cluster backup is DR-critical (R-STATE-03). diff --git a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml index 5e04ed9..a281631 100644 --- a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml +++ b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml @@ -301,3 +301,25 @@ node_baseline_dns_servers: # keycloak.home.keithjackson.dev -> 192.168.2.20 on prod nodes; or a local resolver on the # prod LAN serving the zone; or run prod on the cert kubeconfig and defer OIDC. node_baseline_dns_search: [] + +# --- Flux bootstrap: WHICH overlay this environment reconciles from (R-ENV-02) ----- +# roles/flux_bootstrap defaults flux_bootstrap_env_path to `clusters/dev` (dev is LIVE +# and group_vars/all.yml carries no override, so dev keeps resolving exactly that). +# Prod bootstraps its OWN overlay. Before this seam existed the role could bootstrap +# clusters/dev ONLY — bootstrap/flux/bootstrap.sh's FLUX_ENV_PATH default was never +# exported by the role — so `flux bootstrap ... --path=clusters/prod` (§8 step 4 of +# infra/PROD-PROVISIONING.md) was simply not reachable without editing the role. +# +# Same seam as kube_vip_address / k3s_api_host / k3s_data_dir / cluster_kubeconfig_name / +# node_baseline_backports_kernel above: set once for the [k3s_prod] umbrella group, so a +# FOURTH prod node inherits it from group membership with no per-host or per-run flag. +# +# WHAT clusters/prod STILL NEEDS before this is safe to run — do NOT invoke the +# flux_bootstrap role until these are true (PROD-PROVISIONING.md §7): +# - the remaining TBD-PROD-* placeholders in clusters/prod/kustomization.yaml are real +# (S3_BACKUP_ENDPOINT); they are INVALID on purpose so an early apply fails loudly; +# - the GitRepository url there is the TEMPORARY workstation Gitea +# (http://192.168.2.132:3000/...) and that container is actually running and reachable +# from the prod nodes — it is outside every cluster and can simply be powered off; +# - the `sops-age` Secret is installed out of band on prod (bootstrap/sops). +flux_bootstrap_env_path: "clusters/prod" diff --git a/infra/ansible/roles/flux_bootstrap/defaults/main.yml b/infra/ansible/roles/flux_bootstrap/defaults/main.yml index c44e3dc..de54342 100644 --- a/infra/ansible/roles/flux_bootstrap/defaults/main.yml +++ b/infra/ansible/roles/flux_bootstrap/defaults/main.yml @@ -19,5 +19,22 @@ flux_bootstrap_repo_root: "{{ playbook_dir }}/../../.." flux_bootstrap_sops_cmd: "bash {{ flux_bootstrap_repo_root }}/bootstrap/sops/install-age-key.sh" # Assumed interface of bootstrap/flux (GitOps Engineer, T-1.4): -# runs `flux bootstrap ... --path=clusters/dev` against this cluster. +# runs `flux install` + `kubectl apply -k ` against this cluster. flux_bootstrap_flux_cmd: "bash {{ flux_bootstrap_repo_root }}/bootstrap/flux/bootstrap.sh" + +# --- WHICH ENVIRONMENT OVERLAY THIS BOOTSTRAPS (R-ENV-02) ------------------------- +# The one thing that makes this role environment-agnostic. bootstrap/flux/bootstrap.sh +# reads FLUX_ENV_PATH (default clusters/dev) and ends with `kubectl apply -k `; +# tasks/main.yml exports this value into that variable, so the role NEVER names an +# environment and the task graph is identical for dev and prod. +# +# THE DEFAULT IS DEV ON PURPOSE — dev is LIVE, and group_vars/all.yml carries no +# override, so a dev run resolves exactly the value it always had (clusters/dev) and +# behaves byte-for-byte as before. +# +# PROD OVERRIDES IT in inventory/group_vars/k3s_prod/vars.yml — the same group_vars +# seam as kube_vip_address / k3s_data_dir / cluster_kubeconfig_name / +# node_baseline_backports_kernel. No role fork, no per-run extra-var, no second +# playbook: group membership decides. Before this existed the role could bootstrap +# ONLY clusters/dev, so prod bring-up was impossible without editing the role. +flux_bootstrap_env_path: "clusters/dev" diff --git a/infra/ansible/roles/flux_bootstrap/tasks/main.yml b/infra/ansible/roles/flux_bootstrap/tasks/main.yml index 1a87390..fe4466b 100644 --- a/infra/ansible/roles/flux_bootstrap/tasks/main.yml +++ b/infra/ansible/roles/flux_bootstrap/tasks/main.yml @@ -4,11 +4,17 @@ # — it calls the entrypoints the Security + GitOps Engineers author there. # # Order matters: install the age decryption key FIRST (so Flux can decrypt on its -# very first reconcile), THEN run `flux bootstrap` at clusters/dev. +# very first reconcile), THEN bootstrap Flux at the environment's overlay. # # The two commands are variables (see defaults/main.yml). If the bootstrap workers # expose different entrypoints, override flux_bootstrap_sops_cmd / _flux_cmd — the # task graph does not change. +# +# ENVIRONMENT-AGNOSTIC (R-ENV-02): this role names NO environment. Which overlay gets +# bootstrapped comes from `flux_bootstrap_env_path` (defaults/main.yml, default +# clusters/dev), exported below as FLUX_ENV_PATH — the variable bootstrap/flux/ +# bootstrap.sh already reads. Prod overrides it in group_vars/k3s_prod/vars.yml, the +# same seam as kube_vip_address / k3s_data_dir / cluster_kubeconfig_name. - name: Assert the bootstrap/ subtree exists (authored by the other worker) ansible.builtin.stat: @@ -29,11 +35,14 @@ run_once: true changed_when: true -- name: Bootstrap Flux against clusters/dev (bootstrap/flux) +- name: Bootstrap Flux against this environment's overlay (bootstrap/flux) ansible.builtin.command: cmd: "{{ flux_bootstrap_flux_cmd }}" environment: KUBECONFIG: "{{ flux_bootstrap_kubeconfig }}" + # The ONLY per-environment input. clusters/dev by default (dev unchanged); + # clusters/prod for [k3s_prod] hosts via group_vars. + FLUX_ENV_PATH: "{{ flux_bootstrap_env_path }}" delegate_to: localhost become: false run_once: true diff --git a/infra/ansible/roles/node_baseline/defaults/main.yml b/infra/ansible/roles/node_baseline/defaults/main.yml index d353b71..c2a438f 100644 --- a/infra/ansible/roles/node_baseline/defaults/main.yml +++ b/infra/ansible/roles/node_baseline/defaults/main.yml @@ -183,6 +183,29 @@ node_baseline_protected_mounts: - /home - /tmp +# --- WIRELESS (tasks/main.yml, tag `wireless`) --------------------------------------- +# A CLUSTER NODE GETS EXACTLY ONE PATH ONTO THE NETWORK. ON by default — unlike the +# backports kernel above, which is a prod-only deviation, this is a cluster-wide policy and +# it is a provable NO-OP where it does not apply: the tasks act only on interfaces the node +# reports as real 802.11 devices (a `phy80211` link in sysfs), and dev's Proxmox VMs have +# none, so on dev nothing is written, nothing is unloaded and nothing changes. Defaulting it +# true is what makes a FOURTH node with a wifi card inherit the policy from being a cluster +# node, rather than from someone remembering to set a flag. +# +# WHY IT EXISTS: k3s-prod-3 came up on both eno1 (192.168.2.12, the inventory address) and +# wlp1s0 (192.168.2.195, DHCP on the same /24). The wifi lease's dhcpcd run rewrote +# /etc/resolv.conf three minutes after this role wrote it, and a second address on the LAN +# where kube-vip floats 192.168.2.9 and Cilium L2-announces .20-.30 is a problem in its own +# right. See the `wireless` block in tasks/main.yml for the full reasoning, including why the +# DRIVER MODULE is the layer that gets blocked rather than the link, a service, or rfkill. +# +# SET IT FALSE for a host that must genuinely keep wifi (there is none today). That does NOT +# undo an already-applied block — delete the file below on the node as well. +node_baseline_disable_wireless: true +# Where the persistent block lives. 99- so it is read last and overrides any earlier +# permissive rule; the name says what it does, on a node with no repo checkout to hand. +node_baseline_wireless_modprobe_file: /etc/modprobe.d/99-k3s-no-wireless.conf + # --- DNS resilience (R-DNS-01 / R-DNS-03) ------------------------------------------- # These are the DEV values. Prod overrides them in inventory/group_vars/k3s_prod/vars.yml # (the same seam kube_vip_address / k3s_api_host / cluster_kubeconfig_name use) — NOT in diff --git a/infra/ansible/roles/node_baseline/tasks/main.yml b/infra/ansible/roles/node_baseline/tasks/main.yml index 2008a55..b2fb6a7 100644 --- a/infra/ansible/roles/node_baseline/tasks/main.yml +++ b/infra/ansible/roles/node_baseline/tasks/main.yml @@ -1,6 +1,6 @@ --- -# TAGS — every task here carries one of `packages`, `kernel`, `system`, `disks` or `dns`. An -# untagged run is unchanged (all five run, in this order). The split exists because the PACKAGE floor +# TAGS — every task here carries one of `packages`, `kernel`, `system`, `wireless`, `disks` or +# `dns`. An untagged run is unchanged (all six run, in this order). The split exists because the PACKAGE floor # and the DATA-DISK work have different readiness: packages are safe on any node from # first contact, while the disk tasks need block devices that a new environment may not # have decided yet (prod: infra/PROD-PROVISIONING.md §2, whose decision is itself blocked @@ -16,6 +16,12 @@ # is: it is the ONE thing that must land on a prod node BEFORE k3s is installed (§0b), and # it must be runnable without touching a disk decision that is still open: # ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags kernel +# `wireless` (a cluster node gets exactly ONE path onto the network) is separated for a third +# reason: it must be runnable BEFORE `dns` and without touching packages, the kernel or a disk +# decision, because a node with a second DHCP-managed interface reverts the `dns` write. It is +# placed before the disk work for the same reason, and the pair is the whole remedy for the +# k3s-prod-3 incident: +# ansible-playbook playbooks/node-baseline.yml -e k3s_hosts=k3s_prod --tags wireless,dns - name: Install baseline packages (guest agent, Longhorn/iscsi/nfs prereqs) ansible.builtin.apt: @@ -314,6 +320,304 @@ state: "{{ 'started' if node_baseline_qga_channel.stat.exists else 'stopped' }}" tags: [system] +# ===================================================================================== +# WIRELESS — tag `wireless`: a cluster node gets exactly ONE path onto the network +# ===================================================================================== +# WHAT WENT WRONG (k3s-prod-3, reported by the operator 2026-09-04). The node was up on BOTH +# eno1 (192.168.2.12 — the inventory address) and wlp1s0 (192.168.2.195, DHCP, the SAME /24). +# The wifi lease carried a dhcpcd run with it, and dhcpcd's `20-resolv.conf` hook REWROTE +# /etc/resolv.conf, undoing the resolvers this role had just written: the file read +# "# Generated by dhcpcd from wlp1s0.dhcp, wlp1s0.dhcp6, wlp1s0.ra" with nameserver +# 192.168.2.1 plus two Google IPv6 servers, timestamped THREE MINUTES AFTER the role's own +# write (01:32 -> 01:35). The role had reported `changed` and green. That is exactly the +# "successful run that fixed nothing" the `contended` branch of the DNS block below exists to +# prevent, and the detection blind spot that let it through is closed there. +# +# WHY THE INTERFACE IS REMOVED RATHER THAN JUST REPORTED. Closing the detection hole alone +# would have turned a silent revert into a loud failure — an improvement, but it leaves the +# node with a second, DHCP-managed path onto the LAN, and on a k3s server that is three +# distinct problems beyond DNS: +# * a SECOND DEFAULT ROUTE, whose metric decides which source address the node uses; +# * a SECOND ADDRESS ON THE SAME L2 — the segment where kube-vip floats 192.168.2.9 and +# Cilium L2-announces 192.168.2.20-.30 (§4). One host answering ARP on one subnet from +# two NICs is the exact shape neither mechanism can reason about; +# * an UNMANAGED address: nothing in this repo knows about .195, and the DHCP server is +# free to hand out a different one after any lease expiry. +# None of that is a resolver bug, and none of it is fixed by writing resolv.conf harder. +# +# WHY THE DRIVER LAYER — the four alternatives, and why each is weaker. What has to hold is: +# survives a REBOOT, survives a LEASE RENEWAL, and cannot be undone by a routine `ifup`. +# Measured on the nodes themselves — all three are HP EliteDesk 800 G5 minis, eno1 is +# `e1000e`, the wifi NIC is an Intel AX200 on `iwlwifi`, and the OS is Debian 13 trixie with +# ifupdown + `networking.service`, NO NetworkManager, NO systemd-networkd, NO systemd-resolved: +# `ip link set down` — NOT persistent. Any `ifup`, any hotplug event, any reboot +# undoes it. It is what had already been done by hand on node 3, and the stale dhcpcd +# /etc/resolv.conf outlived it — the link was down and the wrong resolvers were still +# in the file, across a reboot. +# mask the service that manages it — THERE IS NO SUCH SERVICE HERE. dhcpcd on these nodes +# is `dhcpcd-base`: no dhcpcd.service exists at all (`systemctl is-enabled dhcpcd` = +# not-found), it is spawned per-interface by ifupdown. The only unit that can bring +# wireless up is `networking.service` — which ALSO owns eno1's static address, so +# masking it would cut the one SSH path to a machine whose operator is not on site. +# `rfkill` soft-block — weaker on every axis. The netdev still EXISTS, so ifupdown and +# dhcpcd can still be pointed at it; persistence depends on systemd-rfkill restoring +# saved state (the kernel's own `persistent` flag reads 0 on these devices, and the +# `rfkill` CLI is not even installed); and one `rfkill unblock wlan` undoes it. +# the BIOS `M.2 WLAN/BT` toggle — genuinely the deepest layer, and it reads `Enable` on all +# three nodes (hp-bioscfg survey, §1b). NOT taken here: it needs a REBOOT to take +# effect, and a prod reboot is a serialized, supervised operator step (§0b "SECOND +# FAULT" — k3s-prod-2 hung on a warm reboot and needed the power button). It also +# disables Bluetooth on the same M.2 card. Left as an operator recommendation in §4. +# THE DRIVER MODULE — chosen. `blacklist` plus `install /bin/false` in +# /etc/modprobe.d means kmod refuses to load the wifi driver on this boot and every +# future one, so THE NETDEV NEVER EXISTS. No interface means no DHCP lease, no +# resolv.conf hook, no second route, no second address — and every re-activation path +# above (udev hotplug, a hand-run `ifup`, wpa_supplicant, a hand-run dhcpcd) needs a +# device that is not there. `blacklist` alone would still allow a dependency- or +# hand-triggered `modprobe`; the `install ... /bin/false` line is what makes it refuse +# outright. It is ONE declarative file, readable on the node, and undone by deleting +# it. Paired with the runtime unload below, it needs NO reboot to take effect. +# +# SAFE AND IDEMPOTENT ON ALL THREE PROD NODES *AND* ON THE LIVE DEV CLUSTER. Nothing here is +# hostname- or hardware-specific: the block acts only on interfaces the node itself reports as +# real 802.11 devices (a `phy80211` link in sysfs) and on the driver modules behind them. +# * dev (Proxmox VMs, virtio NICs) has no such interface, so the discovery comes back empty, +# every task below is skipped, and NOT EVEN THE modprobe.d FILE IS WRITTEN. The block +# cannot touch dev's networking, which is why it is safe to have on by default. +# * k3s-prod-1 (wlp1s0) and k3s-prod-2 (wlp2s0) already have the link DOWN, so the link-down +# task is skipped by its own `when` and only the modprobe.d file and the module unload +# remain — and both are no-ops on a re-run. +# * k3s-prod-3 (wlp1s0) is the reported node. +# A re-run changes nothing on any of them: the file is content-identical, `ip link set down` +# is gated on the link not already being down, and the unload is gated on the module still +# being loaded. +# +# AND IT REFUSES TO CUT ITS OWN PATH IN. The assert below stops the play if the address +# Ansible is connected to is bound to a wireless interface, or if the default route leaves +# over one, or if the wireless driver module is also the driver of the default-route +# interface. That is the one case where doing this would lock everybody out of a machine +# nobody can walk up to, so it is checked per node, on every run, from that node's own facts. +- name: Survey this node's 802.11 devices, their drivers and any live DHCP client + ansible.builtin.shell: + executable: /bin/sh + cmd: | + set -u + # Flat KEY|value lines so the Ansible side needs one single-group regex per fact and + # no parsing cleverness. A wireless interface is one with a phy80211 link in sysfs — + # that is the kernel's own answer to "is this 802.11", not a name pattern like wlp*. + wifi='' + for d in /sys/class/net/*; do + [ -e "$d/phy80211" ] || continue + i=$(basename "$d") + wifi="$wifi $i" + m=$(basename "$(readlink -f "$d/device/driver" 2>/dev/null || echo unknown)") + state=$(cat "$d/operstate" 2>/dev/null || echo unknown) + printf 'IFACE|%s\n' "$i" + printf 'DETAIL|%s driver=%s operstate=%s\n' "$i" "$m" "$state" + # LIVE means "not already down" — the link-down task keys off this, so a node whose + # radio is already down reports no change. + [ "$state" = down ] || printf 'IFACE_LIVE|%s\n' "$i" + ip -o addr show dev "$i" 2>/dev/null | awk '{print "IFACE_ADDR|" $4}' + [ "$m" = unknown ] || printf 'MODULE|%s\n' "$m" + done + for m in $(printf '%s\n' $wifi | while read -r i; do + [ -n "$i" ] && basename "$(readlink -f "/sys/class/net/$i/device/driver" 2>/dev/null || echo unknown)" + done | sort -u); do + [ "$m" = unknown ] && continue + [ -d "/sys/module/$m" ] || continue + printf 'MODULE_LOADED|%s\n' "$m" + # Holders must be unloaded FIRST: `modprobe -r iwlwifi` fails while iwlmvm still + # holds it. Read from sysfs rather than assumed, so a different wifi driver stack + # (ath*, rtw*, mt76*) works without editing this role. + for h in $(ls "/sys/module/$m/holders" 2>/dev/null); do printf 'MODULE_HOLDER|%s\n' "$h"; done + done + dr=$(ip -o route show default 2>/dev/null | awk '{for (i = 1; i < NF; i++) if ($i == "dev") print $(i + 1)}' | head -1) + printf 'DEFROUTE_IFACE|%s\n' "${dr:-none}" + if [ -n "${dr:-}" ] && [ -e "/sys/class/net/$dr" ]; then + printf 'DEFROUTE_MODULE|%s\n' "$(basename "$(readlink -f "/sys/class/net/$dr/device/driver" 2>/dev/null || echo unknown)")" + fi + # Any DHCP client process, with the interfaces it was started for. Used only to stop a + # client that holds a WIRELESS interface and nothing else. + for p in $(pgrep -x dhcpcd 2>/dev/null; pgrep -x dhclient 2>/dev/null); do + printf 'DHCPCLIENT|%s\n' "$(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null)" + done + exit 0 + register: node_baseline_wifi_scan + changed_when: false + check_mode: false + when: node_baseline_disable_wireless | bool + tags: [wireless] + +- name: Interpret the wireless survey + ansible.builtin.set_fact: + node_baseline_wifi_ifaces: "{{ scan | regex_findall('(?m)^IFACE\\|(.+)$') }}" + node_baseline_wifi_live: "{{ scan | regex_findall('(?m)^IFACE_LIVE\\|(.+)$') }}" + node_baseline_wifi_addrs: "{{ scan | regex_findall('(?m)^IFACE_ADDR\\|(\\S+)') | map('regex_replace', '/.*$', '') | list }}" + node_baseline_wifi_modules: "{{ scan | regex_findall('(?m)^MODULE\\|(.+)$') | unique }}" + node_baseline_wifi_modules_loaded: "{{ scan | regex_findall('(?m)^MODULE_LOADED\\|(.+)$') | unique }}" + node_baseline_wifi_module_holders: "{{ scan | regex_findall('(?m)^MODULE_HOLDER\\|(.+)$') | unique }}" + node_baseline_wifi_detail: "{{ scan | regex_findall('(?m)^DETAIL\\|(.+)$') }}" + node_baseline_wifi_dhcp_clients: "{{ scan | regex_findall('(?m)^DHCPCLIENT\\|(.+)$') }}" + node_baseline_defroute_iface: "{{ (scan | regex_findall('(?m)^DEFROUTE_IFACE\\|(.+)$') | first) | default('none', true) }}" + node_baseline_defroute_module: "{{ (scan | regex_findall('(?m)^DEFROUTE_MODULE\\|(.+)$') | first) | default('unknown', true) }}" + vars: + scan: "{{ node_baseline_wifi_scan.stdout | default('') }}" + when: node_baseline_disable_wireless | bool + tags: [wireless] + +- name: Report this node's wireless hardware + ansible.builtin.debug: + msg: >- + wireless devices = {{ node_baseline_wifi_detail | default([]) | join('; ') | default('(none)', true) }} + | driver modules = {{ node_baseline_wifi_modules | default([]) | join(', ') | default('(none)', true) }} + (loaded: {{ node_baseline_wifi_modules_loaded | default([]) | join(', ') | default('(none)', true) }}, + held by: {{ node_baseline_wifi_module_holders | default([]) | join(', ') | default('(none)', true) }}) + | default route via {{ node_baseline_defroute_iface | default('?') }} + (driver {{ node_baseline_defroute_module | default('?') }}) + when: node_baseline_disable_wireless | bool + tags: [wireless] + +# The one way this block could do real harm is by removing the interface it is being run +# over. Checked from the node's own facts, per node, every run — not assumed from the fact +# that prod "should" be wired. +- name: Refuse to disable wireless if the path to this node runs OVER the wireless interface + ansible.builtin.assert: + that: + - node_baseline_defroute_iface not in node_baseline_wifi_ifaces + - node_baseline_defroute_module not in node_baseline_wifi_modules + - (ansible_host | default(inventory_hostname)) not in node_baseline_wifi_addrs + fail_msg: >- + REFUSING to disable wireless on this node: the connection to it appears to BE the + wireless path. ansible_host={{ ansible_host | default(inventory_hostname) }}, wireless + interfaces={{ node_baseline_wifi_ifaces | join(', ') }} carrying + {{ node_baseline_wifi_addrs | join(', ') | default('no addresses', true) }}, default route + via {{ node_baseline_defroute_iface }} (driver {{ node_baseline_defroute_module }}). + Disabling the driver here would drop this node off the network with no remote way back. + Give the node a wired address, point the inventory at it, and re-run. + quiet: true + when: + - node_baseline_disable_wireless | bool + - node_baseline_wifi_ifaces | length > 0 + tags: [wireless] + +# THE PERSISTENT HALF. This file is the whole reboot story: kmod consults /etc/modprobe.d on +# every load attempt, so from here on the driver cannot be loaded by udev at boot, as a +# dependency, or by hand. Written ONLY when the node actually has a wireless driver, so a +# node without wifi hardware (every dev VM) gets no file and no change at all. +# `update-initramfs` is deliberately NOT run: no wifi module is loaded from the initramfs on +# these nodes (root is a local SATA/NVMe device), the driver is autoloaded by udev after +# switch-root, and re-generating an initramfs on a prod node is a bigger risk than the one it +# would cover. +- name: Block the wireless driver module(s) from ever loading again + ansible.builtin.copy: + dest: "{{ node_baseline_wireless_modprobe_file }}" + owner: root + group: root + mode: "0644" + content: | + # Managed by the Ansible node_baseline role (tags: wireless) — do not edit by hand. + # + # WHY THIS NODE HAS NO WIRELESS. A k3s server gets exactly ONE path onto the network. + # A second, DHCP-managed interface on the same LAN gives the node a second default + # route, a second address nothing in the cluster's config knows about, and a DHCP + # client whose resolv.conf hook silently overwrites the resolvers Ansible manages + # (that is what happened on k3s-prod-3, 2026-09-04). The link is also brought down and + # the module unloaded at the same time, so no reboot is needed for this to take effect. + # + # `blacklist` stops udev's alias-driven autoload; the `install ... /bin/false` line is + # what also refuses a dependency-triggered or hand-run `modprobe`. Both are needed. + # + # TO RE-ENABLE WIRELESS ON THIS NODE: delete this file and `modprobe` the module(s) + # below (or reboot). To stop Ansible putting it back, set + # node_baseline_disable_wireless: false for this host first — otherwise the next + # node-baseline run recreates it. Full rationale: roles/node_baseline/tasks/main.yml, + # the `wireless` block, and infra/PROD-PROVISIONING.md §4. + {% for module in node_baseline_wifi_modules %} + blacklist {{ module }} + install {{ module }} /bin/false + {% endfor %} + when: + - node_baseline_disable_wireless | bool + - node_baseline_wifi_modules | length > 0 + tags: [wireless] + +# THE RUNTIME HALF — everything below makes the change true NOW, so the node does not have to +# be rebooted to stop carrying a second address (this role never reboots: §0b "SECOND FAULT"). +# +# Stop a DHCP client that is holding a wireless interface first, so it releases its lease +# instead of being left holding a device that is about to disappear. HEAVILY GUARDED: it runs +# only for clients whose command line names a wireless interface and names NO non-wireless +# interface, because `dhcpcd -x` against a shared master process would also drop a wired +# lease. On these prod nodes eno1 is a static ifupdown address with no DHCP client at all, so +# this task has nothing to do; it exists so the ordering is correct on a node where wifi is +# still actively leasing. +- name: Stop the dhcpcd instance that holds a wireless interface (and nothing else) + ansible.builtin.command: + argv: [dhcpcd, --exit, "{{ item }}"] + loop: "{{ node_baseline_wifi_ifaces }}" + register: node_baseline_wifi_dhcpcd_exit + changed_when: rc == 0 + failed_when: false + when: + - node_baseline_disable_wireless | bool + - node_baseline_wifi_dhcp_wifi_only | length > 0 + - node_baseline_wifi_dhcp_wifi_only | select('search', 'dhcpcd') | list | length > 0 + vars: + # A client is "wireless-only" when its argv mentions a wireless interface and mentions no + # other interface this node has. Empty interface sets are turned into a pattern that + # cannot match, so an odd node fails towards doing nothing rather than towards guessing. + node_baseline_non_wifi_ifaces: "{{ ansible_facts['interfaces'] | default([]) | difference(node_baseline_wifi_ifaces) | difference(['lo']) }}" + node_baseline_wifi_dhcp_wifi_only: >- + {{ node_baseline_wifi_dhcp_clients + | select('search', '\b(' ~ ((node_baseline_wifi_ifaces | join('|')) | default('__no_such_interface__', true)) ~ ')\b') + | reject('search', '\b(' ~ ((node_baseline_non_wifi_ifaces | join('|')) | default('__no_such_interface__', true)) ~ ')\b') + | list }} + tags: [wireless] + +- name: Take each wireless link down now + ansible.builtin.command: + argv: [ip, link, set, dev, "{{ item }}", down] + loop: "{{ node_baseline_wifi_live }}" + changed_when: true + when: node_baseline_disable_wireless | bool + tags: [wireless] + +# Unload the driver so the netdev is gone immediately — holders (iwlmvm on these nodes) first, +# then the driver itself, because `modprobe -r` refuses a module that is still held. +# NOT fatal on failure: the persistent block above is already in place, so a module that +# cannot be unloaded right now is gone at the next boot regardless. The report below says so +# out loud rather than letting a green play imply the radio is already off. +- name: Unload the wireless driver module(s) now (no reboot needed) + community.general.modprobe: + name: "{{ item }}" + state: absent + loop: "{{ node_baseline_wifi_module_holders + node_baseline_wifi_modules_loaded }}" + register: node_baseline_wifi_unload + failed_when: false + when: node_baseline_disable_wireless | bool + tags: [wireless] + +- name: Report the wireless state this node is left in + ansible.builtin.debug: + msg: >- + {{ 'No 802.11 device on this node — nothing to disable (no modprobe.d file written).' + if (node_baseline_wifi_ifaces | length) == 0 + else ((('WIRELESS DISABLED: ' ~ (node_baseline_wifi_modules | join(', ')) ~ ' blocked in ' + ~ node_baseline_wireless_modprobe_file ~ ' (survives reboot) and unloaded now; ' + ~ (node_baseline_wifi_ifaces | join(', ')) ~ ' is down and the netdev is gone.') + if (failed_unloads | length) == 0 + else ('WIRELESS BLOCKED FOR THE NEXT BOOT BUT STILL LOADED NOW: could not unload ' + ~ (failed_unloads | join(', ')) ~ '. ' ~ node_baseline_wireless_modprobe_file + ~ ' is in place, so the driver will not load again after a reboot, but the ' + ~ 'radio is live until then — check `ip link` and whether something still ' + ~ 'holds the device.'))) }} + vars: + failed_unloads: >- + {{ (node_baseline_wifi_unload.results | default([])) + | selectattr('failed', 'defined') | selectattr('failed') | map(attribute='item') | list }} + when: node_baseline_disable_wireless | bool + tags: [wireless] + # ===================================================================================== # DATA DISKS — tag `disks`, in tasks/disks.yml # ===================================================================================== @@ -377,6 +681,11 @@ # node and the matching seam is written. Adding a third mechanism means adding a branch # here, not rewriting the role. # +# AND DETECTION IS ONLY AS GOOD AS THE SET OF OWNERS IT KNOWS ABOUT. It originally looked for +# resolvconf/openresolv and NetworkManager, missed dhcpcd, and therefore reported a green, +# `changed` run on k3s-prod-3 whose resolvers a wifi lease reverted three minutes later. dhcpcd +# is now part of that set — see the detection task below and the `contended` failure. +# # All of it is `dns`-tagged, so `--tags packages` (the prod package-floor escape hatch # documented at the top of this file) still installs nothing but packages. @@ -396,6 +705,55 @@ check_mode: false tags: [dns] +# THE dhcpcd BLIND SPOT, CLOSED (k3s-prod-3, 2026-09-04). dhcpcd's `20-resolv.conf` hook +# rewrites /etc/resolv.conf on every lease, renewal and carrier change, and it is a THIRD +# way to own that file — neither resolvconf nor NetworkManager. On node 3 it silently reverted +# this role's write three minutes later, and because nothing below looked for it the role had +# already reported `changed` and green: the precise failure the `contended` branch exists to +# make impossible. +# +# WHY THIS LOOKS FOR A LIVE CLIENT AND NOT FOR AN INSTALLED PACKAGE. `dhcpcd-base` is +# installed on all three prod nodes and always will be — it is the DHCP client ifupdown +# reaches for — and there is NO dhcpcd.service on them at all (`systemctl is-active dhcpcd` = +# not-found; it is spawned per-interface by ifupdown). Keying off the package, or off a unit +# that does not exist, would put every prod node permanently in `contended` and leave the DNS +# block unable to configure a node it is perfectly safe to configure. What actually matters is +# whether a DHCP client CAN still write the file, so all three real signals are checked: +# CLIENT| a dhcpcd/dhclient process running right now (with the argv that started it); +# UNIT| a dhcpcd/dhclient unit that IS active or IS enabled, on a distro that has one; +# STANZA| an UNCOMMENTED `iface inet[6] dhcp` in /etc/network/interfaces{,.d} — the +# configuration that makes ifupdown spawn a client at the next boot or `ifup`, even +# when none is running at this instant. +# Any one of them means "another manager is in play" and the play stops below. None of them +# means no client can run, and the file is genuinely ours to own. +- name: Determine whether a DHCP client (dhcpcd/dhclient) can rewrite /etc/resolv.conf + ansible.builtin.shell: + executable: /bin/sh + cmd: | + set -u + for p in $(pgrep -x dhcpcd 2>/dev/null; pgrep -x dhclient 2>/dev/null); do + printf 'CLIENT|%s\n' "$(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null)" + done + for u in dhcpcd dhcpcd5 dhclient isc-dhcp-client; do + a=$(systemctl is-active "$u" 2>/dev/null || true) + e=$(systemctl is-enabled "$u" 2>/dev/null || true) + # Compared exactly, never with a substring match: `is-active` answers "inactive", + # which contains the word "active". + if [ "$a" = active ] || [ "$e" = enabled ] || [ "$e" = enabled-runtime ]; then + printf 'UNIT|%s active=%s enabled=%s\n' "$u" "$a" "$e" + fi + done + # `^[[:space:]]*iface` cannot match a commented-out stanza, which is how the wireless + # stanza is left on these nodes. + grep -rhE '^[[:space:]]*iface[[:space:]]+[^[:space:]]+[[:space:]]+inet6?[[:space:]]+dhcp' \ + /etc/network/interfaces /etc/network/interfaces.d/ 2>/dev/null \ + | sed -e 's/^[[:space:]]*/STANZA|/' + exit 0 + register: node_baseline_dhcp_client_scan + changed_when: false + check_mode: false + tags: [dns] + # follow: false — we need to know whether /etc/resolv.conf IS a symlink and where it # points, which is the strongest single signal of who owns the file. - name: Inspect /etc/resolv.conf itself (symlink or real file, and its target) @@ -426,9 +784,10 @@ # resolved's loopback stub (127.0.0.53 / 127.0.0.54). Writing a resolved drop-in is # then the ONLY thing that changes real behaviour; writing resolv.conf would be # overwritten by resolved (or, worse, land in /run and vanish at reboot). -# contended — nobody's stub, but resolvconf or NetworkManager is present/running and -# would regenerate the file underneath us. We do NOT write. The role fails loudly -# below rather than leaving an operator believing a change stuck when it did not. +# contended — nobody's stub, but resolvconf, NetworkManager or a DHCP client (dhcpcd / +# dhclient) is present/running/configured and would regenerate the file underneath us. +# We do NOT write. The role fails loudly below rather than leaving an operator +# believing a change stuck when it did not. # resolv-conf-file — a plain, directly-managed /etc/resolv.conf (prod's shape today). # Safe to own outright. # resolved running but NOT owning resolv.conf correctly falls through to the file branch: @@ -438,13 +797,21 @@ node_baseline_dns_mechanism: >- {{ 'systemd-resolved' if (node_baseline_resolved_running and node_baseline_resolved_owns_resolv) - else ('contended' if (node_baseline_nm_running or node_baseline_resolvconf_present) + else ('contended' if (node_baseline_nm_running or node_baseline_resolvconf_present + or node_baseline_dhcp_client_live) else 'resolv-conf-file') }} + # Kept as facts (not just task vars) so the failure message, the report and the + # reclaim note below all quote the SAME evidence this decision was made on. + node_baseline_dhcp_client_evidence: "{{ (node_baseline_dhcp_client_scan.stdout | default('')) | regex_findall('(?m)^(?:CLIENT|UNIT|STANZA)\\|(.+)$') }}" + # A file dhcpcd WROTE, versus a dhcpcd that can still write it — two different facts, and + # only the second one is contention. See the reclaim note below. + node_baseline_resolv_dhcpcd_written: "{{ 'Generated by dhcpcd' in (node_baseline_resolv_slurp.content | default('') | b64decode) }}" vars: node_baseline_resolved_running: "{{ (node_baseline_resolved_active.stdout | default('') | trim) == 'active' }}" node_baseline_nm_running: "{{ (node_baseline_nm_active.stdout | default('') | trim) == 'active' }}" node_baseline_resolvconf_present: "{{ node_baseline_resolvconf_dir.stat.exists | default(false) }}" + node_baseline_dhcp_client_live: "{{ (node_baseline_dhcp_client_scan.stdout | default('')) is search('(?m)^(?:CLIENT|UNIT|STANZA)\\|') }}" node_baseline_resolv_text: "{{ (node_baseline_resolv_slurp.content | default('') | b64decode) }}" node_baseline_resolv_ns: "{{ node_baseline_resolv_text | regex_findall('(?m)^\\s*nameserver\\s+(\\S+)') }}" node_baseline_resolved_owns_resolv: >- @@ -461,7 +828,9 @@ NetworkManager is-active={{ node_baseline_nm_active.stdout | default('') | trim }}, /etc/resolv.conf islnk={{ node_baseline_resolv_stat.stat.islnk | default(false) }} target={{ node_baseline_resolv_stat.stat.lnk_source | default('(not a symlink)') }}, - resolvconf dir={{ node_baseline_resolvconf_dir.stat.exists | default(false) }}) + resolvconf dir={{ node_baseline_resolvconf_dir.stat.exists | default(false) }}, + DHCP client evidence={{ node_baseline_dhcp_client_evidence | default([]) | join(' ; ') | default('(none)', true) }}, + resolv.conf written by dhcpcd={{ node_baseline_resolv_dhcpcd_written | default(false) }}) tags: [dns] # Same shape, and the same reason, as the data-disk assert above: this environment's @@ -494,16 +863,45 @@ - name: Refuse to manage /etc/resolv.conf while another resolver manager owns it ansible.builtin.fail: msg: >- - /etc/resolv.conf on this node is not the systemd-resolved stub, but - resolvconf/openresolv (/etc/resolvconf/resolv.conf.d present: - {{ node_baseline_resolvconf_dir.stat.exists | default(false) }}) or NetworkManager - (is-active: {{ node_baseline_nm_active.stdout | default('') | trim }}) is in play and - would regenerate the file underneath this role. Writing it here would look like it - worked and silently revert. Add the matching branch to roles/node_baseline (the - `dns`-tagged block) for whichever manager this node really uses — do not force it. + /etc/resolv.conf on this node is not the systemd-resolved stub, but something else would + regenerate the file underneath this role: resolvconf/openresolv + (/etc/resolvconf/resolv.conf.d present: + {{ node_baseline_resolvconf_dir.stat.exists | default(false) }}), NetworkManager + (is-active: {{ node_baseline_nm_active.stdout | default('') | trim }}), or a DHCP client + that can still write it — dhcpcd/dhclient evidence: + {{ node_baseline_dhcp_client_evidence | default([]) | join(' ; ') | default('(none)', true) }}. + Writing it here would look like it worked and silently revert — which is exactly what + happened on k3s-prod-3 on 2026-09-04, where a wifi lease's dhcpcd hook reverted this + role's resolvers three minutes after a green run. If the evidence above is a DHCP client + on a WIRELESS interface, the fix is the `wireless` block earlier in this role (a cluster + node gets one network path) — run it, then re-run this. If it is a DHCP client on the + node's real interface, or another manager entirely, add the matching branch to + roles/node_baseline for the manager this node really uses — do not force it. when: node_baseline_dns_mechanism == 'contended' tags: [dns] +# THE CORPSE CASE, and why it is NOT a hole in the fail-closed rule above. After the +# `wireless` block disables the radio, k3s-prod-3 is left with a /etc/resolv.conf that dhcpcd +# WROTE (its "# Generated by dhcpcd from wlp1s0.dhcp..." header, the gateway as sole IPv4 +# resolver, two Google IPv6 servers) and NO dhcpcd that can ever run again. `contended` means +# "another manager is live and will undo this write"; a file left behind by a client that no +# longer exists is not a manager, it is the damage — and refusing to write here would leave the +# node permanently mis-resolved, which is the very outcome the operator reported. So the file +# branch below reclaims it, and says so. +- name: Note that a dhcpcd-written /etc/resolv.conf is being reclaimed + ansible.builtin.debug: + msg: >- + /etc/resolv.conf on this node was written by dhcpcd, but no DHCP client can write it any + more (no process, no active/enabled unit, no `inet dhcp` stanza). Taking ownership and + replacing it with the declared upstreams + ({{ node_baseline_dns_servers | join(', ') }}). If a DHCP client ever comes back on this + node, the `contended` check above will stop the play instead of letting it revert + silently. + when: + - node_baseline_dns_mechanism == 'resolv-conf-file' + - node_baseline_resolv_dhcpcd_written | default(false) + tags: [dns] + # --- Mechanism A: systemd-resolved (dev's shape today) ------------------------------- # A drop-in, never an edit of /etc/systemd/resolved.conf, so a package upgrade of that file # never conflicts. `DNS=` is one ORDERED list inside a single resolved scope: resolved uses