Merge IaC hardening (tofu for_each + self-contained rejoin) into main

DECISIONS-LOG.md

@@ -78,8 +78,18 @@ primary play runs, so a limit that excludes the primary yields
   play must have populated in the same run. Then a true single-node rejoin (`--limit k3s-dev-N`) works
   hands-off, which is the T-1.5 durability bar.
 
-Both fixes are **IaC-hardening items for Phase 2** (no cluster change now; the running cluster is
-healthy). Logged here; not yet applied.
+### F1 / F2 — IMPLEMENTED (author-first, 2026-08-16)
+- **F1 done:** `infra/tofu/main.tf` node resources refactored `count` → `for_each` (keyed by node
+  name); ordered-list/`primary` helper locals added; `outputs.tf` updated off the map; **`moved.tf`**
+  adds 12 `moved` blocks (os/data/seed/domain × 3). `tofu validate` OK and **`tofu plan` = `0 to add,
+  0 to change, 0 to destroy`** with all 12 resources reported "has moved to" — a pure state-address
+  rename, no VM churn. **State-persist pending:** a follow-up `tofu apply` (a proven 0-change apply
+  that only rewrites state addresses) is operator-permission-gated; until then the migration re-shows
+  harmlessly on each plan and lands automatically on the next real apply.
+- **F2 done:** `roles/k3s_server/tasks/main.yml` — secondary now fetches the join token via
+  `delegate_to: primary` (with the published fact as a fast path), so `--limit k3s-dev-N` rejoin is
+  self-contained. `ansible-playbook --syntax-check` OK on all playbooks; role lint clean under the CI
+  gate (`make -C ci gate` PASS). No cluster change; the running cluster is untouched.
 
 ## KNOWN ISSUES to fix before Phase-2 un-suspend (logged, not yet fixed)
 | # | Issue | Impact | Fix |

infra/ansible/roles/k3s_server/tasks/main.yml

@@ -47,7 +47,29 @@
     k3s_node_token: "{{ k3s_token_raw.content | b64decode | trim }}"
   when: k3s_server_mode == 'primary'
 
-# --- secondaries: join the existing cluster --------------------------------
+# --- secondaries: obtain the join token, then join the existing cluster -----
+# Fetch the token straight from the primary via delegate_to, falling back to it
+# only when the primary play didn't publish the fact this run. This makes a
+# single-node rejoin (`--limit k3s-dev-N`, primary NOT in the batch) self-
+# contained — delegate_to still reaches the primary even when it isn't a play
+# target, so the join no longer depends on a fact an earlier play had to set.
+# T-1.5 finding F2 (see DECISIONS-LOG.md).
+- name: Fetch the k3s node-token directly from the primary (secondary)
+  ansible.builtin.slurp:
+    src: /var/lib/rancher/k3s/server/token
+  delegate_to: "{{ groups['k3s_primary'][0] }}"
+  register: k3s_primary_token_raw
+  when:
+    - k3s_server_mode == 'secondary'
+    - hostvars[groups['k3s_primary'][0]].k3s_node_token is not defined
+
+- name: Resolve the join token (secondary)
+  ansible.builtin.set_fact:
+    k3s_join_token: >-
+      {{ hostvars[groups['k3s_primary'][0]].k3s_node_token
+         | default(k3s_primary_token_raw.content | b64decode | trim, true) }}
+  when: k3s_server_mode == 'secondary'
+
 - name: Install k3s secondary (join as server)
   ansible.builtin.command:
     cmd: >-
@@ -56,7 +78,7 @@
     creates: /etc/rancher/k3s/k3s.yaml
   environment:
     INSTALL_K3S_VERSION: "{{ k3s_version }}"
-    K3S_TOKEN: "{{ hostvars[groups['k3s_primary'][0]]['k3s_node_token'] }}"
+    K3S_TOKEN: "{{ k3s_join_token }}"
   when: k3s_server_mode == 'secondary'
 
 - name: Wait for the k3s API to be serving locally

infra/tofu/main.tf

@@ -8,15 +8,27 @@ locals {
   # 1 GiB in bytes; libvirt volume sizes are bytes.
   gib = 1073741824
 
-  # Per-node identity/addressing, indexed 0..node_count-1.
-  nodes = [
-    for i in range(var.node_count) : {
+  # Per-node identity/addressing, KEYED BY NODE NAME (map, not list).
+  # for_each over this map gives every node a stable, name-based state address
+  # (libvirt_domain.node["k3s-dev-2"]) instead of a positional one (…node[1]).
+  # Positional count indexes are brittle under `-target`/`-replace`: a single-node
+  # destroy on a count-indexed resource can cascade to siblings (T-1.5 finding F1).
+  # Name keys are individually addressable, so one node can be replaced/destroyed
+  # without disturbing the others. See DECISIONS-LOG.md "T-1.5 … Finding F1".
+  nodes = {
+    for i in range(var.node_count) : "${var.node_name_prefix}${i + 1}" => {
       name     = "${var.node_name_prefix}${i + 1}"
       ip       = var.node_ips[i]
       fqdn     = "${var.node_name_prefix}${i + 1}.${var.node_domain}"
       is_first = i == 0
     }
-  ]
+  }
+
+  # Ordered (node1..nodeN) list + primary view for outputs that need positional
+  # access — the resource for_each uses the map above; these preserve ordering
+  # for the Ansible inventory (primary = cluster-init server, rest = secondaries).
+  node_list = [for i in range(var.node_count) : local.nodes["${var.node_name_prefix}${i + 1}"]]
+  primary   = local.nodes["${var.node_name_prefix}1"]
 }
 
 # --- dedicated NAT network (10.73.0.0/24, gw .1) ---------------------------
@@ -48,8 +60,8 @@ resource "libvirt_volume" "base" {
 
 # --- per-node OS disk: thin qcow2 backed by the base image ------------------
 resource "libvirt_volume" "os" {
-  count          = var.node_count
-  name           = "${local.nodes[count.index].name}-os.qcow2"
+  for_each       = local.nodes
+  name           = "${each.value.name}-os.qcow2"
   pool           = var.pool_name
   base_volume_id = libvirt_volume.base.id
   size           = var.os_disk_gib * local.gib
@@ -58,34 +70,34 @@ resource "libvirt_volume" "os" {
 
 # --- per-node data disk: blank thin qcow2 for Longhorn ----------------------
 resource "libvirt_volume" "data" {
-  count  = var.node_count
-  name   = "${local.nodes[count.index].name}-data.qcow2"
-  pool   = var.pool_name
-  size   = var.data_disk_gib * local.gib
-  format = "qcow2"
+  for_each = local.nodes
+  name     = "${each.value.name}-data.qcow2"
+  pool     = var.pool_name
+  size     = var.data_disk_gib * local.gib
+  format   = "qcow2"
 }
 
 # --- cloud-init seed (user-data + meta-data + static network-config) --------
 # Templates live in the sibling infra/cloud-init/ subtree (T-1.2).
 resource "libvirt_cloudinit_disk" "seed" {
-  count = var.node_count
-  name  = "${local.nodes[count.index].name}-seed.iso"
-  pool  = var.pool_name
+  for_each = local.nodes
+  name     = "${each.value.name}-seed.iso"
+  pool     = var.pool_name
 
   user_data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", {
-    hostname           = local.nodes[count.index].name
-    fqdn               = local.nodes[count.index].fqdn
+    hostname           = each.value.name
+    fqdn               = each.value.fqdn
     username           = var.login_user
     ssh_authorized_key = var.ssh_authorized_key
   })
 
   meta_data = templatefile("${path.module}/../cloud-init/meta-data.yaml.tftpl", {
-    hostname = local.nodes[count.index].name
-    fqdn     = local.nodes[count.index].fqdn
+    hostname = each.value.name
+    fqdn     = each.value.fqdn
   })
 
   network_config = templatefile("${path.module}/../cloud-init/network-config.yaml.tftpl", {
-    ip      = local.nodes[count.index].ip
+    ip      = each.value.ip
     prefix  = var.network_prefix
     gateway = var.network_gateway
   })
@@ -93,10 +105,10 @@ resource "libvirt_cloudinit_disk" "seed" {
 
 # --- the VMs ----------------------------------------------------------------
 resource "libvirt_domain" "node" {
-  count  = var.node_count
-  name   = local.nodes[count.index].name
-  memory = var.memory_mib
-  vcpu   = var.vcpu
+  for_each = local.nodes
+  name     = each.value.name
+  memory   = var.memory_mib
+  vcpu     = var.vcpu
 
   # host-passthrough so nested virt is available for KubeVirt in Phase 5.
   cpu {
@@ -106,7 +118,7 @@ resource "libvirt_domain" "node" {
   # qemu-guest-agent is installed by cloud-init; lets tofu read the lease/agent.
   qemu_agent = true
 
-  cloudinit = libvirt_cloudinit_disk.seed[count.index].id
+  cloudinit = libvirt_cloudinit_disk.seed[each.key].id
 
   # IP + hostname come from cloud-init (DHCP is off on this network), so we do NOT
   # set addresses/hostname here — the provider couples those to DHCP reservations.
@@ -117,11 +129,11 @@ resource "libvirt_domain" "node" {
 
   # OS disk first (vda, boot), Longhorn data disk second (vdb).
   disk {
-    volume_id = libvirt_volume.os[count.index].id
+    volume_id = libvirt_volume.os[each.key].id
   }
 
   disk {
-    volume_id = libvirt_volume.data[count.index].id
+    volume_id = libvirt_volume.data[each.key].id
   }
 
   console {

infra/tofu/moved.tf

@@ -0,0 +1,68 @@
+# moved.tf — state-address migration for the count -> for_each refactor (T-1.5 F1).
+#
+# The node resources changed from positional `count` indexes ([0],[1],[2]) to
+# name-keyed `for_each` (["k3s-dev-1"], …). Without these `moved` blocks OpenTofu
+# would read the address change as "destroy the old, create the new" and rebuild
+# every node. Each block tells tofu the resource is the SAME object under a new
+# address, so the plan is a pure in-state rename: NO destroy, NO recreate.
+#
+# LOCKED mapping (node_count = 3, prefix "k3s-dev-"): index i -> "k3s-dev-${i+1}".
+# `moved` blocks are static config (no for_each), so they are written explicitly.
+# Safe to KEEP indefinitely (harmless once applied) or delete after one apply on
+# every environment that held count-indexed state.
+
+# --- OS disks ---------------------------------------------------------------
+moved {
+  from = libvirt_volume.os[0]
+  to   = libvirt_volume.os["k3s-dev-1"]
+}
+moved {
+  from = libvirt_volume.os[1]
+  to   = libvirt_volume.os["k3s-dev-2"]
+}
+moved {
+  from = libvirt_volume.os[2]
+  to   = libvirt_volume.os["k3s-dev-3"]
+}
+
+# --- data disks -------------------------------------------------------------
+moved {
+  from = libvirt_volume.data[0]
+  to   = libvirt_volume.data["k3s-dev-1"]
+}
+moved {
+  from = libvirt_volume.data[1]
+  to   = libvirt_volume.data["k3s-dev-2"]
+}
+moved {
+  from = libvirt_volume.data[2]
+  to   = libvirt_volume.data["k3s-dev-3"]
+}
+
+# --- cloud-init seeds -------------------------------------------------------
+moved {
+  from = libvirt_cloudinit_disk.seed[0]
+  to   = libvirt_cloudinit_disk.seed["k3s-dev-1"]
+}
+moved {
+  from = libvirt_cloudinit_disk.seed[1]
+  to   = libvirt_cloudinit_disk.seed["k3s-dev-2"]
+}
+moved {
+  from = libvirt_cloudinit_disk.seed[2]
+  to   = libvirt_cloudinit_disk.seed["k3s-dev-3"]
+}
+
+# --- domains (VMs) ----------------------------------------------------------
+moved {
+  from = libvirt_domain.node[0]
+  to   = libvirt_domain.node["k3s-dev-1"]
+}
+moved {
+  from = libvirt_domain.node[1]
+  to   = libvirt_domain.node["k3s-dev-2"]
+}
+moved {
+  from = libvirt_domain.node[2]
+  to   = libvirt_domain.node["k3s-dev-3"]
+}

infra/tofu/outputs.tf

@@ -4,27 +4,27 @@
 
 output "node_names" {
   description = "Ordered VM hostnames."
-  value       = [for n in local.nodes : n.name]
+  value       = [for n in local.node_list : n.name]
 }
 
 output "node_ips" {
   description = "Ordered node static IPs (same order as node_names)."
-  value       = [for n in local.nodes : n.ip]
+  value       = [for n in local.node_list : n.ip]
 }
 
 output "nodes" {
   description = "Per-node map name => ip, for building the Ansible inventory."
-  value       = { for n in local.nodes : n.name => n.ip }
+  value       = { for n in local.node_list : n.name => n.ip }
 }
 
 output "primary_node_ip" {
   description = "First server IP — k3s --cluster-init host and Cilium k8sServiceHost (parity value)."
-  value       = local.nodes[0].ip
+  value       = local.primary.ip
 }
 
 output "k3s_api_endpoint" {
   description = "kube-apiserver endpoint used for k8sServiceHost/Port parity (docs/02)."
-  value       = "${local.nodes[0].ip}:6443"
+  value       = "${local.primary.ip}:6443"
 }
 
 output "cilium_l2_vip_range" {
@@ -40,10 +40,10 @@ output "ansible_inventory" {
   value       = <<-EOT
     # GENERATED from tofu outputs — do not hand-edit; re-render on IP/name change.
     [k3s_primary]
-    ${local.nodes[0].name} ansible_host=${local.nodes[0].ip}
+    ${local.primary.name} ansible_host=${local.primary.ip}
 
     [k3s_secondary]
-    %{for n in slice(local.nodes, 1, length(local.nodes))~}
+    %{for n in slice(local.node_list, 1, length(local.node_list))~}
     ${n.name} ansible_host=${n.ip}
     %{endfor~}
 
@@ -53,6 +53,6 @@ output "ansible_inventory" {
 
     [k3s_servers:vars]
     ansible_user=${var.login_user}
-    k3s_api_host=${local.nodes[0].ip}
+    k3s_api_host=${local.primary.ip}
   EOT
 }