diff --git a/DECISIONS-LOG.md b/DECISIONS-LOG.md index fe6d910..69da140 100644 --- a/DECISIONS-LOG.md +++ b/DECISIONS-LOG.md @@ -78,8 +78,18 @@ primary play runs, so a limit that excludes the primary yields play must have populated in the same run. Then a true single-node rejoin (`--limit k3s-dev-N`) works hands-off, which is the T-1.5 durability bar. -Both fixes are **IaC-hardening items for Phase 2** (no cluster change now; the running cluster is -healthy). Logged here; not yet applied. +### F1 / F2 — IMPLEMENTED (author-first, 2026-08-16) +- **F1 done:** `infra/tofu/main.tf` node resources refactored `count` → `for_each` (keyed by node + name); ordered-list/`primary` helper locals added; `outputs.tf` updated off the map; **`moved.tf`** + adds 12 `moved` blocks (os/data/seed/domain × 3). `tofu validate` OK and **`tofu plan` = `0 to add, + 0 to change, 0 to destroy`** with all 12 resources reported "has moved to" — a pure state-address + rename, no VM churn. **State-persist pending:** a follow-up `tofu apply` (a proven 0-change apply + that only rewrites state addresses) is operator-permission-gated; until then the migration re-shows + harmlessly on each plan and lands automatically on the next real apply. +- **F2 done:** `roles/k3s_server/tasks/main.yml` — secondary now fetches the join token via + `delegate_to: primary` (with the published fact as a fast path), so `--limit k3s-dev-N` rejoin is + self-contained. `ansible-playbook --syntax-check` OK on all playbooks; role lint clean under the CI + gate (`make -C ci gate` PASS). No cluster change; the running cluster is untouched. ## KNOWN ISSUES to fix before Phase-2 un-suspend (logged, not yet fixed) | # | Issue | Impact | Fix | diff --git a/infra/ansible/roles/k3s_server/tasks/main.yml b/infra/ansible/roles/k3s_server/tasks/main.yml index ee19b25..728518e 100644 --- a/infra/ansible/roles/k3s_server/tasks/main.yml +++ b/infra/ansible/roles/k3s_server/tasks/main.yml @@ -47,7 +47,29 @@ k3s_node_token: "{{ k3s_token_raw.content | b64decode | trim }}" when: k3s_server_mode == 'primary' -# --- secondaries: join the existing cluster -------------------------------- +# --- secondaries: obtain the join token, then join the existing cluster ----- +# Fetch the token straight from the primary via delegate_to, falling back to it +# only when the primary play didn't publish the fact this run. This makes a +# single-node rejoin (`--limit k3s-dev-N`, primary NOT in the batch) self- +# contained — delegate_to still reaches the primary even when it isn't a play +# target, so the join no longer depends on a fact an earlier play had to set. +# T-1.5 finding F2 (see DECISIONS-LOG.md). +- name: Fetch the k3s node-token directly from the primary (secondary) + ansible.builtin.slurp: + src: /var/lib/rancher/k3s/server/token + delegate_to: "{{ groups['k3s_primary'][0] }}" + register: k3s_primary_token_raw + when: + - k3s_server_mode == 'secondary' + - hostvars[groups['k3s_primary'][0]].k3s_node_token is not defined + +- name: Resolve the join token (secondary) + ansible.builtin.set_fact: + k3s_join_token: >- + {{ hostvars[groups['k3s_primary'][0]].k3s_node_token + | default(k3s_primary_token_raw.content | b64decode | trim, true) }} + when: k3s_server_mode == 'secondary' + - name: Install k3s secondary (join as server) ansible.builtin.command: cmd: >- @@ -56,7 +78,7 @@ creates: /etc/rancher/k3s/k3s.yaml environment: INSTALL_K3S_VERSION: "{{ k3s_version }}" - K3S_TOKEN: "{{ hostvars[groups['k3s_primary'][0]]['k3s_node_token'] }}" + K3S_TOKEN: "{{ k3s_join_token }}" when: k3s_server_mode == 'secondary' - name: Wait for the k3s API to be serving locally diff --git a/infra/tofu/main.tf b/infra/tofu/main.tf index 516358c..794dc72 100644 --- a/infra/tofu/main.tf +++ b/infra/tofu/main.tf @@ -8,15 +8,27 @@ locals { # 1 GiB in bytes; libvirt volume sizes are bytes. gib = 1073741824 - # Per-node identity/addressing, indexed 0..node_count-1. - nodes = [ - for i in range(var.node_count) : { + # Per-node identity/addressing, KEYED BY NODE NAME (map, not list). + # for_each over this map gives every node a stable, name-based state address + # (libvirt_domain.node["k3s-dev-2"]) instead of a positional one (…node[1]). + # Positional count indexes are brittle under `-target`/`-replace`: a single-node + # destroy on a count-indexed resource can cascade to siblings (T-1.5 finding F1). + # Name keys are individually addressable, so one node can be replaced/destroyed + # without disturbing the others. See DECISIONS-LOG.md "T-1.5 … Finding F1". + nodes = { + for i in range(var.node_count) : "${var.node_name_prefix}${i + 1}" => { name = "${var.node_name_prefix}${i + 1}" ip = var.node_ips[i] fqdn = "${var.node_name_prefix}${i + 1}.${var.node_domain}" is_first = i == 0 } - ] + } + + # Ordered (node1..nodeN) list + primary view for outputs that need positional + # access — the resource for_each uses the map above; these preserve ordering + # for the Ansible inventory (primary = cluster-init server, rest = secondaries). + node_list = [for i in range(var.node_count) : local.nodes["${var.node_name_prefix}${i + 1}"]] + primary = local.nodes["${var.node_name_prefix}1"] } # --- dedicated NAT network (10.73.0.0/24, gw .1) --------------------------- @@ -48,8 +60,8 @@ resource "libvirt_volume" "base" { # --- per-node OS disk: thin qcow2 backed by the base image ------------------ resource "libvirt_volume" "os" { - count = var.node_count - name = "${local.nodes[count.index].name}-os.qcow2" + for_each = local.nodes + name = "${each.value.name}-os.qcow2" pool = var.pool_name base_volume_id = libvirt_volume.base.id size = var.os_disk_gib * local.gib @@ -58,34 +70,34 @@ resource "libvirt_volume" "os" { # --- per-node data disk: blank thin qcow2 for Longhorn ---------------------- resource "libvirt_volume" "data" { - count = var.node_count - name = "${local.nodes[count.index].name}-data.qcow2" - pool = var.pool_name - size = var.data_disk_gib * local.gib - format = "qcow2" + for_each = local.nodes + name = "${each.value.name}-data.qcow2" + pool = var.pool_name + size = var.data_disk_gib * local.gib + format = "qcow2" } # --- cloud-init seed (user-data + meta-data + static network-config) -------- # Templates live in the sibling infra/cloud-init/ subtree (T-1.2). resource "libvirt_cloudinit_disk" "seed" { - count = var.node_count - name = "${local.nodes[count.index].name}-seed.iso" - pool = var.pool_name + for_each = local.nodes + name = "${each.value.name}-seed.iso" + pool = var.pool_name user_data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", { - hostname = local.nodes[count.index].name - fqdn = local.nodes[count.index].fqdn + hostname = each.value.name + fqdn = each.value.fqdn username = var.login_user ssh_authorized_key = var.ssh_authorized_key }) meta_data = templatefile("${path.module}/../cloud-init/meta-data.yaml.tftpl", { - hostname = local.nodes[count.index].name - fqdn = local.nodes[count.index].fqdn + hostname = each.value.name + fqdn = each.value.fqdn }) network_config = templatefile("${path.module}/../cloud-init/network-config.yaml.tftpl", { - ip = local.nodes[count.index].ip + ip = each.value.ip prefix = var.network_prefix gateway = var.network_gateway }) @@ -93,10 +105,10 @@ resource "libvirt_cloudinit_disk" "seed" { # --- the VMs ---------------------------------------------------------------- resource "libvirt_domain" "node" { - count = var.node_count - name = local.nodes[count.index].name - memory = var.memory_mib - vcpu = var.vcpu + for_each = local.nodes + name = each.value.name + memory = var.memory_mib + vcpu = var.vcpu # host-passthrough so nested virt is available for KubeVirt in Phase 5. cpu { @@ -106,7 +118,7 @@ resource "libvirt_domain" "node" { # qemu-guest-agent is installed by cloud-init; lets tofu read the lease/agent. qemu_agent = true - cloudinit = libvirt_cloudinit_disk.seed[count.index].id + cloudinit = libvirt_cloudinit_disk.seed[each.key].id # IP + hostname come from cloud-init (DHCP is off on this network), so we do NOT # set addresses/hostname here — the provider couples those to DHCP reservations. @@ -117,11 +129,11 @@ resource "libvirt_domain" "node" { # OS disk first (vda, boot), Longhorn data disk second (vdb). disk { - volume_id = libvirt_volume.os[count.index].id + volume_id = libvirt_volume.os[each.key].id } disk { - volume_id = libvirt_volume.data[count.index].id + volume_id = libvirt_volume.data[each.key].id } console { diff --git a/infra/tofu/moved.tf b/infra/tofu/moved.tf new file mode 100644 index 0000000..d99a353 --- /dev/null +++ b/infra/tofu/moved.tf @@ -0,0 +1,68 @@ +# moved.tf — state-address migration for the count -> for_each refactor (T-1.5 F1). +# +# The node resources changed from positional `count` indexes ([0],[1],[2]) to +# name-keyed `for_each` (["k3s-dev-1"], …). Without these `moved` blocks OpenTofu +# would read the address change as "destroy the old, create the new" and rebuild +# every node. Each block tells tofu the resource is the SAME object under a new +# address, so the plan is a pure in-state rename: NO destroy, NO recreate. +# +# LOCKED mapping (node_count = 3, prefix "k3s-dev-"): index i -> "k3s-dev-${i+1}". +# `moved` blocks are static config (no for_each), so they are written explicitly. +# Safe to KEEP indefinitely (harmless once applied) or delete after one apply on +# every environment that held count-indexed state. + +# --- OS disks --------------------------------------------------------------- +moved { + from = libvirt_volume.os[0] + to = libvirt_volume.os["k3s-dev-1"] +} +moved { + from = libvirt_volume.os[1] + to = libvirt_volume.os["k3s-dev-2"] +} +moved { + from = libvirt_volume.os[2] + to = libvirt_volume.os["k3s-dev-3"] +} + +# --- data disks ------------------------------------------------------------- +moved { + from = libvirt_volume.data[0] + to = libvirt_volume.data["k3s-dev-1"] +} +moved { + from = libvirt_volume.data[1] + to = libvirt_volume.data["k3s-dev-2"] +} +moved { + from = libvirt_volume.data[2] + to = libvirt_volume.data["k3s-dev-3"] +} + +# --- cloud-init seeds ------------------------------------------------------- +moved { + from = libvirt_cloudinit_disk.seed[0] + to = libvirt_cloudinit_disk.seed["k3s-dev-1"] +} +moved { + from = libvirt_cloudinit_disk.seed[1] + to = libvirt_cloudinit_disk.seed["k3s-dev-2"] +} +moved { + from = libvirt_cloudinit_disk.seed[2] + to = libvirt_cloudinit_disk.seed["k3s-dev-3"] +} + +# --- domains (VMs) ---------------------------------------------------------- +moved { + from = libvirt_domain.node[0] + to = libvirt_domain.node["k3s-dev-1"] +} +moved { + from = libvirt_domain.node[1] + to = libvirt_domain.node["k3s-dev-2"] +} +moved { + from = libvirt_domain.node[2] + to = libvirt_domain.node["k3s-dev-3"] +} diff --git a/infra/tofu/outputs.tf b/infra/tofu/outputs.tf index e892c56..3e1b6ae 100644 --- a/infra/tofu/outputs.tf +++ b/infra/tofu/outputs.tf @@ -4,27 +4,27 @@ output "node_names" { description = "Ordered VM hostnames." - value = [for n in local.nodes : n.name] + value = [for n in local.node_list : n.name] } output "node_ips" { description = "Ordered node static IPs (same order as node_names)." - value = [for n in local.nodes : n.ip] + value = [for n in local.node_list : n.ip] } output "nodes" { description = "Per-node map name => ip, for building the Ansible inventory." - value = { for n in local.nodes : n.name => n.ip } + value = { for n in local.node_list : n.name => n.ip } } output "primary_node_ip" { description = "First server IP — k3s --cluster-init host and Cilium k8sServiceHost (parity value)." - value = local.nodes[0].ip + value = local.primary.ip } output "k3s_api_endpoint" { description = "kube-apiserver endpoint used for k8sServiceHost/Port parity (docs/02)." - value = "${local.nodes[0].ip}:6443" + value = "${local.primary.ip}:6443" } output "cilium_l2_vip_range" { @@ -40,10 +40,10 @@ output "ansible_inventory" { value = <<-EOT # GENERATED from tofu outputs — do not hand-edit; re-render on IP/name change. [k3s_primary] - ${local.nodes[0].name} ansible_host=${local.nodes[0].ip} + ${local.primary.name} ansible_host=${local.primary.ip} [k3s_secondary] - %{for n in slice(local.nodes, 1, length(local.nodes))~} + %{for n in slice(local.node_list, 1, length(local.node_list))~} ${n.name} ansible_host=${n.ip} %{endfor~} @@ -53,6 +53,6 @@ output "ansible_inventory" { [k3s_servers:vars] ansible_user=${var.login_user} - k3s_api_host=${local.nodes[0].ip} + k3s_api_host=${local.primary.ip} EOT }