From 53ca698edcd41b51e2dc294b31001f25520e4a00 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Tue, 25 Aug 2026 10:23:13 -0500 Subject: [PATCH] infra(tofu): migrate node provisioning from libvirt to bpg/proxmox Replace the dmacvicar/libvirt provider + resources with the modern bpg/proxmox provider. The dev cluster moves off the memory-constrained local libvirt host onto the Proxmox VE box. - versions.tf: pin bpg/proxmox ~> 0.66; provider reads auth from env (PROXMOX_VE_*, token never a tofu var) and normalizes a schemeless PROXMOX_VE_ENDPOINT (prepend https://) to avoid the :8006 HTTP 301. - variables.tf: Proxmox placement vars (node=proxmox, datastore=Storage, image_datastore=local, bridge=vmbr0); LAN 192.168.1.0/24 gw .1 DNS .18; node_ips .50/.51/.52; sizing 6 vCPU / 16384 MiB / 40+100 GiB; cpu_type host; Cilium VIP range .53-.60. - main.tf: proxmox_virtual_environment_download_file (cloud image -> local), per-node user-data snippet (proxmox_virtual_environment_file), and proxmox_virtual_environment_vm per node via name-keyed for_each (imported OS disk virtio0, blank data disk virtio1, native initialization ip_config + dns + user_data_file_id). - remove moved.tf (libvirt-specific count->for_each state migration). Authoring only; no apply. Traces R-ENV-01, R-PLAT-01. Co-Authored-By: Claude Opus 4.8 --- infra/tofu/main.tf | 219 +++++++++++++++++++--------------------- infra/tofu/moved.tf | 68 ------------- infra/tofu/variables.tf | 134 +++++++++++++++--------- infra/tofu/versions.tf | 45 +++++++-- 4 files changed, 229 insertions(+), 237 deletions(-) delete mode 100644 infra/tofu/moved.tf diff --git a/infra/tofu/main.tf b/infra/tofu/main.tf index 794dc72..9902e00 100644 --- a/infra/tofu/main.tf +++ b/infra/tofu/main.tf @@ -1,20 +1,22 @@ -# main.tf — dedicated NAT network + 3 cloud-init'd VMs, each with a thin OS disk -# (backed by the cloud image) and a blank thin Longhorn data disk (Phase 1 T-1.1). +# main.tf — 3 cloud-init'd Proxmox VE VMs (k3s-dev-1/2/3), each with an imported +# OS disk (from the Debian cloud image) and a blank Longhorn data disk (R-ENV-01). # # Node lifecycle stage 1 ("Create") per docs/02. cloud-init (infra/cloud-init/) # does stage 2 ("Baseline"); Ansible (infra/ansible/) does stage 3 ("Cluster-join"). +# +# Provider: bpg/proxmox. VMs land on the Proxmox node `var.proxmox_node`, disks on +# `var.datastore` (ZFS "Storage"), NIC on `var.bridge` (vmbr0), static IPs via the +# native cloud-init `initialization` block. The rich first-boot baseline (users, +# packages, kernel modules, Longhorn mount, swap-off) is delivered as a cloud-init +# user-data SNIPPET (see the snippets note in README / operator action below). locals { - # 1 GiB in bytes; libvirt volume sizes are bytes. - gib = 1073741824 - # Per-node identity/addressing, KEYED BY NODE NAME (map, not list). # for_each over this map gives every node a stable, name-based state address - # (libvirt_domain.node["k3s-dev-2"]) instead of a positional one (…node[1]). - # Positional count indexes are brittle under `-target`/`-replace`: a single-node - # destroy on a count-indexed resource can cascade to siblings (T-1.5 finding F1). - # Name keys are individually addressable, so one node can be replaced/destroyed - # without disturbing the others. See DECISIONS-LOG.md "T-1.5 … Finding F1". + # (…vm.node["k3s-dev-2"]) instead of a positional one. Positional count indexes + # are brittle under -target/-replace: a single-node destroy on a count-indexed + # resource can cascade to siblings (F1 finding). Name keys are individually + # addressable, so one node can be replaced/destroyed without disturbing the others. nodes = { for i in range(var.node_count) : "${var.node_name_prefix}${i + 1}" => { name = "${var.node_name_prefix}${i + 1}" @@ -25,132 +27,123 @@ locals { } # Ordered (node1..nodeN) list + primary view for outputs that need positional - # access — the resource for_each uses the map above; these preserve ordering - # for the Ansible inventory (primary = cluster-init server, rest = secondaries). + # access — the resource for_each uses the map above; these preserve ordering for + # the Ansible inventory (primary = cluster-init server, rest = secondaries). node_list = [for i in range(var.node_count) : local.nodes["${var.node_name_prefix}${i + 1}"]] primary = local.nodes["${var.node_name_prefix}1"] } -# --- dedicated NAT network (10.73.0.0/24, gw .1) --------------------------- -# DHCP is OFF: nodes get static IPs from cloud-init network-config, so the -# reserved VIP range (.240-.250) can never be leased to something else. -resource "libvirt_network" "k3s_dev" { - name = var.network_name - mode = "nat" - addresses = [var.network_cidr] - autostart = true +# --- base OS cloud image (downloaded once to `local`, imported per node) ----- +# content_type "iso" stores it on the ISO datastore; the .img file name lets +# Proxmox accept the qcow2 (it still reads it as qcow2 on disk import). +resource "proxmox_virtual_environment_download_file" "debian_cloud_image" { + content_type = "iso" + datastore_id = var.image_datastore + node_name = var.proxmox_node + url = var.image_source + file_name = var.image_file_name + overwrite = false +} - dhcp { - enabled = false +# --- per-node cloud-init user-data snippet ---------------------------------- +# bpg's native initialization.user_account only sets user/keys/password; it can't +# carry packages/write_files/runcmd. To preserve the full k3s/Cilium/Longhorn +# baseline we render the existing user-data.yaml.tftpl and ship it as a SNIPPET, +# referenced by the VM's initialization.user_data_file_id. +# +# OPERATOR ACTION: the `snippets` content type must be enabled on var.image_datastore +# (Datacenter -> Storage -> local -> Content -> add "Snippets"), else this file +# resource fails. See README "Operator prerequisites". +resource "proxmox_virtual_environment_file" "user_data" { + for_each = local.nodes + content_type = "snippets" + datastore_id = var.image_datastore + node_name = var.proxmox_node + + source_raw { + file_name = "${each.value.name}-user-data.yaml" + data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", { + hostname = each.value.name + fqdn = each.value.fqdn + username = var.login_user + ssh_authorized_key = var.ssh_authorized_key + }) } - - dns { - enabled = true - local_only = true - } -} - -# --- base OS volume (downloaded/copied once, shared as a backing file) ------ -resource "libvirt_volume" "base" { - name = "${var.cluster_name}-base.qcow2" - pool = var.pool_name - source = var.image_source - format = "qcow2" -} - -# --- per-node OS disk: thin qcow2 backed by the base image ------------------ -resource "libvirt_volume" "os" { - for_each = local.nodes - name = "${each.value.name}-os.qcow2" - pool = var.pool_name - base_volume_id = libvirt_volume.base.id - size = var.os_disk_gib * local.gib - format = "qcow2" -} - -# --- per-node data disk: blank thin qcow2 for Longhorn ---------------------- -resource "libvirt_volume" "data" { - for_each = local.nodes - name = "${each.value.name}-data.qcow2" - pool = var.pool_name - size = var.data_disk_gib * local.gib - format = "qcow2" -} - -# --- cloud-init seed (user-data + meta-data + static network-config) -------- -# Templates live in the sibling infra/cloud-init/ subtree (T-1.2). -resource "libvirt_cloudinit_disk" "seed" { - for_each = local.nodes - name = "${each.value.name}-seed.iso" - pool = var.pool_name - - user_data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", { - hostname = each.value.name - fqdn = each.value.fqdn - username = var.login_user - ssh_authorized_key = var.ssh_authorized_key - }) - - meta_data = templatefile("${path.module}/../cloud-init/meta-data.yaml.tftpl", { - hostname = each.value.name - fqdn = each.value.fqdn - }) - - network_config = templatefile("${path.module}/../cloud-init/network-config.yaml.tftpl", { - ip = each.value.ip - prefix = var.network_prefix - gateway = var.network_gateway - }) } # --- the VMs ---------------------------------------------------------------- -resource "libvirt_domain" "node" { - for_each = local.nodes - name = each.value.name - memory = var.memory_mib - vcpu = var.vcpu +resource "proxmox_virtual_environment_vm" "node" { + for_each = local.nodes + name = each.value.name + node_name = var.proxmox_node + tags = ["k3s", "k3s-dev", "opentofu"] - # host-passthrough so nested virt is available for KubeVirt in Phase 5. + description = "k3s dev node ${each.value.name} (${each.value.fqdn}) — provisioned by OpenTofu." + on_boot = true + stop_on_destroy = true + + # `host` CPU type passes through the physical CPU so nested virt is available for + # KubeVirt in Phase 5. cpu { - mode = var.cpu_mode + cores = var.vcpu + type = var.cpu_type } - # qemu-guest-agent is installed by cloud-init; lets tofu read the lease/agent. - qemu_agent = true - - cloudinit = libvirt_cloudinit_disk.seed[each.key].id - - # IP + hostname come from cloud-init (DHCP is off on this network), so we do NOT - # set addresses/hostname here — the provider couples those to DHCP reservations. - network_interface { - network_id = libvirt_network.k3s_dev.id - wait_for_lease = false + memory { + dedicated = var.memory_mib } - # OS disk first (vda, boot), Longhorn data disk second (vdb). + # qemu-guest-agent is installed by cloud-init; lets tofu/Proxmox read guest state. + agent { + enabled = true + } + + operating_system { + type = "l26" + } + + # OS disk (virtio0 -> /dev/vda, boot): a copy of the imported cloud image, grown + # to os_disk_gib. Longhorn data disk (virtio1 -> /dev/vdb): blank, data_disk_gib. + # virtio* keeps the guest device names (/dev/vda, /dev/vdb) the cloud-init + # fs_setup/mount for /var/lib/longhorn already expects. disk { - volume_id = libvirt_volume.os[each.key].id + datastore_id = var.datastore + import_from = proxmox_virtual_environment_download_file.debian_cloud_image.id + interface = "virtio0" + size = var.os_disk_gib + discard = "on" } disk { - volume_id = libvirt_volume.data[each.key].id + datastore_id = var.datastore + interface = "virtio1" + size = var.data_disk_gib + discard = "on" } - console { - type = "pty" - target_port = "0" - target_type = "serial" + network_device { + bridge = var.bridge } - console { - type = "pty" - target_port = "1" - target_type = "virtio" + # Static addressing + DNS come from cloud-init (bpg generates the network-config), + # so the libvirt netplan template is gone. User-data (the baseline) is the snippet. + initialization { + datastore_id = var.datastore + + ip_config { + ipv4 { + address = "${each.value.ip}/${var.network_prefix}" + gateway = var.network_gateway + } + } + + dns { + servers = [var.dns_server] + } + + user_data_file_id = proxmox_virtual_environment_file.user_data[each.key].id } - graphics { - type = "spice" - listen_type = "address" - autoport = true - } + # Boot from the OS disk only. + boot_order = ["virtio0"] } diff --git a/infra/tofu/moved.tf b/infra/tofu/moved.tf deleted file mode 100644 index d99a353..0000000 --- a/infra/tofu/moved.tf +++ /dev/null @@ -1,68 +0,0 @@ -# moved.tf — state-address migration for the count -> for_each refactor (T-1.5 F1). -# -# The node resources changed from positional `count` indexes ([0],[1],[2]) to -# name-keyed `for_each` (["k3s-dev-1"], …). Without these `moved` blocks OpenTofu -# would read the address change as "destroy the old, create the new" and rebuild -# every node. Each block tells tofu the resource is the SAME object under a new -# address, so the plan is a pure in-state rename: NO destroy, NO recreate. -# -# LOCKED mapping (node_count = 3, prefix "k3s-dev-"): index i -> "k3s-dev-${i+1}". -# `moved` blocks are static config (no for_each), so they are written explicitly. -# Safe to KEEP indefinitely (harmless once applied) or delete after one apply on -# every environment that held count-indexed state. - -# --- OS disks --------------------------------------------------------------- -moved { - from = libvirt_volume.os[0] - to = libvirt_volume.os["k3s-dev-1"] -} -moved { - from = libvirt_volume.os[1] - to = libvirt_volume.os["k3s-dev-2"] -} -moved { - from = libvirt_volume.os[2] - to = libvirt_volume.os["k3s-dev-3"] -} - -# --- data disks ------------------------------------------------------------- -moved { - from = libvirt_volume.data[0] - to = libvirt_volume.data["k3s-dev-1"] -} -moved { - from = libvirt_volume.data[1] - to = libvirt_volume.data["k3s-dev-2"] -} -moved { - from = libvirt_volume.data[2] - to = libvirt_volume.data["k3s-dev-3"] -} - -# --- cloud-init seeds ------------------------------------------------------- -moved { - from = libvirt_cloudinit_disk.seed[0] - to = libvirt_cloudinit_disk.seed["k3s-dev-1"] -} -moved { - from = libvirt_cloudinit_disk.seed[1] - to = libvirt_cloudinit_disk.seed["k3s-dev-2"] -} -moved { - from = libvirt_cloudinit_disk.seed[2] - to = libvirt_cloudinit_disk.seed["k3s-dev-3"] -} - -# --- domains (VMs) ---------------------------------------------------------- -moved { - from = libvirt_domain.node[0] - to = libvirt_domain.node["k3s-dev-1"] -} -moved { - from = libvirt_domain.node[1] - to = libvirt_domain.node["k3s-dev-2"] -} -moved { - from = libvirt_domain.node[2] - to = libvirt_domain.node["k3s-dev-3"] -} diff --git a/infra/tofu/variables.tf b/infra/tofu/variables.tf index 273531c..62e1788 100644 --- a/infra/tofu/variables.tf +++ b/infra/tofu/variables.tf @@ -1,17 +1,52 @@ -# variables.tf — every sizing / IP / image knob is a variable (Phase 1 T-1.1). -# Defaults encode the LOCKED decisions in infra/PHASE1-SCOPE.md. Override via -# terraform.tfvars (see terraform.tfvars.example). +# variables.tf — every sizing / IP / image knob is a variable (Proxmox migration). +# Defaults encode the migration target: 3 k3s nodes on the Proxmox VE box +# (node "proxmox", 192.168.1.0/24 LAN). Override via terraform.tfvars (see +# terraform.tfvars.example). Traces R-ENV-01 (reproducible node provisioning), +# R-PLAT-01. -variable "libvirt_uri" { - description = "libvirt connection URI." +# --- Proxmox connection / placement ---------------------------------------- + +variable "proxmox_endpoint" { + description = <<-EOT + Proxmox VE API endpoint, e.g. "https://192.168.2.102:8006/". Optional: if unset + the provider reads PROXMOX_VE_ENDPOINT from the environment directly. Set this + (e.g. via `export TF_VAR_proxmox_endpoint="$PROXMOX_VE_ENDPOINT"`) to let tofu + NORMALIZE a schemeless endpoint (prepend https://) — see versions.tf. The API + TOKEN is NOT a variable; it is read from PROXMOX_VE_API_TOKEN by the provider so + it never enters tofu state/vars. + EOT type = string - default = "qemu:///system" + default = null } -variable "pool_name" { - description = "libvirt storage pool for all volumes (base image, OS + data disks, seed ISOs)." +variable "proxmox_insecure" { + description = "Skip TLS verification for the Proxmox self-signed cert (also read from PROXMOX_VE_INSECURE)." + type = bool + default = true +} + +variable "proxmox_node" { + description = "Proxmox VE node name that hosts the VMs (VERIFIED: proxmox)." type = string - default = "default" + default = "proxmox" +} + +variable "datastore" { + description = "Datastore for VM OS + data disks (VERIFIED: Storage, ZFS 4.3TB, rootdir+images)." + type = string + default = "Storage" +} + +variable "image_datastore" { + description = "Datastore that holds the downloaded cloud image / cloud-init snippets (VERIFIED: local, dir storage)." + type = string + default = "local" +} + +variable "bridge" { + description = "Proxmox network bridge for the node NIC (VERIFIED: vmbr0, no VLAN tag)." + type = string + default = "vmbr0" } # --- cluster identity / naming --------------------------------------------- @@ -39,50 +74,44 @@ variable "node_count" { } } -# --- per-node sizing (LOCKED: 4 vCPU / 6 GiB / 20 GiB OS + 40 GiB data) ----- +# --- per-node sizing (Proxmox target: 6 vCPU / 16 GiB / 40 GiB OS + 100 GiB data) -- variable "vcpu" { - description = "vCPUs per node." + description = "vCPUs (cores) per node. Generous — the box has plenty." type = number - default = 4 + default = 6 } variable "memory_mib" { - description = "RAM per node in MiB (6 GiB)." + description = "RAM per node in MiB (16 GiB)." type = number - default = 6144 + default = 16384 } variable "os_disk_gib" { - description = "OS (root) disk size per node, GiB. Thin qcow2 backed by the cloud image." - type = number - default = 20 -} - -variable "data_disk_gib" { - description = "Dedicated Longhorn data disk per node, GiB. Thin qcow2, blank." + description = "OS (root) disk size per node, GiB. Imported from the cloud image, on the Storage datastore." type = number default = 40 } -variable "cpu_mode" { - description = "libvirt CPU mode. host-passthrough exposes nested-virt for KubeVirt (Phase 5)." - type = string - default = "host-passthrough" +variable "data_disk_gib" { + description = "Dedicated Longhorn data disk per node, GiB. Blank, on the Storage datastore." + type = number + default = 100 } -# --- network (LOCKED: dedicated NAT k3s-dev 10.73.0.0/24, gw .1) ------------ - -variable "network_name" { - description = "libvirt NAT network name." +variable "cpu_type" { + description = "Proxmox vCPU type. `host` passes through the physical CPU (nested-virt for KubeVirt, Phase 5)." type = string - default = "k3s-dev" + default = "host" } +# --- network (Proxmox target: main LAN 192.168.1.0/24, gw .1, DNS PiHole .18) --- + variable "network_cidr" { - description = "Cluster NAT subnet in CIDR. Gateway (network_gateway) is the libvirt bridge host IP." + description = "Cluster subnet in CIDR (informational; nodes get static IPs from node_ips)." type = string - default = "10.73.0.0/24" + default = "192.168.1.0/24" } variable "network_prefix" { @@ -92,45 +121,60 @@ variable "network_prefix" { } variable "network_gateway" { - description = "Default gateway / DNS for nodes = libvirt bridge host IP (LOCKED: 10.73.0.1)." + description = "Default gateway for nodes (VERIFIED: 192.168.1.1, pfSense)." type = string - default = "10.73.0.1" + default = "192.168.1.1" +} + +variable "dns_server" { + description = "DNS resolver for nodes (VERIFIED: 192.168.1.18, PiHole)." + type = string + default = "192.168.1.18" } variable "node_ips" { - description = "Static IPs per node, in order (LOCKED: .11/.12/.13)." + description = "Static IPs per node, in order (VERIFIED: .50/.51/.52 on 192.168.1.0/24)." type = list(string) - default = ["10.73.0.11", "10.73.0.12", "10.73.0.13"] + default = ["192.168.1.50", "192.168.1.51", "192.168.1.52"] } -# Reserved for Cilium L2-announcement ingress VIPs (LOCKED: .240-.250). -# Not materialised as libvirt objects; surfaced in outputs and kept OUT of any -# DHCP range so nothing else claims them. Cilium's IP pool (WS-NET, Phase 2) -# consumes this range. +# Reserved for Cilium L2-announcement ingress VIPs (VERIFIED: .53-.60). +# Not materialised as Proxmox objects; surfaced in outputs. Cilium's IP pool +# (platform-config/cilium/l2-ip-pool.yaml) consumes this range. variable "cilium_l2_vip_range_start" { description = "First reserved Cilium L2 VIP." type = string - default = "10.73.0.240" + default = "192.168.1.53" } variable "cilium_l2_vip_range_end" { description = "Last reserved Cilium L2 VIP." type = string - default = "10.73.0.250" + default = "192.168.1.60" } # --- base OS image ---------------------------------------------------------- variable "image_source" { description = <<-EOT - Source for the base OS volume: a URL or local path to a minimal cloud image - (qcow2). Default is Debian 12 (bookworm) genericcloud amd64. Point at a local - path to avoid re-downloading on every apply. + Source URL for the base OS cloud image (qcow2), downloaded to image_datastore + by proxmox_virtual_environment_download_file. Default is Debian 12 (bookworm) + genericcloud amd64 — same image as the pre-migration libvirt module. EOT type = string default = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2" } +variable "image_file_name" { + description = <<-EOT + File name the cloud image is stored as on image_datastore. Proxmox `iso`-content + storage only accepts *.iso / *.img names, so a qcow2 URL must be saved with a + .img extension (Proxmox still reads it as qcow2 on import). + EOT + type = string + default = "debian-12-genericcloud-amd64.img" +} + # --- cloud-init identity ---------------------------------------------------- variable "ssh_authorized_key" { diff --git a/infra/tofu/versions.tf b/infra/tofu/versions.tf index 4a676fc..8cfbe17 100644 --- a/infra/tofu/versions.tf +++ b/infra/tofu/versions.tf @@ -1,25 +1,48 @@ -# versions.tf — OpenTofu + provider pins (Phase 1 T-1.1). +# versions.tf — OpenTofu + provider pins (Proxmox migration, R-ENV-01). # # Toolchain is pinned to match the dev container (infra/devcontainer/versions.env): -# OpenTofu 1.9.0. The dmacvicar/libvirt provider is pinned HERE (not in the -# dev container) per the note in that file. +# OpenTofu 1.9.0. The node provider is now bpg/proxmox (the modern, maintained +# Proxmox VE provider — NOT telmate), pinned HERE, replacing dmacvicar/libvirt. # # AUTHOR-ONLY: this module is written but NOT applied. The PM static-validates -# with `tofu fmt -check` / `tofu validate` in the dev container. +# with `tofu fmt -check` / `tofu validate`, then runs plan/apply against Proxmox. terraform { required_version = "1.9.0" required_providers { - libvirt = { - source = "dmacvicar/libvirt" - version = "~> 0.8.1" + proxmox = { + source = "bpg/proxmox" + version = "~> 0.66" } } } -provider "libvirt" { - # System-level libvirt (qemu:///system) so VMs/pools/networks are host-wide, - # matching the dev-container mount of the host libvirt socket (PHASE1-SCOPE risks). - uri = var.libvirt_uri +locals { + # Endpoint normalization: the operator's env file may export PROXMOX_VE_ENDPOINT + # WITHOUT the https:// scheme (e.g. "192.168.2.102:8006"). If the provider is + # handed a schemeless/HTTP endpoint it talks to :8006 over plain HTTP and eats a + # 301 redirect. So if var.proxmox_endpoint is set we prepend https:// when the + # scheme is missing. When the variable is unset (null/empty) we return null and + # let the provider fall back to its own PROXMOX_VE_ENDPOINT env read. + # + # RUNBOOK: to get normalization, source the env file AND export the tofu var, e.g. + # export TF_VAR_proxmox_endpoint="$PROXMOX_VE_ENDPOINT" + # (or set proxmox_endpoint in terraform.tfvars). See README. + proxmox_endpoint = ( + var.proxmox_endpoint == null || var.proxmox_endpoint == "" + ? null + : (can(regex("^https?://", var.proxmox_endpoint)) ? var.proxmox_endpoint : "https://${var.proxmox_endpoint}") + ) +} + +provider "proxmox" { + # Auth is read from the environment by the provider itself and is NEVER surfaced + # as a tofu variable, so the API token never lands in state, vars, or plan output: + # PROXMOX_VE_API_TOKEN = "user@realm!tokenid=secret" + # + # endpoint/insecure are set explicitly (below) so we control scheme normalization; + # when proxmox_endpoint is null the provider still auto-reads PROXMOX_VE_ENDPOINT. + endpoint = local.proxmox_endpoint + insecure = var.proxmox_insecure }