home-network-k3s-impl/infra/tofu/main.tf

# main.tf — 3 cloud-init'd Proxmox VE VMs (k3s-dev-1/2/3), each with an imported
# OS disk (from the Debian cloud image) and a blank Longhorn data disk (R-ENV-01).
#
# Node lifecycle stage 1 ("Create") per docs/02. cloud-init (infra/cloud-init/)
# does stage 2 ("Baseline"); Ansible (infra/ansible/) does stage 3 ("Cluster-join").
#
# Provider: bpg/proxmox. VMs land on the Proxmox node `var.proxmox_node`, disks on
# `var.datastore` (ZFS "Storage"), NIC on `var.bridge` (vmbr0), static IPs via the
# native cloud-init `initialization` block. Native cloud-init (login user + SSH keys,
# static IP/gateway/DNS) is set via the Proxmox 8006 API. The rich first-boot baseline
# (packages, kernel modules, Longhorn mount, swap-off) is delivered by the Ansible
# `node_baseline` role (infra/ansible), run before k3s — not as a cloud-init snippet.

locals {
  # Per-node identity/addressing, KEYED BY NODE NAME (map, not list).
  # for_each over this map gives every node a stable, name-based state address
  # (…vm.node["k3s-dev-2"]) instead of a positional one. Positional count indexes
  # are brittle under -target/-replace: a single-node destroy on a count-indexed
  # resource can cascade to siblings (F1 finding). Name keys are individually
  # addressable, so one node can be replaced/destroyed without disturbing the others.
  nodes = {
    for i in range(var.node_count) : "${var.node_name_prefix}${i + 1}" => {
      name     = "${var.node_name_prefix}${i + 1}"
      ip       = var.node_ips[i]
      fqdn     = "${var.node_name_prefix}${i + 1}.${var.node_domain}"
      is_first = i == 0
    }
  }

  # Ordered (node1..nodeN) list + primary view for outputs that need positional
  # access — the resource for_each uses the map above; these preserve ordering for
  # the Ansible inventory (primary = cluster-init server, rest = secondaries).
  node_list = [for i in range(var.node_count) : local.nodes["${var.node_name_prefix}${i + 1}"]]
  primary   = local.nodes["${var.node_name_prefix}1"]
}

# --- base OS cloud image (downloaded once to `local`, imported per node) -----
# content_type "import" (PVE 8.2+): the image must be on `import`-content storage to
# be usable as a VM disk `import_from` (uncompressed qcow2 only — ours qualifies).
# OPERATOR ACTION: enable the "Import" content type on var.image_datastore (Datacenter
# -> Storage -> local -> Content -> check "Import"), else the disk import fails with
# "wrong type 'iso' - needs to be 'images' or 'import'".
#
# The Debian cloud image is a bpg `download_file` resource. This intentionally uses the
# LONG (deprecated) type name `proxmox_virtual_environment_download_file`: the R2 backend
# state still holds this resource under the long name, and bpg >= 0.111 dropped the
# cross-type `moved{}` linkage to the short `proxmox_download_file` (ADR-007) — a `moved`
# block between the two now fails plan with "Resource type mismatch" (this broke plan on
# main entirely). Keeping the long name makes config match state, so the plan is clean and
# the ~450 MB image is NOT re-downloaded. Migrating to the short name is a SEPARATE,
# deliberate step (`tofu state mv proxmox_virtual_environment_download_file.debian_cloud_image
# proxmox_download_file.debian_cloud_image`, then rename here) — not bundled with the
# etcd-disk change so this bounce stays focused and low-risk.
resource "proxmox_virtual_environment_download_file" "debian_cloud_image" {
  content_type = "import"
  datastore_id = var.image_datastore
  node_name    = var.proxmox_node
  url          = var.image_source
  file_name    = var.image_file_name
  overwrite    = false
}

# --- cloud-init: NATIVE Proxmox cloud-init only (API-driven) ------------------
# The rich first-boot baseline (packages, kernel modules, Longhorn disk, swap-off)
# is NOT delivered as a custom snippet any more — custom snippets require SSH/SFTP
# to the Proxmox host, which is not reachable here. Instead the VM's `initialization`
# block below uses bpg's native user_account/ip_config/dns, which Proxmox sets via
# the built-in cloud-init API options (ciuser/sshkeys/ipconfig0/nameserver). The
# baseline moved to the Ansible `node_baseline` role (infra/ansible), run before k3s.

# --- the VMs ----------------------------------------------------------------
resource "proxmox_virtual_environment_vm" "node" {
  for_each  = local.nodes
  name      = each.value.name
  node_name = var.proxmox_node
  tags      = ["k3s", "k3s-dev", "opentofu"]

  description     = "k3s dev node ${each.value.name} (${each.value.fqdn}) — provisioned by OpenTofu."
  on_boot         = true
  stop_on_destroy = true

  # `host` CPU type passes through the physical CPU so nested virt is available for
  # KubeVirt in Phase 5.
  cpu {
    cores = var.vcpu
    type  = var.cpu_type
  }

  memory {
    dedicated = var.memory_mib
  }

  # qemu-guest-agent is installed by the Ansible node_baseline role now (not cloud-init),
  # so don't block VM create on the agent — static IPs are assigned, not discovered.
  agent {
    enabled = false
  }

  operating_system {
    type = "l26"
  }

  # Serial console (socket): Debian cloud images boot with console=ttyS0 and can panic
  # on import+resize when no serial device is present (bpg #1639). Standard + harmless
  # for cloud images; the default VGA display is kept so the Proxmox UI console works too.
  serial_device {
    device = "socket"
  }

  # OS disk (virtio0 -> /dev/vda, boot): a copy of the imported cloud image, grown
  # to os_disk_gib. Longhorn data disk (virtio1 -> /dev/vdb): blank, data_disk_gib.
  # virtio* keeps the guest device names (/dev/vda, /dev/vdb) the cloud-init
  # fs_setup/mount for /var/lib/longhorn already expects.
  disk {
    datastore_id = var.datastore
    import_from  = proxmox_virtual_environment_download_file.debian_cloud_image.id
    interface    = "virtio0"
    size         = var.os_disk_gib
    discard      = "on"
  }

  disk {
    datastore_id = var.datastore
    interface    = "virtio1"
    size         = var.data_disk_gib
    discard      = "on"
  }

  # Dedicated SSD etcd disk (virtio2 -> /dev/vdc). Isolates etcd WAL fsync from the HDD
  # `Storage` pool where the OS + Longhorn disks live — the node_baseline role formats
  # it and mounts it at /var/lib/rancher/k3s/server/db before k3s starts. On the SSD
  # `local-zfs` (rpool/data) datastore: an enterprise Micron SATA SSD mirror WITH
  # power-loss protection, so cache=none (honor guest fsync straight through to the ZFS
  # ZIL) is both durable and fast. Kept last (ascending interface) so bpg won't reorder
  # the existing virtio0/virtio1 disks. See docs/plans/ssd-etcd-isolation.md.
  disk {
    datastore_id = var.etcd_ssd_datastore
    interface    = "virtio2"
    size         = var.etcd_disk_gib
    discard      = "on"
    cache        = "none"
    iothread     = false
  }

  network_device {
    bridge = var.bridge
  }

  # Static addressing + DNS come from cloud-init (bpg generates the network-config),
  # so the libvirt netplan template is gone. Native cloud-init (login user + SSH keys)
  # comes from user_account below; the OS baseline is the Ansible node_baseline role.
  initialization {
    datastore_id = var.datastore

    ip_config {
      ipv4 {
        address = "${each.value.ip}/${var.network_prefix}"
        gateway = var.network_gateway
      }
    }

    # DNS resilience (dev only — tofu provisions the dev VMs; prod is bare metal with no
    # cloud-init). A SINGLE upstream was a hard SPOF: PiHole (var.dns_server) answers the
    # local jackson.lan zone but its external recursion flaps, which broke containerd image
    # pulls and CoreDNS forwarding. PiHole stays FIRST — the local zone and network
    # ad-blocking are unchanged — and the pfSense gateway (var.network_gateway) is added
    # purely as a fallback for external recursion. Order is the policy, not a preference.
    # This keeps a freshly built node's cloud-init in parity with what the Ansible
    # node_baseline role's `dns`-tagged block configures on an already-running node
    # (node_baseline_dns_servers), so a rebuild does not regress to one resolver.
    dns {
      servers = [var.dns_server, var.network_gateway]
    }

    user_account {
      username = var.login_user
      keys     = [trimspace(var.ssh_authorized_key)]
    }
  }

  # Boot from the OS disk only.
  boot_order = ["virtio0"]
}