home-network-k3s-impl/infra/tofu/main.tf

# main.tf — 3 cloud-init'd Proxmox VE VMs (k3s-dev-1/2/3), each with an imported
# OS disk (from the Debian cloud image) and a blank Longhorn data disk (R-ENV-01).
#
# Node lifecycle stage 1 ("Create") per docs/02. cloud-init (infra/cloud-init/)
# does stage 2 ("Baseline"); Ansible (infra/ansible/) does stage 3 ("Cluster-join").
#
# Provider: bpg/proxmox. VMs land on the Proxmox node `var.proxmox_node`, disks on
# `var.datastore` (ZFS "Storage"), NIC on `var.bridge` (vmbr0), static IPs via the
# native cloud-init `initialization` block. Native cloud-init (login user + SSH keys,
# static IP/gateway/DNS) is set via the Proxmox 8006 API. The rich first-boot baseline
# (packages, kernel modules, Longhorn mount, swap-off) is delivered by the Ansible
# `node_baseline` role (infra/ansible), run before k3s — not as a cloud-init snippet.

locals {
  # Per-node identity/addressing, KEYED BY NODE NAME (map, not list).
  # for_each over this map gives every node a stable, name-based state address
  # (…vm.node["k3s-dev-2"]) instead of a positional one. Positional count indexes
  # are brittle under -target/-replace: a single-node destroy on a count-indexed
  # resource can cascade to siblings (F1 finding). Name keys are individually
  # addressable, so one node can be replaced/destroyed without disturbing the others.
  nodes = {
    for i in range(var.node_count) : "${var.node_name_prefix}${i + 1}" => {
      name     = "${var.node_name_prefix}${i + 1}"
      ip       = var.node_ips[i]
      fqdn     = "${var.node_name_prefix}${i + 1}.${var.node_domain}"
      is_first = i == 0
    }
  }

  # Ordered (node1..nodeN) list + primary view for outputs that need positional
  # access — the resource for_each uses the map above; these preserve ordering for
  # the Ansible inventory (primary = cluster-init server, rest = secondaries).
  node_list = [for i in range(var.node_count) : local.nodes["${var.node_name_prefix}${i + 1}"]]
  primary   = local.nodes["${var.node_name_prefix}1"]
}

# --- base OS cloud image (downloaded once to `local`, imported per node) -----
# content_type "import" (PVE 8.2+): the image must be on `import`-content storage to
# be usable as a VM disk `import_from` (uncompressed qcow2 only — ours qualifies).
# OPERATOR ACTION: enable the "Import" content type on var.image_datastore (Datacenter
# -> Storage -> local -> Content -> check "Import"), else the disk import fails with
# "wrong type 'iso' - needs to be 'images' or 'import'".
#
# Resource type is the SHORT name `proxmox_download_file` (bpg/proxmox ADR-007). The old
# `proxmox_virtual_environment_download_file` is deprecated ("Use `proxmox_download_file`
# instead. This resource will be removed in v1.0.") — both are the same dual-registered
# Framework resource, so the schema is identical. The `moved` block below migrates existing
# state in place (provider implements MoveState / migration.PrefixMoveState), so no
# re-download of the image and no change to the VMs.
resource "proxmox_download_file" "debian_cloud_image" {
  content_type = "import"
  datastore_id = var.image_datastore
  node_name    = var.proxmox_node
  url          = var.image_source
  file_name    = var.image_file_name
  overwrite    = false
}

# State migration for the deprecated -> short resource-type rename (ADR-007). OpenTofu
# 1.9 (>= 1.8 required) drives the provider's MoveState RPC: the download_file state is
# transferred in place, keeping the same id (datastore_id:content_type/file_name), so the
# already-downloaded Debian image is NOT re-downloaded and the VMs' import_from is unchanged.
moved {
  from = proxmox_virtual_environment_download_file.debian_cloud_image
  to   = proxmox_download_file.debian_cloud_image
}

# --- cloud-init: NATIVE Proxmox cloud-init only (API-driven) ------------------
# The rich first-boot baseline (packages, kernel modules, Longhorn disk, swap-off)
# is NOT delivered as a custom snippet any more — custom snippets require SSH/SFTP
# to the Proxmox host, which is not reachable here. Instead the VM's `initialization`
# block below uses bpg's native user_account/ip_config/dns, which Proxmox sets via
# the built-in cloud-init API options (ciuser/sshkeys/ipconfig0/nameserver). The
# baseline moved to the Ansible `node_baseline` role (infra/ansible), run before k3s.

# --- the VMs ----------------------------------------------------------------
resource "proxmox_virtual_environment_vm" "node" {
  for_each  = local.nodes
  name      = each.value.name
  node_name = var.proxmox_node
  tags      = ["k3s", "k3s-dev", "opentofu"]

  description     = "k3s dev node ${each.value.name} (${each.value.fqdn}) — provisioned by OpenTofu."
  on_boot         = true
  stop_on_destroy = true

  # `host` CPU type passes through the physical CPU so nested virt is available for
  # KubeVirt in Phase 5.
  cpu {
    cores = var.vcpu
    type  = var.cpu_type
  }

  memory {
    dedicated = var.memory_mib
  }

  # qemu-guest-agent is installed by the Ansible node_baseline role now (not cloud-init),
  # so don't block VM create on the agent — static IPs are assigned, not discovered.
  agent {
    enabled = false
  }

  operating_system {
    type = "l26"
  }

  # Serial console (socket): Debian cloud images boot with console=ttyS0 and can panic
  # on import+resize when no serial device is present (bpg #1639). Standard + harmless
  # for cloud images; the default VGA display is kept so the Proxmox UI console works too.
  serial_device {
    device = "socket"
  }

  # OS disk (virtio0 -> /dev/vda, boot): a copy of the imported cloud image, grown
  # to os_disk_gib. Longhorn data disk (virtio1 -> /dev/vdb): blank, data_disk_gib.
  # virtio* keeps the guest device names (/dev/vda, /dev/vdb) the cloud-init
  # fs_setup/mount for /var/lib/longhorn already expects.
  disk {
    datastore_id = var.datastore
    import_from  = proxmox_download_file.debian_cloud_image.id
    interface    = "virtio0"
    size         = var.os_disk_gib
    discard      = "on"
  }

  disk {
    datastore_id = var.datastore
    interface    = "virtio1"
    size         = var.data_disk_gib
    discard      = "on"
  }

  network_device {
    bridge = var.bridge
  }

  # Static addressing + DNS come from cloud-init (bpg generates the network-config),
  # so the libvirt netplan template is gone. Native cloud-init (login user + SSH keys)
  # comes from user_account below; the OS baseline is the Ansible node_baseline role.
  initialization {
    datastore_id = var.datastore

    ip_config {
      ipv4 {
        address = "${each.value.ip}/${var.network_prefix}"
        gateway = var.network_gateway
      }
    }

    # DNS resilience: PiHole (var.dns_server) stays PRIMARY for the local jackson.lan zone +
    # network ad-blocking; the pfSense gateway (var.network_gateway) is an additive FALLBACK so
    # a fresh node's containerd pulls + CoreDNS survive PiHole external-recursion outages (a
    # single upstream was a hard SPOF — see the node_baseline "DNS resilience" task and
    # DECISIONS-LOG). Existing nodes get the same two servers from the node_baseline netplan
    # drop-in; this keeps cloud-init/netplan on rebuild in parity.
    dns {
      servers = [var.dns_server, var.network_gateway]
    }

    user_account {
      username = var.login_user
      keys     = [trimspace(var.ssh_authorized_key)]
    }
  }

  # Boot from the OS disk only.
  boot_order = ["virtio0"]
}