infra(tofu): migrate node provisioning from libvirt to bpg/proxmox
Replace the dmacvicar/libvirt provider + resources with the modern bpg/proxmox provider. The dev cluster moves off the memory-constrained local libvirt host onto the Proxmox VE box. - versions.tf: pin bpg/proxmox ~> 0.66; provider reads auth from env (PROXMOX_VE_*, token never a tofu var) and normalizes a schemeless PROXMOX_VE_ENDPOINT (prepend https://) to avoid the :8006 HTTP 301. - variables.tf: Proxmox placement vars (node=proxmox, datastore=Storage, image_datastore=local, bridge=vmbr0); LAN 192.168.1.0/24 gw .1 DNS .18; node_ips .50/.51/.52; sizing 6 vCPU / 16384 MiB / 40+100 GiB; cpu_type host; Cilium VIP range .53-.60. - main.tf: proxmox_virtual_environment_download_file (cloud image -> local), per-node user-data snippet (proxmox_virtual_environment_file), and proxmox_virtual_environment_vm per node via name-keyed for_each (imported OS disk virtio0, blank data disk virtio1, native initialization ip_config + dns + user_data_file_id). - remove moved.tf (libvirt-specific count->for_each state migration). Authoring only; no apply. Traces R-ENV-01, R-PLAT-01. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 4.8 committed
1 parent
4f28ba6945
commit
53ca698edc
4 files changed
+229
-237
No files matched your search
+106
-113
@@ -1,20 +1,22 @@
|
||||
# main.tf — dedicated NAT network + 3 cloud-init'd VMs, each with a thin OS disk
|
||||
# (backed by the cloud image) and a blank thin Longhorn data disk (Phase 1 T-1.1).
|
||||
# main.tf — 3 cloud-init'd Proxmox VE VMs (k3s-dev-1/2/3), each with an imported
|
||||
# OS disk (from the Debian cloud image) and a blank Longhorn data disk (R-ENV-01).
|
||||
#
|
||||
# Node lifecycle stage 1 ("Create") per docs/02. cloud-init (infra/cloud-init/)
|
||||
# does stage 2 ("Baseline"); Ansible (infra/ansible/) does stage 3 ("Cluster-join").
|
||||
#
|
||||
# Provider: bpg/proxmox. VMs land on the Proxmox node `var.proxmox_node`, disks on
|
||||
# `var.datastore` (ZFS "Storage"), NIC on `var.bridge` (vmbr0), static IPs via the
|
||||
# native cloud-init `initialization` block. The rich first-boot baseline (users,
|
||||
# packages, kernel modules, Longhorn mount, swap-off) is delivered as a cloud-init
|
||||
# user-data SNIPPET (see the snippets note in README / operator action below).
|
||||
|
||||
locals {
|
||||
# 1 GiB in bytes; libvirt volume sizes are bytes.
|
||||
gib = 1073741824
|
||||
|
||||
# Per-node identity/addressing, KEYED BY NODE NAME (map, not list).
|
||||
# for_each over this map gives every node a stable, name-based state address
|
||||
# (libvirt_domain.node["k3s-dev-2"]) instead of a positional one (…node[1]).
|
||||
# Positional count indexes are brittle under `-target`/`-replace`: a single-node
|
||||
# destroy on a count-indexed resource can cascade to siblings (T-1.5 finding F1).
|
||||
# Name keys are individually addressable, so one node can be replaced/destroyed
|
||||
# without disturbing the others. See DECISIONS-LOG.md "T-1.5 … Finding F1".
|
||||
# (…vm.node["k3s-dev-2"]) instead of a positional one. Positional count indexes
|
||||
# are brittle under -target/-replace: a single-node destroy on a count-indexed
|
||||
# resource can cascade to siblings (F1 finding). Name keys are individually
|
||||
# addressable, so one node can be replaced/destroyed without disturbing the others.
|
||||
nodes = {
|
||||
for i in range(var.node_count) : "${var.node_name_prefix}${i + 1}" => {
|
||||
name = "${var.node_name_prefix}${i + 1}"
|
||||
@@ -25,132 +27,123 @@ locals {
|
||||
}
|
||||
|
||||
# Ordered (node1..nodeN) list + primary view for outputs that need positional
|
||||
# access — the resource for_each uses the map above; these preserve ordering
|
||||
# for the Ansible inventory (primary = cluster-init server, rest = secondaries).
|
||||
# access — the resource for_each uses the map above; these preserve ordering for
|
||||
# the Ansible inventory (primary = cluster-init server, rest = secondaries).
|
||||
node_list = [for i in range(var.node_count) : local.nodes["${var.node_name_prefix}${i + 1}"]]
|
||||
primary = local.nodes["${var.node_name_prefix}1"]
|
||||
}
|
||||
|
||||
# --- dedicated NAT network (10.73.0.0/24, gw .1) ---------------------------
|
||||
# DHCP is OFF: nodes get static IPs from cloud-init network-config, so the
|
||||
# reserved VIP range (.240-.250) can never be leased to something else.
|
||||
resource "libvirt_network" "k3s_dev" {
|
||||
name = var.network_name
|
||||
mode = "nat"
|
||||
addresses = [var.network_cidr]
|
||||
autostart = true
|
||||
# --- base OS cloud image (downloaded once to `local`, imported per node) -----
|
||||
# content_type "iso" stores it on the ISO datastore; the .img file name lets
|
||||
# Proxmox accept the qcow2 (it still reads it as qcow2 on disk import).
|
||||
resource "proxmox_virtual_environment_download_file" "debian_cloud_image" {
|
||||
content_type = "iso"
|
||||
datastore_id = var.image_datastore
|
||||
node_name = var.proxmox_node
|
||||
url = var.image_source
|
||||
file_name = var.image_file_name
|
||||
overwrite = false
|
||||
}
|
||||
|
||||
dhcp {
|
||||
enabled = false
|
||||
# --- per-node cloud-init user-data snippet ----------------------------------
|
||||
# bpg's native initialization.user_account only sets user/keys/password; it can't
|
||||
# carry packages/write_files/runcmd. To preserve the full k3s/Cilium/Longhorn
|
||||
# baseline we render the existing user-data.yaml.tftpl and ship it as a SNIPPET,
|
||||
# referenced by the VM's initialization.user_data_file_id.
|
||||
#
|
||||
# OPERATOR ACTION: the `snippets` content type must be enabled on var.image_datastore
|
||||
# (Datacenter -> Storage -> local -> Content -> add "Snippets"), else this file
|
||||
# resource fails. See README "Operator prerequisites".
|
||||
resource "proxmox_virtual_environment_file" "user_data" {
|
||||
for_each = local.nodes
|
||||
content_type = "snippets"
|
||||
datastore_id = var.image_datastore
|
||||
node_name = var.proxmox_node
|
||||
|
||||
source_raw {
|
||||
file_name = "${each.value.name}-user-data.yaml"
|
||||
data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", {
|
||||
hostname = each.value.name
|
||||
fqdn = each.value.fqdn
|
||||
username = var.login_user
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
})
|
||||
}
|
||||
|
||||
dns {
|
||||
enabled = true
|
||||
local_only = true
|
||||
}
|
||||
}
|
||||
|
||||
# --- base OS volume (downloaded/copied once, shared as a backing file) ------
|
||||
resource "libvirt_volume" "base" {
|
||||
name = "${var.cluster_name}-base.qcow2"
|
||||
pool = var.pool_name
|
||||
source = var.image_source
|
||||
format = "qcow2"
|
||||
}
|
||||
|
||||
# --- per-node OS disk: thin qcow2 backed by the base image ------------------
|
||||
resource "libvirt_volume" "os" {
|
||||
for_each = local.nodes
|
||||
name = "${each.value.name}-os.qcow2"
|
||||
pool = var.pool_name
|
||||
base_volume_id = libvirt_volume.base.id
|
||||
size = var.os_disk_gib * local.gib
|
||||
format = "qcow2"
|
||||
}
|
||||
|
||||
# --- per-node data disk: blank thin qcow2 for Longhorn ----------------------
|
||||
resource "libvirt_volume" "data" {
|
||||
for_each = local.nodes
|
||||
name = "${each.value.name}-data.qcow2"
|
||||
pool = var.pool_name
|
||||
size = var.data_disk_gib * local.gib
|
||||
format = "qcow2"
|
||||
}
|
||||
|
||||
# --- cloud-init seed (user-data + meta-data + static network-config) --------
|
||||
# Templates live in the sibling infra/cloud-init/ subtree (T-1.2).
|
||||
resource "libvirt_cloudinit_disk" "seed" {
|
||||
for_each = local.nodes
|
||||
name = "${each.value.name}-seed.iso"
|
||||
pool = var.pool_name
|
||||
|
||||
user_data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", {
|
||||
hostname = each.value.name
|
||||
fqdn = each.value.fqdn
|
||||
username = var.login_user
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
})
|
||||
|
||||
meta_data = templatefile("${path.module}/../cloud-init/meta-data.yaml.tftpl", {
|
||||
hostname = each.value.name
|
||||
fqdn = each.value.fqdn
|
||||
})
|
||||
|
||||
network_config = templatefile("${path.module}/../cloud-init/network-config.yaml.tftpl", {
|
||||
ip = each.value.ip
|
||||
prefix = var.network_prefix
|
||||
gateway = var.network_gateway
|
||||
})
|
||||
}
|
||||
|
||||
# --- the VMs ----------------------------------------------------------------
|
||||
resource "libvirt_domain" "node" {
|
||||
for_each = local.nodes
|
||||
name = each.value.name
|
||||
memory = var.memory_mib
|
||||
vcpu = var.vcpu
|
||||
resource "proxmox_virtual_environment_vm" "node" {
|
||||
for_each = local.nodes
|
||||
name = each.value.name
|
||||
node_name = var.proxmox_node
|
||||
tags = ["k3s", "k3s-dev", "opentofu"]
|
||||
|
||||
# host-passthrough so nested virt is available for KubeVirt in Phase 5.
|
||||
description = "k3s dev node ${each.value.name} (${each.value.fqdn}) — provisioned by OpenTofu."
|
||||
on_boot = true
|
||||
stop_on_destroy = true
|
||||
|
||||
# `host` CPU type passes through the physical CPU so nested virt is available for
|
||||
# KubeVirt in Phase 5.
|
||||
cpu {
|
||||
mode = var.cpu_mode
|
||||
cores = var.vcpu
|
||||
type = var.cpu_type
|
||||
}
|
||||
|
||||
# qemu-guest-agent is installed by cloud-init; lets tofu read the lease/agent.
|
||||
qemu_agent = true
|
||||
|
||||
cloudinit = libvirt_cloudinit_disk.seed[each.key].id
|
||||
|
||||
# IP + hostname come from cloud-init (DHCP is off on this network), so we do NOT
|
||||
# set addresses/hostname here — the provider couples those to DHCP reservations.
|
||||
network_interface {
|
||||
network_id = libvirt_network.k3s_dev.id
|
||||
wait_for_lease = false
|
||||
memory {
|
||||
dedicated = var.memory_mib
|
||||
}
|
||||
|
||||
# OS disk first (vda, boot), Longhorn data disk second (vdb).
|
||||
# qemu-guest-agent is installed by cloud-init; lets tofu/Proxmox read guest state.
|
||||
agent {
|
||||
enabled = true
|
||||
}
|
||||
|
||||
operating_system {
|
||||
type = "l26"
|
||||
}
|
||||
|
||||
# OS disk (virtio0 -> /dev/vda, boot): a copy of the imported cloud image, grown
|
||||
# to os_disk_gib. Longhorn data disk (virtio1 -> /dev/vdb): blank, data_disk_gib.
|
||||
# virtio* keeps the guest device names (/dev/vda, /dev/vdb) the cloud-init
|
||||
# fs_setup/mount for /var/lib/longhorn already expects.
|
||||
disk {
|
||||
volume_id = libvirt_volume.os[each.key].id
|
||||
datastore_id = var.datastore
|
||||
import_from = proxmox_virtual_environment_download_file.debian_cloud_image.id
|
||||
interface = "virtio0"
|
||||
size = var.os_disk_gib
|
||||
discard = "on"
|
||||
}
|
||||
|
||||
disk {
|
||||
volume_id = libvirt_volume.data[each.key].id
|
||||
datastore_id = var.datastore
|
||||
interface = "virtio1"
|
||||
size = var.data_disk_gib
|
||||
discard = "on"
|
||||
}
|
||||
|
||||
console {
|
||||
type = "pty"
|
||||
target_port = "0"
|
||||
target_type = "serial"
|
||||
network_device {
|
||||
bridge = var.bridge
|
||||
}
|
||||
|
||||
console {
|
||||
type = "pty"
|
||||
target_port = "1"
|
||||
target_type = "virtio"
|
||||
# Static addressing + DNS come from cloud-init (bpg generates the network-config),
|
||||
# so the libvirt netplan template is gone. User-data (the baseline) is the snippet.
|
||||
initialization {
|
||||
datastore_id = var.datastore
|
||||
|
||||
ip_config {
|
||||
ipv4 {
|
||||
address = "${each.value.ip}/${var.network_prefix}"
|
||||
gateway = var.network_gateway
|
||||
}
|
||||
}
|
||||
|
||||
dns {
|
||||
servers = [var.dns_server]
|
||||
}
|
||||
|
||||
user_data_file_id = proxmox_virtual_environment_file.user_data[each.key].id
|
||||
}
|
||||
|
||||
graphics {
|
||||
type = "spice"
|
||||
listen_type = "address"
|
||||
autoport = true
|
||||
}
|
||||
# Boot from the OS disk only.
|
||||
boot_order = ["virtio0"]
|
||||
}
|
||||
Reference in new issue
Block a user