tofu(proxmox): API-only native cloud-init; move node baseline to Ansible node_baseline role

Co-Authored-By: Claude Opus 4.8 <[email protected]>

.gitignore

@@ -20,6 +20,9 @@ kubeconfig
 .terraform.lock.hcl
 *.tfvars
 !*.example.tfvars
+# tofu apply/plan run logs (tee'd locally; never commit)
+apply.log
+plan.log
 
 # Test / build artifacts
 /results/

infra/ansible/collections/requirements.yml

@@ -0,0 +1,8 @@
+---
+# Ansible Galaxy collections required by infra/ansible. Install with:
+#   ansible-galaxy collection install -r collections/requirements.yml
+collections:
+  # node_baseline role: community.general.modprobe, community.general.filesystem
+  - name: community.general
+  # node_baseline role: ansible.posix.mount
+  - name: ansible.posix

infra/ansible/playbooks/node-baseline.yml

@@ -0,0 +1,10 @@
+---
+# Node baseline — first-boot floor for k3s/Cilium/Longhorn, moved from cloud-init into
+# Ansible so Proxmox provisioning stays API-only (no SSH/SFTP to the hypervisor). Runs
+# FIRST in site.yml, before k3s. Idempotent; safe to re-run.
+- name: Baseline the k3s nodes (packages, kernel modules, Longhorn disk, swap-off)
+  hosts: k3s_servers
+  become: true
+  gather_facts: true
+  roles:
+    - node_baseline

infra/ansible/playbooks/site.yml

@@ -2,6 +2,9 @@
 # Full dev/test cluster bring-up, in order (docs/02 sequencing; run AFTER
 # `tofu apply` has created and cloud-init has baselined the VMs):
 #   k3s HA install -> Cilium CNI -> verify readiness -> Flux + SOPS hand-off.
+- name: Import - node baseline (packages, kernel modules, Longhorn disk, swap-off)
+  ansible.builtin.import_playbook: node-baseline.yml
+
 - name: Import - install k3s HA control plane
   ansible.builtin.import_playbook: k3s.yml
 

infra/ansible/roles/node_baseline/defaults/main.yml

@@ -0,0 +1,17 @@
+---
+# node_baseline — first-boot floor for k3s/Cilium/Longhorn, moved out of cloud-init
+# (Proxmox provisioning is API-only). Runs before k3s (site.yml). Idempotent.
+node_baseline_packages:
+  - qemu-guest-agent
+  - open-iscsi
+  - nfs-common
+  - curl
+node_baseline_kernel_modules:
+  - overlay
+  - br_netfilter
+  - iscsi_tcp
+  - dm_crypt
+  - nfs
+node_baseline_longhorn_device: /dev/vdb
+node_baseline_longhorn_mount: /var/lib/longhorn
+node_baseline_longhorn_fs_label: longhorn

infra/ansible/roles/node_baseline/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Reload sysctl
+  ansible.builtin.command: sysctl --system
+  changed_when: true

infra/ansible/roles/node_baseline/tasks/main.yml

@@ -0,0 +1,69 @@
+---
+- name: Install baseline packages (guest agent, Longhorn/iscsi/nfs prereqs)
+  ansible.builtin.apt:
+    name: "{{ node_baseline_packages }}"
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Persist kernel modules for k3s/Cilium/Longhorn
+  ansible.builtin.copy:
+    dest: /etc/modules-load.d/k3s-cilium-longhorn.conf
+    owner: root
+    group: root
+    mode: "0644"
+    content: "{{ node_baseline_kernel_modules | join('\n') }}\n"
+
+- name: Load kernel modules now
+  community.general.modprobe:
+    name: "{{ item }}"
+    state: present
+  loop: "{{ node_baseline_kernel_modules }}"
+
+- name: Install sysctl tuning for k3s/Cilium
+  ansible.builtin.copy:
+    dest: /etc/sysctl.d/99-k3s-cilium.conf
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      net.bridge.bridge-nf-call-iptables  = 1
+      net.bridge.bridge-nf-call-ip6tables = 1
+      net.ipv4.ip_forward                 = 1
+      net.ipv6.conf.all.forwarding        = 1
+      fs.inotify.max_user_instances       = 8192
+      fs.inotify.max_user_watches         = 524288
+  notify: Reload sysctl
+
+- name: Enable and start iscsid (Longhorn) and qemu-guest-agent
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - iscsid
+    - qemu-guest-agent
+
+- name: Create ext4 filesystem on the Longhorn data disk (data-safe; skips if present)
+  community.general.filesystem:
+    fstype: ext4
+    dev: "{{ node_baseline_longhorn_device }}"
+    opts: "-L {{ node_baseline_longhorn_fs_label }}"
+
+- name: Mount the Longhorn data disk
+  ansible.posix.mount:
+    path: "{{ node_baseline_longhorn_mount }}"
+    src: "LABEL={{ node_baseline_longhorn_fs_label }}"
+    fstype: ext4
+    opts: "defaults,nofail,x-systemd.device-timeout=5s"
+    state: mounted
+
+- name: Disable swap now
+  ansible.builtin.command: swapoff -a
+  changed_when: false
+
+- name: Neutralize swap entries in fstab (kubelet requires swap off)
+  ansible.builtin.replace:
+    path: /etc/fstab
+    regexp: '^([^#].*\sswap\s.*)$'
+    replace: '# \1  # disabled for k3s/kubelet'

infra/cloud-init/README.md

@@ -1,6 +1,14 @@
 # infra/cloud-init — first-boot baseline templates
 
-The "Baseline" node lifecycle stage (docs/02 stage 2). `user-data.yaml.tftpl` is
+> **RETIRED (API-only migration):** the `user-data.yaml.tftpl` snippet was removed
+> when Proxmox provisioning moved to **API-only native cloud-init**. Custom snippets
+> require SSH/SFTP to the Proxmox host, which is not reachable here, so bpg's native
+> `initialization` block now sets only the login user + SSH keys and static IP/DNS
+> (via the built-in cloud-init API options). The rich first-boot baseline (packages,
+> kernel modules, Longhorn disk, swap-off) now lives in the Ansible **`node_baseline`
+> role** (`infra/ansible`), run before k3s. The notes below are historical context.
+
+The "Baseline" node lifecycle stage (docs/02 stage 2). `user-data.yaml.tftpl` was
 rendered by OpenTofu (`templatefile()`) and shipped to Proxmox as a **cloud-init
 user-data snippet** (`proxmox_virtual_environment_file`, one per node), referenced
 by each VM's `initialization.user_data_file_id`. **Authoring only — never applied.**

infra/cloud-init/user-data.yaml.tftpl

@@ -1,75 +0,0 @@
-#cloud-config
-# user-data — first-boot BASELINE only (docs/02 stage 2 "Baseline"). Rendered by
-# OpenTofu templatefile() ($${...} = tofu vars, NOT cloud-init). Everything here is
-# the node prerequisite floor for k3s / Cilium / Longhorn / KubeVirt. Cluster join,
-# k3s, and Cilium install are ANSIBLE's job (T-1.3) — deliberately NOT done here.
-
-hostname: ${hostname}
-fqdn: ${fqdn}
-preserve_hostname: false
-
-# Login user (also the Ansible ansible_user). Public key only — no passwords.
-users:
-  - name: ${username}
-    groups: [sudo]
-    shell: /bin/bash
-    sudo: "ALL=(ALL) NOPASSWD:ALL"
-    lock_passwd: true
-    ssh_authorized_keys:
-      - ${ssh_authorized_key}
-
-# Baseline packages: guest agent (tofu/Proxmox), and Longhorn's node prerequisites
-# (open-iscsi provides iscsid + iscsiadm; nfs-common provides the NFS client).
-package_update: true
-packages:
-  - qemu-guest-agent
-  - open-iscsi
-  - nfs-common
-  - curl
-
-# Kernel modules required by Cilium (overlay, br_netfilter) and Longhorn
-# (iscsi_tcp, dm_crypt, nfs). Loaded now and persisted for reboots.
-write_files:
-  - path: /etc/modules-load.d/k3s-cilium-longhorn.conf
-    permissions: "0644"
-    content: |
-      overlay
-      br_netfilter
-      iscsi_tcp
-      dm_crypt
-      nfs
-  - path: /etc/sysctl.d/99-k3s-cilium.conf
-    permissions: "0644"
-    content: |
-      net.bridge.bridge-nf-call-iptables  = 1
-      net.bridge.bridge-nf-call-ip6tables = 1
-      net.ipv4.ip_forward                 = 1
-      net.ipv6.conf.all.forwarding        = 1
-      fs.inotify.max_user_instances       = 8192
-      fs.inotify.max_user_watches         = 524288
-
-# Dedicated Longhorn data disk (/dev/vdb, the VM's virtio1 disk) → ext4 mounted at
-# /var/lib/longhorn. nofail so a node still boots if the disk is absent during a rebuild.
-fs_setup:
-  - label: longhorn
-    filesystem: ext4
-    device: /dev/vdb
-    partition: none
-    overwrite: false
-mounts:
-  - ["LABEL=longhorn", "/var/lib/longhorn", "ext4", "defaults,nofail,x-systemd.device-timeout=5s", "0", "2"]
-
-runcmd:
-  # Apply modules + sysctls now (cloud-init runs after they're written).
-  - ["modprobe", "overlay"]
-  - ["modprobe", "br_netfilter"]
-  - ["modprobe", "iscsi_tcp"]
-  - ["sysctl", "--system"]
-  # Longhorn needs iscsid running; guest agent for Proxmox/tofu.
-  - ["systemctl", "enable", "--now", "iscsid"]
-  - ["systemctl", "enable", "--now", "qemu-guest-agent"]
-  # k3s/kubelet baseline: disable swap.
-  - ["swapoff", "-a"]
-  - ["sed", "-i", "/\\sswap\\s/d", "/etc/fstab"]
-
-# NOTE: no reboot here — the baseline is live after runcmd. Ansible connects next.

infra/tofu/main.tf

@@ -6,9 +6,10 @@
 #
 # Provider: bpg/proxmox. VMs land on the Proxmox node `var.proxmox_node`, disks on
 # `var.datastore` (ZFS "Storage"), NIC on `var.bridge` (vmbr0), static IPs via the
-# native cloud-init `initialization` block. The rich first-boot baseline (users,
-# packages, kernel modules, Longhorn mount, swap-off) is delivered as a cloud-init
-# user-data SNIPPET (see the snippets note in README / operator action below).
+# native cloud-init `initialization` block. Native cloud-init (login user + SSH keys,
+# static IP/gateway/DNS) is set via the Proxmox 8006 API. The rich first-boot baseline
+# (packages, kernel modules, Longhorn mount, swap-off) is delivered by the Ansible
+# `node_baseline` role (infra/ansible), run before k3s — not as a cloud-init snippet.
 
 locals {
   # Per-node identity/addressing, KEYED BY NODE NAME (map, not list).
@@ -45,31 +46,13 @@ resource "proxmox_virtual_environment_download_file" "debian_cloud_image" {
   overwrite    = false
 }
 
-# --- per-node cloud-init user-data snippet ----------------------------------
-# bpg's native initialization.user_account only sets user/keys/password; it can't
-# carry packages/write_files/runcmd. To preserve the full k3s/Cilium/Longhorn
-# baseline we render the existing user-data.yaml.tftpl and ship it as a SNIPPET,
-# referenced by the VM's initialization.user_data_file_id.
-#
-# OPERATOR ACTION: the `snippets` content type must be enabled on var.image_datastore
-# (Datacenter -> Storage -> local -> Content -> add "Snippets"), else this file
-# resource fails. See README "Operator prerequisites".
-resource "proxmox_virtual_environment_file" "user_data" {
-  for_each     = local.nodes
-  content_type = "snippets"
-  datastore_id = var.image_datastore
-  node_name    = var.proxmox_node
-
-  source_raw {
-    file_name = "${each.value.name}-user-data.yaml"
-    data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", {
-      hostname           = each.value.name
-      fqdn               = each.value.fqdn
-      username           = var.login_user
-      ssh_authorized_key = var.ssh_authorized_key
-    })
-  }
-}
+# --- cloud-init: NATIVE Proxmox cloud-init only (API-driven) ------------------
+# The rich first-boot baseline (packages, kernel modules, Longhorn disk, swap-off)
+# is NOT delivered as a custom snippet any more — custom snippets require SSH/SFTP
+# to the Proxmox host, which is not reachable here. Instead the VM's `initialization`
+# block below uses bpg's native user_account/ip_config/dns, which Proxmox sets via
+# the built-in cloud-init API options (ciuser/sshkeys/ipconfig0/nameserver). The
+# baseline moved to the Ansible `node_baseline` role (infra/ansible), run before k3s.
 
 # --- the VMs ----------------------------------------------------------------
 resource "proxmox_virtual_environment_vm" "node" {
@@ -93,9 +76,10 @@ resource "proxmox_virtual_environment_vm" "node" {
     dedicated = var.memory_mib
   }
 
-  # qemu-guest-agent is installed by cloud-init; lets tofu/Proxmox read guest state.
+  # qemu-guest-agent is installed by the Ansible node_baseline role now (not cloud-init),
+  # so don't block VM create on the agent — static IPs are assigned, not discovered.
   agent {
-    enabled = true
+    enabled = false
   }
 
   operating_system {
@@ -126,7 +110,8 @@ resource "proxmox_virtual_environment_vm" "node" {
   }
 
   # Static addressing + DNS come from cloud-init (bpg generates the network-config),
-  # so the libvirt netplan template is gone. User-data (the baseline) is the snippet.
+  # so the libvirt netplan template is gone. Native cloud-init (login user + SSH keys)
+  # comes from user_account below; the OS baseline is the Ansible node_baseline role.
   initialization {
     datastore_id = var.datastore
 
@@ -141,7 +126,10 @@ resource "proxmox_virtual_environment_vm" "node" {
       servers = [var.dns_server]
     }
 
-    user_data_file_id = proxmox_virtual_environment_file.user_data[each.key].id
+    user_account {
+      username = var.login_user
+      keys     = [trimspace(var.ssh_authorized_key)]
+    }
   }
 
   # Boot from the OS disk only.