diff --git a/.gitignore b/.gitignore index db86a1d..a787a1a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ kubeconfig .terraform.lock.hcl *.tfvars !*.example.tfvars +# tofu apply/plan run logs (tee'd locally; never commit) +apply.log +plan.log # Test / build artifacts /results/ diff --git a/infra/ansible/collections/requirements.yml b/infra/ansible/collections/requirements.yml new file mode 100644 index 0000000..55fcdf1 --- /dev/null +++ b/infra/ansible/collections/requirements.yml @@ -0,0 +1,8 @@ +--- +# Ansible Galaxy collections required by infra/ansible. Install with: +# ansible-galaxy collection install -r collections/requirements.yml +collections: + # node_baseline role: community.general.modprobe, community.general.filesystem + - name: community.general + # node_baseline role: ansible.posix.mount + - name: ansible.posix diff --git a/infra/ansible/playbooks/node-baseline.yml b/infra/ansible/playbooks/node-baseline.yml new file mode 100644 index 0000000..0b45e17 --- /dev/null +++ b/infra/ansible/playbooks/node-baseline.yml @@ -0,0 +1,10 @@ +--- +# Node baseline — first-boot floor for k3s/Cilium/Longhorn, moved from cloud-init into +# Ansible so Proxmox provisioning stays API-only (no SSH/SFTP to the hypervisor). Runs +# FIRST in site.yml, before k3s. Idempotent; safe to re-run. +- name: Baseline the k3s nodes (packages, kernel modules, Longhorn disk, swap-off) + hosts: k3s_servers + become: true + gather_facts: true + roles: + - node_baseline diff --git a/infra/ansible/playbooks/site.yml b/infra/ansible/playbooks/site.yml index 09cc37e..d471783 100644 --- a/infra/ansible/playbooks/site.yml +++ b/infra/ansible/playbooks/site.yml @@ -2,6 +2,9 @@ # Full dev/test cluster bring-up, in order (docs/02 sequencing; run AFTER # `tofu apply` has created and cloud-init has baselined the VMs): # k3s HA install -> Cilium CNI -> verify readiness -> Flux + SOPS hand-off. +- name: Import - node baseline (packages, kernel modules, Longhorn disk, swap-off) + ansible.builtin.import_playbook: node-baseline.yml + - name: Import - install k3s HA control plane ansible.builtin.import_playbook: k3s.yml diff --git a/infra/ansible/roles/node_baseline/defaults/main.yml b/infra/ansible/roles/node_baseline/defaults/main.yml new file mode 100644 index 0000000..37ce87c --- /dev/null +++ b/infra/ansible/roles/node_baseline/defaults/main.yml @@ -0,0 +1,17 @@ +--- +# node_baseline — first-boot floor for k3s/Cilium/Longhorn, moved out of cloud-init +# (Proxmox provisioning is API-only). Runs before k3s (site.yml). Idempotent. +node_baseline_packages: + - qemu-guest-agent + - open-iscsi + - nfs-common + - curl +node_baseline_kernel_modules: + - overlay + - br_netfilter + - iscsi_tcp + - dm_crypt + - nfs +node_baseline_longhorn_device: /dev/vdb +node_baseline_longhorn_mount: /var/lib/longhorn +node_baseline_longhorn_fs_label: longhorn diff --git a/infra/ansible/roles/node_baseline/handlers/main.yml b/infra/ansible/roles/node_baseline/handlers/main.yml new file mode 100644 index 0000000..15d80f0 --- /dev/null +++ b/infra/ansible/roles/node_baseline/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload sysctl + ansible.builtin.command: sysctl --system + changed_when: true diff --git a/infra/ansible/roles/node_baseline/tasks/main.yml b/infra/ansible/roles/node_baseline/tasks/main.yml new file mode 100644 index 0000000..10e0058 --- /dev/null +++ b/infra/ansible/roles/node_baseline/tasks/main.yml @@ -0,0 +1,69 @@ +--- +- name: Install baseline packages (guest agent, Longhorn/iscsi/nfs prereqs) + ansible.builtin.apt: + name: "{{ node_baseline_packages }}" + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Persist kernel modules for k3s/Cilium/Longhorn + ansible.builtin.copy: + dest: /etc/modules-load.d/k3s-cilium-longhorn.conf + owner: root + group: root + mode: "0644" + content: "{{ node_baseline_kernel_modules | join('\n') }}\n" + +- name: Load kernel modules now + community.general.modprobe: + name: "{{ item }}" + state: present + loop: "{{ node_baseline_kernel_modules }}" + +- name: Install sysctl tuning for k3s/Cilium + ansible.builtin.copy: + dest: /etc/sysctl.d/99-k3s-cilium.conf + owner: root + group: root + mode: "0644" + content: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 + fs.inotify.max_user_instances = 8192 + fs.inotify.max_user_watches = 524288 + notify: Reload sysctl + +- name: Enable and start iscsid (Longhorn) and qemu-guest-agent + ansible.builtin.systemd: + name: "{{ item }}" + enabled: true + state: started + loop: + - iscsid + - qemu-guest-agent + +- name: Create ext4 filesystem on the Longhorn data disk (data-safe; skips if present) + community.general.filesystem: + fstype: ext4 + dev: "{{ node_baseline_longhorn_device }}" + opts: "-L {{ node_baseline_longhorn_fs_label }}" + +- name: Mount the Longhorn data disk + ansible.posix.mount: + path: "{{ node_baseline_longhorn_mount }}" + src: "LABEL={{ node_baseline_longhorn_fs_label }}" + fstype: ext4 + opts: "defaults,nofail,x-systemd.device-timeout=5s" + state: mounted + +- name: Disable swap now + ansible.builtin.command: swapoff -a + changed_when: false + +- name: Neutralize swap entries in fstab (kubelet requires swap off) + ansible.builtin.replace: + path: /etc/fstab + regexp: '^([^#].*\sswap\s.*)$' + replace: '# \1 # disabled for k3s/kubelet' diff --git a/infra/cloud-init/README.md b/infra/cloud-init/README.md index 5dcad4d..53ff3b8 100644 --- a/infra/cloud-init/README.md +++ b/infra/cloud-init/README.md @@ -1,6 +1,14 @@ # infra/cloud-init — first-boot baseline templates -The "Baseline" node lifecycle stage (docs/02 stage 2). `user-data.yaml.tftpl` is +> **RETIRED (API-only migration):** the `user-data.yaml.tftpl` snippet was removed +> when Proxmox provisioning moved to **API-only native cloud-init**. Custom snippets +> require SSH/SFTP to the Proxmox host, which is not reachable here, so bpg's native +> `initialization` block now sets only the login user + SSH keys and static IP/DNS +> (via the built-in cloud-init API options). The rich first-boot baseline (packages, +> kernel modules, Longhorn disk, swap-off) now lives in the Ansible **`node_baseline` +> role** (`infra/ansible`), run before k3s. The notes below are historical context. + +The "Baseline" node lifecycle stage (docs/02 stage 2). `user-data.yaml.tftpl` was rendered by OpenTofu (`templatefile()`) and shipped to Proxmox as a **cloud-init user-data snippet** (`proxmox_virtual_environment_file`, one per node), referenced by each VM's `initialization.user_data_file_id`. **Authoring only — never applied.** diff --git a/infra/cloud-init/user-data.yaml.tftpl b/infra/cloud-init/user-data.yaml.tftpl deleted file mode 100644 index 9e0764f..0000000 --- a/infra/cloud-init/user-data.yaml.tftpl +++ /dev/null @@ -1,75 +0,0 @@ -#cloud-config -# user-data — first-boot BASELINE only (docs/02 stage 2 "Baseline"). Rendered by -# OpenTofu templatefile() ($${...} = tofu vars, NOT cloud-init). Everything here is -# the node prerequisite floor for k3s / Cilium / Longhorn / KubeVirt. Cluster join, -# k3s, and Cilium install are ANSIBLE's job (T-1.3) — deliberately NOT done here. - -hostname: ${hostname} -fqdn: ${fqdn} -preserve_hostname: false - -# Login user (also the Ansible ansible_user). Public key only — no passwords. -users: - - name: ${username} - groups: [sudo] - shell: /bin/bash - sudo: "ALL=(ALL) NOPASSWD:ALL" - lock_passwd: true - ssh_authorized_keys: - - ${ssh_authorized_key} - -# Baseline packages: guest agent (tofu/Proxmox), and Longhorn's node prerequisites -# (open-iscsi provides iscsid + iscsiadm; nfs-common provides the NFS client). -package_update: true -packages: - - qemu-guest-agent - - open-iscsi - - nfs-common - - curl - -# Kernel modules required by Cilium (overlay, br_netfilter) and Longhorn -# (iscsi_tcp, dm_crypt, nfs). Loaded now and persisted for reboots. -write_files: - - path: /etc/modules-load.d/k3s-cilium-longhorn.conf - permissions: "0644" - content: | - overlay - br_netfilter - iscsi_tcp - dm_crypt - nfs - - path: /etc/sysctl.d/99-k3s-cilium.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-iptables = 1 - net.bridge.bridge-nf-call-ip6tables = 1 - net.ipv4.ip_forward = 1 - net.ipv6.conf.all.forwarding = 1 - fs.inotify.max_user_instances = 8192 - fs.inotify.max_user_watches = 524288 - -# Dedicated Longhorn data disk (/dev/vdb, the VM's virtio1 disk) → ext4 mounted at -# /var/lib/longhorn. nofail so a node still boots if the disk is absent during a rebuild. -fs_setup: - - label: longhorn - filesystem: ext4 - device: /dev/vdb - partition: none - overwrite: false -mounts: - - ["LABEL=longhorn", "/var/lib/longhorn", "ext4", "defaults,nofail,x-systemd.device-timeout=5s", "0", "2"] - -runcmd: - # Apply modules + sysctls now (cloud-init runs after they're written). - - ["modprobe", "overlay"] - - ["modprobe", "br_netfilter"] - - ["modprobe", "iscsi_tcp"] - - ["sysctl", "--system"] - # Longhorn needs iscsid running; guest agent for Proxmox/tofu. - - ["systemctl", "enable", "--now", "iscsid"] - - ["systemctl", "enable", "--now", "qemu-guest-agent"] - # k3s/kubelet baseline: disable swap. - - ["swapoff", "-a"] - - ["sed", "-i", "/\\sswap\\s/d", "/etc/fstab"] - -# NOTE: no reboot here — the baseline is live after runcmd. Ansible connects next. diff --git a/infra/tofu/main.tf b/infra/tofu/main.tf index 9902e00..9e324b3 100644 --- a/infra/tofu/main.tf +++ b/infra/tofu/main.tf @@ -6,9 +6,10 @@ # # Provider: bpg/proxmox. VMs land on the Proxmox node `var.proxmox_node`, disks on # `var.datastore` (ZFS "Storage"), NIC on `var.bridge` (vmbr0), static IPs via the -# native cloud-init `initialization` block. The rich first-boot baseline (users, -# packages, kernel modules, Longhorn mount, swap-off) is delivered as a cloud-init -# user-data SNIPPET (see the snippets note in README / operator action below). +# native cloud-init `initialization` block. Native cloud-init (login user + SSH keys, +# static IP/gateway/DNS) is set via the Proxmox 8006 API. The rich first-boot baseline +# (packages, kernel modules, Longhorn mount, swap-off) is delivered by the Ansible +# `node_baseline` role (infra/ansible), run before k3s — not as a cloud-init snippet. locals { # Per-node identity/addressing, KEYED BY NODE NAME (map, not list). @@ -45,31 +46,13 @@ resource "proxmox_virtual_environment_download_file" "debian_cloud_image" { overwrite = false } -# --- per-node cloud-init user-data snippet ---------------------------------- -# bpg's native initialization.user_account only sets user/keys/password; it can't -# carry packages/write_files/runcmd. To preserve the full k3s/Cilium/Longhorn -# baseline we render the existing user-data.yaml.tftpl and ship it as a SNIPPET, -# referenced by the VM's initialization.user_data_file_id. -# -# OPERATOR ACTION: the `snippets` content type must be enabled on var.image_datastore -# (Datacenter -> Storage -> local -> Content -> add "Snippets"), else this file -# resource fails. See README "Operator prerequisites". -resource "proxmox_virtual_environment_file" "user_data" { - for_each = local.nodes - content_type = "snippets" - datastore_id = var.image_datastore - node_name = var.proxmox_node - - source_raw { - file_name = "${each.value.name}-user-data.yaml" - data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", { - hostname = each.value.name - fqdn = each.value.fqdn - username = var.login_user - ssh_authorized_key = var.ssh_authorized_key - }) - } -} +# --- cloud-init: NATIVE Proxmox cloud-init only (API-driven) ------------------ +# The rich first-boot baseline (packages, kernel modules, Longhorn disk, swap-off) +# is NOT delivered as a custom snippet any more — custom snippets require SSH/SFTP +# to the Proxmox host, which is not reachable here. Instead the VM's `initialization` +# block below uses bpg's native user_account/ip_config/dns, which Proxmox sets via +# the built-in cloud-init API options (ciuser/sshkeys/ipconfig0/nameserver). The +# baseline moved to the Ansible `node_baseline` role (infra/ansible), run before k3s. # --- the VMs ---------------------------------------------------------------- resource "proxmox_virtual_environment_vm" "node" { @@ -93,9 +76,10 @@ resource "proxmox_virtual_environment_vm" "node" { dedicated = var.memory_mib } - # qemu-guest-agent is installed by cloud-init; lets tofu/Proxmox read guest state. + # qemu-guest-agent is installed by the Ansible node_baseline role now (not cloud-init), + # so don't block VM create on the agent — static IPs are assigned, not discovered. agent { - enabled = true + enabled = false } operating_system { @@ -126,7 +110,8 @@ resource "proxmox_virtual_environment_vm" "node" { } # Static addressing + DNS come from cloud-init (bpg generates the network-config), - # so the libvirt netplan template is gone. User-data (the baseline) is the snippet. + # so the libvirt netplan template is gone. Native cloud-init (login user + SSH keys) + # comes from user_account below; the OS baseline is the Ansible node_baseline role. initialization { datastore_id = var.datastore @@ -141,7 +126,10 @@ resource "proxmox_virtual_environment_vm" "node" { servers = [var.dns_server] } - user_data_file_id = proxmox_virtual_environment_file.user_data[each.key].id + user_account { + username = var.login_user + keys = [trimspace(var.ssh_authorized_key)] + } } # Boot from the OS disk only.