From 4bb9c8fcbd347ae1c159abb858b38803bd2d569a Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Tue, 25 Aug 2026 10:23:28 -0500 Subject: [PATCH] infra(tofu): docs + example for the Proxmox migration Update tofu README (Proxmox targets table, operator prerequisites: enable Snippets content on `local`, endpoint scheme fix, pfSense NAT), terraform.tfvars.example (Proxmox vars + env-only auth, token never a tofu var), and the devcontainer versions.env provider note. Co-Authored-By: Claude Opus 4.8 --- infra/devcontainer/versions.env | 2 +- infra/tofu/README.md | 93 +++++++++++++++++++---------- infra/tofu/terraform.tfvars.example | 53 +++++++++++----- 3 files changed, 101 insertions(+), 47 deletions(-) diff --git a/infra/devcontainer/versions.env b/infra/devcontainer/versions.env index a63e40e..1fc1d6d 100644 --- a/infra/devcontainer/versions.env +++ b/infra/devcontainer/versions.env @@ -10,4 +10,4 @@ SOPS_VERSION=3.9.4 FLUX_VERSION=2.4.0 KUBECTL_VERSION=1.32.0 HELM_VERSION=3.16.0 -# terraform-libvirt provider (dmacvicar/libvirt) is pinned in the tofu module (T-1.1), not here. +# Proxmox provider (bpg/proxmox) is pinned in the tofu module (infra/tofu/versions.tf), not here. diff --git a/infra/tofu/README.md b/infra/tofu/README.md index 231ad91..436613b 100644 --- a/infra/tofu/README.md +++ b/infra/tofu/README.md @@ -1,39 +1,69 @@ -# infra/tofu — dev/test cluster nodes (OpenTofu + libvirt) +# infra/tofu — dev/test cluster nodes (OpenTofu + Proxmox VE) -Phase 1 **T-1.1**. Provisions the "Create" lifecycle stage (docs/02): a dedicated -NAT network and 3 cloud-init'd KVM VMs, each with a thin OS disk and a blank thin -Longhorn data disk. **Authoring only — nothing here has been applied.** +Provisions the "Create" lifecycle stage (docs/02): 3 cloud-init'd VMs on the +**Proxmox VE** box, each with an imported OS disk and a blank Longhorn data disk. +**Authoring only — nothing here has been applied.** (Migrated from the old +memory-constrained libvirt host; provider is now `bpg/proxmox`.) -Serves **R-ENV-01/02**, **R-PLAT-01**, ADR-0001 (k3s substrate), and the network -half of the LOCKED PHASE1-SCOPE decisions. +Serves **R-ENV-01/02**, **R-PLAT-01**, ADR-0001 (k3s substrate). -## What it builds (defaults = LOCKED decisions) +## What it builds (defaults = VERIFIED Proxmox targets) | Resource | Value | |---|---| -| NAT network | `k3s-dev`, `10.73.0.0/24`, gateway `10.73.0.1`, **DHCP off** (static IPs) | -| Nodes | `k3s-dev-1/2/3` at `10.73.0.11/.12/.13` | -| Per node | 4 vCPU, 6 GiB RAM, **host-passthrough** CPU (nested virt for KubeVirt) | -| OS disk | 20 GiB thin qcow2, backed by the base cloud image | -| Data disk | 40 GiB blank thin qcow2 → `/dev/vdb`, mounted at `/var/lib/longhorn` by cloud-init | -| Seed | `libvirt_cloudinit_disk` wired to `../cloud-init/` templates | -| Reserved | `10.73.0.240–.250` kept out of any DHCP range for Cilium L2 ingress VIPs | +| Proxmox node | `proxmox` | +| Disk datastore | `Storage` (ZFS, 4.3TB) | +| Image/snippet datastore | `local` (dir; content = iso + **snippets**) | +| Bridge | `vmbr0` (no VLAN) | +| Nodes | `k3s-dev-1/2/3` at `192.168.1.50/.51/.52`, gw `192.168.1.1`, DNS `192.168.1.18` | +| Per node | 6 vCPU, 16 GiB RAM, **cpu type `host`** (nested virt for KubeVirt) | +| OS disk | 40 GiB, virtio0 (`/dev/vda`, boot), imported from the Debian 12 cloud image | +| Data disk | 100 GiB, virtio1 (`/dev/vdb`) → `/var/lib/longhorn` (mounted by cloud-init) | +| Cloud-init | native `initialization` (static IP + DNS) + user-data **snippet** for the baseline | +| Reserved | `192.168.1.53–.60` for Cilium L2 ingress VIPs (surfaced in outputs) | -Base OS image is a **variable** (`image_source`), defaulting to Debian 12 -genericcloud amd64. All sizing/IPs/counts are variables — see `variables.tf`. +All sizing/IPs/counts/placement are variables — see `variables.tf`. ## Files -- `versions.tf` — OpenTofu `1.9.0` pin + `dmacvicar/libvirt ~> 0.8.1` provider pin. -- `variables.tf` — every knob (counts, sizing, IPs, image, SSH key). -- `main.tf` — network, base/OS/data volumes, cloud-init seed, domains. -- `outputs.tf` — node IPs/names, `primary_node_ip`, `k3s_api_endpoint`, and a - rendered `ansible_inventory` for `infra/ansible/`. -- `terraform.tfvars.example` — copy to `terraform.tfvars`; only `ssh_authorized_key` is required. +- `versions.tf` — OpenTofu `1.9.0` pin + `bpg/proxmox ~> 0.66`; provider block with + endpoint scheme-normalization (see below). +- `variables.tf` — every knob (Proxmox placement, counts, sizing, IPs, image, SSH key). +- `main.tf` — cloud image download, per-node user-data snippet, and the 3 VMs. +- `outputs.tf` — node IPs/names, `primary_node_ip`, `k3s_api_endpoint`, rendered + `ansible_inventory`. +- `terraform.tfvars.example` — copy to `terraform.tfvars`; only `ssh_authorized_key` required. -## Usage (operator, in the dev container — NOT run here) +## Operator prerequisites (before `tofu apply`) + +1. **Enable `snippets` content on the `local` datastore.** The rich first-boot + baseline (users, packages, kernel modules, Longhorn mount, swap-off) is shipped + as a cloud-init user-data snippet. In the Proxmox UI: *Datacenter → Storage → + `local` → Edit → Content → add **Snippets***. Without this the + `proxmox_virtual_environment_file` (user-data) resource fails. (Alternatively + point `image_datastore` at any datastore that has snippets enabled.) +2. **Endpoint scheme.** If the operator env file exports `PROXMOX_VE_ENDPOINT` + without `https://`, the provider talks HTTP and eats a 301. Either fix the env + file to include the scheme, or export `TF_VAR_proxmox_endpoint="$PROXMOX_VE_ENDPOINT"` + so tofu normalizes it (prepends `https://`). +3. **pfSense SSH NAT** (network-dependent): the PM reaches these nodes and the k3s + API over a pfSense NAT that is not yet created; the Ansible inventory carries a + note to finalize `ansible_host` once the NAT rule exists. + +## Auth (env only — never in tofu vars/state) +The provider reads these from the environment; the **API token is never a tofu +variable**, so it never enters state, vars, or plan output: +``` +export PROXMOX_VE_ENDPOINT="https://192.168.2.102:8006/" +export PROXMOX_VE_API_TOKEN="user@realm!tokenid=SECRET" +export PROXMOX_VE_INSECURE="true" +export TF_VAR_proxmox_endpoint="$PROXMOX_VE_ENDPOINT" # optional; enables normalization +``` + +## Usage (operator — NOT run here) ``` cd infra/tofu cp terraform.tfvars.example terraform.tfvars # set ssh_authorized_key +# source the operator env file (exports the PROXMOX_VE_* above) tofu init tofu plan tofu apply @@ -42,16 +72,17 @@ tofu output -raw ansible_inventory > ../ansible/inventory/hosts.ini ``` ## Hand-off contract -- `primary_node_ip` / `k3s_api_endpoint` (`10.73.0.11:6443`) is the value the - Ansible Cilium bootstrap and the Flux `platform/cilium` HelmRelease must BOTH - use for `k8sServiceHost` / `k8sServicePort` (docs/02 parity contract). -- Destroy/rebuild proof (T-1.5): `tofu destroy -target=libvirt_domain.node[1]` - then `tofu apply` re-creates that node; Ansible re-joins it; Longhorn replicas - reconverge (nodes are cattle — docs/02 statelessness). +- `primary_node_ip` / `k3s_api_endpoint` (`192.168.1.50:6443`) is the value the + Ansible Cilium bootstrap and the Flux `platform/cilium` HelmRelease must BOTH use + for `k8sServiceHost` / `k8sServicePort` (docs/02 parity contract). +- Destroy/rebuild proof: name-keyed `for_each` means + `tofu destroy -target='proxmox_virtual_environment_vm.node["k3s-dev-2"]'` then + `tofu apply` re-creates just that node (`stop_on_destroy` allows the clean stop); + Ansible re-joins it; Longhorn replicas reconverge (nodes are cattle). -## Static validation (PM, in dev container) +## Static validation (PM, offline) ``` tofu fmt -check -recursive -tofu init -backend=false +tofu init # fetches the bpg provider (network, read-only) tofu validate ``` diff --git a/infra/tofu/terraform.tfvars.example b/infra/tofu/terraform.tfvars.example index e107428..94ddc4d 100644 --- a/infra/tofu/terraform.tfvars.example +++ b/infra/tofu/terraform.tfvars.example @@ -4,34 +4,57 @@ # local values never land in git. This *.example file is safe to commit — it # carries NO private material (SSH *public* key only). Never put a private key here. # -# Defaults for everything below already encode the LOCKED PHASE1-SCOPE decisions; -# the ONLY required value is ssh_authorized_key. +# The Proxmox API TOKEN is NEVER a tofu var — it is read from the environment +# (PROXMOX_VE_API_TOKEN) by the provider so it never enters state/vars. Same for +# the endpoint/insecure flag: source the operator env file before running tofu: +# export PROXMOX_VE_ENDPOINT="https://192.168.2.102:8006/" +# export PROXMOX_VE_API_TOKEN="user@realm!tokenid=SECRET" +# export PROXMOX_VE_INSECURE="true" +# # optional — lets tofu normalize a schemeless endpoint (prepend https://): +# export TF_VAR_proxmox_endpoint="$PROXMOX_VE_ENDPOINT" +# +# Defaults for everything below already encode the VERIFIED Proxmox migration +# targets; the ONLY required value is ssh_authorized_key. # REQUIRED — your SSH *public* key (contents of e.g. ~/.ssh/id_ed25519.pub). ssh_authorized_key = "ssh-ed25519 AAAA...replace-with-your-public-key... user@host" # --- optional overrides (defaults shown) ----------------------------------- -# libvirt_uri = "qemu:///system" -# pool_name = "default" +# --- Proxmox placement --- +# proxmox_node = "proxmox" +# datastore = "Storage" # ZFS, OS + data disks +# image_datastore = "local" # cloud image + cloud-init snippets +# bridge = "vmbr0" +# proxmox_insecure = true +# proxmox_endpoint = "https://192.168.2.102:8006/" # or set TF_VAR_proxmox_endpoint +# --- cluster shape --- # node_count = 3 # node_name_prefix = "k3s-dev-" -# node_ips = ["10.73.0.11", "10.73.0.12", "10.73.0.13"] +# node_ips = ["192.168.1.50", "192.168.1.51", "192.168.1.52"] -# vcpu = 4 -# memory_mib = 6144 -# os_disk_gib = 20 -# data_disk_gib = 40 -# cpu_mode = "host-passthrough" +# --- per-node sizing --- +# vcpu = 6 +# memory_mib = 16384 +# os_disk_gib = 40 +# data_disk_gib = 100 +# cpu_type = "host" -# network_name = "k3s-dev" -# network_cidr = "10.73.0.0/24" -# network_gateway = "10.73.0.1" +# --- network --- +# network_cidr = "192.168.1.0/24" +# network_gateway = "192.168.1.1" +# dns_server = "192.168.1.18" # network_prefix = 24 +# --- Cilium L2 VIP reservation (consumed by platform-config/cilium) --- +# cilium_l2_vip_range_start = "192.168.1.53" +# cilium_l2_vip_range_end = "192.168.1.60" + +# --- identity --- # login_user = "k3s" # node_domain = "k3s.dev.local" -# Point at a locally pre-downloaded image to avoid re-fetching on each apply: -# image_source = "/var/lib/libvirt/images/debian-12-genericcloud-amd64.qcow2" +# --- base image --- +# image_source = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2" +# image_file_name = "debian-12-genericcloud-amd64.img"