diff --git a/infra/ansible/.gitignore b/infra/ansible/.gitignore new file mode 100644 index 0000000..8d7676f --- /dev/null +++ b/infra/ansible/.gitignore @@ -0,0 +1,3 @@ +# Runtime cache written by the cilium/verify/flux plays (fetched kubeconfig, +# rendered Cilium values). Never commit — the kubeconfig is a credential. +.cache/ diff --git a/infra/ansible/README.md b/infra/ansible/README.md index d4a2eb8..fd966d4 100644 --- a/infra/ansible/README.md +++ b/infra/ansible/README.md @@ -1,3 +1,70 @@ -# infra/ansible +# infra/ansible — k3s HA + Cilium bootstrap (Phase 1 T-1.3) -_Phase 1 (T-1.x) — to be authored on execution go. See infra/PHASE1-SCOPE.md._ +The "Cluster-join" node lifecycle stage (docs/02 stage 3) and the Flux hand-off +(T-1.4 call-in). Installs a 3-server HA k3s cluster (embedded etcd), Cilium as the +CNI, verifies readiness, then invokes the `bootstrap/` entrypoints. + +> **AUTHOR-ONLY — NOTHING HERE HAS BEEN EXECUTED.** No `ansible-playbook` run, no +> `tofu apply`, no cluster contacted. The PM static-validates (see below). + +Serves **R-ENV-01/02**, **R-PLAT-01**, ADR-0001 (k3s), ADR-0002 (Cilium), +ADR-0004 (Flux hand-off), ADR-0007 (SOPS). + +## Prerequisites +1. `tofu apply` in `../tofu/` has created the VMs; cloud-init (`../cloud-init/`) + has baselined them (modules, sysctls, Longhorn disk, SSH user). +2. Inventory reflects the node IPs. `inventory/hosts.ini` ships aligned with the + LOCKED defaults; regenerate from tofu if overridden: + `cd ../tofu && tofu output -raw ansible_inventory > ../ansible/inventory/hosts.ini` +3. Run from the dev container (has `kubectl` + `helm`, used by the Cilium/verify/ + Flux plays via `delegate_to: localhost`). + +## Run order +``` +cd infra/ansible +ansible-playbook playbooks/site.yml # everything, in order +# — or step by step — +ansible-playbook playbooks/k3s.yml # 1. HA k3s (primary --cluster-init, then joins) +ansible-playbook playbooks/cilium.yml # 2. Cilium CNI (cluster is NotReady until this) +ansible-playbook playbooks/verify.yml # 3. gate #1: 3 nodes Ready, Hubble up, no kube-proxy +ansible-playbook playbooks/flux-bootstrap.yml # 4. hand off to bootstrap/ (SOPS then Flux) +``` + +## What each layer does +| Playbook / role | Action | +|---|---| +| `k3s.yml` / `k3s_server` | k3s server install with `--flannel-backend=none --disable-kube-proxy --disable-network-policy --disable=traefik,servicelb`; primary `--cluster-init`, secondaries join via the primary's node-token (fetched, never pre-stored) | +| `cilium.yml` / `cilium` | Gateway API CRDs, then Helm `cilium` with `kubeProxyReplacement`, `k8sServiceHost/Port`, `gatewayAPI`, `egressGateway`, `hubble`, `l2announcements` | +| `verify.yml` / `cluster_verify` | waits nodes Ready + Cilium/Hubble rollout; asserts kube-proxy absent | +| `flux-bootstrap.yml` / `flux_bootstrap` | calls `bootstrap/` (SOPS age key, then `flux bootstrap --path=clusters/dev`) | + +## Cilium ownership / parity contract (docs/02) +Ansible installs Cilium; the Flux `platform/cilium` HelmRelease adopts config +afterward. The parity-critical values **must match** between this role's +`templates/cilium-values.yaml.j2` and that HelmRelease: +- `kubeProxyReplacement: true` +- `k8sServiceHost: {{ k3s_api_host }}` (= primary node IP, `10.73.0.11` by default) +- `k8sServicePort: 6443` + +The HelmRelease currently has `k8sServiceHost: "REPLACE-ME"` — WS-NET must set it +to the same value this role uses (surfaced by `tofu output primary_node_ip`). + +## Interface assumed of `bootstrap/` (authored by other workers, NOT here) +`roles/flux_bootstrap` invokes two entrypoints via overridable command vars: +- `flux_bootstrap_sops_cmd` → `bootstrap/sops/install-age-key.sh` (Security + Engineer, T-1.4): installs the age **private** key as the `sops-age` Secret in + `flux-system`. Key never committed (`.gitignore` guards `age.key`/`*.key`). +- `flux_bootstrap_flux_cmd` → `bootstrap/flux/bootstrap.sh` (GitOps Engineer, + T-1.4): runs `flux bootstrap ... --path=clusters/dev`. + +**These script names/args are assumptions** — if the bootstrap deliverables +expose different entrypoints, override the two vars in +`roles/flux_bootstrap/defaults/main.yml`; the task graph is unchanged. Flagged +for PM reconciliation. + +## Static validation (PM, in dev container — NOT run here) +``` +yamllint -c ../../ci/yamllint.yaml infra/ansible +ansible-playbook -i inventory/hosts.ini --syntax-check playbooks/site.yml +ansible-lint playbooks/ roles/ # if the profile is enforced for infra/ +``` diff --git a/infra/ansible/ansible.cfg b/infra/ansible/ansible.cfg new file mode 100644 index 0000000..0d8663b --- /dev/null +++ b/infra/ansible/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = inventory/hosts.ini +roles_path = roles +host_key_checking = False +retry_files_enabled = False +stdout_callback = yaml +interpreter_python = auto_silent + +[ssh_connection] +pipelining = True +ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o StrictHostKeyChecking=accept-new diff --git a/infra/ansible/inventory/group_vars/all.yml b/infra/ansible/inventory/group_vars/all.yml new file mode 100644 index 0000000..d4e548b --- /dev/null +++ b/infra/ansible/inventory/group_vars/all.yml @@ -0,0 +1,11 @@ +--- +# Shared connection + cluster facts for every k3s node. + +# cloud-init created this login user (tofu var login_user); it has NOPASSWD sudo. +ansible_user: k3s + +# The kube-apiserver endpoint the whole stack agrees on: the primary node IP. +# Used for k3s --tls-san / secondary --server AND for Cilium k8sServiceHost so the +# Ansible bootstrap and the Flux HelmRelease stay in PARITY (docs/02 contract). +k3s_api_host: "{{ hostvars[groups['k3s_primary'][0]]['ansible_host'] }}" +k3s_api_port: 6443 diff --git a/infra/ansible/inventory/hosts.ini b/infra/ansible/inventory/hosts.ini new file mode 100644 index 0000000..b1c94e6 --- /dev/null +++ b/infra/ansible/inventory/hosts.ini @@ -0,0 +1,17 @@ +# Static inventory — ALIGNED with infra/tofu defaults (LOCKED PHASE1-SCOPE IPs). +# Regenerate from tofu when IPs/names are overridden: +# cd ../tofu && tofu output -raw ansible_inventory > ../ansible/inventory/hosts.ini +# +# k3s_primary = the --cluster-init server (embedded-etcd founder + Cilium k8sServiceHost). +# k3s_secondary = the two servers that join it. All three are servers (HA embedded etcd). + +[k3s_primary] +k3s-dev-1 ansible_host=10.73.0.11 + +[k3s_secondary] +k3s-dev-2 ansible_host=10.73.0.12 +k3s-dev-3 ansible_host=10.73.0.13 + +[k3s_servers:children] +k3s_primary +k3s_secondary diff --git a/infra/ansible/playbooks/cilium.yml b/infra/ansible/playbooks/cilium.yml new file mode 100644 index 0000000..b49a8e1 --- /dev/null +++ b/infra/ansible/playbooks/cilium.yml @@ -0,0 +1,9 @@ +--- +# Install Cilium as the CNI (ADR-0002). The cluster is NotReady until this runs. +# Executed from the controller against a fetched kubeconfig (role handles delegation). +- name: Install Cilium CNI + hosts: k3s_servers + become: true + gather_facts: false + roles: + - role: cilium diff --git a/infra/ansible/playbooks/flux-bootstrap.yml b/infra/ansible/playbooks/flux-bootstrap.yml new file mode 100644 index 0000000..ba14c88 --- /dev/null +++ b/infra/ansible/playbooks/flux-bootstrap.yml @@ -0,0 +1,9 @@ +--- +# FINAL step: hand off to Flux + SOPS (T-1.4). Calls into bootstrap/ (authored by +# the Security + GitOps Engineers) — this play does not author that subtree. +- name: Flux + SOPS bootstrap hand-off + hosts: k3s_servers + become: false + gather_facts: false + roles: + - role: flux_bootstrap diff --git a/infra/ansible/playbooks/k3s.yml b/infra/ansible/playbooks/k3s.yml new file mode 100644 index 0000000..57126d2 --- /dev/null +++ b/infra/ansible/playbooks/k3s.yml @@ -0,0 +1,22 @@ +--- +# Install the HA k3s control plane (embedded etcd, 3 servers) — ADR-0001. +# Primary founds the cluster (--cluster-init); secondaries join it in a second +# play so the primary's node-token fact is available to them. + +- name: k3s primary (cluster-init) + hosts: k3s_primary + become: true + gather_facts: true + roles: + - role: k3s_server + vars: + k3s_server_mode: primary + +- name: k3s secondaries (join) + hosts: k3s_secondary + become: true + gather_facts: true + roles: + - role: k3s_server + vars: + k3s_server_mode: secondary diff --git a/infra/ansible/playbooks/site.yml b/infra/ansible/playbooks/site.yml new file mode 100644 index 0000000..09cc37e --- /dev/null +++ b/infra/ansible/playbooks/site.yml @@ -0,0 +1,15 @@ +--- +# Full dev/test cluster bring-up, in order (docs/02 sequencing; run AFTER +# `tofu apply` has created and cloud-init has baselined the VMs): +# k3s HA install -> Cilium CNI -> verify readiness -> Flux + SOPS hand-off. +- name: Import - install k3s HA control plane + ansible.builtin.import_playbook: k3s.yml + +- name: Import - install Cilium CNI + ansible.builtin.import_playbook: cilium.yml + +- name: Import - verify cluster readiness + ansible.builtin.import_playbook: verify.yml + +- name: Import - Flux + SOPS bootstrap hand-off + ansible.builtin.import_playbook: flux-bootstrap.yml diff --git a/infra/ansible/playbooks/verify.yml b/infra/ansible/playbooks/verify.yml new file mode 100644 index 0000000..b5cbc68 --- /dev/null +++ b/infra/ansible/playbooks/verify.yml @@ -0,0 +1,8 @@ +--- +# Verify Phase-1 exit gate #1: 3 nodes Ready, Cilium/Hubble healthy, no kube-proxy. +- name: Verify cluster readiness + hosts: k3s_servers + become: false + gather_facts: false + roles: + - role: cluster_verify diff --git a/infra/ansible/roles/cilium/defaults/main.yml b/infra/ansible/roles/cilium/defaults/main.yml new file mode 100644 index 0000000..cc930d5 --- /dev/null +++ b/infra/ansible/roles/cilium/defaults/main.yml @@ -0,0 +1,28 @@ +--- +# cilium role defaults (ADR-0002, docs/02 Cilium-ownership contract). +# This role performs the INITIAL Cilium install; the Flux platform/cilium +# HelmRelease adopts config afterward. The parity-critical values +# (kubeProxyReplacement, k8sServiceHost/Port) are templated from k3s_api_host. + +# Pin inside the platform/cilium HelmRelease window (>=1.16.0 <1.17.0). +cilium_version: "1.16.5" +cilium_chart_repo_name: "cilium" +cilium_chart_repo_url: "https://helm.cilium.io/" +cilium_namespace: "kube-system" + +# Gateway API CRDs (standard channel) must exist BEFORE Cilium with gatewayAPI.enabled. +# Cilium 1.16 targets Gateway API v1.2.x. +gateway_api_version: "v1.2.1" +gateway_api_crd_urls: + - "https://github.com/kubernetes-sigs/gateway-api/releases/download/{{ gateway_api_version }}/standard-install.yaml" + # TLSRoute (experimental) is required by Cilium's Gateway API support. + - "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/{{ gateway_api_version }}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml" + +# Local kubeconfig fetched from the primary. *.kubeconfig is gitignored. +cilium_kubeconfig: "{{ playbook_dir }}/../.cache/dev.kubeconfig" +cilium_values_path: "{{ playbook_dir }}/../.cache/cilium-values.yaml" + +# dev: single operator replica to save RAM (PHASE1-SCOPE RAM ceiling). NOT a +# parity field — Flux may differ here; only kubeProxyReplacement/k8sServiceHost +# /k8sServicePort must match the HelmRelease. +cilium_operator_replicas: 1 diff --git a/infra/ansible/roles/cilium/tasks/main.yml b/infra/ansible/roles/cilium/tasks/main.yml new file mode 100644 index 0000000..b3be9a4 --- /dev/null +++ b/infra/ansible/roles/cilium/tasks/main.yml @@ -0,0 +1,88 @@ +--- +# Install Cilium as the CNI (ADR-0002). Runs from the controller (dev container, +# which has kubectl + helm) against a kubeconfig fetched from the primary node. +# The cluster is NotReady until this completes — expected (PHASE1-SCOPE risks). +# run_once: this whole role acts on the cluster once, not per host. + +- name: Fetch the k3s kubeconfig from the primary + ansible.builtin.slurp: + src: /etc/rancher/k3s/k3s.yaml + register: k3s_kubeconfig_raw + delegate_to: "{{ groups['k3s_primary'][0] }}" + run_once: true + +- name: Ensure the local cache dir exists + ansible.builtin.file: + path: "{{ cilium_kubeconfig | dirname }}" + state: directory + mode: "0700" + delegate_to: localhost + become: false + run_once: true + +- name: Write kubeconfig locally (server rewritten to the primary IP) + ansible.builtin.copy: + content: >- + {{ k3s_kubeconfig_raw.content | b64decode + | regex_replace('https://127.0.0.1:6443', + 'https://' ~ k3s_api_host ~ ':' ~ k3s_api_port) }} + dest: "{{ cilium_kubeconfig }}" + mode: "0600" + delegate_to: localhost + become: false + run_once: true + +- name: Install Gateway API CRDs (must precede Cilium gatewayAPI.enabled) + ansible.builtin.command: + cmd: "kubectl apply -f {{ item }}" + environment: + KUBECONFIG: "{{ cilium_kubeconfig }}" + loop: "{{ gateway_api_crd_urls }}" + delegate_to: localhost + become: false + run_once: true + changed_when: true + +- name: Add the Cilium Helm repo + ansible.builtin.command: + cmd: "helm repo add {{ cilium_chart_repo_name }} {{ cilium_chart_repo_url }}" + delegate_to: localhost + become: false + run_once: true + register: cilium_repo_add + changed_when: "'has been added' in cilium_repo_add.stdout" + failed_when: + - cilium_repo_add.rc != 0 + - "'already exists' not in cilium_repo_add.stderr" + +- name: Update Helm repos + ansible.builtin.command: + cmd: "helm repo update {{ cilium_chart_repo_name }}" + delegate_to: localhost + become: false + run_once: true + changed_when: false + +- name: Render the Cilium values file (parity-critical settings from k3s_api_host) + ansible.builtin.template: + src: cilium-values.yaml.j2 + dest: "{{ cilium_values_path }}" + mode: "0644" + delegate_to: localhost + become: false + run_once: true + +- name: Install/upgrade Cilium via Helm + ansible.builtin.command: + cmd: >- + helm upgrade --install cilium {{ cilium_chart_repo_name }}/cilium + --version {{ cilium_version }} + --namespace {{ cilium_namespace }} + --values {{ cilium_values_path }} + --wait --timeout 10m + environment: + KUBECONFIG: "{{ cilium_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + changed_when: true diff --git a/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 b/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 new file mode 100644 index 0000000..e360506 --- /dev/null +++ b/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 @@ -0,0 +1,40 @@ +# Cilium Helm values — rendered by the Ansible bootstrap (T-1.3). +# PARITY CONTRACT (docs/02): the fields marked [parity] MUST equal the +# platform/cilium HelmRelease values, or Flux and the bootstrap fight over the CNI. +--- +# [parity] k3s kube-proxy is disabled → Cilium replaces it (ADR-0001/0002). +kubeProxyReplacement: true +# [parity] = the k3s API endpoint (primary node IP). The Flux HelmRelease's +# k8sServiceHost "REPLACE-ME" must be set to this same value. +k8sServiceHost: "{{ k3s_api_host }}" +k8sServicePort: {{ k3s_api_port }} + +# Ingress via Gateway API — no Traefik (docs/09, ADR-0002). +gatewayAPI: + enabled: true + +# Egress-class steering (ADR-0002; VPN model ADR-0013, Phase 2/3). +egressGateway: + enabled: true + +# Flow observability / isolation audit (ADR-0002). +hubble: + enabled: true + relay: + enabled: true + ui: + enabled: true + +# Ingress VIP announcement on the NAT LAN. IP pool / CiliumL2AnnouncementPolicy +# (from the reserved 10.73.0.240-.250 range) is authored by WS-NET in Phase 2. +l2announcements: + enabled: true + +# k3s ships its own cgroup mount; point Cilium at it instead of auto-mounting. +cgroup: + autoMount: + enabled: false + hostRoot: /run/cilium/cgroupv2 + +operator: + replicas: {{ cilium_operator_replicas }} diff --git a/infra/ansible/roles/cluster_verify/defaults/main.yml b/infra/ansible/roles/cluster_verify/defaults/main.yml new file mode 100644 index 0000000..6c3f707 --- /dev/null +++ b/infra/ansible/roles/cluster_verify/defaults/main.yml @@ -0,0 +1,4 @@ +--- +# cluster_verify role defaults. +cluster_verify_kubeconfig: "{{ playbook_dir }}/../.cache/dev.kubeconfig" +cluster_verify_node_ready_timeout: "300s" diff --git a/infra/ansible/roles/cluster_verify/tasks/main.yml b/infra/ansible/roles/cluster_verify/tasks/main.yml new file mode 100644 index 0000000..637a800 --- /dev/null +++ b/infra/ansible/roles/cluster_verify/tasks/main.yml @@ -0,0 +1,65 @@ +--- +# Verify the cluster reached the Phase-1 exit gate #1: 3 nodes Ready, Cilium +# healthy (Hubble up), no flannel/kube-proxy. Runs from the controller once. + +- name: Wait for all nodes to be Ready (Cilium must be up first) + ansible.builtin.command: + cmd: >- + kubectl wait --for=condition=Ready nodes --all + --timeout={{ cluster_verify_node_ready_timeout }} + environment: + KUBECONFIG: "{{ cluster_verify_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + changed_when: false + +- name: Wait for the Cilium DaemonSet to roll out + ansible.builtin.command: + cmd: >- + kubectl -n kube-system rollout status ds/cilium --timeout=300s + environment: + KUBECONFIG: "{{ cluster_verify_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + changed_when: false + +- name: Wait for hubble-relay to be available + ansible.builtin.command: + cmd: >- + kubectl -n kube-system rollout status deploy/hubble-relay --timeout=300s + environment: + KUBECONFIG: "{{ cluster_verify_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + changed_when: false + +- name: Confirm kube-proxy is absent (Cilium replaces it) + ansible.builtin.command: + cmd: "kubectl -n kube-system get ds kube-proxy" + environment: + KUBECONFIG: "{{ cluster_verify_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + register: kube_proxy_check + changed_when: false + failed_when: kube_proxy_check.rc == 0 + +- name: Summarise cluster state + ansible.builtin.command: + cmd: "kubectl get nodes -o wide" + environment: + KUBECONFIG: "{{ cluster_verify_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + register: cluster_nodes + changed_when: false + +- name: Show nodes + ansible.builtin.debug: + var: cluster_nodes.stdout_lines + run_once: true diff --git a/infra/ansible/roles/flux_bootstrap/defaults/main.yml b/infra/ansible/roles/flux_bootstrap/defaults/main.yml new file mode 100644 index 0000000..afe902a --- /dev/null +++ b/infra/ansible/roles/flux_bootstrap/defaults/main.yml @@ -0,0 +1,22 @@ +--- +# flux_bootstrap role defaults — the hand-off to the OTHER worker's bootstrap/. +# +# THIS ROLE AUTHORS THE CALL, IT DOES NOT OWN bootstrap/. The Security Engineer +# (T-1.4 SOPS/age) and GitOps Engineer (T-1.4 Flux) author bootstrap/sops/ and +# bootstrap/flux/. This role INVOKES their entrypoints. The exact script names / +# args below are ASSUMPTIONS the PM must reconcile with those deliverables — see +# roles/flux_bootstrap/tasks/main.yml and infra/ansible/README.md "Interface". + +flux_bootstrap_kubeconfig: "{{ playbook_dir }}/../.cache/dev.kubeconfig" + +# Repo root, relative to playbooks/ (infra/ansible/playbooks -> ../../..). +flux_bootstrap_repo_root: "{{ playbook_dir }}/../../.." + +# Assumed interface of bootstrap/sops (Security Engineer, T-1.4): +# installs the age private key as the `sops-age` Secret in flux-system so Flux +# can decrypt. Private key stays OUT of git (.gitignore guards age.key/*.key). +flux_bootstrap_sops_cmd: "bash {{ flux_bootstrap_repo_root }}/bootstrap/sops/install-age-key.sh" + +# Assumed interface of bootstrap/flux (GitOps Engineer, T-1.4): +# runs `flux bootstrap ... --path=clusters/dev` against this cluster. +flux_bootstrap_flux_cmd: "bash {{ flux_bootstrap_repo_root }}/bootstrap/flux/bootstrap.sh" diff --git a/infra/ansible/roles/flux_bootstrap/tasks/main.yml b/infra/ansible/roles/flux_bootstrap/tasks/main.yml new file mode 100644 index 0000000..1a87390 --- /dev/null +++ b/infra/ansible/roles/flux_bootstrap/tasks/main.yml @@ -0,0 +1,52 @@ +--- +# FINAL play of the bring-up: hand off to Flux + SOPS (Phase 1 T-1.4, docs/02 +# "hands off to Flux"; ADR-0004, ADR-0007). This role does NOT author bootstrap/ +# — it calls the entrypoints the Security + GitOps Engineers author there. +# +# Order matters: install the age decryption key FIRST (so Flux can decrypt on its +# very first reconcile), THEN run `flux bootstrap` at clusters/dev. +# +# The two commands are variables (see defaults/main.yml). If the bootstrap workers +# expose different entrypoints, override flux_bootstrap_sops_cmd / _flux_cmd — the +# task graph does not change. + +- name: Assert the bootstrap/ subtree exists (authored by the other worker) + ansible.builtin.stat: + path: "{{ flux_bootstrap_repo_root }}/bootstrap" + register: bootstrap_dir + delegate_to: localhost + become: false + run_once: true + failed_when: not bootstrap_dir.stat.exists + +- name: Install the SOPS age key as a cluster Secret (bootstrap/sops) + ansible.builtin.command: + cmd: "{{ flux_bootstrap_sops_cmd }}" + environment: + KUBECONFIG: "{{ flux_bootstrap_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + changed_when: true + +- name: Bootstrap Flux against clusters/dev (bootstrap/flux) + ansible.builtin.command: + cmd: "{{ flux_bootstrap_flux_cmd }}" + environment: + KUBECONFIG: "{{ flux_bootstrap_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + changed_when: true + +- name: Wait for Flux to report the platform Kustomization Ready + ansible.builtin.command: + cmd: >- + kubectl -n flux-system wait kustomization/platform + --for=condition=Ready --timeout=600s + environment: + KUBECONFIG: "{{ flux_bootstrap_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + changed_when: false diff --git a/infra/ansible/roles/k3s_server/defaults/main.yml b/infra/ansible/roles/k3s_server/defaults/main.yml new file mode 100644 index 0000000..1d26380 --- /dev/null +++ b/infra/ansible/roles/k3s_server/defaults/main.yml @@ -0,0 +1,17 @@ +--- +# k3s_server role defaults (ADR-0001). + +# Best-effort version pin (kept near the ci/versions kubectl 1.31.x line). Bump +# deliberately — a version change is a code change. Empty string => latest stable. +k3s_version: "v1.31.5+k3s1" + +k3s_install_url: "https://get.k3s.io" +k3s_install_script: "/usr/local/bin/k3s-install.sh" + +# Components disabled at install so Cilium + Gateway API own networking (ADR-0001/0002). +# flannel + kube-proxy + network-policy are disabled via dedicated flags below; +# traefik + servicelb via --disable. +k3s_disable: "traefik,servicelb" + +# Set per-play: 'primary' (runs --cluster-init) or 'secondary' (joins the primary). +k3s_server_mode: "secondary" diff --git a/infra/ansible/roles/k3s_server/tasks/main.yml b/infra/ansible/roles/k3s_server/tasks/main.yml new file mode 100644 index 0000000..ee19b25 --- /dev/null +++ b/infra/ansible/roles/k3s_server/tasks/main.yml @@ -0,0 +1,66 @@ +--- +# Install k3s as an HA server (embedded etcd) with Cilium-required components +# disabled (ADR-0001, docs/02 stage 3 "Cluster-join"). The primary runs +# --cluster-init; secondaries join it with the shared node-token. + +- name: Download the k3s install script + ansible.builtin.get_url: + url: "{{ k3s_install_url }}" + dest: "{{ k3s_install_script }}" + mode: "0755" + +- name: Compose the common k3s server flags + ansible.builtin.set_fact: + k3s_common_args: >- + server + --flannel-backend=none + --disable-kube-proxy + --disable-network-policy + --disable={{ k3s_disable }} + --write-kubeconfig-mode=644 + --tls-san={{ k3s_api_host }} + --node-ip={{ ansible_host }} + +# --- primary: found the cluster -------------------------------------------- +- name: Install k3s primary (cluster-init, embedded etcd) + ansible.builtin.command: + cmd: "{{ k3s_install_script }} {{ k3s_common_args }} --cluster-init" + creates: /etc/rancher/k3s/k3s.yaml + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + when: k3s_server_mode == 'primary' + +- name: Wait for the k3s node-token to exist (primary) + ansible.builtin.wait_for: + path: /var/lib/rancher/k3s/server/token + timeout: 180 + when: k3s_server_mode == 'primary' + +- name: Read the k3s node-token (primary) + ansible.builtin.slurp: + src: /var/lib/rancher/k3s/server/token + register: k3s_token_raw + when: k3s_server_mode == 'primary' + +- name: Publish the node-token as a fact for secondaries (primary) + ansible.builtin.set_fact: + k3s_node_token: "{{ k3s_token_raw.content | b64decode | trim }}" + when: k3s_server_mode == 'primary' + +# --- secondaries: join the existing cluster -------------------------------- +- name: Install k3s secondary (join as server) + ansible.builtin.command: + cmd: >- + {{ k3s_install_script }} {{ k3s_common_args }} + --server https://{{ k3s_api_host }}:{{ k3s_api_port }} + creates: /etc/rancher/k3s/k3s.yaml + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + K3S_TOKEN: "{{ hostvars[groups['k3s_primary'][0]]['k3s_node_token'] }}" + when: k3s_server_mode == 'secondary' + +- name: Wait for the k3s API to be serving locally + ansible.builtin.wait_for: + host: "127.0.0.1" + port: "{{ k3s_api_port }}" + timeout: 180 diff --git a/infra/cloud-init/README.md b/infra/cloud-init/README.md index c860e7e..8eccefb 100644 --- a/infra/cloud-init/README.md +++ b/infra/cloud-init/README.md @@ -1,3 +1,45 @@ -# infra/cloud-init +# infra/cloud-init — first-boot baseline templates -_Phase 1 (T-1.x) — to be authored on execution go. See infra/PHASE1-SCOPE.md._ +Phase 1 **T-1.2**. The "Baseline" node lifecycle stage (docs/02 stage 2). These +templates are consumed by the OpenTofu `libvirt_cloudinit_disk` in `../tofu/` +(via `templatefile()`), one seed per node. **Authoring only — never applied.** + +Serves **R-ENV-01** (nodes are "created, baselined, cluster-joined, managed as +code") and lays the kernel/OS floor ADR-0001 (k3s) + ADR-0002 (Cilium) + +ADR-0003 (Longhorn) + ADR-0005 (KubeVirt) require. + +## Files +- `user-data.yaml.tftpl` — `#cloud-config`: login user + injected SSH **public** + key, hostname, package baseline, kernel modules, sysctls, Longhorn data-disk + mount, swap-off. +- `meta-data.yaml.tftpl` — NoCloud instance-id / local-hostname. +- `network-config.yaml.tftpl` — netplan v2 **static** IP (DHCP is off on the NAT). + +`${...}` tokens are **OpenTofu** template vars filled by `../tofu/main.tf` +(`hostname`, `fqdn`, `username`, `ssh_authorized_key`, `ip`, `prefix`, `gateway`) +— they are not cloud-init/Jinja. + +## Baseline (cloud-init's job) vs. Ansible's job + +| cloud-init does (here) | Ansible does (T-1.3) | +|---|---| +| user + SSH public key, hostname | install k3s (HA, embedded etcd) | +| static networking | join nodes, disable flannel/kube-proxy/traefik/servicelb | +| `overlay`, `br_netfilter`, `iscsi_tcp`, `dm_crypt`, `nfs` modules | install Cilium (kube-proxy replacement, Gateway API, Hubble) | +| bridge-nf + `ip_forward` sysctls, inotify limits | verify readiness | +| open-iscsi + nfs-common (Longhorn client prereqs) | Flux + SOPS bootstrap handoff | +| mount `/dev/vdb` at `/var/lib/longhorn` | — | +| `swapoff` + strip swap from fstab | — | + +### Why these specifics +- **overlay / br_netfilter + bridge-nf-call sysctls** — pod networking / iptables + bridging Cilium and kube-networking need. +- **ip_forward / forwarding** — routing between pods, nodes, and egress. +- **iscsi_tcp + open-iscsi (iscsid) and nfs / nfs-common** — Longhorn attaches + volumes over iSCSI and uses the NFS client for backup/RWX (ADR-0003). +- **dm_crypt** — headroom for Longhorn volume encryption. +- **swap off** — kubelet/k3s expectation. +- **qemu-guest-agent** — lets libvirt/tofu read guest state. + +Nested-virt for KubeVirt (Phase 5) comes from the tofu `host-passthrough` CPU +mode, not from here. diff --git a/infra/cloud-init/meta-data.yaml.tftpl b/infra/cloud-init/meta-data.yaml.tftpl new file mode 100644 index 0000000..0dd269c --- /dev/null +++ b/infra/cloud-init/meta-data.yaml.tftpl @@ -0,0 +1,3 @@ +# meta-data — NoCloud datasource identity. Rendered by OpenTofu templatefile(). +instance-id: ${hostname} +local-hostname: ${hostname} diff --git a/infra/cloud-init/network-config.yaml.tftpl b/infra/cloud-init/network-config.yaml.tftpl new file mode 100644 index 0000000..e2d01b7 --- /dev/null +++ b/infra/cloud-init/network-config.yaml.tftpl @@ -0,0 +1,19 @@ +# network-config (cloud-init Networking Config v2 / netplan). Rendered by OpenTofu. +# Static addressing on the k3s-dev NAT: DHCP is off on the libvirt network, so each +# node's IP comes from here. `match: name "en*"` avoids hard-coding ens3 vs enp1s0 +# across cloud images. Gateway == libvirt bridge host IP; it also serves DNS. +version: 2 +ethernets: + primary: + match: + name: "en*" + dhcp4: false + dhcp6: false + addresses: + - ${ip}/${prefix} + routes: + - to: default + via: ${gateway} + nameservers: + addresses: + - ${gateway} diff --git a/infra/cloud-init/user-data.yaml.tftpl b/infra/cloud-init/user-data.yaml.tftpl new file mode 100644 index 0000000..b696064 --- /dev/null +++ b/infra/cloud-init/user-data.yaml.tftpl @@ -0,0 +1,75 @@ +#cloud-config +# user-data — first-boot BASELINE only (docs/02 stage 2 "Baseline"). Rendered by +# OpenTofu templatefile() ($${...} = tofu vars, NOT cloud-init). Everything here is +# the node prerequisite floor for k3s / Cilium / Longhorn / KubeVirt. Cluster join, +# k3s, and Cilium install are ANSIBLE's job (T-1.3) — deliberately NOT done here. + +hostname: ${hostname} +fqdn: ${fqdn} +preserve_hostname: false + +# Login user (also the Ansible ansible_user). Public key only — no passwords. +users: + - name: ${username} + groups: [sudo] + shell: /bin/bash + sudo: "ALL=(ALL) NOPASSWD:ALL" + lock_passwd: true + ssh_authorized_keys: + - ${ssh_authorized_key} + +# Baseline packages: guest agent (tofu/libvirt), and Longhorn's node prerequisites +# (open-iscsi provides iscsid + iscsiadm; nfs-common provides the NFS client). +package_update: true +packages: + - qemu-guest-agent + - open-iscsi + - nfs-common + - curl + +# Kernel modules required by Cilium (overlay, br_netfilter) and Longhorn +# (iscsi_tcp, dm_crypt, nfs). Loaded now and persisted for reboots. +write_files: + - path: /etc/modules-load.d/k3s-cilium-longhorn.conf + permissions: "0644" + content: | + overlay + br_netfilter + iscsi_tcp + dm_crypt + nfs + - path: /etc/sysctl.d/99-k3s-cilium.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 + fs.inotify.max_user_instances = 8192 + fs.inotify.max_user_watches = 524288 + +# Dedicated Longhorn data disk (/dev/vdb) → ext4 mounted at /var/lib/longhorn. +# nofail so a node still boots if the disk is absent during a rebuild. +fs_setup: + - label: longhorn + filesystem: ext4 + device: /dev/vdb + partition: none + overwrite: false +mounts: + - ["LABEL=longhorn", "/var/lib/longhorn", "ext4", "defaults,nofail,x-systemd.device-timeout=5s", "0", "2"] + +runcmd: + # Apply modules + sysctls now (cloud-init runs after they're written). + - ["modprobe", "overlay"] + - ["modprobe", "br_netfilter"] + - ["modprobe", "iscsi_tcp"] + - ["sysctl", "--system"] + # Longhorn needs iscsid running; guest agent for libvirt/tofu. + - ["systemctl", "enable", "--now", "iscsid"] + - ["systemctl", "enable", "--now", "qemu-guest-agent"] + # k3s/kubelet baseline: disable swap. + - ["swapoff", "-a"] + - ["sed", "-i", "/\\sswap\\s/d", "/etc/fstab"] + +# NOTE: no reboot here — the baseline is live after runcmd. Ansible connects next. diff --git a/infra/tofu/README.md b/infra/tofu/README.md index 9671c55..231ad91 100644 --- a/infra/tofu/README.md +++ b/infra/tofu/README.md @@ -1,3 +1,57 @@ -# infra/tofu +# infra/tofu — dev/test cluster nodes (OpenTofu + libvirt) -_Phase 1 (T-1.x) — to be authored on execution go. See infra/PHASE1-SCOPE.md._ +Phase 1 **T-1.1**. Provisions the "Create" lifecycle stage (docs/02): a dedicated +NAT network and 3 cloud-init'd KVM VMs, each with a thin OS disk and a blank thin +Longhorn data disk. **Authoring only — nothing here has been applied.** + +Serves **R-ENV-01/02**, **R-PLAT-01**, ADR-0001 (k3s substrate), and the network +half of the LOCKED PHASE1-SCOPE decisions. + +## What it builds (defaults = LOCKED decisions) + +| Resource | Value | +|---|---| +| NAT network | `k3s-dev`, `10.73.0.0/24`, gateway `10.73.0.1`, **DHCP off** (static IPs) | +| Nodes | `k3s-dev-1/2/3` at `10.73.0.11/.12/.13` | +| Per node | 4 vCPU, 6 GiB RAM, **host-passthrough** CPU (nested virt for KubeVirt) | +| OS disk | 20 GiB thin qcow2, backed by the base cloud image | +| Data disk | 40 GiB blank thin qcow2 → `/dev/vdb`, mounted at `/var/lib/longhorn` by cloud-init | +| Seed | `libvirt_cloudinit_disk` wired to `../cloud-init/` templates | +| Reserved | `10.73.0.240–.250` kept out of any DHCP range for Cilium L2 ingress VIPs | + +Base OS image is a **variable** (`image_source`), defaulting to Debian 12 +genericcloud amd64. All sizing/IPs/counts are variables — see `variables.tf`. + +## Files +- `versions.tf` — OpenTofu `1.9.0` pin + `dmacvicar/libvirt ~> 0.8.1` provider pin. +- `variables.tf` — every knob (counts, sizing, IPs, image, SSH key). +- `main.tf` — network, base/OS/data volumes, cloud-init seed, domains. +- `outputs.tf` — node IPs/names, `primary_node_ip`, `k3s_api_endpoint`, and a + rendered `ansible_inventory` for `infra/ansible/`. +- `terraform.tfvars.example` — copy to `terraform.tfvars`; only `ssh_authorized_key` is required. + +## Usage (operator, in the dev container — NOT run here) +``` +cd infra/tofu +cp terraform.tfvars.example terraform.tfvars # set ssh_authorized_key +tofu init +tofu plan +tofu apply +# hand the node IPs to Ansible: +tofu output -raw ansible_inventory > ../ansible/inventory/hosts.ini +``` + +## Hand-off contract +- `primary_node_ip` / `k3s_api_endpoint` (`10.73.0.11:6443`) is the value the + Ansible Cilium bootstrap and the Flux `platform/cilium` HelmRelease must BOTH + use for `k8sServiceHost` / `k8sServicePort` (docs/02 parity contract). +- Destroy/rebuild proof (T-1.5): `tofu destroy -target=libvirt_domain.node[1]` + then `tofu apply` re-creates that node; Ansible re-joins it; Longhorn replicas + reconverge (nodes are cattle — docs/02 statelessness). + +## Static validation (PM, in dev container) +``` +tofu fmt -check -recursive +tofu init -backend=false +tofu validate +``` diff --git a/infra/tofu/main.tf b/infra/tofu/main.tf new file mode 100644 index 0000000..516358c --- /dev/null +++ b/infra/tofu/main.tf @@ -0,0 +1,144 @@ +# main.tf — dedicated NAT network + 3 cloud-init'd VMs, each with a thin OS disk +# (backed by the cloud image) and a blank thin Longhorn data disk (Phase 1 T-1.1). +# +# Node lifecycle stage 1 ("Create") per docs/02. cloud-init (infra/cloud-init/) +# does stage 2 ("Baseline"); Ansible (infra/ansible/) does stage 3 ("Cluster-join"). + +locals { + # 1 GiB in bytes; libvirt volume sizes are bytes. + gib = 1073741824 + + # Per-node identity/addressing, indexed 0..node_count-1. + nodes = [ + for i in range(var.node_count) : { + name = "${var.node_name_prefix}${i + 1}" + ip = var.node_ips[i] + fqdn = "${var.node_name_prefix}${i + 1}.${var.node_domain}" + is_first = i == 0 + } + ] +} + +# --- dedicated NAT network (10.73.0.0/24, gw .1) --------------------------- +# DHCP is OFF: nodes get static IPs from cloud-init network-config, so the +# reserved VIP range (.240-.250) can never be leased to something else. +resource "libvirt_network" "k3s_dev" { + name = var.network_name + mode = "nat" + addresses = [var.network_cidr] + autostart = true + + dhcp { + enabled = false + } + + dns { + enabled = true + local_only = true + } +} + +# --- base OS volume (downloaded/copied once, shared as a backing file) ------ +resource "libvirt_volume" "base" { + name = "${var.cluster_name}-base.qcow2" + pool = var.pool_name + source = var.image_source + format = "qcow2" +} + +# --- per-node OS disk: thin qcow2 backed by the base image ------------------ +resource "libvirt_volume" "os" { + count = var.node_count + name = "${local.nodes[count.index].name}-os.qcow2" + pool = var.pool_name + base_volume_id = libvirt_volume.base.id + size = var.os_disk_gib * local.gib + format = "qcow2" +} + +# --- per-node data disk: blank thin qcow2 for Longhorn ---------------------- +resource "libvirt_volume" "data" { + count = var.node_count + name = "${local.nodes[count.index].name}-data.qcow2" + pool = var.pool_name + size = var.data_disk_gib * local.gib + format = "qcow2" +} + +# --- cloud-init seed (user-data + meta-data + static network-config) -------- +# Templates live in the sibling infra/cloud-init/ subtree (T-1.2). +resource "libvirt_cloudinit_disk" "seed" { + count = var.node_count + name = "${local.nodes[count.index].name}-seed.iso" + pool = var.pool_name + + user_data = templatefile("${path.module}/../cloud-init/user-data.yaml.tftpl", { + hostname = local.nodes[count.index].name + fqdn = local.nodes[count.index].fqdn + username = var.login_user + ssh_authorized_key = var.ssh_authorized_key + }) + + meta_data = templatefile("${path.module}/../cloud-init/meta-data.yaml.tftpl", { + hostname = local.nodes[count.index].name + fqdn = local.nodes[count.index].fqdn + }) + + network_config = templatefile("${path.module}/../cloud-init/network-config.yaml.tftpl", { + ip = local.nodes[count.index].ip + prefix = var.network_prefix + gateway = var.network_gateway + }) +} + +# --- the VMs ---------------------------------------------------------------- +resource "libvirt_domain" "node" { + count = var.node_count + name = local.nodes[count.index].name + memory = var.memory_mib + vcpu = var.vcpu + + # host-passthrough so nested virt is available for KubeVirt in Phase 5. + cpu { + mode = var.cpu_mode + } + + # qemu-guest-agent is installed by cloud-init; lets tofu read the lease/agent. + qemu_agent = true + + cloudinit = libvirt_cloudinit_disk.seed[count.index].id + + # IP + hostname come from cloud-init (DHCP is off on this network), so we do NOT + # set addresses/hostname here — the provider couples those to DHCP reservations. + network_interface { + network_id = libvirt_network.k3s_dev.id + wait_for_lease = false + } + + # OS disk first (vda, boot), Longhorn data disk second (vdb). + disk { + volume_id = libvirt_volume.os[count.index].id + } + + disk { + volume_id = libvirt_volume.data[count.index].id + } + + console { + type = "pty" + target_port = "0" + target_type = "serial" + } + + console { + type = "pty" + target_port = "1" + target_type = "virtio" + } + + graphics { + type = "spice" + listen_type = "address" + autoport = true + } +} diff --git a/infra/tofu/outputs.tf b/infra/tofu/outputs.tf new file mode 100644 index 0000000..e892c56 --- /dev/null +++ b/infra/tofu/outputs.tf @@ -0,0 +1,58 @@ +# outputs.tf — surface the facts the Ansible inventory needs (Phase 1 T-1.1). +# The primary node is the k3s cluster-init server and the k8sServiceHost the +# Cilium bootstrap + Flux HelmRelease must agree on (docs/02 parity contract). + +output "node_names" { + description = "Ordered VM hostnames." + value = [for n in local.nodes : n.name] +} + +output "node_ips" { + description = "Ordered node static IPs (same order as node_names)." + value = [for n in local.nodes : n.ip] +} + +output "nodes" { + description = "Per-node map name => ip, for building the Ansible inventory." + value = { for n in local.nodes : n.name => n.ip } +} + +output "primary_node_ip" { + description = "First server IP — k3s --cluster-init host and Cilium k8sServiceHost (parity value)." + value = local.nodes[0].ip +} + +output "k3s_api_endpoint" { + description = "kube-apiserver endpoint used for k8sServiceHost/Port parity (docs/02)." + value = "${local.nodes[0].ip}:6443" +} + +output "cilium_l2_vip_range" { + description = "Reserved ingress VIP range for Cilium L2 announcement (WS-NET consumes in Phase 2)." + value = "${var.cilium_l2_vip_range_start}-${var.cilium_l2_vip_range_end}" +} + +# Rendered INI inventory for infra/ansible/. The Ansible dir also ships a static +# inventory aligned with these defaults; regenerate it from this output if the +# IPs/names are overridden. `tofu output -raw ansible_inventory > ../ansible/inventory/hosts.ini` +output "ansible_inventory" { + description = "Ready-to-write Ansible INI inventory aligned with tofu outputs." + value = <<-EOT + # GENERATED from tofu outputs — do not hand-edit; re-render on IP/name change. + [k3s_primary] + ${local.nodes[0].name} ansible_host=${local.nodes[0].ip} + + [k3s_secondary] + %{for n in slice(local.nodes, 1, length(local.nodes))~} + ${n.name} ansible_host=${n.ip} + %{endfor~} + + [k3s_servers:children] + k3s_primary + k3s_secondary + + [k3s_servers:vars] + ansible_user=${var.login_user} + k3s_api_host=${local.nodes[0].ip} + EOT +} diff --git a/infra/tofu/terraform.tfvars.example b/infra/tofu/terraform.tfvars.example new file mode 100644 index 0000000..e107428 --- /dev/null +++ b/infra/tofu/terraform.tfvars.example @@ -0,0 +1,37 @@ +# terraform.tfvars.example — copy to terraform.tfvars and edit. +# +# NOTE on secrets/gitignore: real *.tfvars are gitignored (root .gitignore) so +# local values never land in git. This *.example file is safe to commit — it +# carries NO private material (SSH *public* key only). Never put a private key here. +# +# Defaults for everything below already encode the LOCKED PHASE1-SCOPE decisions; +# the ONLY required value is ssh_authorized_key. + +# REQUIRED — your SSH *public* key (contents of e.g. ~/.ssh/id_ed25519.pub). +ssh_authorized_key = "ssh-ed25519 AAAA...replace-with-your-public-key... user@host" + +# --- optional overrides (defaults shown) ----------------------------------- + +# libvirt_uri = "qemu:///system" +# pool_name = "default" + +# node_count = 3 +# node_name_prefix = "k3s-dev-" +# node_ips = ["10.73.0.11", "10.73.0.12", "10.73.0.13"] + +# vcpu = 4 +# memory_mib = 6144 +# os_disk_gib = 20 +# data_disk_gib = 40 +# cpu_mode = "host-passthrough" + +# network_name = "k3s-dev" +# network_cidr = "10.73.0.0/24" +# network_gateway = "10.73.0.1" +# network_prefix = 24 + +# login_user = "k3s" +# node_domain = "k3s.dev.local" + +# Point at a locally pre-downloaded image to avoid re-fetching on each apply: +# image_source = "/var/lib/libvirt/images/debian-12-genericcloud-amd64.qcow2" diff --git a/infra/tofu/variables.tf b/infra/tofu/variables.tf new file mode 100644 index 0000000..273531c --- /dev/null +++ b/infra/tofu/variables.tf @@ -0,0 +1,151 @@ +# variables.tf — every sizing / IP / image knob is a variable (Phase 1 T-1.1). +# Defaults encode the LOCKED decisions in infra/PHASE1-SCOPE.md. Override via +# terraform.tfvars (see terraform.tfvars.example). + +variable "libvirt_uri" { + description = "libvirt connection URI." + type = string + default = "qemu:///system" +} + +variable "pool_name" { + description = "libvirt storage pool for all volumes (base image, OS + data disks, seed ISOs)." + type = string + default = "default" +} + +# --- cluster identity / naming --------------------------------------------- + +variable "cluster_name" { + description = "Logical cluster name; used as a resource-name prefix." + type = string + default = "k3s-dev" +} + +variable "node_name_prefix" { + description = "VM hostname prefix. Nodes are 1, 2, ... (LOCKED: k3s-dev-1/2/3)." + type = string + default = "k3s-dev-" +} + +variable "node_count" { + description = "Number of k3s nodes (LOCKED: 3-server HA embedded etcd)." + type = number + default = 3 + + validation { + condition = var.node_count == length(var.node_ips) + error_message = "node_count must equal the number of entries in node_ips." + } +} + +# --- per-node sizing (LOCKED: 4 vCPU / 6 GiB / 20 GiB OS + 40 GiB data) ----- + +variable "vcpu" { + description = "vCPUs per node." + type = number + default = 4 +} + +variable "memory_mib" { + description = "RAM per node in MiB (6 GiB)." + type = number + default = 6144 +} + +variable "os_disk_gib" { + description = "OS (root) disk size per node, GiB. Thin qcow2 backed by the cloud image." + type = number + default = 20 +} + +variable "data_disk_gib" { + description = "Dedicated Longhorn data disk per node, GiB. Thin qcow2, blank." + type = number + default = 40 +} + +variable "cpu_mode" { + description = "libvirt CPU mode. host-passthrough exposes nested-virt for KubeVirt (Phase 5)." + type = string + default = "host-passthrough" +} + +# --- network (LOCKED: dedicated NAT k3s-dev 10.73.0.0/24, gw .1) ------------ + +variable "network_name" { + description = "libvirt NAT network name." + type = string + default = "k3s-dev" +} + +variable "network_cidr" { + description = "Cluster NAT subnet in CIDR. Gateway (network_gateway) is the libvirt bridge host IP." + type = string + default = "10.73.0.0/24" +} + +variable "network_prefix" { + description = "Netmask prefix length for node static addressing." + type = number + default = 24 +} + +variable "network_gateway" { + description = "Default gateway / DNS for nodes = libvirt bridge host IP (LOCKED: 10.73.0.1)." + type = string + default = "10.73.0.1" +} + +variable "node_ips" { + description = "Static IPs per node, in order (LOCKED: .11/.12/.13)." + type = list(string) + default = ["10.73.0.11", "10.73.0.12", "10.73.0.13"] +} + +# Reserved for Cilium L2-announcement ingress VIPs (LOCKED: .240-.250). +# Not materialised as libvirt objects; surfaced in outputs and kept OUT of any +# DHCP range so nothing else claims them. Cilium's IP pool (WS-NET, Phase 2) +# consumes this range. +variable "cilium_l2_vip_range_start" { + description = "First reserved Cilium L2 VIP." + type = string + default = "10.73.0.240" +} + +variable "cilium_l2_vip_range_end" { + description = "Last reserved Cilium L2 VIP." + type = string + default = "10.73.0.250" +} + +# --- base OS image ---------------------------------------------------------- + +variable "image_source" { + description = <<-EOT + Source for the base OS volume: a URL or local path to a minimal cloud image + (qcow2). Default is Debian 12 (bookworm) genericcloud amd64. Point at a local + path to avoid re-downloading on every apply. + EOT + type = string + default = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2" +} + +# --- cloud-init identity ---------------------------------------------------- + +variable "ssh_authorized_key" { + description = "SSH PUBLIC key injected into the login user (public only — never a private key)." + type = string +} + +variable "login_user" { + description = "Non-root login user created by cloud-init; also the Ansible ansible_user." + type = string + default = "k3s" +} + +variable "node_domain" { + description = "DNS domain suffix for node FQDNs (informational; dev uses split-horizon / hosts)." + type = string + default = "k3s.dev.local" +} diff --git a/infra/tofu/versions.tf b/infra/tofu/versions.tf new file mode 100644 index 0000000..4a676fc --- /dev/null +++ b/infra/tofu/versions.tf @@ -0,0 +1,25 @@ +# versions.tf — OpenTofu + provider pins (Phase 1 T-1.1). +# +# Toolchain is pinned to match the dev container (infra/devcontainer/versions.env): +# OpenTofu 1.9.0. The dmacvicar/libvirt provider is pinned HERE (not in the +# dev container) per the note in that file. +# +# AUTHOR-ONLY: this module is written but NOT applied. The PM static-validates +# with `tofu fmt -check` / `tofu validate` in the dev container. + +terraform { + required_version = "1.9.0" + + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = "~> 0.8.1" + } + } +} + +provider "libvirt" { + # System-level libvirt (qemu:///system) so VMs/pools/networks are host-wide, + # matching the dev-container mount of the host libvirt socket (PHASE1-SCOPE risks). + uri = var.libvirt_uri +}