diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index 0592022..e49618c 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -101,10 +101,37 @@ data: # ship the literal string into the cluster. # k3s API endpoint Cilium's kube-proxy replacement dials (platform/cilium/helmrelease.yaml). - # [parity] MUST equal the Ansible bootstrap's k3s_api_host (infra/ansible/roles/cilium/defaults/ - # main.yml -> cilium-values.yaml.j2, already inventory-parameterized). Wrong value = Cilium cannot - # reach the API server on adopt and the CNI breaks. Port stays 6443 in both envs, so it is NOT a knob. - K3S_API_HOST: "192.168.1.50" + # + # *** THIS IS THE CONTROL-PLANE VIP (kube-vip), NOT A NODE ADDRESS. *** Operator decision + # 2026-09-06. It was each environment's PRIMARY NODE IP until then, which made the CNI's + # control-plane path depend on one machine in a cluster whose entire purpose is HA: verified on + # live prod, all three cilium-agents carried KUBERNETES_SERVICE_HOST=192.168.2.10 (k3s-prod-1's + # own address) while kube-vip held 192.168.2.9. Losing that node leaves the datapath running off + # Cilium's cache while NO service or policy update lands — a silent, unbounded staleness, not an + # outage you would notice. A virtual address that follows the control plane is the fix, and it is + # what kube-vip was stood up to provide. + # + # dev -> 192.168.1.49 (= kube_vip_address, group_vars/all.yml; base default, below) + # prod -> 192.168.2.9 (= kube_vip_address, group_vars/k3s_prod/vars.yml) + # + # THE NAME IS NOW SLIGHTLY WRONG AND IS KEPT ANYWAY: renaming the knob would be a second, + # unrelated change to every substituting Kustomization at the same time as a live CNI endpoint + # move. Read it as "the API host Cilium dials", which is what it always meant. + # + # [parity] MUST equal the Ansible bootstrap's **k3s_apiserver_endpoint** for the SAME environment + # (infra/ansible/inventory/group_vars/all.yml -> roles/cilium/templates/cilium-values.yaml.j2, + # already inventory-parameterized). NOT k3s_api_host — that variable still exists and still holds + # the FOUNDER's address, because the cold-bootstrap `--server` join needs a machine that is + # already up before any VIP exists. The two are deliberately separate; see the TWO API ADDRESSES + # block at the top of group_vars/all.yml. + # + # Wrong value = Cilium cannot reach the API server on adopt and the CNI breaks. Port stays 6443 + # in both envs, so it is NOT a knob. + # + # TLS needs nothing: k3s_tls_sans has always carried kube_vip_address alongside every node IP, so + # both addresses are already in the serving cert (verified live 2026-09-06 — prod answers through + # https://192.168.2.9:6443 under full CA verification). No cert reissue, no k3s restart. + K3S_API_HOST: "192.168.1.49" # CiliumLoadBalancerIPPool `default-pool` (platform-config/cilium/l2-ip-pool.yaml): the VIP range # L2-announced on the flat LAN. Must sit OUTSIDE the node IPs and any DHCP scope, and must match the diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index 1708eab..8c533ea 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -187,11 +187,25 @@ patches: - op: replace path: /data/GATEWAY_VIP value: "192.168.1.53" - # k3s API endpoint Cilium's kube-proxy replacement dials — the primary control-plane node - # (k3s-1). [parity] equals the Ansible inventory's k3s_api_host for dev. + # k3s API endpoint Cilium's kube-proxy replacement dials — DEV'S CONTROL-PLANE VIP, + # the address kube-vip floats, NOT the primary node .50. + # [parity] equals the Ansible inventory's k3s_apiserver_endpoint for dev + # (= kube_vip_address, group_vars/all.yml). See clusters/base/cluster-vars.yaml for why. + # + # *** THE ONE EXCEPTION TO "NEVER LET THIS BLOCK DRIFT FROM THE LIVE CLUSTER" *** + # Every other value in this block is what dev is running right now. This one is not: + # the live dev cilium-agents dial 192.168.1.50 (k3s-1). Changing it here is a + # DELIBERATE, OPERATOR-APPROVED move (2026-09-06) and reconciling it ROLLS THE CILIUM + # AGENT DAEMONSET on dev — one node at a time (updateStrategy.maxUnavailable: 1, kept + # in parity for exactly this reason). Do not push this to dev-live casually, and not + # while dev's DNS migration is mid-flight; the L2 datapath survives an agent restart + # but a wrong value here is the 2026-09-01 incident. .49 is in dev's serving-cert SANs + # already (k3s_tls_sans) and dev's own kubeconfig validates against it via + # tls-server-name, so TLS is not the risk — the risk is reachability, and .49 is on the + # same LAN as the nodes and ARP-announced by kube-vip on every server. - op: replace path: /data/K3S_API_HOST - value: "192.168.1.50" + value: "192.168.1.49" # Cilium LB IP pool = the tofu reservation cilium_l2_vip_range .53-.60, outside the node IPs # (.50/.51/.52) and outside the DHCP scope. .53 Gateway, .54 unifi, .55 crafty; .56-.60 free. - op: replace diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index 3488bbf..da68488 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -356,14 +356,32 @@ patches: - op: replace path: /data/CILIUM_LB_POOL_STOP value: "192.168.2.30" - # --- still UNDECIDED -> loud placeholders (must be real before `flux bootstrap`) ------------ # k3s API endpoint Cilium's kube-proxy replacement dials. This is the prod CONTROL-PLANE VIP - # (kube-vip), NOT .20 (the ingress VIP) and NOT the single node address .10. It is owned - # Ansible-side (prod inventory k3s_api_host) and MUST equal that value — [parity] contract in - # platform/cilium/helmrelease.yaml. Not yet decided. + # (kube-vip), NOT .20 (the ingress VIP) and NOT the single node address .10. + # + # *** DECIDED 2026-09-06 by the operator: the VIP, as intended. *** (This block previously + # read "Not yet decided" while shipping .10 — the founder's own address — and that is what + # the LIVE prod cluster is running: all three cilium-agents carry + # KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip holds .9, verified 2026-09-06. On a + # 3-node HA cluster that makes the CNI's control-plane path depend on k3s-prod-1: lose it and + # the datapath keeps running off Cilium's cache while no service or policy update lands, with + # nothing looking broken. The VIP is a virtual address that reaches the cluster wherever the + # control plane is, which is the whole reason kube-vip exists here.) + # + # It is owned Ansible-side by **k3s_apiserver_endpoint** (group_vars/all.yml, derived from + # this environment's kube_vip_address = 192.168.2.9 in group_vars/k3s_prod/vars.yml) and MUST + # equal that value — [parity] contract in platform/cilium/helmrelease.yaml. NOT k3s_api_host: + # that variable still exists and still holds the FOUNDER's .10, because the cold-bootstrap + # `--server` join needs a host that is up before any VIP is floated. Two addresses, two + # variables, deliberately. + # + # .9 sits OUTSIDE the Cilium LB pool (.20-.30) and is not a node address, so kube-vip and + # LB-IPAM can never be handed the same IP — asserted by + # conformance/tests/l0-static/ha-control-plane-as-code. - op: replace path: /data/K3S_API_HOST - value: "192.168.2.10" + value: "192.168.2.9" + # --- still UNDECIDED -> loud placeholders (must be real before `flux bootstrap`) ------------ # Pinned L4 VIPs, both carved out of the pool by lbipam.cilium.io/ips. Not yet assigned for # prod. Leaving the dev literals in place would hand a prod LoadBalancer a dev-LAN address, so # they stay placeholders; whatever is chosen MUST fall inside [POOL_START, POOL_STOP] and not diff --git a/infra/ansible/README.md b/infra/ansible/README.md index 12bc695..fffd3da 100644 --- a/infra/ansible/README.md +++ b/infra/ansible/README.md @@ -79,11 +79,30 @@ Ansible installs Cilium; the Flux `platform/cilium` HelmRelease adopts config afterward. The parity-critical values **must match** between this role's `templates/cilium-values.yaml.j2` and that HelmRelease: - `kubeProxyReplacement: true` -- `k8sServiceHost: {{ k3s_api_host }}` (= primary node IP, `10.73.0.11` by default) -- `k8sServicePort: 6443` +- `k8sServiceHost: {{ k3s_apiserver_endpoint }}` — **the kube-vip CONTROL-PLANE VIP** + (`= kube_vip_address`: dev `192.168.1.49`, prod `192.168.2.9`), **not** `k3s_api_host` +- `k8sServicePort: {{ k3s_api_port }}` (6443 in both environments) -The HelmRelease currently has `k8sServiceHost: "REPLACE-ME"` — WS-NET must set it -to the same value this role uses (surfaced by `tofu output primary_node_ip`). +### Two API addresses, split 2026-09-06 — do not re-merge them +`k3s_api_host` and `k3s_apiserver_endpoint` were one variable, and the conflation put every +prod `cilium-agent` on `192.168.2.10` (`k3s-prod-1`'s own address) on a cluster whose point is +HA — verified live. Lose that node and the datapath keeps serving from Cilium's cache while no +service or policy update ever lands, which is a silent failure, not an outage. + +| variable | value | consumed by | why it must be that address | +|---|---|---|---| +| `k3s_api_host` | the founder's own node IP (dev `192.168.1.50`, prod `192.168.2.10`) | `roles/k3s_server`: the secondary `--server https://…:6443` join URL, and one `k3s_tls_sans` entry | on a **cold** build no VIP exists yet (kube-vip is a k3s auto-deploy manifest, so it needs a cluster first). The founder just ran `--cluster-init` and is up | +| `k3s_apiserver_endpoint` | `{{ kube_vip_address }}` (dev `192.168.1.49`, prod `192.168.2.9`) | `roles/cilium` → `k8sServiceHost`; any steady-state client | it must survive losing any single server | + +Consequences already handled: `playbooks/site.yml` imports `kube-vip.yml` **between** `k3s.yml` +and `cilium.yml` (the VIP has to be floating before the CNI dials it), and `roles/kube_vip`'s +last task blocks until the VIP answers on 6443. `k3s_tls_sans` already carried both addresses, +so no cert reissue and no k3s restart are involved. + +The Flux side of the same contract is `${K3S_API_HOST}` in `platform/cilium/helmrelease.yaml`, +set per overlay in `clusters//kustomization.yaml`; `[parity]` means it must equal +`k3s_apiserver_endpoint` for the SAME environment. Asserted by +`conformance/tests/l0-static/ha-control-plane-as-code`. ## Interface assumed of `bootstrap/` (authored by other workers, NOT here) `roles/flux_bootstrap` invokes two entrypoints via overridable command vars: @@ -148,7 +167,9 @@ ansible-playbook playbooks/site.yml \ What the roles then resolve **by themselves** for a prod host, from `group_vars/k3s_prod/vars.yml`: `k3s_primary_group=k3s_prod_primary` (the `--cluster-init` founder, the node-token `delegate_to` target, and the node Cilium fetches the kubeconfig from), `k3s_servers_group=k3s_prod` (the tls-san -node-IP list), `k3s_api_host=192.168.2.10`, `kube_vip_address=192.168.2.9`, and +node-IP list), `k3s_api_host=192.168.2.10` (the founder, bootstrap-join only), +`kube_vip_address=192.168.2.9` — and therefore `k3s_apiserver_endpoint=192.168.2.9`, which is what +Cilium dials — and `cluster_kubeconfig_name=prod.kubeconfig` — so a prod run can never overwrite the `.cache/dev.kubeconfig` that `infra/devcontainer/dev.sh` exports as `$KUBECONFIG`. diff --git a/infra/ansible/inventory/group_vars/all.yml b/infra/ansible/inventory/group_vars/all.yml index bd3a241..41f4bab 100644 --- a/infra/ansible/inventory/group_vars/all.yml +++ b/infra/ansible/inventory/group_vars/all.yml @@ -27,19 +27,85 @@ k3s_servers_group: k3s_servers # a kubeconfig is a credential.) cluster_kubeconfig_name: dev.kubeconfig -# The kube-apiserver endpoint the whole stack agrees on: the primary's REAL node IP. +# --- TWO API ADDRESSES, AND THEY ARE NOT INTERCHANGEABLE (split 2026-09-06) ---------- +# This used to be ONE variable, k3s_api_host, consumed by BOTH the cold-bootstrap join and +# Cilium's k8sServiceHost. That conflation is what pointed every cilium-agent on the LIVE +# prod cluster at 192.168.2.10 — k3s-prod-1's own address — on a cluster whose entire +# purpose is HA: verified 2026-09-06, all three agents carried +# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip held 192.168.2.9. Lose that one node +# and no service/policy update reaches the CNI until it returns (the existing datapath +# survives on Cilium's cache, so it fails SILENTLY). The two consumers have genuinely +# different requirements, so they now have two variables. +# +# k3s_api_host THE FOUNDER'S OWN ADDRESS. Bootstrap-time only. +# k3s_apiserver_endpoint THE HA ENDPOINT (the kube-vip VIP). Steady-state. +# +# DO NOT "simplify" these back into one. See each block below for why each half must be +# the address it is. + +# THE FOUNDER'S OWN NODE IP — a BOOTSTRAP-TIME address, deliberately NOT the VIP. # NOTE: ansible_host is now the pfSense NAT SSH target (192.168.2.102:225x), NOT the node -# IP — so we read the real 192.168.1.x from k3s_node_ip instead. Used for k3s --tls-san / -# secondary --server AND for Cilium k8sServiceHost so the Ansible bootstrap and the Flux -# HelmRelease stay in PARITY (docs/02 contract). +# IP — so we read the real 192.168.1.x from k3s_node_ip instead. +# +# EXACTLY TWO CONSUMERS, and both of them need the founder specifically: +# 1. roles/k3s_server — the secondary `--server https://:` JOIN URL. On a COLD +# build the VIP DOES NOT EXIST YET at join time: kube-vip is a k3s auto-deploy manifest +# that only lands once a cluster exists, so a join aimed at the VIP is a join aimed at +# an unanswered address and the cold build is unjoinable. The founder is up (it just ran +# --cluster-init), which is the whole reason this variable is the founder. +# 2. roles/k3s_server — one entry of k3s_tls_sans, so the founder's own address is a SAN +# alongside the VIP and every other server's IP. +# +# ANYTHING THAT RUNS AGAINST A CLUSTER THAT ALREADY EXISTS MUST NOT USE THIS. It is a +# single point of failure by construction. Use k3s_apiserver_endpoint below. k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['k3s_node_ip'] }}" + +# Shared by both halves: k3s serves the API on 6443 in every environment, so the port is +# not a per-environment knob and is not duplicated per address. k3s_api_port: 6443 +# THE HA kube-apiserver ENDPOINT — a VIRTUAL address that reaches the cluster wherever the +# control plane currently is, not a machine. OPERATOR DECISION 2026-09-06: in-cluster +# components dial the VIP, as kube-vip was always intended to provide. +# +# DERIVED from kube_vip_address (below), never restated: an environment overrides the VIP in +# ONE place (group_vars/k3s_prod/vars.yml sets kube_vip_address: 192.168.2.9) and this +# follows automatically, so prod needs no override of its own and a fourth environment +# cannot set one of the two and forget the other. +# +# NAMED FOR THE ROLE, NOT THE MECHANISM. Consumers want "the address that reaches the +# apiserver, HA"; that it is currently floated by kube-vip is an implementation detail. If +# the mechanism is ever something else (an external LB, a DNS name), this one line moves and +# no consumer changes. +# +# CONSUMERS: +# - roles/cilium -> cilium-values.yaml.j2 -> k8sServiceHost. [parity] with the Flux side: +# clusters/base/cluster-vars.yaml K3S_API_HOST (and each overlay's patch) MUST equal +# THIS value for that environment, or Flux and the bootstrap fight over the CNI's +# control-plane path. See platform/cilium/helmrelease.yaml's PARITY CONTRACT header. +# +# TLS IS ALREADY COVERED — verified, not assumed: k3s_tls_sans (roles/k3s_server/defaults) +# has always included kube_vip_address, and on 2026-09-06 the prod apiserver was reached +# through https://192.168.2.9:6443 with FULL CA verification and tls-server-name=192.168.2.9 +# (infra/ansible/.cache/prod.kubeconfig, no insecure-skip-tls-verify). So the serving cert +# needs no change and no k3s restart for this switch. +# +# ORDERING CONSEQUENCE, and it is why playbooks/site.yml changed too: Cilium now needs the +# VIP to be UP before it installs. site.yml therefore imports kube-vip.yml BETWEEN k3s.yml +# and cilium.yml, and roles/kube_vip waits for the VIP to answer before returning. kube-vip +# can precede the CNI because its DaemonSet is hostNetwork with +# KUBERNETES_SERVICE_HOST=127.0.0.1 — it needs neither a CNI nor kube-proxy. +k3s_apiserver_endpoint: "{{ kube_vip_address }}" + # Control-plane HA VIP floated by kube-vip (roles/kube_vip). A stable failover # endpoint for the kube-apiserver, fronted by an operator pfSense port-forward at -# 192.168.1.49:6443. Shared here so BOTH kube_vip AND the k3s_server tls-san list -# agree on the address (the serving cert must carry this IP or TLS via the VIP -# fails to validate). +# 192.168.1.49:6443. Shared here so THREE things agree on the one address: +# - roles/kube_vip — what to ARP and lease-elect for; +# - roles/k3s_server — the tls-san entry (the serving cert must carry this IP or +# TLS via the VIP fails to validate); +# - k3s_apiserver_endpoint — above; what in-cluster components actually dial. +# THIS is the one line an environment overrides (prod: 192.168.2.9, in +# group_vars/k3s_prod/vars.yml); all three follow. kube_vip_address: "192.168.1.49" # Controller (dev container) reaches the kube-API DIRECTLY via the pfSense NAT diff --git a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml index 056ae85..bca8337 100644 --- a/infra/ansible/inventory/group_vars/k3s_prod/vars.yml +++ b/infra/ansible/inventory/group_vars/k3s_prod/vars.yml @@ -51,12 +51,18 @@ k3s_servers_group: k3s_prod # than no kubeconfig at all). .cache/ is gitignored; a kubeconfig is a credential. cluster_kubeconfig_name: prod.kubeconfig -# --- k3s API host — safe to set now, the primary's real IP is already known ------- +# --- k3s API host — THE FOUNDER'S ADDRESS, bootstrap-time only -------------------- # Dev's k3s_api_host (group_vars/all.yml) reads a separate k3s_node_ip hostvar because # dev's ansible_host is the pfSense NAT SSH target, not the node's real address. Prod # has NO NAT hop — ansible_host in hosts.ini IS already the node's real, routable LAN # address (see the KEY DIFFERENCE note there) — so point straight at it instead of # needing a k3s_node_ip split. +# +# WHAT THIS IS *NOT*, since 2026-09-06: it is not the endpoint anything steady-state +# dials. That is k3s_apiserver_endpoint (group_vars/all.yml), which derives from +# kube_vip_address below and therefore needs NO override here. k3s_api_host is consumed +# only by the secondary `--server` cold-join URL and by one tls-san entry. Read the +# TWO API ADDRESSES block at the top of group_vars/all.yml before touching either. k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['ansible_host'] }}" # cluster_controller_tls_server_name (group_vars/all.yml) derives from kube_vip_address # by template reference, so it needs no separate override here — it picks up the real @@ -75,8 +81,15 @@ k3s_api_host: "{{ hostvars[groups[k3s_primary_group][0]]['ansible_host'] }}" # round (control-plane .49, ingress pool .53-.60). # # This is the address kube-vip floats for the API and that k3s tls-san's onto its -# serving cert; secondaries use it as their --server join target. Dev's equivalent is -# kube_vip_address: 192.168.1.49 in group_vars/all.yml. +# serving cert. Dev's equivalent is kube_vip_address: 192.168.1.49 in group_vars/all.yml. +# +# CORRECTION 2026-09-06: an earlier version of this comment claimed "secondaries use it as +# their --server join target". They do not, and they must not — on a COLD build the VIP is +# not up when the secondaries join (kube-vip is a k3s auto-deploy manifest and needs a +# cluster first), so the join uses k3s_api_host, the founder. What DOES use this address is +# k3s_apiserver_endpoint (group_vars/all.yml, derived from this line), i.e. Cilium's +# k8sServiceHost and any other steady-state client. Overriding kube_vip_address here moves +# BOTH the VIP and prod's Cilium endpoint together, which is the point of deriving it. # # HISTORY: an earlier pass here wrongly set this to 192.168.2.20 — that is the INGRESS # VIP, not the control-plane VIP. Corrected. Keep the two distinct. diff --git a/infra/ansible/playbooks/kube-vip.yml b/infra/ansible/playbooks/kube-vip.yml index 7dce62b..fdd0e3b 100644 --- a/infra/ansible/playbooks/kube-vip.yml +++ b/infra/ansible/playbooks/kube-vip.yml @@ -3,6 +3,13 @@ # Control-plane ONLY (svc_enable=false) — Cilium owns Service LoadBalancing. # Writes the manifest into each server's k3s auto-deploy dir; k3s applies it. # +# PART OF THE ONE-COMMAND BRING-UP SINCE 2026-09-06, not a separate operator step: +# playbooks/site.yml imports this play BETWEEN k3s.yml and cilium.yml, because Cilium's +# k8sServiceHost is now this VIP (k3s_apiserver_endpoint, group_vars/all.yml) and the CNI +# cannot be installed against an endpoint nothing announces yet. The role's final task waits +# for the VIP to answer on 6443, so a failure to float it stops the run here with a named +# cause instead of surfacing later as unreachable cilium-agents. +# # TARGET (R-ENV-02): `k3s_hosts` defaults to dev's `k3s_servers`; prod is # ansible-playbook playbooks/kube-vip.yml -e k3s_hosts=k3s_prod # (the VIP itself comes from kube_vip_address, already scoped per environment in diff --git a/infra/ansible/playbooks/site.yml b/infra/ansible/playbooks/site.yml index 8c65f88..e9f8247 100644 --- a/infra/ansible/playbooks/site.yml +++ b/infra/ansible/playbooks/site.yml @@ -1,7 +1,10 @@ --- # Full dev/test cluster bring-up, in order (docs/02 sequencing; run AFTER # `tofu apply` has created and cloud-init has baselined the VMs): -# k3s HA install -> Cilium CNI -> verify readiness -> Flux + SOPS hand-off. +# k3s HA install -> kube-vip control-plane VIP -> Cilium CNI -> verify readiness +# -> Flux + SOPS hand-off. +# (kube-vip joined the chain 2026-09-06 — see the note at its import below; it is a +# prerequisite of the CNI now, not a separate operator step.) # # TARGET (R-ENV-02): every imported play takes its host pattern from a var that # defaults to the dev group, so `ansible-playbook playbooks/site.yml` with no extra @@ -20,6 +23,29 @@ - name: Import - install k3s HA control plane ansible.builtin.import_playbook: k3s.yml +# ADDED 2026-09-06, AND ITS POSITION IS LOAD-BEARING: kube-vip MUST come up between k3s and +# Cilium. Cilium's k8sServiceHost is now the control-plane VIP (k3s_apiserver_endpoint = +# kube_vip_address) rather than the founder's own node IP, so installing Cilium before the +# VIP is announced would point every agent at an address nothing answers on — the exact +# shape of the 2026-09-01 incident, where agents that could not reach the API server +# deadlocked on a cold cluster. +# +# IT CANNOT COME EARLIER. kube-vip ships as a k3s auto-deploy manifest +# (/server/manifests), so a cluster must exist before it can be applied — hence +# after k3s.yml, not before. +# +# IT DOES NOT NEED THE CNI, which is what makes this ordering possible at all: the kube-vip +# DaemonSet is hostNetwork with KUBERNETES_SERVICE_HOST=127.0.0.1 (roles/kube_vip's template +# explains why), so it reaches the node-local apiserver with no CNI and no kube-proxy, and +# DaemonSet pods carry the default not-ready/network-unavailable tolerations that let them +# schedule onto a still-NotReady node. roles/kube_vip blocks until the VIP actually answers +# on 6443, so cilium.yml below cannot start against a VIP that is not up. +# +# Re-running this against the LIVE dev cluster is a no-op for this play: dev already runs +# kube-vip with the same rendered manifest, and the wait finds the VIP already answering. +- name: Import - float the control-plane HA VIP (kube-vip) BEFORE the CNI dials it + ansible.builtin.import_playbook: kube-vip.yml + - name: Import - install Cilium CNI ansible.builtin.import_playbook: cilium.yml diff --git a/infra/ansible/roles/cilium/defaults/main.yml b/infra/ansible/roles/cilium/defaults/main.yml index f5f70eb..dcb12b9 100644 --- a/infra/ansible/roles/cilium/defaults/main.yml +++ b/infra/ansible/roles/cilium/defaults/main.yml @@ -2,7 +2,17 @@ # cilium role defaults (ADR-0002, docs/02 Cilium-ownership contract). # This role performs the INITIAL Cilium install; the Flux platform/cilium # HelmRelease adopts config afterward. The parity-critical values -# (kubeProxyReplacement, k8sServiceHost/Port) are templated from k3s_api_host. +# (kubeProxyReplacement, k8sServiceHost/Port) are templated from +# k3s_apiserver_endpoint / k3s_api_port (group_vars/all.yml). +# +# k8sServiceHost IS THE HA VIP, NOT THE FOUNDER (changed 2026-09-06). It reads +# k3s_apiserver_endpoint — which derives from kube_vip_address, so it is dev 192.168.1.49 / +# prod 192.168.2.9 — and deliberately NOT k3s_api_host, the founder's own node IP. Those two +# were one variable until 2026-09-06, and the conflation is what left all three prod +# cilium-agents dialing k3s-prod-1 on an HA cluster. Do not re-merge them. +# +# THIS ROLE THEREFORE DEPENDS ON THE VIP EXISTING. playbooks/site.yml runs kube-vip.yml +# before cilium.yml and roles/kube_vip blocks until the VIP answers on 6443. # Exact pin — MUST equal the platform/cilium HelmRelease chart.spec.version (1.20.1, an # exact pin, not a range). Renovate bumps both together. diff --git a/infra/ansible/roles/cilium/tasks/main.yml b/infra/ansible/roles/cilium/tasks/main.yml index e84b761..76e8c60 100644 --- a/infra/ansible/roles/cilium/tasks/main.yml +++ b/infra/ansible/roles/cilium/tasks/main.yml @@ -89,7 +89,7 @@ run_once: true changed_when: false -- name: Render the Cilium values file (parity-critical settings from k3s_api_host) +- name: Render the Cilium values file (parity-critical settings from k3s_apiserver_endpoint) ansible.builtin.template: src: cilium-values.yaml.j2 dest: "{{ cilium_values_path }}" diff --git a/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 b/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 index 8a22edf..0971e24 100644 --- a/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 +++ b/infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 @@ -4,9 +4,24 @@ --- # [parity] k3s kube-proxy is disabled → Cilium replaces it (ADR-0001/0002). kubeProxyReplacement: true -# [parity] = the k3s API endpoint (primary node IP). The Flux HelmRelease's -# k8sServiceHost "REPLACE-ME" must be set to this same value. -k8sServiceHost: "{{ k3s_api_host }}" +# [parity] = k3s_apiserver_endpoint, THE HA CONTROL-PLANE VIP (kube_vip_address) — dev +# 192.168.1.49, prod 192.168.2.9. NOT k3s_api_host, which is the FOUNDER'S OWN node IP: +# pointing the CNI's kube-proxy replacement at one machine on a 3-server HA cluster means +# losing that machine stops every service/policy update reaching the datapath, silently +# (the existing datapath keeps working from Cilium's cache, so nothing looks broken). That +# was the live state of prod until 2026-09-06 — all three cilium-agents carried +# KUBERNETES_SERVICE_HOST=192.168.2.10 while kube-vip held .9. See group_vars/all.yml. +# +# [parity] The Flux side is platform/cilium/helmrelease.yaml's k8sServiceHost: +# "${K3S_API_HOST}", whose per-environment value comes from clusters//kustomization.yaml. +# THAT VALUE MUST EQUAL k3s_apiserver_endpoint for the SAME environment. If it does not, +# Flux and this bootstrap render different control-plane paths and fight over the CNI. +# +# THE VIP MUST BE UP BEFORE THIS ROLE RUNS. playbooks/site.yml imports kube-vip.yml between +# k3s.yml and cilium.yml for exactly this reason, and roles/kube_vip blocks until the VIP +# answers. Rendering this against a VIP that nothing announces yet reproduces the +# 2026-09-01 shape: agents that cannot reach the API server on a cold cluster. +k8sServiceHost: "{{ k3s_apiserver_endpoint }}" k8sServicePort: {{ k3s_api_port }} # Ingress via Gateway API — no Traefik (docs/09, ADR-0002). diff --git a/infra/ansible/roles/k3s_server/defaults/main.yml b/infra/ansible/roles/k3s_server/defaults/main.yml index d052d12..31e950f 100644 --- a/infra/ansible/roles/k3s_server/defaults/main.yml +++ b/infra/ansible/roles/k3s_server/defaults/main.yml @@ -59,6 +59,17 @@ k3s_server_mode: "secondary" # serving cert on restart when this set changes. Deduplicated (the primary IP is # also k3s_api_host). # +# THIS LIST COVERS BOTH API ADDRESSES, WHICH IS WHY THE 2026-09-06 SPLIT NEEDED NO CERT +# CHANGE AND NO k3s RESTART. group_vars/all.yml now distinguishes k3s_api_host (the +# founder, for the cold join) from k3s_apiserver_endpoint (= kube_vip_address, what Cilium +# and every steady-state client dial). Both are already in this set, so repointing a +# consumer from one to the other cannot produce a TLS failure. VERIFIED LIVE 2026-09-06, +# not assumed: prod's apiserver answered through https://192.168.2.9:6443 with full CA +# verification and tls-server-name=192.168.2.9. Do not remove either entry — dropping +# kube_vip_address breaks every client of the VIP; dropping k3s_api_host breaks nothing +# today only because it is also in the k3s_servers_group node-IP list below, and that +# coincidence is not a contract. +# # k3s_servers_group — NOT a literal group name — is what makes this role environment- # agnostic (R-ENV-02): group_vars/all.yml sets it to `k3s_servers` (dev) and # group_vars/k3s_prod/vars.yml to `k3s_prod`, so each environment SANs its own servers. diff --git a/infra/ansible/roles/k3s_server/tasks/main.yml b/infra/ansible/roles/k3s_server/tasks/main.yml index 232f3cf..8cfbee1 100644 --- a/infra/ansible/roles/k3s_server/tasks/main.yml +++ b/infra/ansible/roles/k3s_server/tasks/main.yml @@ -155,6 +155,18 @@ | default(k3s_primary_token_raw.content | b64decode | trim, true) }} when: k3s_server_mode == 'secondary' +# THE JOIN TARGET IS k3s_api_host — THE FOUNDER — AND MUST NOT BECOME THE VIP. +# group_vars/all.yml splits the two API addresses (k3s_api_host = the founder's own node IP, +# k3s_apiserver_endpoint = the kube-vip HA VIP) and this is the canonical founder-only +# consumer. On a COLD build the VIP does not exist at this point in the sequence: kube-vip +# is a k3s auto-deploy manifest, so it cannot be applied until a cluster exists, and +# playbooks/site.yml only imports kube-vip.yml AFTER k3s.yml. A `--server` pointed at the +# VIP would therefore dial an address nothing announces and the cold build would be +# unjoinable — with no diagnostic beyond a connection timeout. +# On a WARM single-node rejoin (`--limit k3s-dev-N`) the founder is up and answering too, so +# the founder works in both cases and the VIP works in only one. Cilium's k8sServiceHost is +# the opposite trade (steady state, must survive losing the founder) and uses the endpoint +# variable instead. - name: Install k3s secondary (join as server) ansible.builtin.command: cmd: >- diff --git a/infra/ansible/roles/kube_vip/defaults/main.yml b/infra/ansible/roles/kube_vip/defaults/main.yml index 9d8e677..334dc65 100644 --- a/infra/ansible/roles/kube_vip/defaults/main.yml +++ b/infra/ansible/roles/kube_vip/defaults/main.yml @@ -24,6 +24,16 @@ kube_vip_port: 6443 # only if detection is wrong. kube_vip_interface: "" +# How long to wait for the VIP to actually ANSWER on kube_vip_port after the manifest is +# written (seconds). This is a real gate, not cosmetics: since 2026-09-06 Cilium's +# k8sServiceHost is this VIP (k3s_apiserver_endpoint), and playbooks/site.yml runs this role +# immediately before cilium.yml — so if the VIP never comes up, the run must stop HERE with +# a clear message rather than proceed to install a CNI pointed at an address nothing +# answers. On a cold cluster the budget covers k3s's deploy controller applying the +# manifest, the ghcr.io image pull and leader election; on an already-running cluster the +# check returns immediately. +kube_vip_wait_timeout: 300 + # Leader-election tuning (seconds) — a snappy failover without thrashing. kube_vip_lease_duration: 15 kube_vip_renew_deadline: 10 diff --git a/infra/ansible/roles/kube_vip/tasks/main.yml b/infra/ansible/roles/kube_vip/tasks/main.yml index 3312477..236a436 100644 --- a/infra/ansible/roles/kube_vip/tasks/main.yml +++ b/infra/ansible/roles/kube_vip/tasks/main.yml @@ -21,6 +21,14 @@ executable: /bin/bash register: kube_vip_iface_detect changed_when: false + # RUN THIS EVEN UNDER --check. It is `ip -o -4 addr show`: it reads and changes nothing. + # Without check_mode: false the shell module is SKIPPED in check mode, kube_vip_iface_detect + # has no stdout, and the assert two tasks down aborts the play with "Could not detect the + # interface ... set kube_vip_interface explicitly" — a misleading failure that says nothing + # about the node. That mattered less while this play was an out-of-band operator step; it + # matters now that playbooks/site.yml imports it, because `--check` on the bring-up before a + # cold prod build is exactly the rehearsal an operator should be able to run. + check_mode: false - name: Resolve the kube-vip VIP interface (override wins, else detected) ansible.builtin.set_fact: @@ -52,3 +60,34 @@ owner: root group: root mode: "0600" + +# PROVE THE VIP IS ACTUALLY UP BEFORE THIS PLAY RETURNS (added 2026-09-06). +# +# Templating the manifest is not the same as having a floating endpoint: k3s's deploy +# controller still has to apply it, the image still has to pull, and leader election still +# has to pick an owner. Nothing used to check, which was tolerable while the VIP was only an +# operator convenience. It is not tolerable now — Cilium's k8sServiceHost IS this VIP +# (k3s_apiserver_endpoint) and playbooks/site.yml runs cilium.yml straight after this play, +# so an unannounced VIP would mean installing a CNI that cannot reach the API server. That +# failure is the 2026-09-01 shape and it is very expensive to diagnose from the far side; +# failing here instead names the actual cause. +# +# run_once: the VIP is ONE address for the whole environment, so one probe answers for the +# environment. Probing from a server host (not the controller) is deliberate: the servers +# are on the VIP's own LAN in both environments, whereas the dev controller reaches dev's +# API only through a pfSense NAT — a controller-side probe would have been a property of +# where the play was launched from. READ-ONLY: it opens a TCP connection and changes nothing. +- name: Wait for the control-plane VIP to answer (the CNI's endpoint depends on it) + ansible.builtin.wait_for: + host: "{{ kube_vip_address }}" + port: "{{ kube_vip_port }}" + timeout: "{{ kube_vip_wait_timeout }}" + run_once: true + changed_when: false + vars: + ansible_become: false + # DELIBERATELY NOT check_mode: false, unlike the interface probe above — so a `--check` + # rehearsal SKIPS this and does not sit on a socket for kube_vip_wait_timeout. wait_for + # declares no check-mode support, so the skip is the module's own behaviour and this comment + # is here only so the skip does not read as an oversight. CONSEQUENCE, stated plainly: a + # `--check` run does NOT verify the VIP is up. Only a real run gates the CNI on it. diff --git a/platform/cilium/helmrelease.yaml b/platform/cilium/helmrelease.yaml index 68086d6..46d177b 100644 --- a/platform/cilium/helmrelease.yaml +++ b/platform/cilium/helmrelease.yaml @@ -10,7 +10,8 @@ # # SOURCE OF TRUTH (do not diverge without updating both): # infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 -# infra/ansible/roles/cilium/defaults/main.yml (cilium_version, k3s_api_host/port) +# infra/ansible/roles/cilium/defaults/main.yml (cilium_version) +# infra/ansible/inventory/group_vars/all.yml (k3s_apiserver_endpoint, k3s_api_port) # # Every value tagged [parity] below is asserted identical to the Ansible render. If you change one # here, change it in the Ansible role in the same PR. (docs/02 Cilium-ownership contract.) @@ -45,11 +46,35 @@ spec: # [parity] MUST equal Ansible: kubeProxyReplacement: true kubeProxyReplacement: true # [parity] MUST equal Ansible: k8sServiceHost/k8sServicePort = the k3s API endpoint - # (k3s_api_host/k3s_api_port). This is the primary control-plane node IP. + # (**k3s_apiserver_endpoint** / k3s_api_port, infra/ansible/inventory/group_vars/all.yml). # Setting these wrong breaks Cilium's connection to the API server on adopt. - # ${K3S_API_HOST} (cluster-vars, R-ENV-02): dev 192.168.1.50 | prod set in clusters/prod. - # The `platform` Flux Kustomization substitutes it; the Ansible side is already parameterized - # from inventory (k3s_api_host), so the parity contract holds per-environment, not per-literal. + # + # THIS IS THE CONTROL-PLANE VIP (kube-vip), NOT A NODE ADDRESS — operator decision 2026-09-06. + # ${K3S_API_HOST} (cluster-vars, R-ENV-02): dev 192.168.1.49 | prod 192.168.2.9, each equal to + # that environment's kube_vip_address. The knob's NAME still says "host"; read it as "the API + # host Cilium dials", and see clusters/base/cluster-vars.yaml for why it was not renamed. + # + # WHY, in one line: this value used to be the PRIMARY NODE's IP, so on a 3-server HA cluster the + # CNI's control-plane path depended on one machine. Verified on live prod 2026-09-06 — every + # cilium-agent carried KUBERNETES_SERVICE_HOST=192.168.2.10 (k3s-prod-1) while kube-vip held .9. + # Losing that node does not break the datapath (Cilium keeps serving from cache); it silently + # stops every service and policy update from ever landing, which is worse to detect. + # + # [parity] IS NOW AGAINST k3s_apiserver_endpoint AND NOT k3s_api_host. Those were one Ansible + # variable until 2026-09-06 and the conflation is the defect. k3s_api_host still exists and + # still holds the FOUNDER's own address, because roles/k3s_server's secondary + # `--server https://:6443` join must target a host that is already up — on a COLD build + # no VIP exists yet. So: founder for the join, VIP for the CNI. If you change this line, change + # cilium-values.yaml.j2 in the same PR, and point it at the SAME variable. + # + # The `platform` Flux Kustomization substitutes it; the Ansible side is parameterized from + # inventory (k3s_apiserver_endpoint derives from kube_vip_address), so the parity contract holds + # per-environment, not per-literal. + # + # NEEDS NO CERT WORK: k3s_tls_sans has always carried kube_vip_address alongside every node IP, + # so both addresses are already SANs on the serving cert (verified live 2026-09-06 — prod + # answers through https://192.168.2.9:6443 under full CA verification, no insecure skip). This + # switch reissues nothing and restarts no apiserver. k8sServiceHost: "${K3S_API_HOST}" k8sServicePort: 6443 # --- Gateway API (ingress; replaces Traefik) — docs/09, ADR-0002 --- diff --git a/platform/cilium/values.yaml b/platform/cilium/values.yaml index e9f791b..44a10f5 100644 --- a/platform/cilium/values.yaml +++ b/platform/cilium/values.yaml @@ -6,16 +6,20 @@ # # SOURCE OF TRUTH (all [parity] fields MUST equal these): # infra/ansible/roles/cilium/templates/cilium-values.yaml.j2 -# infra/ansible/roles/cilium/defaults/main.yml (cilium_version: 1.20.1, k3s_api_host/port) +# infra/ansible/roles/cilium/defaults/main.yml (cilium_version: 1.20.1) +# infra/ansible/inventory/group_vars/all.yml (k3s_apiserver_endpoint = the kube-vip VIP, +# k3s_api_port) — NOT k3s_api_host, which is the +# founder's own address and bootstrap-only # # chart version (helmrelease.yaml chart.spec.version): 1.20.1 # [parity] = cilium_version # [parity] k3s kube-proxy disabled → Cilium replaces it (ADR-0001/0002) kubeProxyReplacement: true -# [parity] = the k3s API endpoint (primary control-plane node IP) + port. +# [parity] = the k3s API endpoint + port. THE CONTROL-PLANE VIP (kube-vip), not the primary node — +# changed 2026-09-06; the Ansible half is k3s_apiserver_endpoint, NOT k3s_api_host. # Mirrors the HelmRelease's ${K3S_API_HOST} cluster-vars token (R-ENV-02). This file is NOT a # kustomize resource, so nothing substitutes it — the token is shown with its dev value beside it. -k8sServiceHost: "${K3S_API_HOST}" # dev: 192.168.1.50 +k8sServiceHost: "${K3S_API_HOST}" # the CONTROL-PLANE VIP — dev: 192.168.1.49 | prod: 192.168.2.9 k8sServicePort: 6443 # [parity] ingress via Gateway API, no Traefik (docs/09) gatewayAPI: diff --git a/platform/vpn-egress/egress-steering.yaml b/platform/vpn-egress/egress-steering.yaml index 7542bf7..cf356e7 100644 --- a/platform/vpn-egress/egress-steering.yaml +++ b/platform/vpn-egress/egress-steering.yaml @@ -1,7 +1,10 @@ --- # ============================================================================ # Cilium Egress Gateway steering for `egress-class=vpn` pods — ADR-0015. -# Serves: R-NET-05, R-NET-06, R-NET-07. +# Serves: R-NET-04, R-NET-05, R-NET-06, R-NET-07. +# R-NET-04 added 2026-09-06: this file IS the R-NET-04 datapath — the rule that selects between the +# two egress paths. Omitting it made `run.sh --status` report R-NET-04 as "TEST-ONLY / impl none" +# even though the implementation is right here. # # ########################################################################## # ## LIVE (2026-08-24): wired into kustomization.yaml, reconciled by Flux ## @@ -35,7 +38,7 @@ kind: CiliumEgressGatewayPolicy metadata: name: vpn-egress-steering annotations: - homelab.keithjackson.dev/requirements: R-NET-05,R-NET-06,R-NET-07 + homelab.keithjackson.dev/requirements: R-NET-04,R-NET-05,R-NET-06,R-NET-07 homelab.keithjackson.dev/adr: ADR-0015 spec: # --- Client selection (INVARIANTS 1 & 2) --------------------------------