From 60a58f01ad64d060b400fb56cbab19a64bf4c307 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Tue, 25 Aug 2026 21:52:43 -0500 Subject: [PATCH] feat(infra): kube-vip control-plane HA VIP (192.168.1.49) + API tls-san MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an HA control-plane API VIP for the k3s cluster with kube-vip, and add the VIP + all node IPs to the kube-apiserver serving-cert SANs so TLS via the VIP validates. Prepares the stable failover endpoint the operator fronts with a pfSense port-forward to 192.168.1.49:6443. roles/kube_vip + playbooks/kube-vip.yml: - Templates a single manifest (RBAC + DaemonSet kube-vip-ds) into each server's k3s auto-deploy dir (/var/lib/rancher/k3s/server/manifests) so it is reproducible and independent of Flux; k3s auto-applies it. - Control-plane ONLY (cp_enable=true, svc_enable=false): Cilium keeps ownership of Service LoadBalancing (L2 pool .53-.60) — no ARP conflict. - Runs on control-plane nodes (nodeSelector node-role.kubernetes.io/control-plane), hostNetwork, NET_ADMIN/NET_RAW, image pinned ghcr.io/kube-vip/kube-vip:v0.8.9. - vip_interface auto-detected per node (the iface holding 192.168.1.5x = eth0). - Leader election (plndr-cp-lock) with 15/10/2s lease timings. - This cluster runs without kube-proxy (Cilium replacement), so a hostNetwork pod cannot reach the kubernetes ClusterIP; kube-vip is pointed at the node-local apiserver (KUBERNETES_SERVICE_HOST=127.0.0.1:6443) so leader election works. roles/k3s_server tls-san: - New k3s_tls_sans (VIP + all three node IPs + 127.0.0.1, deduped); install args now emit one --tls-san per entry. - Persist the SAN list to /etc/rancher/k3s/config.yaml (read by a running k3s) and Restart k3s handler so an existing cluster regenerates its serving cert. - k3s.yml secondaries run serial:1 so a SAN change rolls one node at a time, keeping etcd quorum (kube-vip keeps the API reachable during each restart). Verified live: DS 3/3 ready, VIP answers on 192.168.1.49:6443, all 3 certs and an s_client against the VIP carry 192.168.1.49 + .50/.51/.52 + 127.0.0.1, all nodes Ready throughout the rolling restart. Co-Authored-By: Claude Opus 4.8 --- infra/ansible/inventory/group_vars/all.yml | 7 + infra/ansible/playbooks/k3s.yml | 4 + infra/ansible/playbooks/kube-vip.yml | 10 ++ .../roles/k3s_server/defaults/main.yml | 11 ++ .../roles/k3s_server/handlers/main.yml | 17 +++ infra/ansible/roles/k3s_server/tasks/main.yml | 23 +++- .../ansible/roles/kube_vip/defaults/main.yml | 34 +++++ infra/ansible/roles/kube_vip/tasks/main.yml | 45 +++++++ .../roles/kube_vip/templates/kube-vip.yaml.j2 | 126 ++++++++++++++++++ 9 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 infra/ansible/playbooks/kube-vip.yml create mode 100644 infra/ansible/roles/k3s_server/handlers/main.yml create mode 100644 infra/ansible/roles/kube_vip/defaults/main.yml create mode 100644 infra/ansible/roles/kube_vip/tasks/main.yml create mode 100644 infra/ansible/roles/kube_vip/templates/kube-vip.yaml.j2 diff --git a/infra/ansible/inventory/group_vars/all.yml b/infra/ansible/inventory/group_vars/all.yml index 207db75..46911fb 100644 --- a/infra/ansible/inventory/group_vars/all.yml +++ b/infra/ansible/inventory/group_vars/all.yml @@ -12,6 +12,13 @@ ansible_user: k3s k3s_api_host: "{{ hostvars[groups['k3s_primary'][0]]['k3s_node_ip'] }}" k3s_api_port: 6443 +# Control-plane HA VIP floated by kube-vip (roles/kube_vip). A stable failover +# endpoint for the kube-apiserver, fronted by an operator pfSense port-forward at +# 192.168.1.49:6443. Shared here so BOTH kube_vip AND the k3s_server tls-san list +# agree on the address (the serving cert must carry this IP or TLS via the VIP +# fails to validate). +kube_vip_address: "192.168.1.49" + # Controller (dev container) reaches the kube-API through an SSH tunnel # (local 127.0.0.1:16443 -> primary 127.0.0.1:6443), because the API port is not # NAT-forwarded and 192.168.1.50:6443 is unroutable from the dev host. Only the diff --git a/infra/ansible/playbooks/k3s.yml b/infra/ansible/playbooks/k3s.yml index 57126d2..4363a60 100644 --- a/infra/ansible/playbooks/k3s.yml +++ b/infra/ansible/playbooks/k3s.yml @@ -12,9 +12,13 @@ vars: k3s_server_mode: primary +# serial: 1 — process one secondary at a time. Matters when a config change (e.g. +# an updated tls-san set) triggers the Restart k3s handler: restarting both +# secondaries at once would drop etcd below quorum. One-at-a-time keeps 2/3 up. - name: k3s secondaries (join) hosts: k3s_secondary become: true + serial: 1 gather_facts: true roles: - role: k3s_server diff --git a/infra/ansible/playbooks/kube-vip.yml b/infra/ansible/playbooks/kube-vip.yml new file mode 100644 index 0000000..da47279 --- /dev/null +++ b/infra/ansible/playbooks/kube-vip.yml @@ -0,0 +1,10 @@ +--- +# Deploy kube-vip as the control-plane HA VIP (192.168.1.49) for the kube-API. +# Control-plane ONLY (svc_enable=false) — Cilium owns Service LoadBalancing. +# Writes the manifest into each server's k3s auto-deploy dir; k3s applies it. +- name: Deploy kube-vip control-plane VIP + hosts: k3s_servers + become: true + gather_facts: true + roles: + - role: kube_vip diff --git a/infra/ansible/roles/k3s_server/defaults/main.yml b/infra/ansible/roles/k3s_server/defaults/main.yml index 716c388..967ba0b 100644 --- a/infra/ansible/roles/k3s_server/defaults/main.yml +++ b/infra/ansible/roles/k3s_server/defaults/main.yml @@ -19,3 +19,14 @@ k3s_disable: "traefik,servicelb,local-storage" # Set per-play: 'primary' (runs --cluster-init) or 'secondary' (joins the primary). k3s_server_mode: "secondary" + +# Subject Alternative Names baked into the kube-apiserver serving cert. Must include +# the kube-vip control-plane VIP (kube_vip_address) so TLS via the VIP validates, +# every server's real node IP (so the cert is valid whichever node answers behind +# the VIP), and 127.0.0.1 (local kubeconfig / health probes). k3s regenerates the +# serving cert on restart when this set changes. Deduplicated (the primary IP is +# also k3s_api_host). +k3s_tls_sans: >- + {{ ([k3s_api_host, kube_vip_address, '127.0.0.1'] + + (groups['k3s_servers'] | map('extract', hostvars, 'k3s_node_ip') | list)) + | unique }} diff --git a/infra/ansible/roles/k3s_server/handlers/main.yml b/infra/ansible/roles/k3s_server/handlers/main.yml new file mode 100644 index 0000000..2070f87 --- /dev/null +++ b/infra/ansible/roles/k3s_server/handlers/main.yml @@ -0,0 +1,17 @@ +--- +# Restart k3s so a changed config.yaml (e.g. an updated tls-san set) takes effect +# and the serving cert is regenerated. RUN THIS ROLLING (serial: 1) — restarting +# more than one server at once would break etcd quorum. The wait below gates the +# next node until this one's API is serving again. +- name: Restart k3s + ansible.builtin.systemd_service: + name: k3s + state: restarted + listen: Restart k3s + +- name: Wait for the k3s API to be serving locally after restart + ansible.builtin.wait_for: + host: "127.0.0.1" + port: "{{ k3s_api_port }}" + timeout: 180 + listen: Restart k3s diff --git a/infra/ansible/roles/k3s_server/tasks/main.yml b/infra/ansible/roles/k3s_server/tasks/main.yml index d56740e..ea570d8 100644 --- a/infra/ansible/roles/k3s_server/tasks/main.yml +++ b/infra/ansible/roles/k3s_server/tasks/main.yml @@ -20,9 +20,30 @@ --disable-network-policy --disable={{ k3s_disable }} --write-kubeconfig-mode=644 - --tls-san={{ k3s_api_host }} + {% for san in k3s_tls_sans %}--tls-san={{ san }} {% endfor %} --node-ip={{ k3s_node_ip }} +# On an ALREADY-installed cluster the install task above is idempotent (creates: +# k3s.yaml) and won't re-apply new --tls-san flags. Persist the SAN list in +# /etc/rancher/k3s/config.yaml, which a running k3s reads on the next restart — +# it then regenerates the serving cert when the SAN set changes. Apply the +# restart ROLLING, one node at a time (kube-vip keeps the API reachable; etcd +# quorum holds at 2/3). See playbooks note: run with `serial: 1`. +- name: Write /etc/rancher/k3s/config.yaml with the tls-san list + ansible.builtin.copy: + dest: /etc/rancher/k3s/config.yaml + owner: root + group: root + mode: "0600" + content: | + # Managed by Ansible (roles/k3s_server). Read by k3s in addition to the + # systemd unit flags; tls-san entries are merged into the serving cert. + tls-san: + {% for san in k3s_tls_sans %} + - "{{ san }}" + {% endfor %} + notify: Restart k3s + # --- primary: found the cluster -------------------------------------------- - name: Install k3s primary (cluster-init, embedded etcd) ansible.builtin.command: diff --git a/infra/ansible/roles/kube_vip/defaults/main.yml b/infra/ansible/roles/kube_vip/defaults/main.yml new file mode 100644 index 0000000..a89bb9a --- /dev/null +++ b/infra/ansible/roles/kube_vip/defaults/main.yml @@ -0,0 +1,34 @@ +--- +# kube_vip role defaults — control-plane HA VIP for the k3s kube-API. +# +# kube-vip runs as a DaemonSet on the control-plane nodes and floats a single +# virtual IP (kube_vip_address) that always points at the current leader, giving +# the kube-apiserver a stable failover endpoint (an operator port-forward fronts +# it at 192.168.1.49:6443). It is CONTROL-PLANE ONLY (svc_enable=false): Cilium +# owns Service LoadBalancing (L2 pool .53-.60), so kube-vip must not touch +# Services or the two would fight over ARP for LoadBalancer IPs. + +# The floating control-plane VIP. Defined in group_vars/all.yml so k3s_server's +# tls-san list and this role stay in parity; this default is the fallback. +kube_vip_address: "192.168.1.49" + +# Image pinned deliberately — a version bump is a code change (verified tag on +# ghcr.io). v0.8.9 is a current v0.8.x release. +kube_vip_version: "v0.8.9" +kube_vip_image: "ghcr.io/kube-vip/kube-vip:{{ kube_vip_version }}" + +kube_vip_namespace: "kube-system" +kube_vip_port: 6443 + +# Empty => auto-detected per host (the interface holding 192.168.1.5x); override +# only if detection is wrong. +kube_vip_interface: "" + +# Leader-election tuning (seconds) — a snappy failover without thrashing. +kube_vip_lease_duration: 15 +kube_vip_renew_deadline: 10 +kube_vip_retry_period: 2 + +# k3s auto-deploy manifests dir: k3s applies anything here on start, so kube-vip +# is reproducible and independent of Flux/Helm. +kube_vip_manifest_path: "/var/lib/rancher/k3s/server/manifests/kube-vip.yaml" diff --git a/infra/ansible/roles/kube_vip/tasks/main.yml b/infra/ansible/roles/kube_vip/tasks/main.yml new file mode 100644 index 0000000..db63d7e --- /dev/null +++ b/infra/ansible/roles/kube_vip/tasks/main.yml @@ -0,0 +1,45 @@ +--- +# Deploy kube-vip (control-plane HA VIP) via the k3s auto-deploy manifests dir. +# k3s applies /var/lib/rancher/k3s/server/manifests/*.yaml on start, so this is +# reproducible and independent of Flux. The same manifest is written on every +# server so the DaemonSet survives the loss of any single node. + +- name: Detect the LAN interface holding the node's 192.168.1.5x address + ansible.builtin.shell: >- + set -o pipefail; + ip -o -4 addr show | awk '/192\.168\.1\.5[0-2]\//{print $2; exit}' + args: + executable: /bin/bash + register: kube_vip_iface_detect + changed_when: false + +- name: Resolve the kube-vip VIP interface (override wins, else detected) + ansible.builtin.set_fact: + kube_vip_vip_interface: >- + {{ kube_vip_interface if (kube_vip_interface | length > 0) + else (kube_vip_iface_detect.stdout | trim) }} + +- name: Fail early if no LAN interface could be resolved + ansible.builtin.assert: + that: + - kube_vip_vip_interface is defined + - kube_vip_vip_interface | length > 0 + fail_msg: >- + Could not detect the 192.168.1.5x LAN interface for kube-vip on + {{ inventory_hostname }} — set kube_vip_interface explicitly. + +- name: Ensure the k3s auto-deploy manifests directory exists + ansible.builtin.file: + path: "{{ kube_vip_manifest_path | dirname }}" + state: directory + owner: root + group: root + mode: "0700" + +- name: Template the kube-vip control-plane manifest (RBAC + DaemonSet) + ansible.builtin.template: + src: kube-vip.yaml.j2 + dest: "{{ kube_vip_manifest_path }}" + owner: root + group: root + mode: "0600" diff --git a/infra/ansible/roles/kube_vip/templates/kube-vip.yaml.j2 b/infra/ansible/roles/kube_vip/templates/kube-vip.yaml.j2 new file mode 100644 index 0000000..460297c --- /dev/null +++ b/infra/ansible/roles/kube_vip/templates/kube-vip.yaml.j2 @@ -0,0 +1,126 @@ +{{ ansible_managed | comment }} +# kube-vip — control-plane HA VIP for the kube-apiserver ({{ kube_vip_address }}:{{ kube_vip_port }}). +# Deployed via the k3s auto-deploy manifests dir (reproducible, Flux-independent). +# CONTROL-PLANE ONLY: svc_enable=false so Cilium keeps ownership of Service LB (L2 pool .53-.60). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: {{ kube_vip_namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services", "services/status", "nodes"] + verbs: ["list", "get", "watch", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: + - kind: ServiceAccount + name: kube-vip + namespace: {{ kube_vip_namespace }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-vip-ds + namespace: {{ kube_vip_namespace }} + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: {{ kube_vip_version }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: {{ kube_vip_version }} + spec: + # Run only on control-plane nodes (verified label on these k3s servers). + nodeSelector: + node-role.kubernetes.io/control-plane: "true" + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + containers: + - name: kube-vip + image: {{ kube_vip_image }} + imagePullPolicy: IfNotPresent + args: + - manager + env: + # This cluster runs WITHOUT kube-proxy (Cilium kube-proxy replacement, + # ADR-0001/0002). A hostNetwork pod like kube-vip cannot reach the + # kubernetes ClusterIP (10.43.0.1) — Cilium's socket-LB does not redirect + # host-namespace traffic to ClusterIPs — so client-go's default in-cluster + # endpoint times out. Point kube-vip at the node-local apiserver instead + # (hostNetwork => 127.0.0.1:6443 is this node's k3s API; its serving cert + # carries the 127.0.0.1 SAN and is signed by the cluster CA, so the SA + # token validates). Without this the leader lease never gets acquired. + - name: KUBERNETES_SERVICE_HOST + value: "127.0.0.1" + - name: KUBERNETES_SERVICE_PORT + value: "{{ kube_vip_port }}" + - name: vip_interface + value: "{{ kube_vip_vip_interface }}" + - name: address + value: "{{ kube_vip_address }}" + - name: port + value: "{{ kube_vip_port }}" + - name: vip_cidr + value: "32" + - name: vip_arp + value: "true" + # Control-plane VIP ON, Service LB OFF (Cilium owns Services). + - name: cp_enable + value: "true" + - name: cp_namespace + value: "{{ kube_vip_namespace }}" + - name: svc_enable + value: "false" + # Leader election so exactly one node owns the VIP at a time. + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "{{ kube_vip_lease_duration }}" + - name: vip_renewdeadline + value: "{{ kube_vip_renew_deadline }}" + - name: vip_retryperiod + value: "{{ kube_vip_retry_period }}" + - name: prometheus_server + value: ":2112" + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + resources: + requests: + cpu: 25m + memory: 32Mi