Add an HA control-plane API VIP for the k3s cluster with kube-vip, and add the VIP + all node IPs to the kube-apiserver serving-cert SANs so TLS via the VIP validates. Prepares the stable failover endpoint the operator fronts with a pfSense port-forward to 192.168.1.49:6443. roles/kube_vip + playbooks/kube-vip.yml: - Templates a single manifest (RBAC + DaemonSet kube-vip-ds) into each server's k3s auto-deploy dir (/var/lib/rancher/k3s/server/manifests) so it is reproducible and independent of Flux; k3s auto-applies it. - Control-plane ONLY (cp_enable=true, svc_enable=false): Cilium keeps ownership of Service LoadBalancing (L2 pool .53-.60) — no ARP conflict. - Runs on control-plane nodes (nodeSelector node-role.kubernetes.io/control-plane), hostNetwork, NET_ADMIN/NET_RAW, image pinned ghcr.io/kube-vip/kube-vip:v0.8.9. - vip_interface auto-detected per node (the iface holding 192.168.1.5x = eth0). - Leader election (plndr-cp-lock) with 15/10/2s lease timings. - This cluster runs without kube-proxy (Cilium replacement), so a hostNetwork pod cannot reach the kubernetes ClusterIP; kube-vip is pointed at the node-local apiserver (KUBERNETES_SERVICE_HOST=127.0.0.1:6443) so leader election works. roles/k3s_server tls-san: - New k3s_tls_sans (VIP + all three node IPs + 127.0.0.1, deduped); install args now emit one --tls-san per entry. - Persist the SAN list to /etc/rancher/k3s/config.yaml (read by a running k3s) and Restart k3s handler so an existing cluster regenerates its serving cert. - k3s.yml secondaries run serial:1 so a SAN change rolls one node at a time, keeping etcd quorum (kube-vip keeps the API reachable during each restart). Verified live: DS 3/3 ready, VIP answers on 192.168.1.49:6443, all 3 certs and an s_client against the VIP carry 192.168.1.49 + .50/.51/.52 + 127.0.0.1, all nodes Ready throughout the rolling restart. Co-Authored-By: Claude Opus 4.8 <[email protected]>
127 lines
4.3 KiB
Django/Jinja
127 lines
4.3 KiB
Django/Jinja
{{ ansible_managed | comment }}
|
|
# kube-vip — control-plane HA VIP for the kube-apiserver ({{ kube_vip_address }}:{{ kube_vip_port }}).
|
|
# Deployed via the k3s auto-deploy manifests dir (reproducible, Flux-independent).
|
|
# CONTROL-PLANE ONLY: svc_enable=false so Cilium keeps ownership of Service LB (L2 pool .53-.60).
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: kube-vip
|
|
namespace: {{ kube_vip_namespace }}
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
annotations:
|
|
rbac.authorization.kubernetes.io/autoupdate: "true"
|
|
name: system:kube-vip-role
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["services", "services/status", "nodes"]
|
|
verbs: ["list", "get", "watch", "update"]
|
|
- apiGroups: ["coordination.k8s.io"]
|
|
resources: ["leases"]
|
|
verbs: ["list", "get", "watch", "update", "create"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: system:kube-vip-binding
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: system:kube-vip-role
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: kube-vip
|
|
namespace: {{ kube_vip_namespace }}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: kube-vip-ds
|
|
namespace: {{ kube_vip_namespace }}
|
|
labels:
|
|
app.kubernetes.io/name: kube-vip-ds
|
|
app.kubernetes.io/version: {{ kube_vip_version }}
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: kube-vip-ds
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: kube-vip-ds
|
|
app.kubernetes.io/version: {{ kube_vip_version }}
|
|
spec:
|
|
# Run only on control-plane nodes (verified label on these k3s servers).
|
|
nodeSelector:
|
|
node-role.kubernetes.io/control-plane: "true"
|
|
hostNetwork: true
|
|
serviceAccountName: kube-vip
|
|
tolerations:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node-role.kubernetes.io/master
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: kube-vip
|
|
image: {{ kube_vip_image }}
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- manager
|
|
env:
|
|
# This cluster runs WITHOUT kube-proxy (Cilium kube-proxy replacement,
|
|
# ADR-0001/0002). A hostNetwork pod like kube-vip cannot reach the
|
|
# kubernetes ClusterIP (10.43.0.1) — Cilium's socket-LB does not redirect
|
|
# host-namespace traffic to ClusterIPs — so client-go's default in-cluster
|
|
# endpoint times out. Point kube-vip at the node-local apiserver instead
|
|
# (hostNetwork => 127.0.0.1:6443 is this node's k3s API; its serving cert
|
|
# carries the 127.0.0.1 SAN and is signed by the cluster CA, so the SA
|
|
# token validates). Without this the leader lease never gets acquired.
|
|
- name: KUBERNETES_SERVICE_HOST
|
|
value: "127.0.0.1"
|
|
- name: KUBERNETES_SERVICE_PORT
|
|
value: "{{ kube_vip_port }}"
|
|
- name: vip_interface
|
|
value: "{{ kube_vip_vip_interface }}"
|
|
- name: address
|
|
value: "{{ kube_vip_address }}"
|
|
- name: port
|
|
value: "{{ kube_vip_port }}"
|
|
- name: vip_cidr
|
|
value: "32"
|
|
- name: vip_arp
|
|
value: "true"
|
|
# Control-plane VIP ON, Service LB OFF (Cilium owns Services).
|
|
- name: cp_enable
|
|
value: "true"
|
|
- name: cp_namespace
|
|
value: "{{ kube_vip_namespace }}"
|
|
- name: svc_enable
|
|
value: "false"
|
|
# Leader election so exactly one node owns the VIP at a time.
|
|
- name: vip_leaderelection
|
|
value: "true"
|
|
- name: vip_leaseduration
|
|
value: "{{ kube_vip_lease_duration }}"
|
|
- name: vip_renewdeadline
|
|
value: "{{ kube_vip_renew_deadline }}"
|
|
- name: vip_retryperiod
|
|
value: "{{ kube_vip_retry_period }}"
|
|
- name: prometheus_server
|
|
value: ":2112"
|
|
securityContext:
|
|
capabilities:
|
|
add:
|
|
- NET_ADMIN
|
|
- NET_RAW
|
|
resources:
|
|
requests:
|
|
cpu: 25m
|
|
memory: 32Mi
|