From 81ef78295a1b299cd30bba378ebb4662428910f6 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Tue, 25 Aug 2026 22:10:27 -0500 Subject: [PATCH] infra: direct kube-API access via kube-vip VIP + NAT (retire SSH tunnel) Controller now reaches the kube-API directly through the pfSense NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443), validating TLS against the VIP via tls-server-name. Removes the dev.sh SSH-tunnel scaffolding and updates the ansible source-of-truth so a rebuild regenerates a direct-access kubeconfig. Co-Authored-By: Claude Opus 4.8 --- infra/ansible/inventory/group_vars/all.yml | 12 ++++---- infra/ansible/roles/cilium/tasks/main.yml | 26 +++++++++++++++++- infra/devcontainer/dev.sh | 32 ++++++++-------------- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/infra/ansible/inventory/group_vars/all.yml b/infra/ansible/inventory/group_vars/all.yml index 46911fb..91d6307 100644 --- a/infra/ansible/inventory/group_vars/all.yml +++ b/infra/ansible/inventory/group_vars/all.yml @@ -19,8 +19,10 @@ k3s_api_port: 6443 # fails to validate). kube_vip_address: "192.168.1.49" -# Controller (dev container) reaches the kube-API through an SSH tunnel -# (local 127.0.0.1:16443 -> primary 127.0.0.1:6443), because the API port is not -# NAT-forwarded and 192.168.1.50:6443 is unroutable from the dev host. Only the -# controller kubeconfig uses this; k8sServiceHost (in-cluster) stays the real node IP. -cluster_controller_api_server: "https://127.0.0.1:16443" +# Controller (dev container) reaches the kube-API DIRECTLY via the pfSense NAT +# (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443) — no SSH tunnel. TLS is +# validated against the VIP via tls-server-name (the VIP IP is in the serving cert's +# tls-san list, so TLS through the NAT validates). Only the controller kubeconfig uses +# these; k8sServiceHost (in-cluster) stays the real node IP. +cluster_controller_api_server: "https://192.168.2.50:6443" +cluster_controller_tls_server_name: "{{ kube_vip_address }}" diff --git a/infra/ansible/roles/cilium/tasks/main.yml b/infra/ansible/roles/cilium/tasks/main.yml index 7393b4c..a84a8f9 100644 --- a/infra/ansible/roles/cilium/tasks/main.yml +++ b/infra/ansible/roles/cilium/tasks/main.yml @@ -20,7 +20,7 @@ become: false run_once: true -- name: Write kubeconfig locally (server rewritten to the primary IP) +- name: Write kubeconfig locally (server rewritten to the direct NAT endpoint) ansible.builtin.copy: content: >- {{ k3s_kubeconfig_raw.content | b64decode @@ -31,6 +31,30 @@ become: false run_once: true +- name: Read the cluster name from the written kubeconfig + ansible.builtin.command: + cmd: "kubectl config view -o jsonpath={.clusters[0].name}" + environment: + KUBECONFIG: "{{ cilium_kubeconfig }}" + register: cilium_kubeconfig_cluster + delegate_to: localhost + become: false + run_once: true + changed_when: false + +- name: Pin tls-server-name to the kube-vip VIP so TLS validates through the NAT + ansible.builtin.command: + cmd: >- + kubectl config set + clusters.{{ cilium_kubeconfig_cluster.stdout }}.tls-server-name + {{ cluster_controller_tls_server_name }} + environment: + KUBECONFIG: "{{ cilium_kubeconfig }}" + delegate_to: localhost + become: false + run_once: true + changed_when: true + - name: Install Gateway API CRDs (must precede Cilium gatewayAPI.enabled) ansible.builtin.command: cmd: "kubectl apply -f {{ item }}" diff --git a/infra/devcontainer/dev.sh b/infra/devcontainer/dev.sh index 94d23a8..19652fb 100755 --- a/infra/devcontainer/dev.sh +++ b/infra/devcontainer/dev.sh @@ -3,14 +3,15 @@ # # Mounts the repo so OpenTofu (bpg/proxmox) and Ansible drive the Proxmox-hosted k3s cluster # from inside the container. Rootless podman by default (ENGINE=docker to switch); SELinux -# label is disabled for the run. On each run it also opens a best-effort SSH tunnel to the -# cluster's kube-API over the pfSense SSH NAT, so kubectl/flux/helm reach the Proxmox cluster -# transparently (the API port itself is not NAT-forwarded). Set DEV_NO_TUNNEL=1 to skip it. +# label is disabled for the run. kubectl/flux/helm reach the kube-API DIRECTLY via the pfSense +# NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443) — no SSH tunnel; the kubeconfig +# validates TLS against the VIP via tls-server-name. The pfSense SSH NAT (2250/2251/2252) is +# retained ONLY for Ansible node access, not for the kube-API. # # ./dev.sh build # build the image only -# ./dev.sh # build (if needed) + open a shell (kube tunnel up) +# ./dev.sh # build (if needed) + open a shell # ./dev.sh run tofu -version # run one command in the toolbox -# ./dev.sh run kubectl get nodes # talks to the Proxmox k3s cluster via the tunnel +# ./dev.sh run kubectl get nodes # talks to the Proxmox k3s cluster directly (no tunnel) set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${HERE}/../.." && pwd)" @@ -19,14 +20,10 @@ ENGINE="${ENGINE:-podman}" # shellcheck disable=SC1091 source "${HERE}/versions.env" -# k3s API access: the dev cluster runs on Proxmox (nodes on 192.168.1.0/24), whose kube-API -# port is NOT directly routable from here — dev.sh opens an SSH tunnel over the pfSense SSH NAT -# (local :${K3S_API_LOCAL_PORT} -> primary node 127.0.0.1:6443) so kubectl/flux/helm work -# transparently. infra/ansible/.cache/dev.kubeconfig targets https://127.0.0.1:${K3S_API_LOCAL_PORT}. -K3S_SSH_HOST="${K3S_SSH_HOST:-192.168.2.102}" # pfSense NAT front for the primary node SSH -K3S_SSH_PORT="${K3S_SSH_PORT:-2250}" # forwards to k3s-dev-1 (192.168.1.50):22 -K3S_SSH_USER="${K3S_SSH_USER:-k3s}" -K3S_API_LOCAL_PORT="${K3S_API_LOCAL_PORT:-16443}" +# k3s API access: the dev cluster runs on Proxmox, and its kube-API is reached DIRECTLY over +# the pfSense NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443). No SSH tunnel is +# opened. infra/ansible/.cache/dev.kubeconfig targets https://192.168.2.50:6443 with +# tls-server-name=192.168.1.49 so TLS validates against the VIP's SAN through the NAT. build() { "$ENGINE" build \ @@ -53,13 +50,6 @@ run() { # SOPS-encrypted secret). SOPS_AGE_KEY_FILE defaults to ~/.config/sops/age/keys.txt, # which is this mount. Create it first so the bind source exists. mkdir -p "${HOME}/.config/sops/age" - # In-container prelude (runs before your command): bring up the kube-API SSH tunnel so - # kubectl/flux/helm reach the Proxmox cluster. Best-effort — if the cluster is unreachable - # it warns and continues (tofu/ansible don't need it). Skip with DEV_NO_TUNNEL=1. - local prelude=":" - if [ "${DEV_NO_TUNNEL:-0}" != "1" ]; then - prelude="if [ -f /root/.ssh/home_network ]; then ssh -fN -L ${K3S_API_LOCAL_PORT}:127.0.0.1:6443 -p ${K3S_SSH_PORT} -i /root/.ssh/home_network -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ExitOnForwardFailure=yes -o ConnectTimeout=5 -o ServerAliveInterval=30 ${K3S_SSH_USER}@${K3S_SSH_HOST} 2>/dev/null || echo '[dev.sh] k3s API tunnel not up (cluster down or NAT changed?) — kubectl/flux/helm will not reach the cluster' >&2; fi" - fi # Spec repo (home-network-k3s) mount, read-only, for ci/spec-immutability.sh (R-TEST-06). # That script reads SPEC_REPO (default ${REPO_ROOT}/../home-network-k3s); inside the # container REPO_ROOT is /work, so the default resolves to /home-network-k3s. Mount the @@ -80,7 +70,7 @@ run() { -v "${HOME}/.config/sops:/root/.config/sops" \ -v "${HOME}/.config/home-k3s/ansible-vault-pass:/root/.config/home-k3s/ansible-vault-pass:ro" \ -e KUBECONFIG="${DEV_KUBECONFIG:-/work/infra/ansible/.cache/dev.kubeconfig}" \ - "$IMAGE" bash -lc "${prelude}"'; if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi' dev.sh "$@" + "$IMAGE" bash -lc 'if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi' dev.sh "$@" } case "${1:-shell}" in