infra: direct kube-API access via kube-vip VIP + NAT (retire SSH tunnel)
Controller now reaches the kube-API directly through the pfSense NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443), validating TLS against the VIP via tls-server-name. Removes the dev.sh SSH-tunnel scaffolding and updates the ansible source-of-truth so a rebuild regenerates a direct-access kubeconfig. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
jakman85andClaude Opus 4.8 committed
1 parent
60a58f01ad
commit
81ef78295a
3 files changed
+43
-27
No files matched your search
@@ -19,8 +19,10 @@ k3s_api_port: 6443
|
||||
# fails to validate).
|
||||
kube_vip_address: "192.168.1.49"
|
||||
|
||||
# Controller (dev container) reaches the kube-API through an SSH tunnel
|
||||
# (local 127.0.0.1:16443 -> primary 127.0.0.1:6443), because the API port is not
|
||||
# NAT-forwarded and 192.168.1.50:6443 is unroutable from the dev host. Only the
|
||||
# controller kubeconfig uses this; k8sServiceHost (in-cluster) stays the real node IP.
|
||||
cluster_controller_api_server: "https://127.0.0.1:16443"
|
||||
# Controller (dev container) reaches the kube-API DIRECTLY via the pfSense NAT
|
||||
# (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443) — no SSH tunnel. TLS is
|
||||
# validated against the VIP via tls-server-name (the VIP IP is in the serving cert's
|
||||
# tls-san list, so TLS through the NAT validates). Only the controller kubeconfig uses
|
||||
# these; k8sServiceHost (in-cluster) stays the real node IP.
|
||||
cluster_controller_api_server: "https://192.168.2.50:6443"
|
||||
cluster_controller_tls_server_name: "{{ kube_vip_address }}"
|
||||
@@ -20,7 +20,7 @@
|
||||
become: false
|
||||
run_once: true
|
||||
|
||||
- name: Write kubeconfig locally (server rewritten to the primary IP)
|
||||
- name: Write kubeconfig locally (server rewritten to the direct NAT endpoint)
|
||||
ansible.builtin.copy:
|
||||
content: >-
|
||||
{{ k3s_kubeconfig_raw.content | b64decode
|
||||
@@ -31,6 +31,30 @@
|
||||
become: false
|
||||
run_once: true
|
||||
|
||||
- name: Read the cluster name from the written kubeconfig
|
||||
ansible.builtin.command:
|
||||
cmd: "kubectl config view -o jsonpath={.clusters[0].name}"
|
||||
environment:
|
||||
KUBECONFIG: "{{ cilium_kubeconfig }}"
|
||||
register: cilium_kubeconfig_cluster
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
run_once: true
|
||||
changed_when: false
|
||||
|
||||
- name: Pin tls-server-name to the kube-vip VIP so TLS validates through the NAT
|
||||
ansible.builtin.command:
|
||||
cmd: >-
|
||||
kubectl config set
|
||||
clusters.{{ cilium_kubeconfig_cluster.stdout }}.tls-server-name
|
||||
{{ cluster_controller_tls_server_name }}
|
||||
environment:
|
||||
KUBECONFIG: "{{ cilium_kubeconfig }}"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
run_once: true
|
||||
changed_when: true
|
||||
|
||||
- name: Install Gateway API CRDs (must precede Cilium gatewayAPI.enabled)
|
||||
ansible.builtin.command:
|
||||
cmd: "kubectl apply -f {{ item }}"
|
||||
|
||||
+11
-21
@@ -3,14 +3,15 @@
|
||||
#
|
||||
# Mounts the repo so OpenTofu (bpg/proxmox) and Ansible drive the Proxmox-hosted k3s cluster
|
||||
# from inside the container. Rootless podman by default (ENGINE=docker to switch); SELinux
|
||||
# label is disabled for the run. On each run it also opens a best-effort SSH tunnel to the
|
||||
# cluster's kube-API over the pfSense SSH NAT, so kubectl/flux/helm reach the Proxmox cluster
|
||||
# transparently (the API port itself is not NAT-forwarded). Set DEV_NO_TUNNEL=1 to skip it.
|
||||
# label is disabled for the run. kubectl/flux/helm reach the kube-API DIRECTLY via the pfSense
|
||||
# NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443) — no SSH tunnel; the kubeconfig
|
||||
# validates TLS against the VIP via tls-server-name. The pfSense SSH NAT (2250/2251/2252) is
|
||||
# retained ONLY for Ansible node access, not for the kube-API.
|
||||
#
|
||||
# ./dev.sh build # build the image only
|
||||
# ./dev.sh # build (if needed) + open a shell (kube tunnel up)
|
||||
# ./dev.sh # build (if needed) + open a shell
|
||||
# ./dev.sh run tofu -version # run one command in the toolbox
|
||||
# ./dev.sh run kubectl get nodes # talks to the Proxmox k3s cluster via the tunnel
|
||||
# ./dev.sh run kubectl get nodes # talks to the Proxmox k3s cluster directly (no tunnel)
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${HERE}/../.." && pwd)"
|
||||
@@ -19,14 +20,10 @@ ENGINE="${ENGINE:-podman}"
|
||||
# shellcheck disable=SC1091
|
||||
source "${HERE}/versions.env"
|
||||
|
||||
# k3s API access: the dev cluster runs on Proxmox (nodes on 192.168.1.0/24), whose kube-API
|
||||
# port is NOT directly routable from here — dev.sh opens an SSH tunnel over the pfSense SSH NAT
|
||||
# (local :${K3S_API_LOCAL_PORT} -> primary node 127.0.0.1:6443) so kubectl/flux/helm work
|
||||
# transparently. infra/ansible/.cache/dev.kubeconfig targets https://127.0.0.1:${K3S_API_LOCAL_PORT}.
|
||||
K3S_SSH_HOST="${K3S_SSH_HOST:-192.168.2.102}" # pfSense NAT front for the primary node SSH
|
||||
K3S_SSH_PORT="${K3S_SSH_PORT:-2250}" # forwards to k3s-dev-1 (192.168.1.50):22
|
||||
K3S_SSH_USER="${K3S_SSH_USER:-k3s}"
|
||||
K3S_API_LOCAL_PORT="${K3S_API_LOCAL_PORT:-16443}"
|
||||
# k3s API access: the dev cluster runs on Proxmox, and its kube-API is reached DIRECTLY over
|
||||
# the pfSense NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443). No SSH tunnel is
|
||||
# opened. infra/ansible/.cache/dev.kubeconfig targets https://192.168.2.50:6443 with
|
||||
# tls-server-name=192.168.1.49 so TLS validates against the VIP's SAN through the NAT.
|
||||
|
||||
build() {
|
||||
"$ENGINE" build \
|
||||
@@ -53,13 +50,6 @@ run() {
|
||||
# SOPS-encrypted secret). SOPS_AGE_KEY_FILE defaults to ~/.config/sops/age/keys.txt,
|
||||
# which is this mount. Create it first so the bind source exists.
|
||||
mkdir -p "${HOME}/.config/sops/age"
|
||||
# In-container prelude (runs before your command): bring up the kube-API SSH tunnel so
|
||||
# kubectl/flux/helm reach the Proxmox cluster. Best-effort — if the cluster is unreachable
|
||||
# it warns and continues (tofu/ansible don't need it). Skip with DEV_NO_TUNNEL=1.
|
||||
local prelude=":"
|
||||
if [ "${DEV_NO_TUNNEL:-0}" != "1" ]; then
|
||||
prelude="if [ -f /root/.ssh/home_network ]; then ssh -fN -L ${K3S_API_LOCAL_PORT}:127.0.0.1:6443 -p ${K3S_SSH_PORT} -i /root/.ssh/home_network -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ExitOnForwardFailure=yes -o ConnectTimeout=5 -o ServerAliveInterval=30 ${K3S_SSH_USER}@${K3S_SSH_HOST} 2>/dev/null || echo '[dev.sh] k3s API tunnel not up (cluster down or NAT changed?) — kubectl/flux/helm will not reach the cluster' >&2; fi"
|
||||
fi
|
||||
# Spec repo (home-network-k3s) mount, read-only, for ci/spec-immutability.sh (R-TEST-06).
|
||||
# That script reads SPEC_REPO (default ${REPO_ROOT}/../home-network-k3s); inside the
|
||||
# container REPO_ROOT is /work, so the default resolves to /home-network-k3s. Mount the
|
||||
@@ -80,7 +70,7 @@ run() {
|
||||
-v "${HOME}/.config/sops:/root/.config/sops" \
|
||||
-v "${HOME}/.config/home-k3s/ansible-vault-pass:/root/.config/home-k3s/ansible-vault-pass:ro" \
|
||||
-e KUBECONFIG="${DEV_KUBECONFIG:-/work/infra/ansible/.cache/dev.kubeconfig}" \
|
||||
"$IMAGE" bash -lc "${prelude}"'; if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi' dev.sh "$@"
|
||||
"$IMAGE" bash -lc 'if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi' dev.sh "$@"
|
||||
}
|
||||
|
||||
case "${1:-shell}" in
|
||||
|
||||
Reference in new issue
Block a user