jakman85andClaude Opus 5
708a05dfc1
infra/devcontainer/kubeconfig.sh was never staged into the delivery-A commit, so the branch built on main carried main's single-cluster tooling and `--context k3s-prod` failed with 'context does not exist' mid-incident. Not Flux-reconciled, so it never affected the cluster — but it breaks prod diagnosis from a checkout of this branch, which is when you need it most. Carries the concurrency fix: the merged kubeconfig holds both contexts and NO current-context, with each invocation getting a per-invocation selector file prepended to KUBECONFIG. Two concurrent dev.sh runs with different CLUSTER values previously overwrote each other's current-context mid-command — reproduced live, a CLUSTER=dev reader returned k3s-prod nodes on 3 of 7 iterations. Co-Authored-By: Claude Opus 5 <[email protected]>
157 lines
8.3 KiB
Bash
Executable File
157 lines
8.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# infra/devcontainer/dev.sh — build & enter the pinned dev toolbox.
|
|
#
|
|
# Mounts the repo so OpenTofu (bpg/proxmox) and Ansible drive the Proxmox-hosted k3s cluster
|
|
# from inside the container. Rootless podman by default (ENGINE=docker to switch); SELinux
|
|
# label is disabled for the run. kubectl/flux/helm reach the kube-API DIRECTLY via the pfSense
|
|
# NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443) — no SSH tunnel; the kubeconfig
|
|
# validates TLS against the VIP via tls-server-name. The pfSense SSH NAT (2250/2251/2252) is
|
|
# retained ONLY for Ansible node access, not for the kube-API.
|
|
#
|
|
# ./dev.sh build # build the image only
|
|
# ./dev.sh # build (if needed) + open a shell
|
|
# ./dev.sh run tofu -version # run one command in the toolbox
|
|
# ./dev.sh run kubectl get nodes # talks to the DEV k3s cluster directly (no tunnel)
|
|
#
|
|
# CHOOSING A CLUSTER — `CLUSTER=<dev|prod>`, defaulting to dev:
|
|
#
|
|
# ./dev.sh run kubectl get nodes # dev (the default; unchanged behaviour)
|
|
# CLUSTER=prod ./dev.sh run kubectl get nodes # prod
|
|
# CLUSTER=prod ./dev.sh # interactive shell pointed at prod
|
|
#
|
|
# The selector is an env-var PREFIX so the target is visible on the same line as the command, and
|
|
# every invocation prints a banner naming the cluster it is about to talk to. Inside the container
|
|
# KUBECONFIG is a two-file CHAIN, "<per-invocation selector>:<shared merged config>":
|
|
#
|
|
# * the shared merged config holds BOTH clusters under unambiguous names — contexts `k3s-dev`
|
|
# and `k3s-prod`, never `default` — so `--context k3s-prod` actually works, and it carries NO
|
|
# current-context of its own;
|
|
# * the selector is a private per-invocation file holding only `current-context: k3s-$CLUSTER`.
|
|
#
|
|
# So `kubectl config current-context` still answers the "which cluster am I on?" question honestly,
|
|
# but the answer is now PER INVOCATION. That is the point: the previous design stamped
|
|
# current-context into the one shared merged file, so two concurrent dev.sh runs with different
|
|
# CLUSTER values overwrote each other's default MID-COMMAND — a `CLUSTER=prod` script could read
|
|
# prod for its first half and DEV for its second. See infra/devcontainer/kubeconfig.sh for the full
|
|
# write-up, and for why the naive `KUBECONFIG=dev:prod` merge is dangerous in the first place (both
|
|
# source files name everything `default`).
|
|
#
|
|
# Concurrency note: `CLUSTER=dev ./dev.sh run ...` and `CLUSTER=prod ./dev.sh run ...` may now run
|
|
# at the same time and neither can affect the other's default cluster.
|
|
#
|
|
# Requesting a cluster whose kubeconfig is missing is a hard ERROR — it never falls back to the
|
|
# other cluster.
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${HERE}/../.." && pwd)"
|
|
IMAGE="${IMAGE:-home-k3s-dev:local}"
|
|
ENGINE="${ENGINE:-podman}"
|
|
# shellcheck disable=SC1091
|
|
source "${HERE}/versions.env"
|
|
|
|
# k3s API access: the dev cluster runs on Proxmox, and its kube-API is reached DIRECTLY over
|
|
# the pfSense NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443). No SSH tunnel is
|
|
# opened. infra/ansible/.cache/dev.kubeconfig targets https://192.168.2.50:6443 with
|
|
# tls-server-name=192.168.1.49 so TLS validates against the VIP's SAN through the NAT.
|
|
# Prod is reached the same way via infra/ansible/.cache/prod.kubeconfig (https://192.168.2.9:6443).
|
|
|
|
# ------------------------------------------------------------------ cluster selection
|
|
CLUSTER="${CLUSTER:-dev}"
|
|
case "${CLUSTER}" in
|
|
dev|prod) ;;
|
|
*) printf '\033[31mERROR: CLUSTER=%s is not a known cluster (dev|prod).\033[0m\n' "${CLUSTER}" >&2; exit 2 ;;
|
|
esac
|
|
CLUSTER_KUBECONFIG="${REPO_ROOT}/infra/ansible/.cache/${CLUSTER}.kubeconfig"
|
|
|
|
build() {
|
|
"$ENGINE" build \
|
|
--build-arg FEDORA_VERSION="$FEDORA_VERSION" \
|
|
--build-arg OPENTOFU_VERSION="$OPENTOFU_VERSION" \
|
|
--build-arg AGE_VERSION="$AGE_VERSION" \
|
|
--build-arg SOPS_VERSION="$SOPS_VERSION" \
|
|
--build-arg FLUX_VERSION="$FLUX_VERSION" \
|
|
--build-arg KUBECTL_VERSION="$KUBECTL_VERSION" \
|
|
--build-arg HELM_VERSION="$HELM_VERSION" \
|
|
--build-arg KYVERNO_VERSION="$KYVERNO_VERSION" \
|
|
--build-arg KYVERNO_JSON_VERSION="$KYVERNO_JSON_VERSION" \
|
|
--build-arg CHAINSAW_VERSION="$CHAINSAW_VERSION" \
|
|
--build-arg YAMLLINT_VERSION="$YAMLLINT_VERSION" \
|
|
--build-arg YQ_VERSION="$YQ_VERSION" \
|
|
--build-arg CHECK_JSONSCHEMA_VERSION="$CHECK_JSONSCHEMA_VERSION" \
|
|
-t "$IMAGE" -f "${HERE}/Containerfile" "${HERE}"
|
|
}
|
|
|
|
run() {
|
|
# Persist the SOPS age keyring on the HOST so the age PRIVATE key generated by
|
|
# bootstrap/sops/age-keygen.sh survives this --rm container (otherwise it lives only
|
|
# in the ephemeral container fs and is lost on exit — you'd be locked out of every
|
|
# SOPS-encrypted secret). SOPS_AGE_KEY_FILE defaults to ~/.config/sops/age/keys.txt,
|
|
# which is this mount. Create it first so the bind source exists.
|
|
mkdir -p "${HOME}/.config/sops/age"
|
|
# Spec repo (home-network-k3s) mount, read-only, for ci/spec-immutability.sh (R-TEST-06).
|
|
# That script reads SPEC_REPO (default ${REPO_ROOT}/../home-network-k3s); inside the
|
|
# container REPO_ROOT is /work, so the default resolves to /home-network-k3s. Mount the
|
|
# sibling spec repo there RO when it exists; skip the mount otherwise so the container
|
|
# still runs (the gate then honestly SKIPS immutability — exit 3, never faked green).
|
|
local -a spec_mount=()
|
|
local spec_host
|
|
spec_host="$(cd "${REPO_ROOT}/../home-network-k3s" 2>/dev/null && pwd || true)"
|
|
if [ -n "${spec_host}" ] && [ -d "${spec_host}/.git" ]; then
|
|
spec_mount=(-v "${spec_host}:/home-network-k3s:ro" -e SPEC_REPO=/home-network-k3s)
|
|
fi
|
|
|
|
# Which cluster, said out loud, before anything runs. CLUSTER can be inherited from an exported
|
|
# shell variable, so the banner is what stops "I thought I was on dev".
|
|
if [ -n "${DEV_KUBECONFIG:-}" ]; then
|
|
# SUPERSEDED but still honoured: DEV_KUBECONFIG used to be the only way to reach a cluster other
|
|
# than the (then non-existent) default dev path. It wins over CLUSTER so nothing that depends on
|
|
# it breaks, but it is opaque about its target — prefer CLUSTER=dev|prod.
|
|
printf '\033[33m==> dev.sh: KUBECONFIG pinned by DEV_KUBECONFIG=%s (CLUSTER=%s ignored; DEV_KUBECONFIG is superseded — use CLUSTER=dev|prod)\033[0m\n' \
|
|
"${DEV_KUBECONFIG}" "${CLUSTER}" >&2
|
|
else
|
|
# Fail here, on the host, before a container is even started: a missing kubeconfig must never
|
|
# degrade into "whatever config we could find", because that is how a prod command lands on dev.
|
|
if [ ! -f "${CLUSTER_KUBECONFIG}" ]; then
|
|
printf '\033[31mERROR: dev.sh: CLUSTER=%s but its kubeconfig is MISSING: %s\n Refusing to run. dev.sh does NOT fall back to the other cluster.\033[0m\n' \
|
|
"${CLUSTER}" "${CLUSTER_KUBECONFIG}" >&2
|
|
exit 1
|
|
fi
|
|
printf '\033[1m==> dev.sh: cluster=%s (kube context k3s-%s)\033[0m\n' "${CLUSTER}" "${CLUSTER}" >&2
|
|
fi
|
|
|
|
"$ENGINE" run --rm -it \
|
|
--network host \
|
|
--security-opt label=disable \
|
|
-v "${REPO_ROOT}:/work" -w /work \
|
|
"${spec_mount[@]}" \
|
|
-v "${HOME}/.ssh:/root/.ssh:ro" \
|
|
-v "${HOME}/.config/sops:/root/.config/sops" \
|
|
-v "${HOME}/.config/home-k3s/ansible-vault-pass:/root/.config/home-k3s/ansible-vault-pass:ro" \
|
|
-e CLUSTER="${CLUSTER}" \
|
|
-e DEV_KUBECONFIG="${DEV_KUBECONFIG:-}" \
|
|
"$IMAGE" bash -lc '
|
|
set -e
|
|
if [ -n "${DEV_KUBECONFIG:-}" ]; then
|
|
export KUBECONFIG="${DEV_KUBECONFIG}"
|
|
else
|
|
# Regenerated every run: cheap, and it means an updated dev/prod.kubeconfig is picked up
|
|
# without anyone remembering to refresh a hand-made merged file. Exits non-zero (set -e)
|
|
# if the requested cluster is missing — no silent fallback.
|
|
#
|
|
# This prints a KUBECONFIG CHAIN ("<per-invocation selector>:<shared merged config>"), not a
|
|
# single path. Export it verbatim; do not split it or take only the last element, or this
|
|
# invocation loses its private default context and inherits whatever is shared.
|
|
KUBECONFIG="$(/work/infra/devcontainer/kubeconfig.sh "${CLUSTER}")"
|
|
export KUBECONFIG
|
|
fi
|
|
if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi
|
|
' dev.sh "$@"
|
|
}
|
|
|
|
case "${1:-shell}" in
|
|
build) build ;;
|
|
run) shift; run "$@" ;;
|
|
shell) build; shift 2>/dev/null || true; run ;;
|
|
*) build; run "$@" ;;
|
|
esac
|