home-network-k3s-impl/infra/devcontainer/dev.sh

#!/usr/bin/env bash
# infra/devcontainer/dev.sh — build & enter the pinned dev toolbox.
#
# Mounts the repo so OpenTofu (bpg/proxmox) and Ansible drive the Proxmox-hosted k3s cluster
# from inside the container. Rootless podman by default (ENGINE=docker to switch); SELinux
# label is disabled for the run. kubectl/flux/helm reach the kube-API DIRECTLY via the pfSense
# NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443) — no SSH tunnel; the kubeconfig
# validates TLS against the VIP via tls-server-name. The pfSense SSH NAT (2250/2251/2252) is
# retained ONLY for Ansible node access, not for the kube-API.
#
#   ./dev.sh build                    # build the image only
#   ./dev.sh                          # build (if needed) + open a shell
#   ./dev.sh run tofu -version        # run one command in the toolbox
#   ./dev.sh run kubectl get nodes    # talks to the PROD k3s cluster directly (no tunnel)
#
# CHOOSING A CLUSTER — `CLUSTER=<prod|dev>`, defaulting to PROD (dev was decommissioned):
#
#   ./dev.sh run kubectl get nodes                 # prod (the default since 2026-09-12)
#   CLUSTER=dev ./dev.sh run kubectl get nodes     # dev — DECOMMISSIONED, warns; VMs are down
#   ./dev.sh                                       # interactive shell pointed at prod
#
# The selector is an env-var PREFIX so the target is visible on the same line as the command, and
# every invocation prints a banner naming the cluster it is about to talk to. Inside the container
# KUBECONFIG is a two-file CHAIN, "<per-invocation selector>:<shared merged config>":
#
#   * the shared merged config holds BOTH clusters under unambiguous names — contexts `k3s-dev`
#     and `k3s-prod`, never `default` — so `--context k3s-prod` actually works, and it carries NO
#     current-context of its own;
#   * the selector is a private per-invocation file holding only `current-context: k3s-$CLUSTER`.
#
# So `kubectl config current-context` still answers the "which cluster am I on?" question honestly,
# but the answer is now PER INVOCATION. That is the point: the previous design stamped
# current-context into the one shared merged file, so two concurrent dev.sh runs with different
# CLUSTER values overwrote each other's default MID-COMMAND — a `CLUSTER=prod` script could read
# prod for its first half and DEV for its second. See infra/devcontainer/kubeconfig.sh for the full
# write-up, and for why the naive `KUBECONFIG=dev:prod` merge is dangerous in the first place (both
# source files name everything `default`).
#
# Concurrency note: `CLUSTER=dev ./dev.sh run ...` and `CLUSTER=prod ./dev.sh run ...` may now run
# at the same time and neither can affect the other's default cluster.
#
# Requesting a cluster whose kubeconfig is missing is a hard ERROR — it never falls back to the
# other cluster.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${HERE}/../.." && pwd)"
IMAGE="${IMAGE:-home-k3s-dev:local}"
ENGINE="${ENGINE:-podman}"
# shellcheck disable=SC1091
source "${HERE}/versions.env"

# k3s API access: the dev cluster runs on Proxmox, and its kube-API is reached DIRECTLY over
# the pfSense NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443). No SSH tunnel is
# opened. infra/ansible/.cache/dev.kubeconfig targets https://192.168.2.50:6443 with
# tls-server-name=192.168.1.49 so TLS validates against the VIP's SAN through the NAT.
# Prod is reached the same way via infra/ansible/.cache/prod.kubeconfig (https://192.168.2.9:6443).

# ------------------------------------------------------------------ cluster selection
# Defaults to PROD because dev was decommissioned on 2026-09-12 (full migration: dev out, prod
# onto the apex — see DECISIONS-LOG). Defaulting to a cluster that no longer answers made every
# unprefixed `./dev.sh run kubectl ...` a silent no-op against a dead context, which is a trap:
# the banner named dev, the command hung or errored, and nothing said "that cluster is gone".
# `CLUSTER=dev` is still ACCEPTED, not removed — the dev VM disks were deliberately kept so the
# environment is reversible — but it warns, because a working dev cluster is no longer the default
# assumption anywhere in this repo.
CLUSTER="${CLUSTER:-prod}"
case "${CLUSTER}" in
  prod) ;;
  dev)  printf '\033[33mWARNING: CLUSTER=dev — dev was DECOMMISSIONED 2026-09-12; this context will not\nanswer unless the dev VMs have been deliberately brought back up.\033[0m\n' >&2 ;;
  *) printf '\033[31mERROR: CLUSTER=%s is not a known cluster (dev|prod).\033[0m\n' "${CLUSTER}" >&2; exit 2 ;;
esac
CLUSTER_KUBECONFIG="${REPO_ROOT}/infra/ansible/.cache/${CLUSTER}.kubeconfig"

build() {
  "$ENGINE" build \
    --build-arg FEDORA_VERSION="$FEDORA_VERSION" \
    --build-arg OPENTOFU_VERSION="$OPENTOFU_VERSION" \
    --build-arg AGE_VERSION="$AGE_VERSION" \
    --build-arg SOPS_VERSION="$SOPS_VERSION" \
    --build-arg FLUX_VERSION="$FLUX_VERSION" \
    --build-arg KUBECTL_VERSION="$KUBECTL_VERSION" \
    --build-arg HELM_VERSION="$HELM_VERSION" \
    --build-arg KYVERNO_VERSION="$KYVERNO_VERSION" \
    --build-arg KYVERNO_JSON_VERSION="$KYVERNO_JSON_VERSION" \
    --build-arg CHAINSAW_VERSION="$CHAINSAW_VERSION" \
    --build-arg YAMLLINT_VERSION="$YAMLLINT_VERSION" \
    --build-arg YQ_VERSION="$YQ_VERSION" \
    --build-arg CHECK_JSONSCHEMA_VERSION="$CHECK_JSONSCHEMA_VERSION" \
    --build-arg PROMTOOL_VERSION="$PROMTOOL_VERSION" \
    -t "$IMAGE" -f "${HERE}/Containerfile" "${HERE}"
}

run() {
  # Persist the SOPS age keyring on the HOST so the age PRIVATE key generated by
  # bootstrap/sops/age-keygen.sh survives this --rm container (otherwise it lives only
  # in the ephemeral container fs and is lost on exit — you'd be locked out of every
  # SOPS-encrypted secret). SOPS_AGE_KEY_FILE defaults to ~/.config/sops/age/keys.txt,
  # which is this mount. Create it first so the bind source exists.
  mkdir -p "${HOME}/.config/sops/age"
  # Spec repo (home-network-k3s) mount, read-only, for ci/spec-immutability.sh (R-TEST-06).
  # That script reads SPEC_REPO (default ${REPO_ROOT}/../home-network-k3s); inside the
  # container REPO_ROOT is /work, so the default resolves to /home-network-k3s. Mount the
  # sibling spec repo there RO when it exists; skip the mount otherwise so the container
  # still runs (the gate then honestly SKIPS immutability — exit 3, never faked green).
  local -a spec_mount=()
  local spec_host
  spec_host="$(cd "${REPO_ROOT}/../home-network-k3s" 2>/dev/null && pwd || true)"
  if [ -n "${spec_host}" ] && [ -d "${spec_host}/.git" ]; then
    spec_mount=(-v "${spec_host}:/home-network-k3s:ro" -e SPEC_REPO=/home-network-k3s)
  fi

  # Which cluster, said out loud, before anything runs. CLUSTER can be inherited from an exported
  # shell variable, so the banner is what stops "I thought I was on dev".
  if [ -n "${DEV_KUBECONFIG:-}" ]; then
    # SUPERSEDED but still honoured: DEV_KUBECONFIG used to be the only way to reach a cluster other
    # than the (then non-existent) default dev path. It wins over CLUSTER so nothing that depends on
    # it breaks, but it is opaque about its target — prefer CLUSTER=dev|prod.
    printf '\033[33m==> dev.sh: KUBECONFIG pinned by DEV_KUBECONFIG=%s (CLUSTER=%s ignored; DEV_KUBECONFIG is superseded — use CLUSTER=dev|prod)\033[0m\n' \
      "${DEV_KUBECONFIG}" "${CLUSTER}" >&2
  else
    # Fail here, on the host, before a container is even started: a missing kubeconfig must never
    # degrade into "whatever config we could find", because that is how a prod command lands on dev.
    if [ ! -f "${CLUSTER_KUBECONFIG}" ]; then
      printf '\033[31mERROR: dev.sh: CLUSTER=%s but its kubeconfig is MISSING: %s\n       Refusing to run. dev.sh does NOT fall back to the other cluster.\033[0m\n' \
        "${CLUSTER}" "${CLUSTER_KUBECONFIG}" >&2
      exit 1
    fi
    printf '\033[1m==> dev.sh: cluster=%s  (kube context k3s-%s)\033[0m\n' "${CLUSTER}" "${CLUSTER}" >&2
  fi

  "$ENGINE" run --rm -it \
    --network host \
    --security-opt label=disable \
    -v "${REPO_ROOT}:/work" -w /work \
    "${spec_mount[@]}" \
    -v "${HOME}/.ssh:/root/.ssh:ro" \
    -v "${HOME}/.config/sops:/root/.config/sops" \
    -v "${HOME}/.config/home-k3s/ansible-vault-pass:/root/.config/home-k3s/ansible-vault-pass:ro" \
    -e CLUSTER="${CLUSTER}" \
    -e DEV_KUBECONFIG="${DEV_KUBECONFIG:-}" \
    "$IMAGE" bash -lc '
      set -e
      if [ -n "${DEV_KUBECONFIG:-}" ]; then
        export KUBECONFIG="${DEV_KUBECONFIG}"
      else
        # Regenerated every run: cheap, and it means an updated dev/prod.kubeconfig is picked up
        # without anyone remembering to refresh a hand-made merged file. Exits non-zero (set -e)
        # if the requested cluster is missing — no silent fallback.
        #
        # This prints a KUBECONFIG CHAIN ("<per-invocation selector>:<shared merged config>"), not a
        # single path. Export it verbatim; do not split it or take only the last element, or this
        # invocation loses its private default context and inherits whatever is shared.
        KUBECONFIG="$(/work/infra/devcontainer/kubeconfig.sh "${CLUSTER}")"
        export KUBECONFIG
      fi
      if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi
    ' dev.sh "$@"
}

case "${1:-shell}" in
  build) build ;;
  run)   shift; run "$@" ;;
  shell) build; shift 2>/dev/null || true; run ;;
  *)     build; run "$@" ;;
esac