diff --git a/apps/wave-0/nextcloud-db-backup-credentials.sops.yaml b/apps/wave-0/nextcloud-db-backup-credentials.sops.yaml index 3cef69f..d1e3625 100644 --- a/apps/wave-0/nextcloud-db-backup-credentials.sops.yaml +++ b/apps/wave-0/nextcloud-db-backup-credentials.sops.yaml @@ -5,8 +5,8 @@ metadata: namespace: nextcloud-db type: Opaque stringData: - ACCESS_KEY_ID: ENC[AES256_GCM,data:gS533S1h2f5uWQ==,iv:XmcNCYdlkAIewn7Br8Kk4CHcZWbMQxJE0Mlge9/oGRY=,tag:+krC/C+EMQl5pYgwkuut1g==,type:str] - ACCESS_SECRET_KEY: ENC[AES256_GCM,data:NTcBY05zUy8ngV3phNF18FBHRCEKQGu2KQ2yQGu4YiuJasllznZ+YvCBn/3i0Z7/,iv:uXyenHmNqBRb5CC/ELgVbFXV/W59ZPOrSeGwIAi8Hz0=,tag:9ZRcKL0ijOPBi4QUbDZyoQ==,type:str] + ACCESS_KEY_ID: ENC[AES256_GCM,data:LkBD4YP2NXJQNDcxwukoHoawmnVHF+cOZID+T4QSfIU=,iv:mVDSC02WczRjSor+V0dTHDVX3XpWuaAgwvOE45CSu/Q=,tag:n60oC4oW1sHIx3nKUAPnKA==,type:str] + ACCESS_SECRET_KEY: ENC[AES256_GCM,data:IbzhRTEoJUTe/uEguMhaRwtkbw4HCmbrnLg3H4DcfgCo/63LVaq7dWogJaXhvA+P78aO7sRdFW/g/NCThofU+g==,iv:E6gjP/v5WNoXLKTF+Td5szwJ/Q2kPcnRYXda9Ku4nCw=,tag:FYns9o1Y4rtHFXUD8FNICw==,type:str] sops: kms: [] gcp_kms: [] @@ -22,8 +22,8 @@ sops: ODAyK2g4bGlXeXV0Q1FMbXNiSEpUYncKvwviYgI/XjCBJG5kpS8tOHLTLsnXv1bl b4E4rigNesmreSYxAyhGw9bgQNY6ba/F3Ecexrdx+iV/tYYc/VoDKA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-08-29T03:31:48Z" - mac: ENC[AES256_GCM,data:ks4i7zjrJQajKdncnnV5dD4Ap1Q1xdKyiQAiqhao2USXu9S+4flX2kzc9lv8lFG/dTiWvLu6Tqgmj46ssiqtd0E8DnrASNEuKz/Xlw1iCka2008C5BhE/k8oy84kXXDgEuo7GiYdhJU/mAaAnY4p10ZQkvTKddQMmSO65YRwRMo=,iv:skZoS+bhAOnLlY5R5OfZvFV8yCEiEKtLac4mVbdmZ2s=,tag:6T1JIroe5hI8RXWAyzEKLw==,type:str] + lastmodified: "2026-09-07T05:41:34Z" + mac: ENC[AES256_GCM,data:dO1Z1YF2wYThTMRBUJbg9bMuiDAtDpN2ERpkC2mpvLvH09rDBmb3IWzC7iHKm2GQ9v/8yZ6WEN42RMAQBxD7MtVRtjiTDuuIeOObFuw69FDOfRFx2NR7C6LoXxrUm8/8Q/KcZM9PRHQMM7jpGRIWTyFyeA/2EuUdRooIN7MpIkA=,iv:9hpzLEZ2UdRgVYVEqDY7bjHfWUdjf+kflGgeYx19LBE=,tag:A7ihm4K9rLBgoQ4uvx5Ebg==,type:str] pgp: [] encrypted_regex: ^(data|stringData)$ version: 3.9.4 diff --git a/apps/wave-0/semaphore-db-backup-credentials.sops.yaml b/apps/wave-0/semaphore-db-backup-credentials.sops.yaml index d7a4574..fc97f8e 100644 --- a/apps/wave-0/semaphore-db-backup-credentials.sops.yaml +++ b/apps/wave-0/semaphore-db-backup-credentials.sops.yaml @@ -5,8 +5,8 @@ metadata: namespace: semaphore-db type: Opaque stringData: - ACCESS_KEY_ID: ENC[AES256_GCM,data:jmhu9U7Lyo0ugA==,iv:EyU67JLLO10za1tM/peju9tvBBINJ7ZDn6F9Zu0g6H8=,tag:9M/adv8Gdu3XNBJ/G4xfeA==,type:str] - ACCESS_SECRET_KEY: ENC[AES256_GCM,data:f9qW7BX5OQ2L7FqZ6pr6lqdEhTy+7xabLSMTnHZ/UYj1RnQ9asL+/HNkEWsMatJC,iv:hS2Q5l4EsvMBop/sBnld83tm2MhsTTI5N72BLGLMdtk=,tag:MwfC205a/djUA6HNqobcUw==,type:str] + ACCESS_KEY_ID: ENC[AES256_GCM,data:sIVd7sJEWKBpu3PYxxmSCZ8OsZQLrys5SiCj0iPor28=,iv:hLmkR7TYyEAWrtuq41RatwqQR1RdUz9Q54BfcGAJ5xI=,tag:oRSTMdOd9SJUf8+7l+YsNw==,type:str] + ACCESS_SECRET_KEY: ENC[AES256_GCM,data:8sBtArxeBu7cDBX2Y18oy9mMigukmJc5NhbUD56ddSPB1eZgJbznZmMwel59+/IkqsMZSsaKDqT8PvVMWPrbxQ==,iv:OSfNZtl+ByJlZBPNUSV2NaF5SEDJLuhAtFpbVdRk4ko=,tag:Aow0/AKLkARx1NiIqc7LKA==,type:str] sops: kms: [] gcp_kms: [] @@ -22,8 +22,8 @@ sops: cGpweDl4VjNsNHM0Sm8zVDBWcGFJanMKpBBV9cnSZRVmIGjEi9YBMlbm4df253cS osMRBCKizDIz1Q0Awq952y/L97ljUnSE6JWnlKTriHoGtLDct4mg3Q== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-08-29T03:31:48Z" - mac: ENC[AES256_GCM,data:uqtBwfubrUqaRN5r5qfMlITBvKxfbdyxZGDcZgyWtmDvX1OulKBvuOYBZaDW1MdIrq++nbKhocQK2PkuAmqZI3NbfUZ5mrGWsIvD3mImrfyiOnASvnCMwIlMTHlcJbJTIY+8JLmzbr+2TPbZ7zVechR4EoX9tEykY+GZml7cTic=,iv:7qoBRE+JlmArMejf++orrccAPDOpJsDp4vA+UHYCkoA=,tag:IwE7rZCXB+UhiP0yWWIFGA==,type:str] + lastmodified: "2026-09-07T05:41:34Z" + mac: ENC[AES256_GCM,data:QUGHd8pmgMrcdQKaoQDq41VstAoV1Dhb8SmihRAr+p62KNbWTLf7L10mAnwopTSy8ucA+p52bRgcMx6//ZvM7j/DjilmC+NMnMIFqRthVtJ3WWUGbWmlWAGHHg0XyEdims0F3fh1CIkZIx0ONgiOpUT3F/s2VajJhrjKq37peMA=,iv:sgfrDaAsChQlrcSlqD9rZ+8JPCCa+dhYOm6L+hnPfcY=,tag:b879Nimwl+Fdw9NRvbEe+g==,type:str] pgp: [] encrypted_regex: ^(data|stringData)$ version: 3.9.4 diff --git a/infra/PHASE1-STATUS.md b/infra/PHASE1-STATUS.md index e3b7a79..f754d2f 100644 --- a/infra/PHASE1-STATUS.md +++ b/infra/PHASE1-STATUS.md @@ -84,6 +84,9 @@ toolchain: yamllint, check-jsonschema, kyverno, chainsaw, yq) lives in the image infra/devcontainer/dev.sh build # once infra/devcontainer/dev.sh run make -C ci gate # exactly what CI runs infra/devcontainer/dev.sh # interactive shell at /work + +infra/devcontainer/dev.sh run kubectl get nodes # DEV (the default) +CLUSTER=prod infra/devcontainer/dev.sh run kubectl get nodes # PROD ``` Unlike the libvirt era, **tofu and ansible now run fine inside the container** — the @@ -95,10 +98,23 @@ which is what broke the old `dmacvicar/libvirt` provider in a container. file, and the sibling spec repo read-only at `/home-network-k3s` so the spec-immutability check runs instead of silently skipping. -> **Known gap.** `dev.sh` defaults `KUBECONFIG` to `/work/infra/ansible/.cache/dev.kubeconfig`, -> a libvirt-era path that no longer exists, and does not mount `~/.kube`. In-container `kubectl` -> therefore needs an explicit `DEV_KUBECONFIG=` — the OIDC -> context cannot work there anyway (no browser). Not yet fixed. +> **Closed (was a known gap): `dev.sh` can now target either cluster.** It used to hard-wire +> `KUBECONFIG` to `dev.kubeconfig`, leaving an undocumented — and misleadingly named — +> `DEV_KUBECONFIG=` as the only route to prod. Select with **`CLUSTER=dev|prod`**; +> **dev remains the default**, so every existing invocation means what it always meant. Inside the +> container `KUBECONFIG` is a chain of a tiny per-invocation selector file over a merged config +> carrying both clusters as contexts **`k3s-dev`** and **`k3s-prod`** (never `default`), regenerated +> each run by [`infra/devcontainer/kubeconfig.sh`](devcontainer/kubeconfig.sh) into the gitignored +> `infra/ansible/.cache/clusters.kubeconfig`. The merged file deliberately carries **no** +> `current-context`: it is shared by every container, and stamping the current cluster into it made +> concurrent `CLUSTER=dev` and `CLUSTER=prod` runs silently swap clusters mid-command. Naming matters: k3s writes *both* source kubeconfigs +> with context, cluster and user all called `default`, and kubectl's merge keeps the first +> duplicate — so a plain `KUBECONFIG=dev:prod` would silently resolve `default` to **dev** while you +> believed you were on prod. Requesting a cluster whose kubeconfig is absent is a hard error, never +> a fallback. `DEV_KUBECONFIG` still works and still wins, but is **superseded** — it is opaque +> about its target and yields an ambiguous `default` context. `~/.kube` is still not mounted and the +> OIDC context still cannot work in a container (no browser); use the break-glass cert kubeconfigs +> above. Details: [`devcontainer/README.md`](devcontainer/README.md). ## Off-repo state you must not lose diff --git a/infra/devcontainer/README.md b/infra/devcontainer/README.md index 189f4df..61b3cb9 100644 --- a/infra/devcontainer/README.md +++ b/infra/devcontainer/README.md @@ -33,6 +33,84 @@ cd infra/devcontainer ``` `ENGINE=docker ./dev.sh` to use docker instead of rootless podman. +## Choosing a cluster — `CLUSTER=dev|prod` + +```bash +./dev.sh run kubectl get nodes # DEV — the default, unchanged +CLUSTER=prod ./dev.sh run kubectl get nodes # PROD +CLUSTER=prod ./dev.sh # interactive shell pointed at prod +``` + +An env-var **prefix**, so the target sits on the same line as the command. Every invocation prints +a banner naming the cluster before the container starts, because `CLUSTER` can also be inherited +from an exported shell variable. + +Inside the container `KUBECONFIG` is a two-file **chain**, +`:`: + +| file | shared? | holds | +|---|---|---| +| selector, in a private `mktemp` dir | no — one per `dev.sh` invocation | only `current-context: k3s-$CLUSTER`; no credentials | +| `infra/ansible/.cache/clusters.kubeconfig` | yes — bind-mounted, read by every container | both clusters under unambiguous names; **no `current-context`** | + +kubectl merges the chain first-wins, so the selector supplies this invocation's default: + +``` +CURRENT NAME CLUSTER AUTHINFO +* k3s-dev k3s-dev k3s-dev-admin + k3s-prod k3s-prod k3s-prod-admin +``` + +So `kubectl config current-context` answers "which cluster am I on?" honestly, and +`kubectl --context k3s-prod …` works from a dev-defaulted shell. + +**Why the default context is not stored in the merged file.** It used to be, and that was a +cross-cluster data-corruption bug. The merged file lives in the repo's `.cache`, which every +container bind-mounts, so `current-context` was **shared mutable state**: two concurrent `dev.sh` +runs with different `CLUSTER` values overwrote each other's default *mid-command*. A +`CLUSTER=prod` script read prod for its first half and **dev** for its second; it was caught only +because a dev-only PVC name showed up in a prod query. Moving selection into a per-invocation file +removes the shared field rather than locking it — there is no window left to lose. Read on its own +the merged file now resolves to nothing (`error: current-context is not set`), which is loud and +safe; `--context k3s-dev` / `--context k3s-prod` against it are unaffected and remain the pattern +to prefer in scripts. Concurrent `CLUSTER=dev` and `CLUSTER=prod` invocations are now safe. + +Consequences worth knowing: + +- Export what `kubeconfig.sh` prints **verbatim** — it is a colon-joined chain, not one path. +- `kubectl config use-context …` in an interactive toolbox shell writes to the **first** file in + the chain (the selector), so a manual switch stays local to that container. + +**Why a merge script and not `KUBECONFIG=dev.kubeconfig:prod.kubeconfig`.** k3s names the context, +the cluster *and* the user in **both** files `default`. kubectl's merge keeps the **first** +occurrence of a duplicated name, so a colon-joined `KUBECONFIG` silently resolves `default` to +**dev** and `--context default` can never reach prod — you would believe you were on prod while +issuing commands against the live dev cluster. `kubeconfig.sh` therefore **renames on merge** +(`k3s-dev` / `k3s-prod`, `*-admin` users) with `yq`, structurally — not with `sed`, which corrupts +these files because the cluster and the user entry are both `- name: default` at identical indent. + +- Generator: [`kubeconfig.sh`](kubeconfig.sh). Output: `infra/ansible/.cache/clusters.kubeconfig`, + mode 0600, **regenerated on every `dev.sh run`** — a build artifact, never hand-edited, never + committed (`.cache/` and `*.kubeconfig` are both gitignored; it embeds cluster-admin certs). + Its content no longer depends on which cluster was asked for, so concurrent regenerations write + identical bytes; the write is published by atomic rename so a concurrent reader can never load a + half-written kubeconfig. +- A cluster whose source kubeconfig is missing is simply omitted from the merge, so a machine with + only `dev.kubeconfig` still works. +- **Asking for a cluster whose kubeconfig is absent is a hard error.** `dev.sh` refuses on the host + before starting a container and never falls back to the other cluster — that fallback is exactly + how a prod command lands on dev. +- `bootstrap/standup.sh` is unaffected: it exports its own `KUBECONFIG` from the mandatory + `--kubeconfig` flag, which overrides what `dev.sh` sets. Its `--expect-server` guard remains the + thing that makes it safe, not `CLUSTER`. + +### `DEV_KUBECONFIG` — superseded, still honoured + +`DEV_KUBECONFIG= ./dev.sh run …` still pins `KUBECONFIG` verbatim and takes precedence over +`CLUSTER` (nothing depending on it breaks), but it is **deprecated**: the name says "dev" while the +established workaround set it to a *prod* path, and it produces a single-cluster config whose +context is the ambiguous `default`. Use `CLUSTER=dev|prod`. Invocations that use it print a warning. + ## What `dev.sh` mounts, and why each one matters - `--network host` — Ansible SSHes to the nodes through the pfSense NAT (`192.168.2.102:225x`) and @@ -61,18 +139,16 @@ SSH, which `--network host` provides. So: run everything in here. The host needs only podman (or docker). -## kubectl inside the container — known gap +## kubectl inside the container — was a known gap, now fixed -`dev.sh` sets `KUBECONFIG` to `/work/infra/ansible/.cache/dev.kubeconfig`, a **libvirt-era path -that no longer exists**, and does not mount `~/.kube`. Override it explicitly: +`dev.sh` used to hard-wire `KUBECONFIG` to `dev.kubeconfig` with no supported way to reach prod, and +it does not mount `~/.kube`. It now selects between `infra/ansible/.cache/dev.kubeconfig` and +`prod.kubeconfig` via `CLUSTER=dev|prod` (see above) and merges both under distinct context names. -```bash -DEV_KUBECONFIG=/work/path/to/break-glass.kubeconfig ./dev.sh run kubectl get nodes -``` - -Use a **break-glass cert kubeconfig**, not the `k3s-oidc` context — OIDC login needs a browser, -which a container shell does not have. See [`../PHASE1-STATUS.md`](../PHASE1-STATUS.md) for both -access paths. (Not yet fixed in `dev.sh`.) +The container still needs a **break-glass cert kubeconfig**, not the `k3s-oidc` context — OIDC login +needs a browser, which a container shell does not have. See +[`../PHASE1-STATUS.md`](../PHASE1-STATUS.md) for both access paths and for how to regenerate these +two files from `/etc/rancher/k3s/k3s.yaml`. ## Notes diff --git a/infra/devcontainer/dev.sh b/infra/devcontainer/dev.sh index 19652fb..578ead9 100755 --- a/infra/devcontainer/dev.sh +++ b/infra/devcontainer/dev.sh @@ -11,7 +11,36 @@ # ./dev.sh build # build the image only # ./dev.sh # build (if needed) + open a shell # ./dev.sh run tofu -version # run one command in the toolbox -# ./dev.sh run kubectl get nodes # talks to the Proxmox k3s cluster directly (no tunnel) +# ./dev.sh run kubectl get nodes # talks to the DEV k3s cluster directly (no tunnel) +# +# CHOOSING A CLUSTER — `CLUSTER=`, defaulting to dev: +# +# ./dev.sh run kubectl get nodes # dev (the default; unchanged behaviour) +# CLUSTER=prod ./dev.sh run kubectl get nodes # prod +# CLUSTER=prod ./dev.sh # interactive shell pointed at prod +# +# The selector is an env-var PREFIX so the target is visible on the same line as the command, and +# every invocation prints a banner naming the cluster it is about to talk to. Inside the container +# KUBECONFIG is a two-file CHAIN, ":": +# +# * the shared merged config holds BOTH clusters under unambiguous names — contexts `k3s-dev` +# and `k3s-prod`, never `default` — so `--context k3s-prod` actually works, and it carries NO +# current-context of its own; +# * the selector is a private per-invocation file holding only `current-context: k3s-$CLUSTER`. +# +# So `kubectl config current-context` still answers the "which cluster am I on?" question honestly, +# but the answer is now PER INVOCATION. That is the point: the previous design stamped +# current-context into the one shared merged file, so two concurrent dev.sh runs with different +# CLUSTER values overwrote each other's default MID-COMMAND — a `CLUSTER=prod` script could read +# prod for its first half and DEV for its second. See infra/devcontainer/kubeconfig.sh for the full +# write-up, and for why the naive `KUBECONFIG=dev:prod` merge is dangerous in the first place (both +# source files name everything `default`). +# +# Concurrency note: `CLUSTER=dev ./dev.sh run ...` and `CLUSTER=prod ./dev.sh run ...` may now run +# at the same time and neither can affect the other's default cluster. +# +# Requesting a cluster whose kubeconfig is missing is a hard ERROR — it never falls back to the +# other cluster. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${HERE}/../.." && pwd)" @@ -24,6 +53,15 @@ source "${HERE}/versions.env" # the pfSense NAT (192.168.2.50:6443 -> kube-vip VIP 192.168.1.49:6443). No SSH tunnel is # opened. infra/ansible/.cache/dev.kubeconfig targets https://192.168.2.50:6443 with # tls-server-name=192.168.1.49 so TLS validates against the VIP's SAN through the NAT. +# Prod is reached the same way via infra/ansible/.cache/prod.kubeconfig (https://192.168.2.9:6443). + +# ------------------------------------------------------------------ cluster selection +CLUSTER="${CLUSTER:-dev}" +case "${CLUSTER}" in + dev|prod) ;; + *) printf '\033[31mERROR: CLUSTER=%s is not a known cluster (dev|prod).\033[0m\n' "${CLUSTER}" >&2; exit 2 ;; +esac +CLUSTER_KUBECONFIG="${REPO_ROOT}/infra/ansible/.cache/${CLUSTER}.kubeconfig" build() { "$ENGINE" build \ @@ -61,6 +99,26 @@ run() { if [ -n "${spec_host}" ] && [ -d "${spec_host}/.git" ]; then spec_mount=(-v "${spec_host}:/home-network-k3s:ro" -e SPEC_REPO=/home-network-k3s) fi + + # Which cluster, said out loud, before anything runs. CLUSTER can be inherited from an exported + # shell variable, so the banner is what stops "I thought I was on dev". + if [ -n "${DEV_KUBECONFIG:-}" ]; then + # SUPERSEDED but still honoured: DEV_KUBECONFIG used to be the only way to reach a cluster other + # than the (then non-existent) default dev path. It wins over CLUSTER so nothing that depends on + # it breaks, but it is opaque about its target — prefer CLUSTER=dev|prod. + printf '\033[33m==> dev.sh: KUBECONFIG pinned by DEV_KUBECONFIG=%s (CLUSTER=%s ignored; DEV_KUBECONFIG is superseded — use CLUSTER=dev|prod)\033[0m\n' \ + "${DEV_KUBECONFIG}" "${CLUSTER}" >&2 + else + # Fail here, on the host, before a container is even started: a missing kubeconfig must never + # degrade into "whatever config we could find", because that is how a prod command lands on dev. + if [ ! -f "${CLUSTER_KUBECONFIG}" ]; then + printf '\033[31mERROR: dev.sh: CLUSTER=%s but its kubeconfig is MISSING: %s\n Refusing to run. dev.sh does NOT fall back to the other cluster.\033[0m\n' \ + "${CLUSTER}" "${CLUSTER_KUBECONFIG}" >&2 + exit 1 + fi + printf '\033[1m==> dev.sh: cluster=%s (kube context k3s-%s)\033[0m\n' "${CLUSTER}" "${CLUSTER}" >&2 + fi + "$ENGINE" run --rm -it \ --network host \ --security-opt label=disable \ @@ -69,8 +127,25 @@ run() { -v "${HOME}/.ssh:/root/.ssh:ro" \ -v "${HOME}/.config/sops:/root/.config/sops" \ -v "${HOME}/.config/home-k3s/ansible-vault-pass:/root/.config/home-k3s/ansible-vault-pass:ro" \ - -e KUBECONFIG="${DEV_KUBECONFIG:-/work/infra/ansible/.cache/dev.kubeconfig}" \ - "$IMAGE" bash -lc 'if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi' dev.sh "$@" + -e CLUSTER="${CLUSTER}" \ + -e DEV_KUBECONFIG="${DEV_KUBECONFIG:-}" \ + "$IMAGE" bash -lc ' + set -e + if [ -n "${DEV_KUBECONFIG:-}" ]; then + export KUBECONFIG="${DEV_KUBECONFIG}" + else + # Regenerated every run: cheap, and it means an updated dev/prod.kubeconfig is picked up + # without anyone remembering to refresh a hand-made merged file. Exits non-zero (set -e) + # if the requested cluster is missing — no silent fallback. + # + # This prints a KUBECONFIG CHAIN (":"), not a + # single path. Export it verbatim; do not split it or take only the last element, or this + # invocation loses its private default context and inherits whatever is shared. + KUBECONFIG="$(/work/infra/devcontainer/kubeconfig.sh "${CLUSTER}")" + export KUBECONFIG + fi + if [ "$#" -gt 0 ]; then exec "$@"; else exec bash -l; fi + ' dev.sh "$@" } case "${1:-shell}" in diff --git a/infra/devcontainer/kubeconfig.sh b/infra/devcontainer/kubeconfig.sh new file mode 100755 index 0000000..3c433ed --- /dev/null +++ b/infra/devcontainer/kubeconfig.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# +# infra/devcontainer/kubeconfig.sh — build ONE merged kubeconfig that can address BOTH clusters +# under unambiguous names, and select which of them is current WITHOUT mutating shared state. +# +# kubeconfig.sh # regenerate the merged file, mint a per-invocation selector, +# # print the KUBECONFIG *chain* to use (selector:merged) +# +# WHY THIS EXISTS — the name collision that makes the obvious fix dangerous +# infra/ansible/.cache/dev.kubeconfig and .cache/prod.kubeconfig each name their context, their +# cluster AND their user `default`. k3s writes them that way. So the "obvious" fix — +# KUBECONFIG=dev.kubeconfig:prod.kubeconfig — is actively unsafe: kubectl's merge keeps the FIRST +# occurrence of a duplicated name, so `default` silently resolves to DEV and `--context default` +# can NEVER select prod. An operator (or an agent) would believe they were on prod while issuing +# commands against the live dev cluster. +# +# Therefore we RENAME on merge rather than relying on file order: +# dev.kubeconfig -> cluster/context `k3s-dev`, user `k3s-dev-admin` +# prod.kubeconfig -> cluster/context `k3s-prod`, user `k3s-prod-admin` +# The rename is done with `yq` (structural, YAML-aware). It is NOT done with sed: the cluster and +# the user entry are both `- name: default` at identical indent, so a textual rename corrupts the +# file. The two renamed copies are then merged by kubectl itself, which is safe once the names are +# distinct. +# +# WHY THE MERGED FILE HAS NO `current-context` — the concurrency bug this shape fixes +# This script used to stamp `current-context: k3s-` into the ONE shared merged file. That +# file lives in the repo's .cache, which every dev.sh container bind-mounts, so it is shared +# mutable state across concurrent invocations. Two `dev.sh` runs with different CLUSTER values +# therefore raced: `CLUSTER=prod dev.sh run