home-network-k3s-impl/bootstrap/standup.sh

#!/usr/bin/env bash
#
# standup.sh — bring a bootstrapped cluster's reconciliation graph all the way up.
#
# ENVIRONMENT-AGNOSTIC (R-ENV-02): it reads everything it needs from the cluster it is pointed at and
# from the authored manifests. There is no dev/prod fork. What makes it safe to run against prod is
# the MANDATORY --expect-server guard, not a name in the filename.
#
# WHY THIS EXISTS
#   `bootstrap/flux/bootstrap.sh` installs Flux and applies the graph. It does NOT finish the job when
#   tiers have been deliberately suspended (a DNS/TLS migration window, an incident recovery) or when
#   a HelmRelease has wedged. Those last steps were living in shell history and in a human's head,
#   which is the same failure the Kyverno CRD pre-apply had. This is that sequence, in git, ordered,
#   idempotent, and gated.
#
# THE ORDERING IS LOAD-BEARING. Read infra/PROD-STANDUP.md before deviating from it.
#
# USAGE
#   bootstrap/standup.sh --kubeconfig <path> --expect-server <url> [--confirm-destructive] <step...>
#
#   steps:  preflight  crds  unwedge  externaldns  s3check  resume  verify   (or: all)
#
#   --expect-server is REQUIRED and is compared against the live API server before ANY command runs.
#   Pointing this at the wrong cluster is the one mistake with unbounded blast radius: dev is live and
#   shares a Cloudflare zone with prod. The guard is not optional and there is no override flag.
#
#   --confirm-destructive is required by `unwedge` only. Everything else is additive or reversible.
#
set -euo pipefail

# ---------------------------------------------------------------------------- args
KUBECONFIG_ARG="" EXPECT_SERVER="" CONFIRM_DESTRUCTIVE=0 S3_ENDPOINT_OVERRIDE=""
STEPS=()
while [ $# -gt 0 ]; do
  case "$1" in
    --kubeconfig)          KUBECONFIG_ARG="${2:?--kubeconfig needs a value}"; shift 2 ;;
    --expect-server)       EXPECT_SERVER="${2:?--expect-server needs a value}"; shift 2 ;;
    --confirm-destructive) CONFIRM_DESTRUCTIVE=1; shift ;;
    --s3-endpoint)         S3_ENDPOINT_OVERRIDE="${2:?--s3-endpoint needs a value}"; shift 2 ;;
    -h|--help)             sed -n '2,30p' "$0"; exit 0 ;;
    -*)                    echo "ERROR: unknown flag '$1'" >&2; exit 2 ;;
    *)                     STEPS+=("$1"); shift ;;
  esac
done

[ -n "${KUBECONFIG_ARG}" ] || { echo "ERROR: --kubeconfig is required." >&2; exit 2; }
[ -n "${EXPECT_SERVER}" ]  || { echo "ERROR: --expect-server is required (safety guard)." >&2; exit 2; }
[ "${#STEPS[@]}" -gt 0 ]   || { echo "ERROR: no step given. One of: preflight crds unwedge externaldns s3check resume verify all" >&2; exit 2; }

script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/.." && pwd))"
cd "${repo_root}"

export KUBECONFIG="${KUBECONFIG_ARG}"
[ -f "${KUBECONFIG}" ] || { echo "ERROR: kubeconfig not found: ${KUBECONFIG}" >&2; exit 1; }

# Where DNS snapshots go. This MUST be a path that survives between invocations: each `dev.sh run` is
# a FRESH container, so a $TMPDIR default silently threw the baseline away and made `verify`'s
# before/after comparison — the check that detects damage to the OTHER cluster's records — impossible
# to run across steps. The repo is bind-mounted, so a repo-relative dir persists on the host.
# Gitignored: these are run artifacts, not authored state.
SNAP_DIR="${STANDUP_SNAP_DIR:-${repo_root}/.standup-dns}"
mkdir -p "${SNAP_DIR}"

say()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
info() { printf '    %s\n' "$*"; }
die()  { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
warn() { printf '\033[33mWARN: %s\033[0m\n' "$*" >&2; }

for bin in kubectl flux helm yq sops; do
  command -v "$bin" >/dev/null 2>&1 || die "'$bin' not found — run this inside infra/devcontainer."
done

# ------------------------------------------------------------------ THE SAFETY GUARD
# Checked once, before any step, and re-checked by every mutating helper. The failure mode this
# prevents is running a prod bring-up against the LIVE dev cluster, whose public DNS records prod
# has already destroyed once (2026-09-04).
live_server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
[ "${live_server}" = "${EXPECT_SERVER}" ] || die \
  "TARGET MISMATCH — refusing to run.
       --expect-server : ${EXPECT_SERVER}
       live API server : ${live_server}
       The kubeconfig does not point where you think it does. Nothing has been changed."
say "target confirmed: ${live_server}"

# Dependency order. Flux enforces the edges itself via dependsOn; resuming in this order simply means
# each wait is meaningful instead of blocking on a dependency that is still suspended.
#   flux-system -> cilium -> platform -> platform-config -> {leaf tiers, apps-wave-0} -> wave-1 -> wave-2
TIER_ORDER=(platform platform-config oauth2-proxy headlamp homepage crafty apps-wave-0 apps-wave-1 apps-wave-2 apps-vms)
# velero is NOT in that list on purpose — see step `resume`: it consumes S3_BACKUP_ENDPOINT, which on
# prod is an intentionally-invalid TBD placeholder pending an operator decision.

cvar() { kubectl -n flux-system get cm cluster-vars -o jsonpath="{.data.$1}" 2>/dev/null || true; }

k_ready()  { kubectl -n flux-system get kustomization "$1" \
               -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true; }
k_msg()    { kubectl -n flux-system get kustomization "$1" \
               -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true; }
k_susp()   { kubectl -n flux-system get kustomization "$1" -o jsonpath='{.spec.suspend}' 2>/dev/null || true; }

wait_ready() { # <kustomization> <timeout-seconds>
  local n="$1" t="${2:-600}" i=0
  while [ "$i" -lt "$t" ]; do
    [ "$(k_ready "$n")" = "True" ] && { info "$n Ready (${i}s)"; return 0; }
    sleep 10; i=$((i+10))
  done
  warn "$n did NOT reach Ready in ${t}s: $(k_msg "$n")"
  return 1
}

# Snapshot the Cloudflare zone. This is how damage to the OTHER cluster's records becomes detectable.
# Queried against the Cloudflare API, never a local resolver: the workstation's own dnsmasq wildcard
# (address=/.home.keithjackson.dev/) answers the whole tree locally and produced a false
# healthy reading during the 2026-09-04 incident and hid the damage for a while.
dns_snapshot() { # <output-file>
  local out="$1" tok zid page tp
  tok="$(sops -d platform/externaldns/cloudflare-api-token.sops.yaml \
         | yq '.stringData."api-token"' | tr -d '"')"
  [ -n "${tok}" ] && [ "${tok}" != "null" ] || die "could not read the Cloudflare token (SOPS/age key available?)"
  zid="$(curl -sS -H "Authorization: Bearer ${tok}" \
        'https://api.cloudflare.com/client/v4/zones?name=keithjackson.dev' | yq '.result[0].id' | tr -d '"')"
  [ -n "${zid}" ] && [ "${zid}" != "null" ] || die "Cloudflare zone lookup failed"
  page=1; : > "${out}"
  while :; do
    local r; r="$(curl -sS -H "Authorization: Bearer ${tok}" \
      "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?per_page=100&page=${page}")"
    printf '%s' "${r}" | yq -o=json '.result' \
      | yq -r '.[] | (.type + "|" + .name + "|" + .content)' | tr -d '"' >> "${out}"
    tp="$(printf '%s' "${r}" | yq '.result_info.total_pages' | tr -d '"')"
    [ "${page}" -ge "${tp:-1}" ] && break
    page=$((page+1))
  done
  # Strip CRs, and sort under a FIXED collation. Both matter, and both were learned the hard way:
  #   - a stray \r makes every line differ from a baseline captured without one;
  #   - locale-dependent sort ordering puts `*.keithjackson.dev` in a different position on the host
  #     than in the container, so a diff reports pairs of IDENTICAL lines as changed. That produced a
  #     "RECORDS OUTSIDE ... CHANGED" alarm on a zone where nothing had changed at all.
  # A regression check that cries wolf is worse than none — it trains you to skip the one alarm that
  # matters. LC_ALL=C makes the ordering reproducible everywhere.
  tr -d '\r' < "${out}" | grep -vE '^\s*$' | LC_ALL=C sort -o "${out}"
  info "$(wc -l < "${out}") records -> ${out}"
}

dns_diff() { # <before> <after>  — returns non-zero if anything changed
  if diff -u "$1" "$2" > "${SNAP_DIR}/dns.diff"; then
    info "Cloudflare zone IDENTICAL — nothing created, modified or deleted."
    return 0
  fi
  warn "Cloudflare zone CHANGED:"; cat "${SNAP_DIR}/dns.diff"
  return 1
}

# ============================================================================ steps
step_preflight() {
  say "PREFLIGHT (read-only)"
  info "nodes Ready: $(kubectl get nodes --no-headers | awk '$2=="Ready"' | wc -l)/$(kubectl get nodes --no-headers | wc -l)"
  info "flux controllers Running: $(kubectl -n flux-system get pods --no-headers | awk '$3=="Running"' | wc -l)"
  # Count BOTH groups the Kyverno chart ships. Grepping only 'kyverno.io' undercounts by 2 — the
  # policy-report types are in the wgpolicyk8s.io group — which reads as "CRDs are missing" when the
  # pre-apply in fact succeeded. Expect 22 for kyverno 3.9.x.
  info "kyverno CRDs present: $(kubectl get crd --no-headers 2>/dev/null | grep -cE 'kyverno\.io|wgpolicyk8s\.io' || true) (expect 22 for kyverno 3.9.x)"

  local base owner filter s3 acme
  base="$(cvar BASE_DOMAIN)"; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
  filter="$(cvar EXTERNALDNS_DOMAIN_FILTER)"; s3="$(cvar S3_BACKUP_ENDPOINT)"; acme="$(cvar ACME_CLUSTER_ISSUER)"
  say "cluster-vars guards"
  info "BASE_DOMAIN                = ${base}"
  info "EXTERNALDNS_TXT_OWNER_ID   = ${owner}"
  info "EXTERNALDNS_DOMAIN_FILTER  = ${filter}"
  info "ACME_CLUSTER_ISSUER        = ${acme}"
  info "S3_BACKUP_ENDPOINT         = ${s3}"

  # THE GUARD THAT MATTERS. Two clusters sharing one Cloudflare zone are isolated by txtOwnerId and
  # by serving disjoint hostnames — NOT by domainFilters (finding F-DNS1: a subdomain filter drops the
  # parent zone and publishes nothing). Both clusters inheriting the chart default `default` is the
  # single root cause of the 2026-09-04 incident, in which prod deleted every one of dev's records.
  # preflight is a DIAGNOSTIC: it must always finish and report. The only thing it refuses over is the
  # target guard, because that one means you are pointed at the wrong cluster. Everything else is
  # reported and warned about here, and REFUSED on the mutating path (step_externaldns) where acting
  # on a bad value is what actually causes harm. Dying here instead hid the rest of dev's health
  # report behind a condition that was, for dev, simply the long-standing status quo.
  if [ -z "${owner}" ]; then
    warn "EXTERNALDNS_TXT_OWNER_ID is EMPTY — external-dns falls back to the chart default 'default'.
       Safe only if this is the sole cluster publishing into the zone. 'externaldns' will refuse."
  elif [ "${owner}" = "unset-see-cluster-vars" ]; then
    warn "EXTERNALDNS_TXT_OWNER_ID is still the base sentinel — the overlay did not apply.
       'externaldns' will refuse until a real per-environment value is set."
  elif [ "${owner}" = "default" ]; then
    # NOT fatal here, because `default` is DEV's legitimate, established owner id — its live records
    # are registered under it and cannot be renamed without orphaning them. Dying on it would make
    # this script refuse to run against dev at all, which would be an environment fork (R-ENV-02) in
    # a script whose whole claim is that it has none. preflight only READS, so it warns; the
    # MUTATING path (step_externaldns) still refuses outright.
    warn "EXTERNALDNS_TXT_OWNER_ID is 'default' — the Helm chart's default value.
       That is correct for DEV (its records are registered under it) and WRONG for any second cluster
       sharing this zone: two clusters both defaulting to 'default' is the exact root cause of the
       2026-09-04 incident, in which prod treated dev's TXT registry entries as its own and
       policy: sync deleted the records behind them."
  else
    info "owner id is distinct and non-default: OK"
  fi

  case "${s3}" in
    ""|TBD-*) warn "S3_BACKUP_ENDPOINT is unset/placeholder ('${s3}'). velero AND apps-wave-0 consume it.
       velero will be SKIPPED by the resume step. apps-wave-0 will still apply (the token IS defined,
       so strict substitution succeeds) but anything writing to that endpoint fails at RUNTIME, not at
       apply time — so a green apps-wave-0 does NOT mean backups work. Operator decision required." ;;
    *) info "S3 backup endpoint set: OK" ;;
  esac

  # COLD-START CRD ORDERING. Flux server-side dry-runs EVERY object in a Kustomization before applying
  # any of them, so a single unknown `kind` aborts the whole tier. platform-config applies the CDI CR
  # *and* CDI StorageProfiles in the same pass — but the StorageProfile CRD does not exist until the
  # cdi-operator has reconciled that CR. On an established cluster the CRDs are long since there and
  # this is invisible; on a COLD one it is a deadlock, and it is the same class of bug as Kyverno's
  # 5.7 MiB CRD bundle. Report it here so `resume` does not burn 15 minutes discovering it.
  say "CRDs that later tiers apply resources against"
  local crd missing=0
  for crd in storageprofiles.cdi.kubevirt.io cdis.cdi.kubevirt.io kubevirts.kubevirt.io \
             volumesnapshotclasses.snapshot.storage.k8s.io; do
    if kubectl get crd "${crd}" >/dev/null 2>&1; then
      printf '    %-52s present\n' "${crd}"
    else
      printf '    \033[33m%-52s MISSING\033[0m\n' "${crd}"; missing=1
    fi
  done
  if [ "${missing}" = "1" ]; then
    warn "a MISSING crd above will fail platform-config's dry-run and block every tier below it.
       CDI's own CRDs (StorageProfile, DataVolume, ...) are created by cdi-operator only AFTER it
       reconciles the CDI CR. Check that the CDI CR is applied and cdi-operator is Available; if the
       CRD appeared only recently this can also be a stale discovery cache in kustomize-controller,
       which a forced reconcile clears."
    kubectl -n cdi get cdi,deploy 2>/dev/null | head -12 || info "(no cdi namespace yet)"
  fi

  say "tiers"
  local n
  for n in flux-system cilium "${TIER_ORDER[@]}" velero; do
    printf '    %-16s suspend=%-6s ready=%-6s %s\n' "$n" "$(k_susp "$n")" "$(k_ready "$n")" "$(k_msg "$n" | head -c 90)"
  done

  say "HelmReleases not Ready"
  kubectl get helmrelease -A \
    -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,MSG:.status.conditions[?\(@.type==\"Ready\"\)].message \
    --no-headers 2>/dev/null | grep -v ' True ' || info "(all Ready)"

  # SUSPENDED releases are invisible above: a suspended HelmRelease keeps whatever Ready condition it
  # last had, so a STALE `Ready=True` hides the fact that Flux is not managing it at all. external-dns
  # is deliberately in this state until `standup.sh externaldns` runs.
  say "HelmReleases SUSPENDED (Flux is not reconciling these — a stale Ready=True means nothing)"
  kubectl get helmrelease -A \
    -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend --no-headers 2>/dev/null \
    | awk '$3=="true"' || true
}

step_crds() {
  say "CRD PRE-APPLY (idempotent)"
  # Charts whose CRD bundle exceeds kubectl's 256 KiB annotation limit set crds.install=false, so
  # nothing installs them and `platform` can never go Ready. See bootstrap/flux/preapply-crds.sh.
  bash "${script_dir}/flux/preapply-crds.sh"

  # ---------------------------------------------------------------- the CDI cold-start deadlock
  # SAME CLASS OF BUG, DIFFERENT OPERATOR. `platform-config` applies the CDI CR *and*
  # platform/kubevirt/cdi-storageprofiles.yaml in ONE pass. But StorageProfile's CRD does not exist
  # until cdi-operator has reconciled that CR — and Flux server-side dry-runs EVERY object before
  # applying ANY of them, so the unknown kind aborts the whole tier, the CR is never applied, the CRD
  # is never created, and platform-config can never converge. A true deadlock, invisible on an
  # established cluster (the CRDs are long since there) and fatal on a cold one.
  #
  # Breaking it needs exactly one thing: get the CDI CR in. Then cdi-operator creates the remaining
  # CDI CRDs and platform-config's next reconcile dry-runs clean. We apply ONLY the CR — the minimum
  # — and leave every other object to Flux. The CR carries no ${} tokens, so no substitution is
  # needed and applying it here cannot diverge from what Flux would render.
  local cdi_cr="${repo_root}/platform/kubevirt/cdi-cr.yaml"
  if kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1; then
    info "storageprofiles CRD already present — CDI is past its cold start"
  elif [ ! -f "${cdi_cr}" ]; then
    warn "expected ${cdi_cr} and it is missing — skipping the CDI unblock"
  else
    say "CDI cold-start unblock: applying the CDI CR so cdi-operator can create its CRDs"
    kubectl get ns cdi >/dev/null 2>&1 || die "namespace 'cdi' absent — the platform tier has not applied the CDI operator yet. Run 'resume' far enough to make platform Ready first."
    kubectl apply --server-side --force-conflicts -f "${cdi_cr}"
    local i=0
    while [ "$i" -lt 300 ]; do
      kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 && break
      sleep 10; i=$((i+10))
    done
    kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 \
      || die "cdi-operator did not create storageprofiles.cdi.kubevirt.io within ${i}s.
       Inspect: kubectl -n cdi get cdi,pods,deploy — platform-config stays blocked until it exists."
    kubectl wait --for=condition=Established --timeout=60s crd/storageprofiles.cdi.kubevirt.io >/dev/null
    info "storageprofiles.cdi.kubevirt.io created and Established (${i}s)"
  fi
}

step_unwedge() {
  say "UNWEDGE STUCK HELM RELEASES"
  # A Helm release left in uninstalling/pending-* cannot be advanced by helm-controller: it reports
  # StateError ("unable to determine state for release with status 'uninstalling'") forever, which
  # keeps `platform` (wait: true) not-Ready and every tier below it blocked.
  #
  # How prod got here: Kyverno's install timed out because its pods were crash-looping with no CRDs;
  # install.remediation.retries:3 then fired an uninstall, and that uninstall timed out too.
  #
  # Pre-applied CRDs live OUTSIDE the Helm release and SURVIVE this — that is precisely why the
  # pre-apply is a separate step and must run before this one.
  local found=0 ns rel st
  while read -r ns rel st; do
    [ -n "${ns:-}" ] || continue
    found=1
    warn "${ns}/${rel} is '${st}'"
    [ "${CONFIRM_DESTRUCTIVE}" = "1" ] || {
      info "DRY RUN. This DELETES the release's Kubernetes objects and lets Flux install cold."
      info "Re-run with --confirm-destructive to act."
      continue
    }
    say "clearing ${ns}/${rel}"
    # --no-hooks and --wait=false: the WAIT is what hung the original remediation, so do not repeat it.
    helm uninstall "${rel}" -n "${ns}" --no-hooks --wait=false || warn "helm uninstall returned non-zero; continuing to check state"
    if helm list -n "${ns}" --all -q 2>/dev/null | grep -qx "${rel}"; then
      warn "${ns}/${rel} still present after uninstall — inspect manually, NOT deleting release secrets by hand."
    else
      info "${ns}/${rel} cleared"
      flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile of ${ns}/${rel} did not complete in 10m"
    fi
  done < <(
    for ns in $(kubectl get helmrelease -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"\n"}{end}' | sort -u); do
      helm list -n "${ns}" --all -o json 2>/dev/null \
        | yq -r '.[] | select(.status | test("uninstalling|pending-install|pending-upgrade|pending-rollback")) | "'"${ns}"' " + .name + " " + .status' 2>/dev/null || true
    done
  )
  [ "${found}" = "1" ] || info "no wedged releases found."
}

step_externaldns() {
  say "EXTERNAL-DNS — SAFE UPGRADE (prove the owner id BEFORE the pod can run)"
  # THE HAZARD: a Helm upgrade sets replicas back to the chart default (1) and applies the new args in
  # ONE operation. There is no window in which to check the result first. If --txt-owner-id is missing
  # the pod starts as owner `default`, claims the other cluster's TXT registry entries, and under
  # `policy: sync` DELETES the records behind them. That is the 2026-09-04 incident.
  #
  # THE FIX IS TO VERIFY IN ADVANCE, NOT AFTER: render the chart with the LIVE HelmRelease values and
  # assert the flag appears. Rendering is deterministic and touches nothing, so this converts a race
  # into a precondition.
  local ns=external-dns rel=external-dns
  local owner chart ver repo vals
  owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
  [ -n "${owner}" ] && [ "${owner}" != "default" ] && [ "${owner}" != "unset-see-cluster-vars" ] \
    || die "refusing: EXTERNALDNS_TXT_OWNER_ID is '${owner}'"

  chart="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.chart}')"
  ver="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.version}')"
  repo="$(yq eval-all --no-doc 'select(.kind == "HelmRepository") | .spec.url' platform/externaldns/helmrepository.yaml | tr -d '"')"
  [ -n "${repo}" ] || die "could not derive the external-dns chart repo url"

  vals="${SNAP_DIR}/ed-values.yaml"
  kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.values}' | yq -P > "${vals}"
  info "chart=${chart} version=${ver} repo=${repo}"
  info "live spec.values.txtOwnerId = $(yq '.txtOwnerId' "${vals}")"

  say "rendering the chart with the LIVE values to prove the flag"
  local rendered="${SNAP_DIR}/ed-rendered.yaml"
  helm template "${rel}" "${chart}" --repo "${repo}" --version "${ver}" -n "${ns}" -f "${vals}" > "${rendered}"
  local args; args="$(yq -r 'select(.kind == "Deployment") | .spec.template.spec.containers[0].args[]' "${rendered}")"
  printf '%s\n' "${args}" | sed 's/^/      /'
  printf '%s\n' "${args}" | grep -qx -- "--txt-owner-id=${owner}" \
    || die "the render does NOT contain --txt-owner-id=${owner}.
       An upgrade would start external-dns as owner 'default' and delete the other cluster's records.
       REFUSING. Fix the chart values key before retrying."
  info "PROVEN: the upgrade will carry --txt-owner-id=${owner}"

  say "snapshotting the Cloudflare zone (baseline for the post-check)"
  dns_snapshot "${SNAP_DIR}/dns-before.txt"

  say "resuming + forcing the upgrade"
  kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.suspend}' | grep -q true \
    && flux resume helmrelease "${rel}" -n "${ns}" || info "(HelmRelease was not suspended)"
  flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile did not complete in 10m"

  say "verifying the LIVE deployment args"
  local live; live="$(kubectl -n "${ns}" get deploy "${rel}" -o jsonpath='{.spec.template.spec.containers[0].args}' 2>/dev/null || true)"
  printf '    %s\n' "${live}"
  printf '%s' "${live}" | grep -q -- "--txt-owner-id=${owner}" \
    || { kubectl -n "${ns}" scale deploy "${rel}" --replicas=0 || true
         die "live args LACK --txt-owner-id=${owner}. Scaled to 0. Investigate before scaling up."; }
  info "live deployment carries --txt-owner-id=${owner}"

  say "post-check: did anything in the shared zone change?"
  sleep 45   # external-dns --interval=1m; give it a full cycle to act before judging it safe
  dns_snapshot "${SNAP_DIR}/dns-after.txt"
  dns_diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-after.txt" \
    || warn "records changed — expected ONLY additions under this cluster's own BASE_DOMAIN ($(cvar BASE_DOMAIN)).
       Any change to a name outside it, or to a TXT entry owned by another cluster, is a REGRESSION."
}

step_s3check() {
  say "S3 BACKUP ENDPOINT — reachability FROM INSIDE THE CLUSTER"
  # Velero runs in-cluster, so the only reachability that counts is the pods' own. Probing from the
  # workstation proves nothing either way: prod sits on the SOHO network (pfSense's WAN) while the
  # dev LAN's MinIO sits inside pfSense, so host and pod egress differ; a healthy reading there
  # network, and a healthy result there has already misled this project once.
  local ep live; live="$(cvar S3_BACKUP_ENDPOINT)"
  info "live S3_BACKUP_ENDPOINT = ${live:-<unset>}"
  if [ -n "${S3_ENDPOINT_OVERRIDE}" ]; then
    # Probe a CANDIDATE endpoint before adopting it. The whole point is to find out whether a proposed
    # backup target is reachable from this cluster's pods BEFORE authoring it into cluster-vars —
    # otherwise the first evidence is silently-missing backups.
    ep="${S3_ENDPOINT_OVERRIDE}"
    info "probing CANDIDATE endpoint from --s3-endpoint: ${ep}"
  else
    ep="${live}"
  fi
  case "${ep}" in
    ""|TBD-*) die "no usable endpoint to probe (live='${live}'). Pass --s3-endpoint <url> to test a candidate." ;;
  esac

  local ns=velero pod=s3check-$$
  kubectl get ns "${ns}" >/dev/null 2>&1 || ns=default
  info "probing from a throwaway pod in namespace '${ns}'"
  # --restart=Never + explicit delete in a trap: a probe must never leave a pod behind on a cluster.
  trap 'kubectl -n "${ns}" delete pod "${pod}" --ignore-not-found --wait=false >/dev/null 2>&1 || true' RETURN
  if kubectl -n "${ns}" run "${pod}" --restart=Never --image=curlimages/curl:8.11.1 --command --quiet \
       -- curl -sS -o /dev/null -w '%{http_code}' --max-time 15 "${ep}/minio/health/live" >/dev/null 2>&1; then
    local i=0 phase=""
    while [ "$i" -lt 90 ]; do
      phase="$(kubectl -n "${ns}" get pod "${pod}" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
      case "${phase}" in Succeeded|Failed) break ;; esac
      sleep 5; i=$((i+5))
    done
    local out; out="$(kubectl -n "${ns}" logs "${pod}" 2>/dev/null || true)"
    info "pod phase=${phase:-unknown} response=${out:-<none>}"
    case "${out}" in
      200|2??) info "REACHABLE: ${ep} answered from inside the cluster." ;;
      *) warn "NOT REACHABLE from inside the cluster (phase=${phase}, body='${out}').
       Velero would install cleanly and then fail every backup at RUNTIME — a tier that looks healthy
       while no backup exists. Fix routing, or give this environment its own endpoint, before
       resuming velero." ;;
    esac
  else
    warn "could not create the probe pod (admission policy? image pull?). Reachability UNKNOWN."
  fi
}

step_resume() {
  say "RESUME TIERS (dependency order)"
  local n s3; s3="$(cvar S3_BACKUP_ENDPOINT)"
  for n in "${TIER_ORDER[@]}"; do
    if [ "$(k_susp "$n")" = "true" ]; then
      info "resuming $n"; flux resume kustomization "$n" --timeout=1s >/dev/null 2>&1 || true
    else
      info "$n already unsuspended"
    fi
    wait_ready "$n" 900 || { warn "stopping at $n — a partial, understood bring-up beats a forced green."; return 1; }
  done

  case "${s3}" in
    ""|TBD-*) warn "velero left SUSPENDED: S3_BACKUP_ENDPOINT='${s3}' is not a usable endpoint.
       Resuming it would create a BackupStorageLocation that can never reach a bucket, i.e. a tier
       that reports Ready-ish while backups silently do not exist. Set the real prod endpoint first." ;;
    *) info "resuming velero"; flux resume kustomization velero --timeout=1s >/dev/null 2>&1 || true
       wait_ready velero 900 || warn "velero did not converge" ;;
  esac
}

step_verify() {
  say "VERIFY"
  local base; base="$(cvar BASE_DOMAIN)"
  info "BASE_DOMAIN = ${base}"
  say "kustomizations"
  kubectl get kustomization -A \
    -o custom-columns=NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status --no-headers
  say "certificates (must be Ready before any hostname is trusted)"
  kubectl get certificate -A --no-headers 2>/dev/null || info "(none yet)"
  say "pods not Running/Completed"
  kubectl get pods -A --no-headers | awk '$4!="Running" && $4!="Completed"' || true
  info "total pods: $(kubectl get pods -A --no-headers | wc -l)"
  say "gateway / LoadBalancer VIPs"
  kubectl get svc -A --field-selector spec.type=LoadBalancer --no-headers 2>/dev/null || true
  say "published DNS for this cluster's own domain"
  local owner; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
  dns_snapshot "${SNAP_DIR}/dns-verify.txt"
  info "records under ${base}: $(grep -c "${base}" "${SNAP_DIR}/dns-verify.txt" || true)"

  # Every TXT registry entry this cluster publishes must carry OUR owner id. One reading another
  # cluster's id means we have claimed its records and `policy: sync` will delete them next cycle.
  local foreign
  foreign="$(grep "${base}" "${SNAP_DIR}/dns-verify.txt" | grep 'heritage=external-dns' \
             | grep -v "external-dns/owner=${owner}" || true)"
  if [ -n "${foreign}" ]; then
    warn "TXT entries under ${base} NOT owned by '${owner}' — this cluster has claimed records it does not own:"
    printf '%s\n' "${foreign}" | sed 's/^/      /'
  else
    info "every TXT registry entry under ${base} reads owner=${owner}: OK"
  fi

  # THE REGRESSION CHECK. Additions under our own BASE_DOMAIN are expected. ANY change to a name
  # outside it means we have touched the other cluster's zone — the 2026-09-04 incident.
  if [ -f "${SNAP_DIR}/dns-before.txt" ]; then
    say "regression check against the pre-standup baseline"
    local outside
    outside="$(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" \
               | grep -E '^[<>]' | grep -v "${base}" || true)"
    if [ -n "${outside}" ]; then
      warn "RECORDS OUTSIDE ${base} CHANGED — investigate before doing anything else:"
      printf '%s\n' "${outside}" | sed 's/^/      /'
    else
      info "no record outside ${base} was added, changed or removed: OK"
      info "added under ${base}: $(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" | grep -c '^>' || true)"
    fi
  else
    warn "no ${SNAP_DIR}/dns-before.txt baseline — cannot prove the other cluster's records are untouched.
       Take one BEFORE the next standup so this check can run."
  fi
}

# ============================================================================ dispatch
for s in "${STEPS[@]}"; do
  case "$s" in
    preflight)   step_preflight ;;
    s3check)     step_s3check ;;
    crds)        step_crds ;;
    unwedge)     step_unwedge ;;
    externaldns) step_externaldns ;;
    resume)      step_resume ;;
    verify)      step_verify ;;
    all)         step_preflight; step_crds; step_unwedge; step_externaldns; step_resume; step_verify ;;
    *)           die "unknown step '$s' (preflight crds unwedge externaldns s3check resume verify all)" ;;
  esac
done

say "done: ${STEPS[*]}"