#!/usr/bin/env bash
#
# standup.sh — bring a bootstrapped cluster's reconciliation graph all the way up.
#
# ENVIRONMENT-AGNOSTIC (R-ENV-02): it reads everything it needs from the cluster it is pointed at and
# from the authored manifests. There is no dev/prod fork. What makes it safe to run against prod is
# the MANDATORY --expect-server guard, not a name in the filename.
#
# WHY THIS EXISTS
#   `bootstrap/flux/bootstrap.sh` installs Flux and applies the graph. It does NOT finish the job when
#   tiers have been deliberately suspended (a DNS/TLS migration window, an incident recovery) or when
#   a HelmRelease has wedged. Those last steps were living in shell history and in a human's head,
#   which is the same failure the Kyverno CRD pre-apply had. This is that sequence, in git, ordered,
#   idempotent, and gated.
#
# THE ORDERING IS LOAD-BEARING. Read infra/PROD-STANDUP.md before deviating from it.
#
# USAGE
#   bootstrap/standup.sh --kubeconfig <path> --expect-server <url> [--confirm-destructive] <step...>
#
#   steps:  preflight  crds  unwedge  externaldns  s3check  resume  verify   (or: all)
#
#   --expect-server is REQUIRED and is compared against the live API server before ANY command runs.
#   Pointing this at the wrong cluster is the one mistake with unbounded blast radius: dev is live and
#   shares a Cloudflare zone with prod. The guard is not optional and there is no override flag.
#
#   --confirm-destructive is required by `unwedge` only. Everything else is additive or reversible.
#
#   --s3-endpoint <url> makes `s3check` probe a CANDIDATE backup endpoint instead of the live
#   cluster-vars value, so a proposed target can be PROVEN before it is authored into cluster-vars.
#   `s3check` writes and then deletes ONE throwaway object under `<prefix>/.s3check/`; it changes
#   nothing else and applies nothing. See infra/BACKUP-R2.md.
#
set -euo pipefail

# ---------------------------------------------------------------------------- args
KUBECONFIG_ARG="" EXPECT_SERVER="" CONFIRM_DESTRUCTIVE=0 S3_ENDPOINT_OVERRIDE=""
STEPS=()
while [ $# -gt 0 ]; do
  case "$1" in
    --kubeconfig)          KUBECONFIG_ARG="${2:?--kubeconfig needs a value}"; shift 2 ;;
    --expect-server)       EXPECT_SERVER="${2:?--expect-server needs a value}"; shift 2 ;;
    --confirm-destructive) CONFIRM_DESTRUCTIVE=1; shift ;;
    --s3-endpoint)         S3_ENDPOINT_OVERRIDE="${2:?--s3-endpoint needs a value}"; shift 2 ;;
    -h|--help)             sed -n '2,30p' "$0"; exit 0 ;;
    -*)                    echo "ERROR: unknown flag '$1'" >&2; exit 2 ;;
    *)                     STEPS+=("$1"); shift ;;
  esac
done

[ -n "${KUBECONFIG_ARG}" ] || { echo "ERROR: --kubeconfig is required." >&2; exit 2; }
[ -n "${EXPECT_SERVER}" ]  || { echo "ERROR: --expect-server is required (safety guard)." >&2; exit 2; }
[ "${#STEPS[@]}" -gt 0 ]   || { echo "ERROR: no step given. One of: preflight crds unwedge externaldns s3check resume verify all" >&2; exit 2; }

script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/.." && pwd))"
cd "${repo_root}"

export KUBECONFIG="${KUBECONFIG_ARG}"
[ -f "${KUBECONFIG}" ] || { echo "ERROR: kubeconfig not found: ${KUBECONFIG}" >&2; exit 1; }

# Where DNS snapshots go. This MUST be a path that survives between invocations: each `dev.sh run` is
# a FRESH container, so a $TMPDIR default silently threw the baseline away and made `verify`'s
# before/after comparison — the check that detects damage to the OTHER cluster's records — impossible
# to run across steps. The repo is bind-mounted, so a repo-relative dir persists on the host.
# Gitignored: these are run artifacts, not authored state.
SNAP_DIR="${STANDUP_SNAP_DIR:-${repo_root}/.standup-dns}"
mkdir -p "${SNAP_DIR}"

say()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
info() { printf '    %s\n' "$*"; }
die()  { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
warn() { printf '\033[33mWARN: %s\033[0m\n' "$*" >&2; }

for bin in kubectl flux helm yq sops; do
  command -v "$bin" >/dev/null 2>&1 || die "'$bin' not found — run this inside infra/devcontainer."
done

# ------------------------------------------------------------------ THE SAFETY GUARD
# Checked once, before any step, and re-checked by every mutating helper. The failure mode this
# prevents is running a prod bring-up against the LIVE dev cluster, whose public DNS records prod
# has already destroyed once (2026-09-04).
live_server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
[ "${live_server}" = "${EXPECT_SERVER}" ] || die \
  "TARGET MISMATCH — refusing to run.
       --expect-server : ${EXPECT_SERVER}
       live API server : ${live_server}
       The kubeconfig does not point where you think it does. Nothing has been changed."
say "target confirmed: ${live_server}"

# Dependency order. Flux enforces the edges itself via dependsOn; resuming in this order simply means
# each wait is meaningful instead of blocking on a dependency that is still suspended.
#   flux-system -> cilium -> platform -> platform-config -> platform-config-cdi
#                                     -> {leaf tiers, apps-wave-0} -> wave-1 -> wave-2 -> apps-vms
#
# `platform-config-cdi` MUST stay in this list and MUST stay before apps-vms. It carries the CDI
# StorageProfiles, which were split out of platform-config because StorageProfile's CRD does not exist
# until cdi-operator has reconciled the CDI CR (clusters/base/platform-config-cdi.yaml). apps-vms
# dependsOn it: apps/vms/unifi provisions on `longhorn-vm`, and without the profile applied CDI infers
# ReadWriteOnce instead of the authored ReadWriteMany — permanently non-live-migratable for that disk.
# Omitting it here would make `resume` return success without ever waiting on the tier that guarantees it.
TIER_ORDER=(platform platform-config platform-config-cdi oauth2-proxy headlamp homepage crafty apps-wave-0 apps-wave-1 apps-wave-2 apps-vms)
# velero is NOT in that list on purpose — see step `resume`: it consumes S3_BACKUP_ENDPOINT, which is
# an intentionally-invalid placeholder until the operator supplies the real R2 endpoint.

cvar() { kubectl -n flux-system get cm cluster-vars -o jsonpath="{.data.$1}" 2>/dev/null || true; }

k_ready()  { kubectl -n flux-system get kustomization "$1" \
               -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true; }
k_msg()    { kubectl -n flux-system get kustomization "$1" \
               -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true; }
k_susp()   { kubectl -n flux-system get kustomization "$1" -o jsonpath='{.spec.suspend}' 2>/dev/null || true; }

wait_ready() { # <kustomization> <timeout-seconds>
  local n="$1" t="${2:-600}" i=0
  while [ "$i" -lt "$t" ]; do
    [ "$(k_ready "$n")" = "True" ] && { info "$n Ready (${i}s)"; return 0; }
    sleep 10; i=$((i+10))
  done
  warn "$n did NOT reach Ready in ${t}s: $(k_msg "$n")"
  return 1
}

# Snapshot the Cloudflare zone. This is how damage to the OTHER cluster's records becomes detectable.
# Queried against the Cloudflare API, never a local resolver: the workstation's own dnsmasq wildcard
# (address=/.home.keithjackson.dev/) answers the whole tree locally and produced a false
# healthy reading during the 2026-09-04 incident and hid the damage for a while.
dns_snapshot() { # <output-file>
  local out="$1" tok zid page tp
  tok="$(sops -d platform/externaldns/cloudflare-api-token.sops.yaml \
         | yq '.stringData."api-token"' | tr -d '"')"
  [ -n "${tok}" ] && [ "${tok}" != "null" ] || die "could not read the Cloudflare token (SOPS/age key available?)"
  zid="$(curl -sS -H "Authorization: Bearer ${tok}" \
        'https://api.cloudflare.com/client/v4/zones?name=keithjackson.dev' | yq '.result[0].id' | tr -d '"')"
  [ -n "${zid}" ] && [ "${zid}" != "null" ] || die "Cloudflare zone lookup failed"
  page=1; : > "${out}"
  while :; do
    local r; r="$(curl -sS -H "Authorization: Bearer ${tok}" \
      "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?per_page=100&page=${page}")"
    printf '%s' "${r}" | yq -o=json '.result' \
      | yq -r '.[] | (.type + "|" + .name + "|" + .content)' | tr -d '"' >> "${out}"
    tp="$(printf '%s' "${r}" | yq '.result_info.total_pages' | tr -d '"')"
    [ "${page}" -ge "${tp:-1}" ] && break
    page=$((page+1))
  done
  # Strip CRs, and sort under a FIXED collation. Both matter, and both were learned the hard way:
  #   - a stray \r makes every line differ from a baseline captured without one;
  #   - locale-dependent sort ordering puts `*.keithjackson.dev` in a different position on the host
  #     than in the container, so a diff reports pairs of IDENTICAL lines as changed. That produced a
  #     "RECORDS OUTSIDE ... CHANGED" alarm on a zone where nothing had changed at all.
  # A regression check that cries wolf is worse than none — it trains you to skip the one alarm that
  # matters. LC_ALL=C makes the ordering reproducible everywhere.
  tr -d '\r' < "${out}" | grep -vE '^\s*$' | LC_ALL=C sort -o "${out}"
  info "$(wc -l < "${out}") records -> ${out}"
}

dns_diff() { # <before> <after>  — returns non-zero if anything changed
  if diff -u "$1" "$2" > "${SNAP_DIR}/dns.diff"; then
    info "Cloudflare zone IDENTICAL — nothing created, modified or deleted."
    return 0
  fi
  warn "Cloudflare zone CHANGED:"; cat "${SNAP_DIR}/dns.diff"
  return 1
}

# ======================================================================= s3check plumbing
# WHY THIS IS THIS BIG. The previous version of `s3check` curl'd /minio/health/live and reported
# REACHABLE. That probe could not pass against Cloudflare R2 (no such path), and reachability is not
# the property anyone cares about: a reachable endpoint with a wrong key, a missing bucket, or R2's
# checksum rejection still yields a velero tier that reports Ready while NO backup exists. What must
# be proven is the write path Velero and CNPG-barman actually use, with the credentials they actually
# hold, from the network position they actually occupy.

# Pinned. The AWS CLI is a deliberate choice over `minio/mc`:
#   - it prints the S3 ERROR CODE (NoSuchBucket / AccessDenied / InvalidAccessKeyId / InvalidArgument),
#     so a failure can be NAMED and turned into a fix instead of guessed at from an HTTP status;
#   - it exposes the FLEXIBLE-CHECKSUM behaviour that is the R2 trap
#     (AWS_REQUEST_CHECKSUM_CALCULATION), so the trap can be reproduced on purpose here instead of
#     discovered by a silently-failing backup at 03:00;
#   - its image carries bash + coreutils. `minio/mc` ships no sed/awk/diff — that minimalism has
#     already broken one probe in this repo — and it cannot be made to send the checksum headers, so
#     it cannot test for the trap at all.
S3CHECK_IMAGE="amazon/aws-cli:2.36.40"
S3CHECK_RUN_ID="$(date +%s)"

# Throwaway probe pods, tracked so they are removed on EVERY exit path (including `die`). A probe
# must never leave a pod behind on a live cluster.
S3CHECK_PODS=()
s3_cleanup_pods() {
  local p
  for p in ${S3CHECK_PODS[@]+"${S3CHECK_PODS[@]}"}; do
    kubectl -n "${p%%/*}" delete pod "${p#*/}" --ignore-not-found --wait=false >/dev/null 2>&1 || true
  done
  S3CHECK_PODS=()
}
trap s3_cleanup_pods EXIT

# TRUE when the endpoint is not a usable target. `TBD-*` alone was WRONG once the placeholder became
# `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`: that does not START with TBD-, so the old
# prefix pattern matched nothing and `resume` would have resumed velero straight onto a placeholder
# hostname — the exact silent-no-backups failure the guard exists to prevent. Match anywhere.
s3_is_placeholder() { # <endpoint>
  case "$1" in
    ""|*TBD*|*tbd-*|*REPLACE-ME*|*replace-me*|*CHANGE-ME*|*change-me*) return 0 ;;
  esac
  return 1
}

# Refuse any value that is not a plain URL/name fragment. This is BOTH a safety guard — these values
# are interpolated into the probe pod's JSON overrides AND into a shell script that runs inside it, so
# a quote, a backslash or a newline could rewrite either — and a correctness guard: a cluster-vars
# value with a trailing space or a stray quote is a defect, not something to paper over.
s3_safe_token() { # <label> <value>
  case "$2" in
    *[!A-Za-z0-9:/._-]*) die "refusing to probe: $1 contains a character outside [A-Za-z0-9:/._-].
       That is either a corrupt cluster-vars value or an injection attempt. Value (quoted): $(printf '%q' "$2")" ;;
  esac
}

# ---------------------------------------------------------------- the in-pod probe
# Emitted to the pod base64-encoded, so nothing in it has to survive JSON or shell quoting.
# CONTRACT WITH THE CALLER (this is what makes a silent no-op impossible):
#   - every proven property prints exactly one `S3CHECK-OK[n] ...` line and increments n;
#   - every failure prints `S3CHECK-FAIL ...` and is counted;
#   - the LAST line is `S3CHECK-RESULT PASS|FAIL assertions=<n> failures=<n>`.
# The caller independently knows how many assertions this environment must produce and re-counts the
# OK lines itself, so neither a truncated log, a skipped bucket, an empty log, nor a lying result line
# can be read as success. (A CRD guard in this repo once counted a blank line as "1 CRD" and sailed
# past; that class of bug is designed out here rather than commented about.)
s3_probe_script() {
  cat <<'PROBE'
set -u
umask 077
A=0; F=0
ok()   { A=$((A+1)); printf 'S3CHECK-OK[%d] %s\n' "$A" "$*"; }
bad()  { F=$((F+1)); printf 'S3CHECK-FAIL %s\n' "$*"; }
note() { printf 'S3CHECK-NOTE %s\n' "$*"; }
verdict() {
  if [ "${F}" -eq 0 ] && [ "${A}" -gt 0 ]; then
    printf 'S3CHECK-RESULT PASS assertions=%d failures=0\n' "${A}"; exit 0
  fi
  printf 'S3CHECK-RESULT FAIL assertions=%d failures=%d\n' "${A}" "${F}"; exit 1
}

for v in S3EP S3REGION S3TARGETS S3KEY CRED_MODE; do
  eval "x=\${${v}-}"
  [ -n "${x}" ] || { bad "required env ${v} is empty — the probe was launched wrong"; verdict; }
done

export AWS_EC2_METADATA_DISABLED=true
export AWS_PAGER=""
export AWS_DEFAULT_REGION="${S3REGION}"
export AWS_REGION="${S3REGION}"
export AWS_MAX_ATTEMPTS=2
export AWS_RETRY_MODE=standard
export AWS_CONFIG_FILE=/tmp/s3check-aws-config
# Path-style addressing, because both targets need it: MinIO on a bare IP has no virtual-host names,
# and R2's S3 API is documented path-style. Checksums OFF, mirroring `checksumAlgorithm: ""` on the
# Velero BSL — this run must exercise the same wire shape the real backup will use. Written to the
# CONFIG FILE as well as the environment so the setting holds even if one channel is ignored by a
# future CLI version.
{
  echo '[default]'
  echo 'request_checksum_calculation = when_required'
  echo 'response_checksum_validation = when_required'
  echo 's3 ='
  echo '    addressing_style = path'
} > "${AWS_CONFIG_FILE}"

# ---- (1) the credential loads at all -------------------------------------------------
# NEVER print a credential value, in either `key: value` or `key=value` shape. Only booleans,
# byte counts and lengths leave this pod.
case "${CRED_MODE}" in
  file)
    cf="${AWS_SHARED_CREDENTIALS_FILE-}"
    if [ -z "${cf}" ] || [ ! -s "${cf}" ]; then
      bad "credential file '${cf:-<unset>}' is missing or empty — the Secret key did not mount"
    elif ! grep -Eq '^[[:space:]]*\[default\][[:space:]]*$' "${cf}"; then
      bad "credential file has no [default] profile. velero-plugin-for-aws reads an AWS INI file from
        the 'cloud' key of velero-cloud-credentials; without [default] every backup fails at RUNTIME"
    elif ! grep -Eq '^[[:space:]]*aws_access_key_id[[:space:]]*=[[:space:]]*[^[:space:]]' "${cf}" \
      || ! grep -Eq '^[[:space:]]*aws_secret_access_key[[:space:]]*=[[:space:]]*[^[:space:]]' "${cf}"; then
      bad "credential file [default] lacks a non-empty aws_access_key_id / aws_secret_access_key"
    else
      ok "credential LOADS: AWS INI with [default] + both keys ($(wc -c < "${cf}") bytes; values never printed)"
    fi
    ;;
  env)
    if [ -z "${AWS_ACCESS_KEY_ID-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY-}" ]; then
      bad "AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY did not arrive from the Secret (missing ACCESS_KEY_ID / ACCESS_SECRET_KEY key?)"
    else
      ok "credential LOADS: access key id present (${#AWS_ACCESS_KEY_ID} chars; values never printed)"
    fi
    ;;
  *) bad "unknown CRED_MODE '${CRED_MODE}'" ;;
esac
[ "${F}" -eq 0 ] || verdict     # no point talking to S3 with no credential

OUT=""; RC=0
awsx() { OUT="$(aws --endpoint-url "${S3EP}" "$@" 2>&1)"; RC=$?; return 0; }

detail() { printf '%s\n' "${OUT}" | head -n 12 | while IFS= read -r l; do printf 'S3CHECK-DETAIL %s\n' "${l}"; done; }

# Every failure is NAMED and given the fix. An unclassified failure is still a failure.
diagnose() { # <operation label>
  local op="$1"
  case "${OUT}" in
    *InvalidArgument*|*checksum*|*Checksum*)
      bad "${op}: 400 InvalidArgument — THE R2 CHECKSUM TRAP. velero-plugin-for-aws v1.14+ bundles an
        AWS SDK that sends x-amz-sdk-checksum-algorithm / x-amz-checksum-crc32 on every PUT, and
        Cloudflare R2 REJECTS those headers with HTTP 400 InvalidArgument, so every Velero backup and
        every kopia blob upload fails. FIX: checksumAlgorithm: \"\" must be present in the
        backupStorageLocation config in platform/velero/helmrelease.yaml. This probe already disables
        checksums, so seeing this here means the client sent one anyway — treat it as a hard blocker." ;;
    *"Could not connect"*|*EndpointConnectionError*|*ConnectTimeoutError*|*"Failed to establish"*|\
    *"Name or service not known"*|*"nodename nor servname"*|*"Temporary failure in name resolution"*|\
    *"Connection reset"*|*"Read timeout"*)
      bad "${op}: this POD cannot reach ${S3EP} (connect / DNS / timeout). Velero and CNPG would
        install perfectly clean and then fail every backup at RUNTIME — a tier that looks healthy
        while no backup exists. This is the reading that matters: the workstation's own reachability
        is irrelevant (prod sits on the SOHO net, pfSense's WAN; the dev LAN's MinIO sits INSIDE
        pfSense) and a host-side probe has already misled this project once. Fix pod egress/routing,
        or give this environment its own endpoint." ;;
    *NoSuchBucket*|*"(404)"*|*"Not Found"*)
      bad "${op}: the bucket does not exist at this endpoint. Create it before enabling velero/CNPG
        (R2: dashboard -> R2 -> Create bucket; see infra/BACKUP-R2.md)." ;;
    *InvalidAccessKeyId*|*SignatureDoesNotMatch*|*"(401)"*|*Unauthorized*)
      bad "${op}: the endpoint REJECTED these credentials. Wrong key, or a key belonging to a
        different R2 account / MinIO. Re-encrypt the credential Secrets — infra/BACKUP-R2.md." ;;
    *AccessDenied*|*"(403)"*|*Forbidden*)
      bad "${op}: authenticated but FORBIDDEN. The token is not scoped to this bucket. An R2 API token
        needs Object Read & Write on BOTH the velero and cnpg buckets." ;;
    *) bad "${op}: unclassified failure (rc=${RC}). Treated as a FAILURE, never as unknown-so-fine." ;;
  esac
  detail
}

PAY=/tmp/s3check.payload
GOT=/tmp/s3check.got
head -c 4096 /dev/urandom | base64 > "${PAY}"
[ -s "${PAY}" ] || { bad "could not generate a probe payload inside the pod"; verdict; }

TRAP_DONE=0
for t in ${S3TARGETS}; do
  B="${t%%:*}"; P="${t#*:}"
  [ "${P}" = "${B}" ] && P=""
  if [ -n "${P}" ]; then K="${P}/.s3check/${S3KEY}"; else K=".s3check/${S3KEY}"; fi

  # ---- (2) the bucket exists and is LISTABLE. This is the operation Velero's BSL sync and Barman's
  #          WAL scan both perform; ListBuckets deliberately is NOT used, because neither consumer
  #          needs it and a correctly bucket-scoped R2 token may not grant it.
  args=(s3api list-objects-v2 --bucket "${B}" --max-keys 1)
  [ -n "${P}" ] && args+=(--prefix "${P}/")
  awsx "${args[@]}"
  if [ "${RC}" -eq 0 ]; then
    ok "${B}: bucket EXISTS, AUTHENTICATES and is LISTABLE (prefix '${P:-<bucket root>}')"
  else
    diagnose "LIST ${B}"
    continue      # a short assertion count is how the caller learns this target was abandoned
  fi

  # ---- (3) PUT -> GET -> byte-compare -> DELETE. A real write, not a HEAD: HEAD on a bucket proves
  #          nothing about whether an object can be STORED (wrong scope, checksum rejection and
  #          read-only tokens all pass a HEAD and fail a backup).
  awsx s3api put-object --bucket "${B}" --key "${K}" --body "${PAY}"
  if [ "${RC}" -eq 0 ]; then ok "${B}: PUT ${K} accepted (genuine write path)"
  else diagnose "PUT ${B}/${K}"; continue; fi

  rm -f "${GOT}"
  awsx s3api get-object --bucket "${B}" --key "${K}" "${GOT}"
  if [ "${RC}" -eq 0 ]; then ok "${B}: GET ${K} returned an object"
  else diagnose "GET ${B}/${K}"
       aws --endpoint-url "${S3EP}" s3api delete-object --bucket "${B}" --key "${K}" >/dev/null 2>&1 || true
       continue; fi

  sp="$(wc -c < "${PAY}")"; sg="$(wc -c < "${GOT}" 2>/dev/null || echo 0)"
  hp="$(sha256sum < "${PAY}")"; hp="${hp%% *}"
  hg="$(sha256sum < "${GOT}")"; hg="${hg%% *}"
  if [ "${sp}" = "${sg}" ] && [ -n "${hp}" ] && [ "${hp}" = "${hg}" ]; then
    ok "${B}: round trip is BYTE-IDENTICAL (${sp} bytes, sha256 matches)"
  else
    bad "${B}: round trip CORRUPTED — wrote ${sp} bytes (sha256 ${hp}), read back ${sg} bytes (sha256 ${hg})"
  fi

  awsx s3api delete-object --bucket "${B}" --key "${K}"
  if [ "${RC}" -eq 0 ]; then ok "${B}: DELETE ${K} accepted"
  else diagnose "DELETE ${B}/${K}"; fi

  awsx s3api head-object --bucket "${B}" --key "${K}"
  if [ "${RC}" -ne 0 ]; then ok "${B}: scratch object is GONE — the probe left nothing behind"
  else bad "${B}: scratch object ${K} STILL EXISTS after DELETE. Remove it by hand; the probe leaked."; fi

  # ---- (4) the R2 checksum trap, probed ON PURPOSE. Informational by design: the authored BSL sets
  #          checksumAlgorithm: "", so a rejection here does NOT mean backups are broken — it means
  #          that line is LOAD-BEARING. Reported either way so the operator learns which world this
  #          endpoint lives in before, not after, enabling velero.
  if [ "${TRAP_DONE}" -eq 0 ]; then
    TRAP_DONE=1
    TK="${K}.checksum-trap-probe"
    OUT="$(AWS_REQUEST_CHECKSUM_CALCULATION=when_supported \
           aws --endpoint-url "${S3EP}" s3api put-object --bucket "${B}" --key "${TK}" --body "${PAY}" 2>&1)"; RC=$?
    if [ "${RC}" -eq 0 ]; then
      note "checksum probe: this endpoint ACCEPTS a checksum-bearing PUT — the R2 checksum trap is NOT
        present here (expected for MinIO). checksumAlgorithm: \"\" stays harmless."
      aws --endpoint-url "${S3EP}" s3api delete-object --bucket "${B}" --key "${TK}" >/dev/null 2>&1 \
        || note "checksum probe: could NOT delete ${TK} — remove it by hand"
    else
      case "${OUT}" in
        *InvalidArgument*)
          note "checksum probe: R2 CHECKSUM TRAP CONFIRMED at this endpoint — a PUT carrying
        x-amz-sdk-checksum-algorithm / x-amz-checksum-crc32 is rejected with 400 InvalidArgument.
        checksumAlgorithm: \"\" in platform/velero/helmrelease.yaml is LOAD-BEARING: delete it, or bump
        velero-plugin-for-aws without re-checking it, and EVERY backup breaks." ;;
        *) note "checksum probe: inconclusive (rc=${RC}) — neither confirms nor rules out the trap." ;;
      esac
    fi
  fi
done

verdict
PROBE
}

# ---------------------------------------------------------------- run one probe pod
# <ns> <file|env> <secret> <space-separated bucket:prefix targets>  <endpoint> <region>
s3_probe() {
  local ns="$1" mode="$2" sec="$3" targets="$4" ep="$5" region="$6"
  # $$ alone is NOT unique here: inside the dev container this script is frequently PID 1, so two runs
  # would reuse the same pod name AND the same scratch object key. Mix in the epoch second.
  local pod="s3check-$$-${S3CHECK_RUN_ID}-${mode}"

  # What this environment MUST produce: 1 credential assertion + 6 per bucket (list, put, get,
  # byte-compare, delete, delete-verified). Computed HERE, by the caller, from the target list — the
  # pod's own claim is never the only source of the expected count.
  local t n=0
  for t in ${targets}; do n=$((n+1)); done
  local min=$((1 + 6 * n))

  local volsjson="" mountsjson="" creds=""
  case "${mode}" in
    file)  # exactly how velero consumes it: the `cloud` key as an AWS INI credentials FILE
      volsjson='"volumes":[{"name":"s3creds","secret":{"secretName":"'"${sec}"'","items":[{"key":"cloud","path":"credentials"}],"defaultMode":256}}],'
      mountsjson='"volumeMounts":[{"name":"s3creds","mountPath":"/s3creds","readOnly":true}],'
      creds='{"name":"AWS_SHARED_CREDENTIALS_FILE","value":"/s3creds/credentials"},{"name":"CRED_MODE","value":"file"}' ;;
    env)   # exactly how CNPG/barman consumes it: discrete ACCESS_KEY_ID / ACCESS_SECRET_KEY keys
      creds='{"name":"AWS_ACCESS_KEY_ID","valueFrom":{"secretKeyRef":{"name":"'"${sec}"'","key":"ACCESS_KEY_ID"}}},{"name":"AWS_SECRET_ACCESS_KEY","valueFrom":{"secretKeyRef":{"name":"'"${sec}"'","key":"ACCESS_SECRET_KEY"}}},{"name":"CRED_MODE","value":"env"}' ;;
    *) die "internal: unknown probe mode '${mode}'" ;;
  esac

  local b64; b64="$(s3_probe_script | base64 | tr -d '\n')"
  [ -n "${b64}" ] || die "internal: probe script encoded to nothing"

  # egress-class label: harmless here (the velero namespace is not an app namespace) but required by
  # the require-egress-class-label ClusterPolicy in any namespace labelled app-namespace=true, which
  # the CNPG credential namespaces are. Without it the probe pod would be REJECTED by admission and
  # the check would report "unknown" instead of a result.
  local overrides
  overrides='{"apiVersion":"v1","metadata":{"labels":{"app.kubernetes.io/name":"s3check","homelab.keithjackson.dev/egress-class":"internet"}},"spec":{"restartPolicy":"Never","automountServiceAccountToken":false,'"${volsjson}"'"containers":[{"name":"s3check","image":"'"${S3CHECK_IMAGE}"'","imagePullPolicy":"IfNotPresent","command":["/bin/bash","-c","printf %s \"$PROBE_B64\" | base64 -d > /tmp/probe.sh && exec bash /tmp/probe.sh"],'"${mountsjson}"'"env":['"${creds}"',{"name":"PROBE_B64","value":"'"${b64}"'"},{"name":"S3EP","value":"'"${ep}"'"},{"name":"S3REGION","value":"'"${region}"'"},{"name":"S3TARGETS","value":"'"${targets}"'"},{"name":"S3KEY","value":"'"${pod}"'.probe"},{"name":"AWS_REQUEST_CHECKSUM_CALCULATION","value":"when_required"},{"name":"AWS_RESPONSE_CHECKSUM_VALIDATION","value":"when_required"}],"resources":{"requests":{"cpu":"50m","memory":"64Mi"},"limits":{"memory":"256Mi"}}}]}}'

  say "probe: ns=${ns} secret=${sec} (${mode}) targets='${targets}'  → ${min} assertions required"
  kubectl -n "${ns}" delete pod "${pod}" --ignore-not-found >/dev/null 2>&1 || true
  S3CHECK_PODS+=("${ns}/${pod}")
  kubectl -n "${ns}" run "${pod}" --restart=Never --image="${S3CHECK_IMAGE}" \
      --overrides="${overrides}" >/dev/null 2>&1 \
    || die "could NOT create the probe pod ${ns}/${pod} (admission policy? RBAC? quota?).
       That is a FAILURE, not an unknown: nothing about ${ep} has been proven."

  local i=0 phase=""
  while [ "${i}" -lt 300 ]; do
    phase="$(kubectl -n "${ns}" get pod "${pod}" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
    case "${phase}" in Succeeded|Failed) break ;; esac
    sleep 5; i=$((i+5))
  done
  case "${phase}" in
    Succeeded|Failed) ;;
    *) local reason
       reason="$(kubectl -n "${ns}" get pod "${pod}" \
                 -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || true)"
       die "probe pod ${ns}/${pod} did not finish in ${i}s (phase=${phase:-<none>}, waiting=${reason:-none}).
       NOT a pass. If waiting=ErrImagePull/ImagePullBackOff this cluster cannot pull ${S3CHECK_IMAGE}." ;;
  esac

  local log; log="$(kubectl -n "${ns}" logs "${pod}" 2>/dev/null || true)"
  [ -n "${log}" ] || die "probe pod ${ns}/${pod} produced NO output (phase=${phase}). Failing closed —
       an empty log is the shape of a probe that never ran, not of a healthy backup target."
  printf '%s\n' "${log}" | while IFS= read -r l; do printf '    %s\n' "${l}"; done

  # ---- the four independent accounting checks that make a silent no-op impossible ----
  local okcount failcount resline nres verd a f
  okcount="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-OK\[' || true)"
  failcount="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-FAIL' || true)"
  nres="$(printf '%s\n' "${log}" | grep -c '^S3CHECK-RESULT ' || true)"
  [ "${nres}" = "1" ] || die "expected exactly ONE S3CHECK-RESULT line from ${ns}/${pod}, found ${nres}.
       A missing verdict line means the probe died mid-way; it is never a pass."
  resline="$(printf '%s\n' "${log}" | grep '^S3CHECK-RESULT ')"
  read -r _ verd a f <<<"${resline}"
  a="${a#assertions=}"; f="${f#failures=}"
  case "${a}" in ''|*[!0-9]*) die "malformed verdict from ${ns}/${pod}: '${resline}'" ;; esac
  case "${f}" in ''|*[!0-9]*) die "malformed verdict from ${ns}/${pod}: '${resline}'" ;; esac
  [ "${verd}" = "PASS" ] || die "S3 ROUND TRIP FAILED against ${ep} (${ns}/${pod}: ${f} failure(s)).
       Read the S3CHECK-FAIL lines above — each one names the cause and the fix. Do NOT enable velero
       or CNPG backups against this endpoint until they are all gone."
  [ "${f}" = "0" ]          || die "${ns}/${pod} claimed PASS while reporting ${f} failures — refusing to believe it."
  [ "${failcount}" = "0" ]  || die "${ns}/${pod} claimed PASS but the log carries ${failcount} S3CHECK-FAIL line(s)."
  [ "${a}" -ge "${min}" ]   || die "${ns}/${pod} claimed PASS with only ${a} assertions; ${min} are required
       (1 credential + 6 per bucket). A short count means steps were SKIPPED — exactly the
       degrade-to-a-successful-no-op this check exists to prevent."
  [ "${okcount}" = "${a}" ] || die "assertion accounting mismatch for ${ns}/${pod}: ${okcount} S3CHECK-OK
       lines in the log vs a claimed ${a}. Refusing to pass on a self-report that does not add up."
  [ "${phase}" = "Succeeded" ] || die "${ns}/${pod} printed PASS but the pod ended ${phase}."
  info "PROVEN: ${a} assertions, 0 failures, pod ${phase}, no object left behind."
}

# ============================================================================ steps
step_preflight() {
  say "PREFLIGHT (read-only)"
  info "nodes Ready: $(kubectl get nodes --no-headers | awk '$2=="Ready"' | wc -l)/$(kubectl get nodes --no-headers | wc -l)"
  info "flux controllers Running: $(kubectl -n flux-system get pods --no-headers | awk '$3=="Running"' | wc -l)"
  # Count BOTH groups the Kyverno chart ships. Grepping only 'kyverno.io' undercounts by 2 — the
  # policy-report types are in the wgpolicyk8s.io group — which reads as "CRDs are missing" when the
  # pre-apply in fact succeeded. Expect 22 for kyverno 3.9.x.
  info "kyverno CRDs present: $(kubectl get crd --no-headers 2>/dev/null | grep -cE 'kyverno\.io|wgpolicyk8s\.io' || true) (expect 22 for kyverno 3.9.x)"

  local base owner filter s3 acme
  base="$(cvar BASE_DOMAIN)"; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
  filter="$(cvar EXTERNALDNS_DOMAIN_FILTER)"; s3="$(cvar S3_BACKUP_ENDPOINT)"; acme="$(cvar ACME_CLUSTER_ISSUER)"
  say "cluster-vars guards"
  info "BASE_DOMAIN                = ${base}"
  info "EXTERNALDNS_TXT_OWNER_ID   = ${owner}"
  info "EXTERNALDNS_DOMAIN_FILTER  = ${filter}"
  info "ACME_CLUSTER_ISSUER        = ${acme}"
  info "S3_BACKUP_ENDPOINT         = ${s3}"

  # THE GUARD THAT MATTERS. Two clusters sharing one Cloudflare zone are isolated by txtOwnerId and
  # by serving disjoint hostnames — NOT by domainFilters (finding F-DNS1: a subdomain filter drops the
  # parent zone and publishes nothing). Both clusters inheriting the chart default `default` is the
  # single root cause of the 2026-09-04 incident, in which prod deleted every one of dev's records.
  # preflight is a DIAGNOSTIC: it must always finish and report. The only thing it refuses over is the
  # target guard, because that one means you are pointed at the wrong cluster. Everything else is
  # reported and warned about here, and REFUSED on the mutating path (step_externaldns) where acting
  # on a bad value is what actually causes harm. Dying here instead hid the rest of dev's health
  # report behind a condition that was, for dev, simply the long-standing status quo.
  if [ -z "${owner}" ]; then
    warn "EXTERNALDNS_TXT_OWNER_ID is EMPTY — external-dns falls back to the chart default 'default'.
       Safe only if this is the sole cluster publishing into the zone. 'externaldns' will refuse."
  elif [ "${owner}" = "unset-see-cluster-vars" ]; then
    warn "EXTERNALDNS_TXT_OWNER_ID is still the base sentinel — the overlay did not apply.
       'externaldns' will refuse until a real per-environment value is set."
  elif [ "${owner}" = "default" ]; then
    # NOT fatal here, because `default` is DEV's legitimate, established owner id — its live records
    # are registered under it and cannot be renamed without orphaning them. Dying on it would make
    # this script refuse to run against dev at all, which would be an environment fork (R-ENV-02) in
    # a script whose whole claim is that it has none. preflight only READS, so it warns; the
    # MUTATING path (step_externaldns) still refuses outright.
    warn "EXTERNALDNS_TXT_OWNER_ID is 'default' — the Helm chart's default value.
       That is correct for DEV (its records are registered under it) and WRONG for any second cluster
       sharing this zone: two clusters both defaulting to 'default' is the exact root cause of the
       2026-09-04 incident, in which prod treated dev's TXT registry entries as its own and
       policy: sync deleted the records behind them."
  else
    info "owner id is distinct and non-default: OK"
  fi

  if s3_is_placeholder "${s3}"; then
    warn "S3_BACKUP_ENDPOINT is unset/placeholder ('${s3}'). velero AND apps-wave-0 consume it.
       velero will be SKIPPED by the resume step. apps-wave-0 will still apply (the token IS defined,
       so strict substitution succeeds) but anything writing to that endpoint fails at RUNTIME, not at
       apply time — so a green apps-wave-0 does NOT mean backups work. Operator decision required."
  else
    info "S3 backup endpoint set: OK"
  fi

  # COLD-START CRD ORDERING. Flux server-side dry-runs EVERY object in a Kustomization before applying
  # any of them, so a single unknown `kind` aborts the whole tier. platform-config applies the CDI CR
  # *and* CDI StorageProfiles in the same pass — but the StorageProfile CRD does not exist until the
  # cdi-operator has reconciled that CR. On an established cluster the CRDs are long since there and
  # this is invisible; on a COLD one it is a deadlock, and it is the same class of bug as Kyverno's
  # 5.7 MiB CRD bundle. Report it here so `resume` does not burn 15 minutes discovering it.
  say "CRDs that later tiers apply resources against"
  local crd missing=0
  for crd in storageprofiles.cdi.kubevirt.io cdis.cdi.kubevirt.io kubevirts.kubevirt.io \
             volumesnapshotclasses.snapshot.storage.k8s.io; do
    if kubectl get crd "${crd}" >/dev/null 2>&1; then
      printf '    %-52s present\n' "${crd}"
    else
      printf '    \033[33m%-52s MISSING\033[0m\n' "${crd}"; missing=1
    fi
  done
  if [ "${missing}" = "1" ]; then
    warn "a MISSING crd above will fail platform-config's dry-run and block every tier below it.
       CDI's own CRDs (StorageProfile, DataVolume, ...) are created by cdi-operator only AFTER it
       reconciles the CDI CR. Check that the CDI CR is applied and cdi-operator is Available; if the
       CRD appeared only recently this can also be a stale discovery cache in kustomize-controller,
       which a forced reconcile clears."
    kubectl -n cdi get cdi,deploy 2>/dev/null | head -12 || info "(no cdi namespace yet)"
  fi

  say "tiers"
  local n
  for n in flux-system cilium "${TIER_ORDER[@]}" velero; do
    printf '    %-16s suspend=%-6s ready=%-6s %s\n' "$n" "$(k_susp "$n")" "$(k_ready "$n")" "$(k_msg "$n" | head -c 90)"
  done

  say "HelmReleases not Ready"
  kubectl get helmrelease -A \
    -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,MSG:.status.conditions[?\(@.type==\"Ready\"\)].message \
    --no-headers 2>/dev/null | grep -v ' True ' || info "(all Ready)"

  # SUSPENDED releases are invisible above: a suspended HelmRelease keeps whatever Ready condition it
  # last had, so a STALE `Ready=True` hides the fact that Flux is not managing it at all. external-dns
  # is deliberately in this state until `standup.sh externaldns` runs.
  say "HelmReleases SUSPENDED (Flux is not reconciling these — a stale Ready=True means nothing)"
  kubectl get helmrelease -A \
    -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend --no-headers 2>/dev/null \
    | awk '$3=="true"' || true
}

step_crds() {
  say "CRD PRE-APPLY (idempotent)"
  # Charts whose CRD bundle exceeds kubectl's 256 KiB annotation limit set crds.install=false, so
  # nothing installs them and `platform` can never go Ready. See bootstrap/flux/preapply-crds.sh.
  bash "${script_dir}/flux/preapply-crds.sh"

  # ---------------------------------------------------------------- the CDI cold-start deadlock
  # SAME CLASS OF BUG, DIFFERENT OPERATOR. `platform-config` applies the CDI CR *and*
  # platform/kubevirt/cdi-storageprofiles.yaml in ONE pass. But StorageProfile's CRD does not exist
  # until cdi-operator has reconciled that CR — and Flux server-side dry-runs EVERY object before
  # applying ANY of them, so the unknown kind aborts the whole tier, the CR is never applied, the CRD
  # is never created, and platform-config can never converge. A true deadlock, invisible on an
  # established cluster (the CRDs are long since there) and fatal on a cold one.
  #
  # Breaking it needs exactly one thing: get the CDI CR in. Then cdi-operator creates the remaining
  # CDI CRDs and platform-config's next reconcile dry-runs clean. We apply ONLY the CR — the minimum
  # — and leave every other object to Flux. The CR carries no ${} tokens, so no substitution is
  # needed and applying it here cannot diverge from what Flux would render.
  local cdi_cr="${repo_root}/platform/kubevirt/cdi-cr.yaml"
  if kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1; then
    info "storageprofiles CRD already present — CDI is past its cold start"
  elif [ ! -f "${cdi_cr}" ]; then
    warn "expected ${cdi_cr} and it is missing — skipping the CDI unblock"
  else
    say "CDI cold-start unblock: applying the CDI CR so cdi-operator can create its CRDs"
    kubectl get ns cdi >/dev/null 2>&1 || die "namespace 'cdi' absent — the platform tier has not applied the CDI operator yet. Run 'resume' far enough to make platform Ready first."
    kubectl apply --server-side --force-conflicts -f "${cdi_cr}"
    local i=0
    while [ "$i" -lt 300 ]; do
      kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 && break
      sleep 10; i=$((i+10))
    done
    kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 \
      || die "cdi-operator did not create storageprofiles.cdi.kubevirt.io within ${i}s.
       Inspect: kubectl -n cdi get cdi,pods,deploy — platform-config stays blocked until it exists."
    kubectl wait --for=condition=Established --timeout=60s crd/storageprofiles.cdi.kubevirt.io >/dev/null
    info "storageprofiles.cdi.kubevirt.io created and Established (${i}s)"
  fi
}

step_unwedge() {
  say "UNWEDGE STUCK HELM RELEASES"
  # A Helm release left in uninstalling/pending-* cannot be advanced by helm-controller: it reports
  # StateError ("unable to determine state for release with status 'uninstalling'") forever, which
  # keeps `platform` (wait: true) not-Ready and every tier below it blocked.
  #
  # How prod got here: Kyverno's install timed out because its pods were crash-looping with no CRDs;
  # install.remediation.retries:3 then fired an uninstall, and that uninstall timed out too.
  #
  # Pre-applied CRDs live OUTSIDE the Helm release and SURVIVE this — that is precisely why the
  # pre-apply is a separate step and must run before this one.
  local found=0 ns rel st
  while read -r ns rel st; do
    [ -n "${ns:-}" ] || continue
    found=1
    warn "${ns}/${rel} is '${st}'"
    [ "${CONFIRM_DESTRUCTIVE}" = "1" ] || {
      info "DRY RUN. This DELETES the release's Kubernetes objects and lets Flux install cold."
      info "Re-run with --confirm-destructive to act."
      continue
    }
    say "clearing ${ns}/${rel}"
    # --no-hooks and --wait=false: the WAIT is what hung the original remediation, so do not repeat it.
    helm uninstall "${rel}" -n "${ns}" --no-hooks --wait=false || warn "helm uninstall returned non-zero; continuing to check state"
    if helm list -n "${ns}" --all -q 2>/dev/null | grep -qx "${rel}"; then
      warn "${ns}/${rel} still present after uninstall — inspect manually, NOT deleting release secrets by hand."
    else
      info "${ns}/${rel} cleared"
      flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile of ${ns}/${rel} did not complete in 10m"
    fi
  done < <(
    for ns in $(kubectl get helmrelease -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"\n"}{end}' | sort -u); do
      helm list -n "${ns}" --all -o json 2>/dev/null \
        | yq -r '.[] | select(.status | test("uninstalling|pending-install|pending-upgrade|pending-rollback")) | "'"${ns}"' " + .name + " " + .status' 2>/dev/null || true
    done
  )
  [ "${found}" = "1" ] || info "no wedged releases found."
}

step_externaldns() {
  say "EXTERNAL-DNS — SAFE UPGRADE (prove the owner id BEFORE the pod can run)"
  # THE HAZARD: a Helm upgrade sets replicas back to the chart default (1) and applies the new args in
  # ONE operation. There is no window in which to check the result first. If --txt-owner-id is missing
  # the pod starts as owner `default`, claims the other cluster's TXT registry entries, and under
  # `policy: sync` DELETES the records behind them. That is the 2026-09-04 incident.
  #
  # THE FIX IS TO VERIFY IN ADVANCE, NOT AFTER: render the chart with the LIVE HelmRelease values and
  # assert the flag appears. Rendering is deterministic and touches nothing, so this converts a race
  # into a precondition.
  local ns=external-dns rel=external-dns
  local owner chart ver repo vals
  owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
  [ -n "${owner}" ] && [ "${owner}" != "default" ] && [ "${owner}" != "unset-see-cluster-vars" ] \
    || die "refusing: EXTERNALDNS_TXT_OWNER_ID is '${owner}'"

  chart="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.chart}')"
  ver="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.version}')"
  repo="$(yq eval-all --no-doc 'select(.kind == "HelmRepository") | .spec.url' platform/externaldns/helmrepository.yaml | tr -d '"')"
  [ -n "${repo}" ] || die "could not derive the external-dns chart repo url"

  vals="${SNAP_DIR}/ed-values.yaml"
  kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.values}' | yq -P > "${vals}"
  info "chart=${chart} version=${ver} repo=${repo}"
  info "live spec.values.txtOwnerId = $(yq '.txtOwnerId' "${vals}")"

  say "rendering the chart with the LIVE values to prove the flag"
  local rendered="${SNAP_DIR}/ed-rendered.yaml"
  helm template "${rel}" "${chart}" --repo "${repo}" --version "${ver}" -n "${ns}" -f "${vals}" > "${rendered}"
  local args; args="$(yq -r 'select(.kind == "Deployment") | .spec.template.spec.containers[0].args[]' "${rendered}")"
  printf '%s\n' "${args}" | sed 's/^/      /'
  printf '%s\n' "${args}" | grep -qx -- "--txt-owner-id=${owner}" \
    || die "the render does NOT contain --txt-owner-id=${owner}.
       An upgrade would start external-dns as owner 'default' and delete the other cluster's records.
       REFUSING. Fix the chart values key before retrying."
  info "PROVEN: the upgrade will carry --txt-owner-id=${owner}"

  say "snapshotting the Cloudflare zone (baseline for the post-check)"
  dns_snapshot "${SNAP_DIR}/dns-before.txt"

  say "resuming + forcing the upgrade"
  kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.suspend}' | grep -q true \
    && flux resume helmrelease "${rel}" -n "${ns}" || info "(HelmRelease was not suspended)"
  flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile did not complete in 10m"

  say "verifying the LIVE deployment args"
  local live; live="$(kubectl -n "${ns}" get deploy "${rel}" -o jsonpath='{.spec.template.spec.containers[0].args}' 2>/dev/null || true)"
  printf '    %s\n' "${live}"
  printf '%s' "${live}" | grep -q -- "--txt-owner-id=${owner}" \
    || { kubectl -n "${ns}" scale deploy "${rel}" --replicas=0 || true
         die "live args LACK --txt-owner-id=${owner}. Scaled to 0. Investigate before scaling up."; }
  info "live deployment carries --txt-owner-id=${owner}"

  say "post-check: did anything in the shared zone change?"
  sleep 45   # external-dns --interval=1m; give it a full cycle to act before judging it safe
  dns_snapshot "${SNAP_DIR}/dns-after.txt"
  dns_diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-after.txt" \
    || warn "records changed — expected ONLY additions under this cluster's own BASE_DOMAIN ($(cvar BASE_DOMAIN)).
       Any change to a name outside it, or to a TXT entry owned by another cluster, is a REGRESSION."
}

step_s3check() {
  say "S3 BACKUP TARGET — PROVE A REAL ROUND TRIP FROM INSIDE THE CLUSTER"
  # Velero and CNPG run in-cluster, so pod egress is the only reachability that counts: prod sits on
  # the SOHO network (pfSense's WAN) while the dev LAN's MinIO sits INSIDE pfSense, so host and pod
  # egress differ — and a healthy host-side reading has already misled this project once.
  #
  # WHAT THIS PROVES (and it fails closed on every one of them):
  #   1. the committed credential loads and AUTHENTICATES, in the exact shape each consumer reads it
  #      (velero: an AWS INI file from the `cloud` key; CNPG: discrete ACCESS_KEY_ID/ACCESS_SECRET_KEY);
  #   2. each required bucket EXISTS and is LISTABLE under this environment's own prefix;
  #   3. PUT -> GET -> byte-compare -> DELETE of a throwaway object under that prefix — the genuine
  #      write path, not a HEAD (a HEAD passes with a read-only token and with R2 rejecting writes);
  #   4. whether the R2 CHECKSUM TRAP is present at this endpoint, named explicitly either way.
  #
  # WHAT IT STILL DOES NOT PROVE: that Velero's kopia uploader and CNPG's barman-cloud themselves work
  # against this target. Both are UNPROVEN against R2 in this repo. See infra/BACKUP-R2.md.
  local ep live region
  live="$(cvar S3_BACKUP_ENDPOINT)"
  info "live S3_BACKUP_ENDPOINT = ${live:-<unset>}"
  if [ -n "${S3_ENDPOINT_OVERRIDE}" ]; then
    # Probe a CANDIDATE endpoint before adopting it. The whole point is to find out whether a proposed
    # backup target works from this cluster's pods BEFORE authoring it into cluster-vars — otherwise
    # the first evidence is silently-missing backups.
    ep="${S3_ENDPOINT_OVERRIDE}"
    info "probing CANDIDATE endpoint from --s3-endpoint: ${ep}"
  else
    ep="${live}"
  fi
  if s3_is_placeholder "${ep}"; then
    die "no usable endpoint to probe (endpoint='${ep:-<unset>}').
       cluster-vars still carries the loud placeholder. Pass --s3-endpoint https://<account-id>.r2.cloudflarestorage.com
       to test a CANDIDATE before authoring it. See infra/BACKUP-R2.md."
  fi
  s3_safe_token S3_BACKUP_ENDPOINT "${ep}"
  case "${ep}" in
    http://*|https://*) ;;
    *) die "endpoint '${ep}' carries no http:// or https:// scheme — the AWS SDK needs one." ;;
  esac

  # Region is a knob, not a constant: MinIO ignores it, R2 requires exactly `auto`, and an EMPTY value
  # in cluster-vars silently builds a BSL with no region.
  region="$(cvar S3_BACKUP_REGION)"
  case "${ep}" in
    *.r2.cloudflarestorage.com*)
      if [ "${region}" != "auto" ]; then
        warn "endpoint is Cloudflare R2 but cluster-vars S3_BACKUP_REGION='${region:-<unset>}'. R2 requires
       exactly 'auto'. Probing with 'auto' anyway; FIX clusters/base/cluster-vars.yaml before enabling
       velero, or the BackupStorageLocation is built with the wrong region."
      fi
      region=auto ;;
    *)
      if [ -z "${region}" ]; then
        region="us-east-1"
        warn "cluster-vars S3_BACKUP_REGION is EMPTY on this cluster. Probing with 'us-east-1' (MinIO
       ignores it) — but the BSL would be rendered with region: \"\". Set it explicitly."
      fi ;;
  esac
  s3_safe_token S3_BACKUP_REGION "${region}"

  # Buckets come from the AUTHORED manifests + cluster-vars, never from a hardcoded name here: the
  # whole point is to probe what this environment will really write to (R-ENV-02, no dev/prod fork).
  local vbucket vprefix cpath cbucket cprefix rest
  vbucket="$(yq -r '.spec.values.configuration.backupStorageLocation[0].bucket' \
             platform/velero/helmrelease.yaml 2>/dev/null | tr -d '"')"
  [ -n "${vbucket}" ] && [ "${vbucket}" != "null" ] \
    || die "could not read the Velero bucket name from platform/velero/helmrelease.yaml"
  vprefix="$(cvar VELERO_BACKUP_PREFIX)"

  cpath="$(cvar CNPG_BACKUP_PATH)"
  if [ -z "${cpath}" ]; then
    warn "cluster-vars CNPG_BACKUP_PATH is EMPTY on this cluster. barmanObjectStore.destinationPath would
       substitute to the empty string and EVERY WAL archive would fail at runtime. Probing the historical
       default s3://cnpg so the round trip is still proven — but SET THIS VALUE."
    cpath="s3://cnpg"
  fi
  case "${cpath}" in s3://*) ;; *) die "CNPG_BACKUP_PATH='${cpath}' is not an s3:// URL." ;; esac
  rest="${cpath#s3://}"; cbucket="${rest%%/*}"
  if [ "${rest}" = "${cbucket}" ]; then cprefix=""; else cprefix="${rest#*/}"; fi
  cprefix="${cprefix%/}"
  [ -n "${cbucket}" ] || die "CNPG_BACKUP_PATH='${cpath}' names no bucket."

  s3_safe_token velero-bucket "${vbucket}"; s3_safe_token VELERO_BACKUP_PREFIX "${vprefix}"
  s3_safe_token cnpg-bucket   "${cbucket}"; s3_safe_token CNPG_BACKUP_PATH-prefix "${cprefix}"

  say "what is about to be proven"
  info "endpoint    ${ep}"
  info "region      ${region}"
  info "velero      bucket '${vbucket}'  prefix '${vprefix:-<bucket root>}'   (authored bucket + VELERO_BACKUP_PREFIX)"
  info "cnpg        bucket '${cbucket}'  prefix '${cprefix:-<bucket root>}'   (from CNPG_BACKUP_PATH)"
  info "scratch key <prefix>/.s3check/<pod>.probe — written, read back, byte-compared, DELETED"

  local ran=0 keys
  # (a) Velero's own credential, from its own namespace, mounted the way the aws plugin reads it.
  if kubectl get ns velero >/dev/null 2>&1 \
     && kubectl -n velero get secret velero-cloud-credentials >/dev/null 2>&1; then
    # List KEY NAMES only. Never pipe or print a value (a careless redaction has leaked a password
    # into a transcript on this project already). go-template, not jsonpath: kubectl's jsonpath
    # cannot range over a map's keys (`range $k,$v :=` is rejected, U+002C) and returns EMPTY, which
    # silently looked like "the key is missing".
    keys="$(kubectl -n velero get secret velero-cloud-credentials \
            -o go-template='{{range $k,$v := .data}}{{$k}}{{"\n"}}{{end}}' 2>/dev/null || true)"
    printf '%s\n' "${keys}" | grep -qx 'cloud' \
      || die "Secret velero/velero-cloud-credentials carries no 'cloud' key (keys: $(printf '%s' "${keys}" | tr '\n' ' ')).
       velero-plugin-for-aws reads exactly that key as an AWS INI file; without it every backup fails
       at RUNTIME while the HelmRelease reports Ready."
    s3_probe velero file velero-cloud-credentials "${vbucket}:${vprefix}" "${ep}" "${region}"
    ran=$((ran+1))
  else
    warn "no Secret velero/velero-cloud-credentials on this cluster — Velero's own credential could NOT
       be proven. That is expected before the velero tier has ever reconciled here; it is not a pass."
  fi

  # (b) A CNPG/barman credential, discovered rather than hardcoded (R-ENV-02: no per-environment list
  #     of app names in this script). Any one of them proves the key + the cnpg bucket + the discrete
  #     ACCESS_KEY_ID/ACCESS_SECRET_KEY shape; they are all kept in sync by design.
  local pick cns csec
  pick="$(kubectl get secret -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{"\n"}{end}' 2>/dev/null \
          | grep -- '-db-backup-credentials$' | head -n 1 || true)"
  if [ -n "${pick}" ]; then
    cns="${pick%%/*}"; csec="${pick#*/}"
    s3_probe "${cns}" env "${csec}" "${cbucket}:${cprefix}" "${ep}" "${region}"
    ran=$((ran+1))
  else
    warn "no *-db-backup-credentials Secret found in any namespace — the CNPG/barman credential could
       NOT be proven."
  fi

  [ "${ran}" -gt 0 ] || die "NO backup credential Secret exists on this cluster, so NOTHING could be
       proven about ${ep}. That is a FAILURE, not a skip: a green s3check must never mean 'I found
       nothing to check'."

  say "S3 ROUND TRIP PROVEN from inside the cluster: ${ep} (${ran} credential path(s))"
  info "STILL UNPROVEN by this step: Velero's kopia uploader and CNPG barman-cloud against this target,"
  info "and a RESTORE. A backup that has never been restored is not a backup — see infra/BACKUP-R2.md."
}

step_resume() {
  say "RESUME TIERS (dependency order)"
  local n s3; s3="$(cvar S3_BACKUP_ENDPOINT)"
  for n in "${TIER_ORDER[@]}"; do
    if [ "$(k_susp "$n")" = "true" ]; then
      info "resuming $n"; flux resume kustomization "$n" --timeout=1s >/dev/null 2>&1 || true
    else
      info "$n already unsuspended"
    fi
    wait_ready "$n" 900 || { warn "stopping at $n — a partial, understood bring-up beats a forced green."; return 1; }
  done

  # The placeholder test MUST be substring-based. It was `TBD-*` (a PREFIX match) while the placeholder
  # was `TBD-PROD-…`; the R2 move changed it to `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`,
  # which does not START with TBD- — so this case arm silently stopped matching and `resume` would have
  # RESUMED velero onto a placeholder hostname on prod. That is the precise failure this guard exists
  # to prevent, reintroduced by a value change in another file. s3_is_placeholder matches anywhere.
  if s3_is_placeholder "${s3}"; then
    warn "velero left SUSPENDED: S3_BACKUP_ENDPOINT='${s3}' is not a usable endpoint.
       Resuming it would create a BackupStorageLocation that can never reach a bucket, i.e. a tier
       that reports Ready-ish while backups silently do not exist. Set the real endpoint first, prove
       it with \`s3check\`, and only then resume (infra/BACKUP-R2.md)."
  else
    info "resuming velero"; flux resume kustomization velero --timeout=1s >/dev/null 2>&1 || true
    wait_ready velero 900 || warn "velero did not converge"
  fi
}

step_verify() {
  say "VERIFY"
  local base; base="$(cvar BASE_DOMAIN)"
  info "BASE_DOMAIN = ${base}"
  say "kustomizations"
  kubectl get kustomization -A \
    -o custom-columns=NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status --no-headers
  say "certificates (must be Ready before any hostname is trusted)"
  kubectl get certificate -A --no-headers 2>/dev/null || info "(none yet)"
  say "pods not Running/Completed"
  kubectl get pods -A --no-headers | awk '$4!="Running" && $4!="Completed"' || true
  info "total pods: $(kubectl get pods -A --no-headers | wc -l)"
  say "gateway / LoadBalancer VIPs"
  kubectl get svc -A --field-selector spec.type=LoadBalancer --no-headers 2>/dev/null || true
  say "published DNS for this cluster's own domain"
  local owner; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
  dns_snapshot "${SNAP_DIR}/dns-verify.txt"
  info "records under ${base}: $(grep -c "${base}" "${SNAP_DIR}/dns-verify.txt" || true)"

  # Every TXT registry entry this cluster publishes must carry OUR owner id. One reading another
  # cluster's id means we have claimed its records and `policy: sync` will delete them next cycle.
  local foreign
  foreign="$(grep "${base}" "${SNAP_DIR}/dns-verify.txt" | grep 'heritage=external-dns' \
             | grep -v "external-dns/owner=${owner}" || true)"
  if [ -n "${foreign}" ]; then
    warn "TXT entries under ${base} NOT owned by '${owner}' — this cluster has claimed records it does not own:"
    printf '%s\n' "${foreign}" | sed 's/^/      /'
  else
    info "every TXT registry entry under ${base} reads owner=${owner}: OK"
  fi

  # THE REGRESSION CHECK. Additions under our own BASE_DOMAIN are expected. ANY change to a name
  # outside it means we have touched the other cluster's zone — the 2026-09-04 incident.
  if [ -f "${SNAP_DIR}/dns-before.txt" ]; then
    say "regression check against the pre-standup baseline"
    local outside
    outside="$(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" \
               | grep -E '^[<>]' | grep -v "${base}" || true)"
    if [ -n "${outside}" ]; then
      warn "RECORDS OUTSIDE ${base} CHANGED — investigate before doing anything else:"
      printf '%s\n' "${outside}" | sed 's/^/      /'
    else
      info "no record outside ${base} was added, changed or removed: OK"
      info "added under ${base}: $(diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-verify.txt" | grep -c '^>' || true)"
    fi
  else
    warn "no ${SNAP_DIR}/dns-before.txt baseline — cannot prove the other cluster's records are untouched.
       Take one BEFORE the next standup so this check can run."
  fi
}

# ============================================================================ dispatch
for s in "${STEPS[@]}"; do
  case "$s" in
    preflight)   step_preflight ;;
    s3check)     step_s3check ;;
    crds)        step_crds ;;
    unwedge)     step_unwedge ;;
    externaldns) step_externaldns ;;
    resume)      step_resume ;;
    verify)      step_verify ;;
    all)         step_preflight; step_crds; step_unwedge; step_externaldns; step_resume; step_verify ;;
    *)           die "unknown step '$s' (preflight crds unwedge externaldns s3check resume verify all)" ;;
  esac
done

say "done: ${STEPS[*]}"