home-network-k3s-impl/bootstrap/standup.sh

#!/usr/bin/env bash
#
# standup.sh — bring a bootstrapped cluster's reconciliation graph all the way up.
#
# ENVIRONMENT-AGNOSTIC (R-ENV-02): it reads everything it needs from the cluster it is pointed at and
# from the authored manifests. There is no dev/prod fork. What makes it safe to run against prod is
# the MANDATORY --expect-server guard, not a name in the filename.
#
# WHY THIS EXISTS
#   `bootstrap/flux/bootstrap.sh` installs Flux and applies the graph. It does NOT finish the job when
#   tiers have been deliberately suspended (a DNS/TLS migration window, an incident recovery) or when
#   a HelmRelease has wedged. Those last steps were living in shell history and in a human's head,
#   which is the same failure the Kyverno CRD pre-apply had. This is that sequence, in git, ordered,
#   idempotent, and gated.
#
# THE ORDERING IS LOAD-BEARING. Read infra/PROD-STANDUP.md before deviating from it.
#
# USAGE
#   bootstrap/standup.sh --kubeconfig <path> --expect-server <url> [--confirm-destructive] <step...>
#
#   steps:  preflight  crds  unwedge  externaldns  resume  verify   (or: all)
#
#   --expect-server is REQUIRED and is compared against the live API server before ANY command runs.
#   Pointing this at the wrong cluster is the one mistake with unbounded blast radius: dev is live and
#   shares a Cloudflare zone with prod. The guard is not optional and there is no override flag.
#
#   --confirm-destructive is required by `unwedge` only. Everything else is additive or reversible.
#
set -euo pipefail

# ---------------------------------------------------------------------------- args
KUBECONFIG_ARG="" EXPECT_SERVER="" CONFIRM_DESTRUCTIVE=0
STEPS=()
while [ $# -gt 0 ]; do
  case "$1" in
    --kubeconfig)          KUBECONFIG_ARG="${2:?--kubeconfig needs a value}"; shift 2 ;;
    --expect-server)       EXPECT_SERVER="${2:?--expect-server needs a value}"; shift 2 ;;
    --confirm-destructive) CONFIRM_DESTRUCTIVE=1; shift ;;
    -h|--help)             sed -n '2,30p' "$0"; exit 0 ;;
    -*)                    echo "ERROR: unknown flag '$1'" >&2; exit 2 ;;
    *)                     STEPS+=("$1"); shift ;;
  esac
done

[ -n "${KUBECONFIG_ARG}" ] || { echo "ERROR: --kubeconfig is required." >&2; exit 2; }
[ -n "${EXPECT_SERVER}" ]  || { echo "ERROR: --expect-server is required (safety guard)." >&2; exit 2; }
[ "${#STEPS[@]}" -gt 0 ]   || { echo "ERROR: no step given. One of: preflight crds unwedge externaldns resume verify all" >&2; exit 2; }

script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo_root="$(git -C "${script_dir}" rev-parse --show-toplevel 2>/dev/null || (cd "${script_dir}/.." && pwd))"
cd "${repo_root}"

export KUBECONFIG="${KUBECONFIG_ARG}"
[ -f "${KUBECONFIG}" ] || { echo "ERROR: kubeconfig not found: ${KUBECONFIG}" >&2; exit 1; }

# Where DNS snapshots go. Deliberately OUTSIDE the repo: they are run artifacts, not authored state.
SNAP_DIR="${STANDUP_SNAP_DIR:-${TMPDIR:-/tmp}/standup-dns}"
mkdir -p "${SNAP_DIR}"

say()  { printf '\n\033[1m==> %s\033[0m\n' "$*"; }
info() { printf '    %s\n' "$*"; }
die()  { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; }
warn() { printf '\033[33mWARN: %s\033[0m\n' "$*" >&2; }

for bin in kubectl flux helm yq sops; do
  command -v "$bin" >/dev/null 2>&1 || die "'$bin' not found — run this inside infra/devcontainer."
done

# ------------------------------------------------------------------ THE SAFETY GUARD
# Checked once, before any step, and re-checked by every mutating helper. The failure mode this
# prevents is running a prod bring-up against the LIVE dev cluster, whose public DNS records prod
# has already destroyed once (2026-09-04).
live_server="$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')"
[ "${live_server}" = "${EXPECT_SERVER}" ] || die \
  "TARGET MISMATCH — refusing to run.
       --expect-server : ${EXPECT_SERVER}
       live API server : ${live_server}
       The kubeconfig does not point where you think it does. Nothing has been changed."
say "target confirmed: ${live_server}"

# Dependency order. Flux enforces the edges itself via dependsOn; resuming in this order simply means
# each wait is meaningful instead of blocking on a dependency that is still suspended.
#   flux-system -> cilium -> platform -> platform-config -> {leaf tiers, apps-wave-0} -> wave-1 -> wave-2
TIER_ORDER=(platform platform-config oauth2-proxy headlamp homepage crafty apps-wave-0 apps-wave-1 apps-wave-2 apps-vms)
# velero is NOT in that list on purpose — see step `resume`: it consumes S3_BACKUP_ENDPOINT, which on
# prod is an intentionally-invalid TBD placeholder pending an operator decision.

cvar() { kubectl -n flux-system get cm cluster-vars -o jsonpath="{.data.$1}" 2>/dev/null || true; }

k_ready()  { kubectl -n flux-system get kustomization "$1" \
               -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true; }
k_msg()    { kubectl -n flux-system get kustomization "$1" \
               -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true; }
k_susp()   { kubectl -n flux-system get kustomization "$1" -o jsonpath='{.spec.suspend}' 2>/dev/null || true; }

wait_ready() { # <kustomization> <timeout-seconds>
  local n="$1" t="${2:-600}" i=0
  while [ "$i" -lt "$t" ]; do
    [ "$(k_ready "$n")" = "True" ] && { info "$n Ready (${i}s)"; return 0; }
    sleep 10; i=$((i+10))
  done
  warn "$n did NOT reach Ready in ${t}s: $(k_msg "$n")"
  return 1
}

# Snapshot the Cloudflare zone. This is how damage to the OTHER cluster's records becomes detectable.
# Queried against the Cloudflare API, never a local resolver: a pfSense override produced a false
# healthy reading during the 2026-09-04 incident and hid the damage for a while.
dns_snapshot() { # <output-file>
  local out="$1" tok zid page tp
  tok="$(sops -d platform/externaldns/cloudflare-api-token.sops.yaml \
         | yq '.stringData."api-token"' | tr -d '"')"
  [ -n "${tok}" ] && [ "${tok}" != "null" ] || die "could not read the Cloudflare token (SOPS/age key available?)"
  zid="$(curl -sS -H "Authorization: Bearer ${tok}" \
        'https://api.cloudflare.com/client/v4/zones?name=keithjackson.dev' | yq '.result[0].id' | tr -d '"')"
  [ -n "${zid}" ] && [ "${zid}" != "null" ] || die "Cloudflare zone lookup failed"
  page=1; : > "${out}"
  while :; do
    local r; r="$(curl -sS -H "Authorization: Bearer ${tok}" \
      "https://api.cloudflare.com/client/v4/zones/${zid}/dns_records?per_page=100&page=${page}")"
    printf '%s' "${r}" | yq -o=json '.result' \
      | yq -r '.[] | (.type + "|" + .name + "|" + .content)' | tr -d '"' >> "${out}"
    tp="$(printf '%s' "${r}" | yq '.result_info.total_pages' | tr -d '"')"
    [ "${page}" -ge "${tp:-1}" ] && break
    page=$((page+1))
  done
  grep -vE '^\s*$' "${out}" | sort -o "${out}"
  info "$(wc -l < "${out}") records -> ${out}"
}

dns_diff() { # <before> <after>  — returns non-zero if anything changed
  if diff -u "$1" "$2" > "${SNAP_DIR}/dns.diff"; then
    info "Cloudflare zone IDENTICAL — nothing created, modified or deleted."
    return 0
  fi
  warn "Cloudflare zone CHANGED:"; cat "${SNAP_DIR}/dns.diff"
  return 1
}

# ============================================================================ steps
step_preflight() {
  say "PREFLIGHT (read-only)"
  info "nodes Ready: $(kubectl get nodes --no-headers | awk '$2=="Ready"' | wc -l)/$(kubectl get nodes --no-headers | wc -l)"
  info "flux controllers Running: $(kubectl -n flux-system get pods --no-headers | awk '$3=="Running"' | wc -l)"
  # Count BOTH groups the Kyverno chart ships. Grepping only 'kyverno.io' undercounts by 2 — the
  # policy-report types are in the wgpolicyk8s.io group — which reads as "CRDs are missing" when the
  # pre-apply in fact succeeded. Expect 22 for kyverno 3.9.x.
  info "kyverno CRDs present: $(kubectl get crd --no-headers 2>/dev/null | grep -cE 'kyverno\.io|wgpolicyk8s\.io' || true) (expect 22 for kyverno 3.9.x)"

  local base owner filter s3 acme
  base="$(cvar BASE_DOMAIN)"; owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
  filter="$(cvar EXTERNALDNS_DOMAIN_FILTER)"; s3="$(cvar S3_BACKUP_ENDPOINT)"; acme="$(cvar ACME_CLUSTER_ISSUER)"
  say "cluster-vars guards"
  info "BASE_DOMAIN                = ${base}"
  info "EXTERNALDNS_TXT_OWNER_ID   = ${owner}"
  info "EXTERNALDNS_DOMAIN_FILTER  = ${filter}"
  info "ACME_CLUSTER_ISSUER        = ${acme}"
  info "S3_BACKUP_ENDPOINT         = ${s3}"

  # THE GUARD THAT MATTERS. Two clusters sharing one Cloudflare zone are isolated by txtOwnerId and
  # by serving disjoint hostnames — NOT by domainFilters (finding F-DNS1: a subdomain filter drops the
  # parent zone and publishes nothing). Both clusters inheriting the chart default `default` is the
  # single root cause of the 2026-09-04 incident, in which prod deleted every one of dev's records.
  [ -n "${owner}" ]                        || die "EXTERNALDNS_TXT_OWNER_ID is EMPTY — external-dns would fall back to 'default', which is dev's owner."
  [ "${owner}" != "unset-see-cluster-vars" ] || die "EXTERNALDNS_TXT_OWNER_ID is still the base sentinel — the overlay did not apply."
  [ "${owner}" != "default" ]               || die "EXTERNALDNS_TXT_OWNER_ID is 'default' — that is DEV's owner id. Refusing."
  info "owner id is distinct and non-default: OK"

  case "${s3}" in
    ""|TBD-*) warn "S3_BACKUP_ENDPOINT is unset/placeholder ('${s3}'). velero AND apps-wave-0 consume it.
       velero will be SKIPPED by the resume step. apps-wave-0 will still apply (the token IS defined,
       so strict substitution succeeds) but anything writing to that endpoint fails at RUNTIME, not at
       apply time — so a green apps-wave-0 does NOT mean backups work. Operator decision required." ;;
    *) info "S3 backup endpoint set: OK" ;;
  esac

  say "tiers"
  local n
  for n in flux-system cilium "${TIER_ORDER[@]}" velero; do
    printf '    %-16s suspend=%-6s ready=%-6s %s\n' "$n" "$(k_susp "$n")" "$(k_ready "$n")" "$(k_msg "$n" | head -c 90)"
  done

  say "HelmReleases not Ready"
  kubectl get helmrelease -A \
    -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,MSG:.status.conditions[?\(@.type==\"Ready\"\)].message \
    --no-headers 2>/dev/null | grep -v ' True ' || info "(all Ready)"

  # SUSPENDED releases are invisible above: a suspended HelmRelease keeps whatever Ready condition it
  # last had, so a STALE `Ready=True` hides the fact that Flux is not managing it at all. external-dns
  # is deliberately in this state until `standup.sh externaldns` runs.
  say "HelmReleases SUSPENDED (Flux is not reconciling these — a stale Ready=True means nothing)"
  kubectl get helmrelease -A \
    -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,SUSPEND:.spec.suspend --no-headers 2>/dev/null \
    | awk '$3=="true"' || true
}

step_crds() {
  say "CRD PRE-APPLY (idempotent)"
  # Charts whose CRD bundle exceeds kubectl's 256 KiB annotation limit set crds.install=false, so
  # nothing installs them and `platform` can never go Ready. See bootstrap/flux/preapply-crds.sh.
  bash "${script_dir}/flux/preapply-crds.sh"
}

step_unwedge() {
  say "UNWEDGE STUCK HELM RELEASES"
  # A Helm release left in uninstalling/pending-* cannot be advanced by helm-controller: it reports
  # StateError ("unable to determine state for release with status 'uninstalling'") forever, which
  # keeps `platform` (wait: true) not-Ready and every tier below it blocked.
  #
  # How prod got here: Kyverno's install timed out because its pods were crash-looping with no CRDs;
  # install.remediation.retries:3 then fired an uninstall, and that uninstall timed out too.
  #
  # Pre-applied CRDs live OUTSIDE the Helm release and SURVIVE this — that is precisely why the
  # pre-apply is a separate step and must run before this one.
  local found=0 ns rel st
  while read -r ns rel st; do
    [ -n "${ns:-}" ] || continue
    found=1
    warn "${ns}/${rel} is '${st}'"
    [ "${CONFIRM_DESTRUCTIVE}" = "1" ] || {
      info "DRY RUN. This DELETES the release's Kubernetes objects and lets Flux install cold."
      info "Re-run with --confirm-destructive to act."
      continue
    }
    say "clearing ${ns}/${rel}"
    # --no-hooks and --wait=false: the WAIT is what hung the original remediation, so do not repeat it.
    helm uninstall "${rel}" -n "${ns}" --no-hooks --wait=false || warn "helm uninstall returned non-zero; continuing to check state"
    if helm list -n "${ns}" --all -q 2>/dev/null | grep -qx "${rel}"; then
      warn "${ns}/${rel} still present after uninstall — inspect manually, NOT deleting release secrets by hand."
    else
      info "${ns}/${rel} cleared"
      flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile of ${ns}/${rel} did not complete in 10m"
    fi
  done < <(
    for ns in $(kubectl get helmrelease -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"\n"}{end}' | sort -u); do
      helm list -n "${ns}" --all -o json 2>/dev/null \
        | yq -r '.[] | select(.status | test("uninstalling|pending-install|pending-upgrade|pending-rollback")) | "'"${ns}"' " + .name + " " + .status' 2>/dev/null || true
    done
  )
  [ "${found}" = "1" ] || info "no wedged releases found."
}

step_externaldns() {
  say "EXTERNAL-DNS — SAFE UPGRADE (prove the owner id BEFORE the pod can run)"
  # THE HAZARD: a Helm upgrade sets replicas back to the chart default (1) and applies the new args in
  # ONE operation. There is no window in which to check the result first. If --txt-owner-id is missing
  # the pod starts as owner `default`, claims the other cluster's TXT registry entries, and under
  # `policy: sync` DELETES the records behind them. That is the 2026-09-04 incident.
  #
  # THE FIX IS TO VERIFY IN ADVANCE, NOT AFTER: render the chart with the LIVE HelmRelease values and
  # assert the flag appears. Rendering is deterministic and touches nothing, so this converts a race
  # into a precondition.
  local ns=external-dns rel=external-dns
  local owner chart ver repo vals
  owner="$(cvar EXTERNALDNS_TXT_OWNER_ID)"
  [ -n "${owner}" ] && [ "${owner}" != "default" ] && [ "${owner}" != "unset-see-cluster-vars" ] \
    || die "refusing: EXTERNALDNS_TXT_OWNER_ID is '${owner}'"

  chart="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.chart}')"
  ver="$(kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.chart.spec.version}')"
  repo="$(yq eval-all --no-doc 'select(.kind == "HelmRepository") | .spec.url' platform/externaldns/helmrepository.yaml | tr -d '"')"
  [ -n "${repo}" ] || die "could not derive the external-dns chart repo url"

  vals="${SNAP_DIR}/ed-values.yaml"
  kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.values}' | yq -P > "${vals}"
  info "chart=${chart} version=${ver} repo=${repo}"
  info "live spec.values.txtOwnerId = $(yq '.txtOwnerId' "${vals}")"

  say "rendering the chart with the LIVE values to prove the flag"
  local rendered="${SNAP_DIR}/ed-rendered.yaml"
  helm template "${rel}" "${chart}" --repo "${repo}" --version "${ver}" -n "${ns}" -f "${vals}" > "${rendered}"
  local args; args="$(yq -r 'select(.kind == "Deployment") | .spec.template.spec.containers[0].args[]' "${rendered}")"
  printf '%s\n' "${args}" | sed 's/^/      /'
  printf '%s\n' "${args}" | grep -qx -- "--txt-owner-id=${owner}" \
    || die "the render does NOT contain --txt-owner-id=${owner}.
       An upgrade would start external-dns as owner 'default' and delete the other cluster's records.
       REFUSING. Fix the chart values key before retrying."
  info "PROVEN: the upgrade will carry --txt-owner-id=${owner}"

  say "snapshotting the Cloudflare zone (baseline for the post-check)"
  dns_snapshot "${SNAP_DIR}/dns-before.txt"

  say "resuming + forcing the upgrade"
  kubectl -n "${ns}" get helmrelease "${rel}" -o jsonpath='{.spec.suspend}' | grep -q true \
    && flux resume helmrelease "${rel}" -n "${ns}" || info "(HelmRelease was not suspended)"
  flux reconcile helmrelease "${rel}" -n "${ns}" --force --timeout=10m || warn "reconcile did not complete in 10m"

  say "verifying the LIVE deployment args"
  local live; live="$(kubectl -n "${ns}" get deploy "${rel}" -o jsonpath='{.spec.template.spec.containers[0].args}' 2>/dev/null || true)"
  printf '    %s\n' "${live}"
  printf '%s' "${live}" | grep -q -- "--txt-owner-id=${owner}" \
    || { kubectl -n "${ns}" scale deploy "${rel}" --replicas=0 || true
         die "live args LACK --txt-owner-id=${owner}. Scaled to 0. Investigate before scaling up."; }
  info "live deployment carries --txt-owner-id=${owner}"

  say "post-check: did anything in the shared zone change?"
  sleep 45   # external-dns --interval=1m; give it a full cycle to act before judging it safe
  dns_snapshot "${SNAP_DIR}/dns-after.txt"
  dns_diff "${SNAP_DIR}/dns-before.txt" "${SNAP_DIR}/dns-after.txt" \
    || warn "records changed — expected ONLY additions under this cluster's own BASE_DOMAIN ($(cvar BASE_DOMAIN)).
       Any change to a name outside it, or to a TXT entry owned by another cluster, is a REGRESSION."
}

step_resume() {
  say "RESUME TIERS (dependency order)"
  local n s3; s3="$(cvar S3_BACKUP_ENDPOINT)"
  for n in "${TIER_ORDER[@]}"; do
    if [ "$(k_susp "$n")" = "true" ]; then
      info "resuming $n"; flux resume kustomization "$n" --timeout=1s >/dev/null 2>&1 || true
    else
      info "$n already unsuspended"
    fi
    wait_ready "$n" 900 || { warn "stopping at $n — a partial, understood bring-up beats a forced green."; return 1; }
  done

  case "${s3}" in
    ""|TBD-*) warn "velero left SUSPENDED: S3_BACKUP_ENDPOINT='${s3}' is not a usable endpoint.
       Resuming it would create a BackupStorageLocation that can never reach a bucket, i.e. a tier
       that reports Ready-ish while backups silently do not exist. Set the real prod endpoint first." ;;
    *) info "resuming velero"; flux resume kustomization velero --timeout=1s >/dev/null 2>&1 || true
       wait_ready velero 900 || warn "velero did not converge" ;;
  esac
}

step_verify() {
  say "VERIFY"
  local base; base="$(cvar BASE_DOMAIN)"
  info "BASE_DOMAIN = ${base}"
  say "kustomizations"
  kubectl get kustomization -A \
    -o custom-columns=NAME:.metadata.name,SUSPEND:.spec.suspend,READY:.status.conditions[?\(@.type==\"Ready\"\)].status --no-headers
  say "certificates (must be Ready before any hostname is trusted)"
  kubectl get certificate -A --no-headers 2>/dev/null || info "(none yet)"
  say "pods not Running/Completed"
  kubectl get pods -A --no-headers | awk '$4!="Running" && $4!="Completed"' || true
  info "total pods: $(kubectl get pods -A --no-headers | wc -l)"
  say "gateway / LoadBalancer VIPs"
  kubectl get svc -A --field-selector spec.type=LoadBalancer --no-headers 2>/dev/null || true
  say "published DNS for this cluster's own domain"
  dns_snapshot "${SNAP_DIR}/dns-verify.txt"
  info "records under ${base}: $(grep -c "${base}" "${SNAP_DIR}/dns-verify.txt" || true)"
  warn "confirm every TXT registry entry for ${base} reads this cluster's owner id ($(cvar EXTERNALDNS_TXT_OWNER_ID)),
       and that NO record outside ${base} was touched. Compare against ${SNAP_DIR}/dns-before.txt."
}

# ============================================================================ dispatch
for s in "${STEPS[@]}"; do
  case "$s" in
    preflight)   step_preflight ;;
    crds)        step_crds ;;
    unwedge)     step_unwedge ;;
    externaldns) step_externaldns ;;
    resume)      step_resume ;;
    verify)      step_verify ;;
    all)         step_preflight; step_crds; step_unwedge; step_externaldns; step_resume; step_verify ;;
    *)           die "unknown step '$s' (preflight crds unwedge externaldns resume verify all)" ;;
  esac
done

say "done: ${STEPS[*]}"