diff --git a/bootstrap/standup.sh b/bootstrap/standup.sh index b7f9e3e..366849a 100755 --- a/bootstrap/standup.sh +++ b/bootstrap/standup.sh @@ -26,6 +26,11 @@ # # --confirm-destructive is required by `unwedge` only. Everything else is additive or reversible. # +# --s3-endpoint makes `s3check` probe a CANDIDATE backup endpoint instead of the live +# cluster-vars value, so a proposed target can be PROVEN before it is authored into cluster-vars. +# `s3check` writes and then deletes ONE throwaway object under `/.s3check/`; it changes +# nothing else and applies nothing. See infra/BACKUP-R2.md. +# set -euo pipefail # ---------------------------------------------------------------------------- args @@ -85,10 +90,18 @@ say "target confirmed: ${live_server}" # Dependency order. Flux enforces the edges itself via dependsOn; resuming in this order simply means # each wait is meaningful instead of blocking on a dependency that is still suspended. -# flux-system -> cilium -> platform -> platform-config -> {leaf tiers, apps-wave-0} -> wave-1 -> wave-2 -TIER_ORDER=(platform platform-config oauth2-proxy headlamp homepage crafty apps-wave-0 apps-wave-1 apps-wave-2 apps-vms) -# velero is NOT in that list on purpose — see step `resume`: it consumes S3_BACKUP_ENDPOINT, which on -# prod is an intentionally-invalid TBD placeholder pending an operator decision. +# flux-system -> cilium -> platform -> platform-config -> platform-config-cdi +# -> {leaf tiers, apps-wave-0} -> wave-1 -> wave-2 -> apps-vms +# +# `platform-config-cdi` MUST stay in this list and MUST stay before apps-vms. It carries the CDI +# StorageProfiles, which were split out of platform-config because StorageProfile's CRD does not exist +# until cdi-operator has reconciled the CDI CR (clusters/base/platform-config-cdi.yaml). apps-vms +# dependsOn it: apps/vms/unifi provisions on `longhorn-vm`, and without the profile applied CDI infers +# ReadWriteOnce instead of the authored ReadWriteMany — permanently non-live-migratable for that disk. +# Omitting it here would make `resume` return success without ever waiting on the tier that guarantees it. +TIER_ORDER=(platform platform-config platform-config-cdi oauth2-proxy headlamp homepage crafty apps-wave-0 apps-wave-1 apps-wave-2 apps-vms) +# velero is NOT in that list on purpose — see step `resume`: it consumes S3_BACKUP_ENDPOINT, which is +# an intentionally-invalid placeholder until the operator supplies the real R2 endpoint. cvar() { kubectl -n flux-system get cm cluster-vars -o jsonpath="{.data.$1}" 2>/dev/null || true; } @@ -150,6 +163,354 @@ dns_diff() { # — returns non-zero if anything changed return 1 } +# ======================================================================= s3check plumbing +# WHY THIS IS THIS BIG. The previous version of `s3check` curl'd /minio/health/live and reported +# REACHABLE. That probe could not pass against Cloudflare R2 (no such path), and reachability is not +# the property anyone cares about: a reachable endpoint with a wrong key, a missing bucket, or R2's +# checksum rejection still yields a velero tier that reports Ready while NO backup exists. What must +# be proven is the write path Velero and CNPG-barman actually use, with the credentials they actually +# hold, from the network position they actually occupy. + +# Pinned. The AWS CLI is a deliberate choice over `minio/mc`: +# - it prints the S3 ERROR CODE (NoSuchBucket / AccessDenied / InvalidAccessKeyId / InvalidArgument), +# so a failure can be NAMED and turned into a fix instead of guessed at from an HTTP status; +# - it exposes the FLEXIBLE-CHECKSUM behaviour that is the R2 trap +# (AWS_REQUEST_CHECKSUM_CALCULATION), so the trap can be reproduced on purpose here instead of +# discovered by a silently-failing backup at 03:00; +# - its image carries bash + coreutils. `minio/mc` ships no sed/awk/diff — that minimalism has +# already broken one probe in this repo — and it cannot be made to send the checksum headers, so +# it cannot test for the trap at all. +S3CHECK_IMAGE="amazon/aws-cli:2.36.40" +S3CHECK_RUN_ID="$(date +%s)" + +# Throwaway probe pods, tracked so they are removed on EVERY exit path (including `die`). A probe +# must never leave a pod behind on a live cluster. +S3CHECK_PODS=() +s3_cleanup_pods() { + local p + for p in ${S3CHECK_PODS[@]+"${S3CHECK_PODS[@]}"}; do + kubectl -n "${p%%/*}" delete pod "${p#*/}" --ignore-not-found --wait=false >/dev/null 2>&1 || true + done + S3CHECK_PODS=() +} +trap s3_cleanup_pods EXIT + +# TRUE when the endpoint is not a usable target. `TBD-*` alone was WRONG once the placeholder became +# `https://TBD-R2-ACCOUNT-ID.r2.cloudflarestorage.com`: that does not START with TBD-, so the old +# prefix pattern matched nothing and `resume` would have resumed velero straight onto a placeholder +# hostname — the exact silent-no-backups failure the guard exists to prevent. Match anywhere. +s3_is_placeholder() { # + case "$1" in + ""|*TBD*|*tbd-*|*REPLACE-ME*|*replace-me*|*CHANGE-ME*|*change-me*) return 0 ;; + esac + return 1 +} + +# Refuse any value that is not a plain URL/name fragment. This is BOTH a safety guard — these values +# are interpolated into the probe pod's JSON overrides AND into a shell script that runs inside it, so +# a quote, a backslash or a newline could rewrite either — and a correctness guard: a cluster-vars +# value with a trailing space or a stray quote is a defect, not something to paper over. +s3_safe_token() { #