diff --git a/bootstrap/standup.sh b/bootstrap/standup.sh index f654839..b38025d 100755 --- a/bootstrap/standup.sh +++ b/bootstrap/standup.sh @@ -174,6 +174,31 @@ step_preflight() { *) info "S3 backup endpoint set: OK" ;; esac + # COLD-START CRD ORDERING. Flux server-side dry-runs EVERY object in a Kustomization before applying + # any of them, so a single unknown `kind` aborts the whole tier. platform-config applies the CDI CR + # *and* CDI StorageProfiles in the same pass — but the StorageProfile CRD does not exist until the + # cdi-operator has reconciled that CR. On an established cluster the CRDs are long since there and + # this is invisible; on a COLD one it is a deadlock, and it is the same class of bug as Kyverno's + # 5.7 MiB CRD bundle. Report it here so `resume` does not burn 15 minutes discovering it. + say "CRDs that later tiers apply resources against" + local crd missing=0 + for crd in storageprofiles.cdi.kubevirt.io cdis.cdi.kubevirt.io kubevirts.kubevirt.io \ + volumesnapshotclasses.snapshot.storage.k8s.io; do + if kubectl get crd "${crd}" >/dev/null 2>&1; then + printf ' %-52s present\n' "${crd}" + else + printf ' \033[33m%-52s MISSING\033[0m\n' "${crd}"; missing=1 + fi + done + if [ "${missing}" = "1" ]; then + warn "a MISSING crd above will fail platform-config's dry-run and block every tier below it. + CDI's own CRDs (StorageProfile, DataVolume, ...) are created by cdi-operator only AFTER it + reconciles the CDI CR. Check that the CDI CR is applied and cdi-operator is Available; if the + CRD appeared only recently this can also be a stale discovery cache in kustomize-controller, + which a forced reconcile clears." + kubectl -n cdi get cdi,deploy 2>/dev/null | head -12 || info "(no cdi namespace yet)" + fi + say "tiers" local n for n in flux-system cilium "${TIER_ORDER[@]}" velero; do @@ -199,6 +224,39 @@ step_crds() { # Charts whose CRD bundle exceeds kubectl's 256 KiB annotation limit set crds.install=false, so # nothing installs them and `platform` can never go Ready. See bootstrap/flux/preapply-crds.sh. bash "${script_dir}/flux/preapply-crds.sh" + + # ---------------------------------------------------------------- the CDI cold-start deadlock + # SAME CLASS OF BUG, DIFFERENT OPERATOR. `platform-config` applies the CDI CR *and* + # platform/kubevirt/cdi-storageprofiles.yaml in ONE pass. But StorageProfile's CRD does not exist + # until cdi-operator has reconciled that CR — and Flux server-side dry-runs EVERY object before + # applying ANY of them, so the unknown kind aborts the whole tier, the CR is never applied, the CRD + # is never created, and platform-config can never converge. A true deadlock, invisible on an + # established cluster (the CRDs are long since there) and fatal on a cold one. + # + # Breaking it needs exactly one thing: get the CDI CR in. Then cdi-operator creates the remaining + # CDI CRDs and platform-config's next reconcile dry-runs clean. We apply ONLY the CR — the minimum + # — and leave every other object to Flux. The CR carries no ${} tokens, so no substitution is + # needed and applying it here cannot diverge from what Flux would render. + local cdi_cr="${repo_root}/platform/kubevirt/cdi-cr.yaml" + if kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1; then + info "storageprofiles CRD already present — CDI is past its cold start" + elif [ ! -f "${cdi_cr}" ]; then + warn "expected ${cdi_cr} and it is missing — skipping the CDI unblock" + else + say "CDI cold-start unblock: applying the CDI CR so cdi-operator can create its CRDs" + kubectl get ns cdi >/dev/null 2>&1 || die "namespace 'cdi' absent — the platform tier has not applied the CDI operator yet. Run 'resume' far enough to make platform Ready first." + kubectl apply --server-side --force-conflicts -f "${cdi_cr}" + local i=0 + while [ "$i" -lt 300 ]; do + kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 && break + sleep 10; i=$((i+10)) + done + kubectl get crd storageprofiles.cdi.kubevirt.io >/dev/null 2>&1 \ + || die "cdi-operator did not create storageprofiles.cdi.kubevirt.io within ${i}s. + Inspect: kubectl -n cdi get cdi,pods,deploy — platform-config stays blocked until it exists." + kubectl wait --for=condition=Established --timeout=60s crd/storageprofiles.cdi.kubevirt.io >/dev/null + info "storageprofiles.cdi.kubevirt.io created and Established (${i}s)" + fi } step_unwedge() { diff --git a/infra/PROD-STANDUP.md b/infra/PROD-STANDUP.md index 1d995ee..209fb84 100644 --- a/infra/PROD-STANDUP.md +++ b/infra/PROD-STANDUP.md @@ -115,13 +115,37 @@ timeout 300 infra/devcontainer/dev.sh run bootstrap/standup.sh \ --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 preflight ``` -### C2 — pre-apply chart CRDs (idempotent) +### C2 — pre-apply chart CRDs, and break the CDI cold-start deadlock (idempotent) ```bash timeout 600 infra/devcontainer/dev.sh run bootstrap/standup.sh \ --kubeconfig infra/ansible/.cache/prod.kubeconfig --expect-server https://192.168.2.9:6443 crds ``` +Two cold-start problems, same root cause: **Flux server-side dry-runs every object in a Kustomization +before applying any of them, so one unknown `kind` aborts the entire tier.** + +1. **Kyverno** — its ~5.6 MiB CRD bundle exceeds kubectl's 256 KiB annotation limit, so the chart sets + `crds.install: false` and nothing else installs them. `bootstrap/flux/preapply-crds.sh` applies them + server-side, deriving the version from the HelmRelease Flux reconciles (no second pin). + +2. **CDI — a true deadlock.** `platform-config` applies the CDI CR *and* + `platform/kubevirt/cdi-storageprofiles.yaml` in the **same pass**. But `StorageProfile`'s CRD does + not exist until `cdi-operator` has reconciled that CR. So the dry-run fails on the unknown kind → + the CR is never applied → the CRD is never created → `platform-config` can never converge, and + every tier below it stays blocked. **Invisible on an established cluster** (dev's CRDs have been + there for months) **and fatal on a cold one.** Observed on prod 2026-09-05: `cdi-operator` + Available for 12h with **no CDI CR** and no `storageprofiles` CRD. + + The step breaks it by applying **only the CR** — the minimum — and leaving everything else to + Flux. It carries no `${}` tokens, so applying it here cannot diverge from what Flux would render. + The CRD appeared in 10s. + + **The durable fix is a repo change, not this script:** move `cdi-storageprofiles.yaml` out of + `platform-config` into its own Kustomization with `dependsOn: platform-config`, so the CR lands one + reconcile before the resources that need its CRDs. Until that lands, C2 is a prerequisite for C5 on + any cold cluster. + ### C3 — unwedge stuck Helm releases A release left in `uninstalling` / `pending-*` cannot be advanced by helm-controller: it reports