diff --git a/clusters/base/cilium.yaml b/clusters/base/cilium.yaml new file mode 100644 index 0000000..2ba18e1 --- /dev/null +++ b/clusters/base/cilium.yaml @@ -0,0 +1,60 @@ +--- +# Flux Kustomization: CILIUM (the CNI) — reconciles ./platform/cilium. +# Serves R-PLAT-01 (platform components reconciled by GitOps) and R-GITOPS-01/03. +# +# WHY CILIUM HAS ITS OWN KUSTOMIZATION (split out of `platform`, 2026-09-01) +# +# This is the MIRROR IMAGE of the oauth2-proxy / headlamp splits, not a repeat of them. Those are +# leaves that must NEVER gate a tier, so they left `platform` and run `wait: false` below it. +# Cilium is the one component that legitimately SHOULD gate everything — but inside the shared +# `platform` Kustomization its health was indistinguishable from eleven unrelated components: +# +# * `platform`'s Ready condition could not tell you "the CNI is down" from "longhorn is slow". +# * The only emergency valve was `flux suspend kustomization platform` — freezing the ENTIRE +# tier (storage, TLS, DNS, ingress, monitoring, Keycloak, KubeVirt) to stop touching the CNI. +# `flux suspend kustomization cilium` now freezes exactly the CNI and nothing else, which is +# the valve the 2026-09-01 substitution incident wanted and did not have. +# * `platform`'s 10m interval + 5m timeout are wrong for a CNI: a 3-node DaemonSet rollout with +# `wait: true` can exceed 5m, and re-rendering the CNI every 10m buys nothing. +# +# GATING IS PRESERVED, NOT WEAKENED: `wait: true` here plus `platform.dependsOn: [cilium]` means +# nothing in the graph reconciles until the CNI is healthy — the same guarantee as before, now +# expressed as an explicit edge instead of an accident of co-tenancy. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cilium + namespace: flux-system +spec: + # A CNI does not need a 10m re-render cadence. A git push still reconciles immediately — + # kustomize-controller is source-triggered; `interval` is only the drift-correction floor. + interval: 1h + retryInterval: 2m + # 10m (not the 5m `platform` uses): a 3-node Cilium DaemonSet rollout under `wait: true`, + # rolled ONE NODE AT A TIME (maxUnavailable: 1, P2), legitimately exceeds 5m. + timeout: 10m + prune: true + # TRUE, deliberately: this is the ONE component whose health should gate the whole platform. + wait: true + # Same revision-gating reason `platform` has it (2026-09-01 incident): `cluster-vars` is APPLIED + # by the root `flux-system` Kustomization, and this Kustomization SUBSTITUTES from it. `dependsOn` + # is revision-gated (kustomize-controller v1.4.0 `checkDependencies`), so this cannot render + # revision N of helmrelease.yaml against the revision N-1 `cluster-vars`. That race is exactly how + # `k8sServiceHost: ""` once reached the Cilium HelmRelease and took down 2/3 agents. + dependsOn: + - name: flux-system # the ROOT Kustomization (sync.yaml) that applies cluster-vars + sourceRef: + kind: GitRepository + name: flux-system + path: ./platform/cilium + # MANDATORY, not decorative: platform/cilium/helmrelease.yaml references ${K3S_API_HOST}. + # kustomize-controller now runs --feature-gates=StrictPostBuildSubstitutions=true, so without + # this stanza an undefined ${K3S_API_HOST} FAILS this Kustomization outright (it no longer + # renders empty). That is the intended fail-closed behaviour — but it means the stanza is + # load-bearing for the CNI coming up at all. Do not remove it. + postBuild: + substituteFrom: + - kind: ConfigMap + name: cluster-vars + # No `decryption` stanza: platform/cilium carries no SOPS-encrypted resource (unlike + # oauth2-proxy/headlamp/crafty/velero, whose overlays patch one in by name). diff --git a/clusters/base/kustomization.yaml b/clusters/base/kustomization.yaml index 85aa5a6..b1db32b 100644 --- a/clusters/base/kustomization.yaml +++ b/clusters/base/kustomization.yaml @@ -5,6 +5,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - flux-system.yaml # GitRepository (reconciliation source) + # Cilium (the CNI) — its OWN Kustomization, split OUT of `platform` (2026-09-01). The mirror image + # of the oauth2-proxy/headlamp splits: those are leaves that must never gate a tier (wait:false); + # Cilium is the one component that legitimately SHOULD gate everything, so it keeps wait:true and + # `platform` dependsOn it. Splitting it out buys a per-CNI Ready condition, CNI-appropriate + # interval/timeout, and `flux suspend kustomization cilium` as an emergency valve that does not + # freeze the other eleven platform components. See clusters/base/cilium.yaml. + - cilium.yaml # Flux Kustomization -> ./platform/cilium (dependsOn flux-system, wait:true) - platform.yaml # Flux Kustomization -> ./platform (CONTROLLERS: HelmReleases / CRDs) - platform-config.yaml # Flux Kustomization -> ./platform-config (CONFIGS: CRs; dependsOn platform) # App tier, split into dependency-ordered WAVES (was the single ./apps `apps` Kustomization). diff --git a/clusters/base/platform.yaml b/clusters/base/platform.yaml index 02251e1..394f0cd 100644 --- a/clusters/base/platform.yaml +++ b/clusters/base/platform.yaml @@ -11,7 +11,27 @@ spec: interval: 10m retryInterval: 2m timeout: 5m - prune: true # drift correction: delete resources removed from git (R-GITOPS-03) + # ======================= TEMPORARY — prune:false TRANSITION STATE ======================= + # RESTORE TO `prune: true` IN THE VERY NEXT MERGE. This is step 1 of the documented Flux + # procedure for MOVING RESOURCES BETWEEN KUSTOMIZATIONS (fluxcd.io/flux/faq: "set spec.prune to + # false, sync, move the manifests, then re-enable pruning"), used here to move ./platform/cilium + # out of this tier and into its own `cilium` Kustomization (clusters/base/cilium.yaml). + # + # WHY IT IS NOT OPTIONAL: `platform/cilium/namespace.yaml` is the ONLY declaration of + # Namespace/kube-system anywhere under ./platform, and the live object carries + # `kustomize.toolkit.fluxcd.io/name=platform`. The moment ./platform/cilium leaves this + # Kustomization's rendered path, Namespace/kube-system + the Cilium HelmRepository/HelmRelease + # become STALE in this Kustomization's inventory. With prune:true that is not "uninstall the + # CNI" — it is `kubectl delete namespace kube-system`, which takes the whole cluster. + # + # A SECOND, INDEPENDENT PROTECTION IS ALSO IN PLACE (belt and braces, deliberately): + # kustomize-controller's pruner passes `Inclusions: GetOwnerLabels(platform, flux-system)`, so it + # only deletes objects whose LIVE labels still say they belong to `platform`. The new `cilium` + # Kustomization relabels all three objects to `name=cilium` when it applies them, and the + # `dependsOn: [cilium]` edge below is REVISION-gated — `platform` cannot reconcile revision N + # until `cilium` has APPLIED revision N. So the relabel provably precedes the prune. Neither + # mechanism is trusted alone for an object of this blast radius. + prune: false # <-- TEMPORARY; see block above. Normal value: true (R-GITOPS-03) wait: true # block `apps` until platform is healthy # ORDERING FIX (2026-09-01 incident: a substitution race that failed OPEN broke Cilium). # @@ -40,6 +60,13 @@ spec: # health-gates its own children (see clusters/dev/sync.yaml). dependsOn: - name: flux-system # the ROOT Kustomization (sync.yaml) that applies cluster-vars + # THE CNI GATES THE TIER. Cilium was split out of ./platform into its own Kustomization + # (clusters/base/cilium.yaml, wait:true) so its health has its own Ready condition and its own + # suspend valve. This edge preserves the ordering that co-tenancy used to provide implicitly: + # nothing in `platform` — and therefore nothing anywhere downstream — reconciles until the CNI + # is healthy. It is also revision-gated, which is what makes the prune-safety argument above + # deterministic rather than a race. + - name: cilium sourceRef: kind: GitRepository name: flux-system diff --git a/platform/kustomization.yaml b/platform/kustomization.yaml index b0dcedd..78a07e0 100644 --- a/platform/kustomization.yaml +++ b/platform/kustomization.yaml @@ -12,7 +12,15 @@ kind: Kustomization resources: # ---- Phase 2 — Platform Core: Network, Storage, TLS, DNS, Ingress ---- - kyverno # admission enforcer FIRST — policies present before other components reconcile - - cilium # CNI (adopts the running Ansible-installed release; parity verified) + # cilium (the CNI) is DELIBERATELY NOT here: it now reconciles in its OWN Flux Kustomization + # (clusters/base/cilium.yaml, dependsOn flux-system, wait:true), and `platform` dependsOn IT. + # Not the headlamp/oauth2-proxy rationale (a leaf that must not gate) but its mirror image: + # the CNI is the one component that SHOULD gate everything, and inside this shared tier its + # health was indistinguishable from eleven unrelated components, the only emergency valve was + # freezing the whole tier, and the tier's 10m/5m interval/timeout are wrong for a 3-node + # DaemonSet rollout. Gating is preserved via the explicit `platform.dependsOn: [cilium]` edge. + # NOTE: platform/cilium/namespace.yaml is the ONLY declaration of Namespace/kube-system in + # this tree — that is why the move required a prune-disabled transition. See DECISIONS-LOG. - longhorn # replicated block storage / CSI - external-snapshotter # CSI snapshot CRDs + snapshot-controller (VolumeSnapshot API; CDI smart-clone) - cert-manager # Let's Encrypt DNS-01 wildcard — needs SOPS `cloudflare-api-token`