From 00d895d25f1b8baf0db2341dd89357f83ba9cc23 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sun, 30 Aug 2026 12:08:29 -0500 Subject: [PATCH] kps: 20m timeout + no-rollback remediation for the 65->88 catch-up The 65->88 multi-major upgrade exceeded the default 5m Helm timeout (full operator/KSM/ node-exporter + Prometheus/Alertmanager StatefulSet rollout), so Flux rolled back to 65.8.1 four times and Stalled, blocking the platform kustomization and everything downstream. CRDs pre-applied server-side at operator v0.93.1. Co-Authored-By: Claude Opus 4.8 --- platform/monitoring/kube-prometheus-stack.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/platform/monitoring/kube-prometheus-stack.yaml b/platform/monitoring/kube-prometheus-stack.yaml index e957224..767635e 100644 --- a/platform/monitoring/kube-prometheus-stack.yaml +++ b/platform/monitoring/kube-prometheus-stack.yaml @@ -14,6 +14,13 @@ metadata: namespace: monitoring spec: interval: 30m + # The 65->88 catch-up is a large multi-major jump (operator + KSM + node-exporter roll, plus the + # Prometheus/Alertmanager StatefulSets). The default 5m Helm timeout is too short for the full rollout + # to reach Ready, so Flux logged "context deadline exceeded" and rolled back to 65.8.1 four times. + # A generous timeout lets one attempt finish; remediateLastFailure:false keeps partial progress instead + # of yo-yoing back to the old chart (a re-reconcile then drives it forward). CRDs are pre-applied + # server-side (Helm never updates crds/ on upgrade), so schema is already at operator v0.93.1. + timeout: 20m0s chart: spec: chart: kube-prometheus-stack @@ -27,7 +34,9 @@ spec: retries: 3 upgrade: remediation: - retries: 3 + retries: 2 + remediateLastFailure: false + cleanupOnFail: true values: # --- k3s trim: control-plane components are embedded in the k3s server process and are not # scrapable as discrete endpoints. Disabling avoids permanently-firing "target down" noise.