From 6bc4406dca8f29db94b4f6639198e500d15eac8c Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sun, 30 Aug 2026 01:04:15 -0500 Subject: [PATCH] W5 hardening: Longhorn HR timeout 15m + no downgrade-rollback Root cause of a W5 wedge: default 5m Helm timeout on slow-HDD Longhorn upgrades caused a rollback attempt after the data had already migrated to the new minor, which the old manager fatally refuses. Give the upgrade room and never auto-rollback (fail forward). Co-Authored-By: Claude Opus 4.8 --- platform/longhorn/helmrelease.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/platform/longhorn/helmrelease.yaml b/platform/longhorn/helmrelease.yaml index 090abc2..a434558 100644 --- a/platform/longhorn/helmrelease.yaml +++ b/platform/longhorn/helmrelease.yaml @@ -11,6 +11,10 @@ metadata: namespace: longhorn-system spec: interval: 30m + # Longhorn upgrades roll every workload + engine image on slow HDD-backed nodes and can exceed the + # 5m Helm default; a short timeout caused a W5 mid-upgrade wedge (data migrated to the new minor but + # workloads rolled back, and the new-version data refuses the old-version manager). Give it room. + timeout: 15m0s chart: spec: chart: longhorn @@ -26,6 +30,10 @@ spec: upgrade: remediation: retries: 3 + # NEVER auto-rollback a Longhorn minor upgrade: once the on-disk data migrates to the new minor, + # the older manager fatally refuses it ("downgrading from vX to vY is not supported"). Rolling + # back the workloads while data is already migrated wedges the cluster. Fail forward instead. + remediateLastFailure: false values: # We own the StorageClasses (storageclasses.yaml) so the default 3-replica class and the bulk # tier are explicit and GitOps-managed. Disable the chart's built-in default SC so there is