From 5c64e06354849a590009a983449135337db71ee0 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sat, 29 Aug 2026 15:03:03 -0500 Subject: [PATCH] fix(longhorn): cap concurrent-replica-rebuild-per-node-limit=1 (HDD etcd protection) The k3s VM disks are HDD-backed, so a full Longhorn replica rebuild's sequential I/O can saturate the spindle and starve etcd's WAL fsync, crash-looping etcd and flapping the kube-vip VIP (observed during a rolling node resize). Cap rebuilds to one replica per node at a time so a node bounce or replica replenishment can never storm the disks. Set via HelmRelease defaultSettings.concurrentReplicaRebuildPerNodeLimit (GitOps-managed, permanent). replica-replenishment-wait-interval stays at the Longhorn default of 600s. Reference values.yaml kept in sync. Co-Authored-By: Claude Opus 4.8 --- platform/longhorn/helmrelease.yaml | 9 +++++++++ platform/longhorn/values.yaml | 1 + 2 files changed, 10 insertions(+) diff --git a/platform/longhorn/helmrelease.yaml b/platform/longhorn/helmrelease.yaml index df7c409..c27091c 100644 --- a/platform/longhorn/helmrelease.yaml +++ b/platform/longhorn/helmrelease.yaml @@ -56,6 +56,15 @@ spec: # never blocks scheduling). replicaAutoBalance: "best-effort" + # --- rebuild I/O throttle for slow disks (etcd WAL fsync protection) --- + # The k3s VM disks sit on HDDs, so a full-replica rebuild's sequential I/O can saturate the + # spindle and starve etcd's WAL fsync — which crash-loops etcd and flaps the kube-vip VIP. + # Cap Longhorn to rebuilding ONE replica per node at a time so a node bounce / replica + # replenishment can never storm the HDDs. Deliberately conservative for this slow-disk cluster + # (Longhorn's chart default is higher). Pair with replica-replenishment-wait-interval=600 + # (Longhorn default) so brief node bounces don't trigger replenishment at all. + concurrentReplicaRebuildPerNodeLimit: 1 + # --- capacity guardrails (docs/04 "reserve disk headroom") --- # Reserve 20% of the dedicated data disk so a node never fills to 100% (which wedges the # kubelet and Longhorn alike). Dev disks are small, so headroom matters. diff --git a/platform/longhorn/values.yaml b/platform/longhorn/values.yaml index 116f756..a061442 100644 --- a/platform/longhorn/values.yaml +++ b/platform/longhorn/values.yaml @@ -13,6 +13,7 @@ # replicaSoftAntiAffinity: false # HARD anti-affinity: one replica per node # replicaZoneSoftAntiAffinity: true # single zone (3 mini-PCs, no failure zones) # replicaAutoBalance: best-effort # rebalance toward one-per-node when a node returns +# concurrentReplicaRebuildPerNodeLimit: 1 # throttle rebuild I/O so it can't starve etcd WAL fsync (HDD-backed) # storageReservedPercentageForDefaultDisk: "20" # reserve disk headroom (docs/04 guardrail) # storageMinimalAvailablePercentage: "25" # stop scheduling below 25% free # storageOverProvisioningPercentage: "100" # dev: no thin over-provisioning (1:1)