277 lines
20 KiB
YAML
277 lines
20 KiB
YAML
---
|
|
# Longhorn — distributed replicated block storage across the 3 nodes (ADR-0003).
|
|
# Serves R-STATE-01 (state survives destroy/rebuild) and R-STATE-02 (3-way replication).
|
|
#
|
|
# Dev-appropriate values. Environment differences enter ONLY via $${...} substitution from
|
|
# cluster-vars (R-ENV-02); everything else is identical base config for dev and prod.
|
|
apiVersion: helm.toolkit.fluxcd.io/v2
|
|
kind: HelmRelease
|
|
metadata:
|
|
name: longhorn
|
|
namespace: longhorn-system
|
|
spec:
|
|
interval: 30m
|
|
# Longhorn upgrades roll every workload + engine image on slow HDD-backed nodes and can exceed the
|
|
# 5m Helm default; a short timeout caused a W5 mid-upgrade wedge (data migrated to the new minor but
|
|
# workloads rolled back, and the new-version data refuses the old-version manager). Give it room.
|
|
timeout: 15m0s
|
|
chart:
|
|
spec:
|
|
chart: longhorn
|
|
version: "<1.14.0" # W5 upgrade 1.11->1.12 (resolves 1.12.1; supports k8s 1.36). Renovate maintains
|
|
sourceRef:
|
|
kind: HelmRepository
|
|
name: longhorn
|
|
namespace: longhorn-system
|
|
install:
|
|
createNamespace: false # namespace.yaml owns it (GitOps-managed, labelled)
|
|
remediation:
|
|
retries: 3
|
|
upgrade:
|
|
remediation:
|
|
retries: 3
|
|
# NEVER auto-rollback a Longhorn minor upgrade: once the on-disk data migrates to the new minor,
|
|
# the older manager fatally refuses it ("downgrading from vX to vY is not supported"). Rolling
|
|
# back the workloads while data is already migrated wedges the cluster. Fail forward instead.
|
|
remediateLastFailure: false
|
|
values:
|
|
# We own the StorageClasses (storageclasses.yaml) so the default 3-replica class and the bulk
|
|
# tier are explicit and GitOps-managed. Disable the chart's built-in default SC so there is
|
|
# exactly one default-class annotation in the cluster.
|
|
persistence:
|
|
defaultClass: false
|
|
|
|
defaultSettings:
|
|
# --- data path (R-STATE-02, R-ENV-02) ---
|
|
# The on-node directory every replica is written into. A cluster-vars KNOB, not a literal,
|
|
# because the two environments mount their data disk differently and a hardcoded path would
|
|
# put one of them on the OS filesystem:
|
|
# dev /var/lib/longhorn — a dedicated ext4 disk (/dev/vdb) mounted there by
|
|
# roles/node_baseline. The mount root IS the dataPath: one disk,
|
|
# one tenant. 18 live volumes; this value must never change.
|
|
# prod /data/longhorn — a SUBDIRECTORY of the claimed ~424 GB partition mounted at
|
|
# /data, whose sibling /data/k3s holds k3s's entire data dir
|
|
# (etcd + the containerd image store, moved off the 11.8 GiB
|
|
# /var). Two tenants, so Longhorn gets a subdirectory rather
|
|
# than the mount root. infra/PROD-PROVISIONING.md §2.
|
|
# [parity] MUST equal roles/node_baseline's node_baseline_longhorn_data_path for the same
|
|
# environment — Ansible creates and mounts the directory, Helm tells Longhorn to use it.
|
|
# Set in BOTH overlays (clusters/dev, clusters/prod); base default is the dev value.
|
|
defaultDataPath: "${LONGHORN_DATA_PATH}"
|
|
|
|
# --- replication (R-STATE-02) ---
|
|
# Global fallback replica count. The `longhorn` StorageClass sets numberOfReplicas explicitly
|
|
# (authoritative); this keeps any volume created without an explicit count consistent.
|
|
# Substituted from cluster-vars.longhorn_default_replica_count (=3 in dev).
|
|
defaultReplicaCount: ${LONGHORN_DEFAULT_REPLICA_COUNT}
|
|
|
|
# --- replica anti-affinity across nodes (R-STATE-02 durability guarantee) ---
|
|
# HARD anti-affinity: never place two replicas of the same volume on the same node, so a
|
|
# single node loss cannot take down more than one replica. On a 3-node / 3-replica cluster
|
|
# this yields exactly one replica per node.
|
|
replicaSoftAntiAffinity: false
|
|
# Single zone (3 mini-PCs, no failure zones): allow replicas within the one zone.
|
|
replicaZoneSoftAntiAffinity: true
|
|
# When a downed node returns, rebalance replicas back toward one-per-node (best-effort so it
|
|
# never blocks scheduling).
|
|
replicaAutoBalance: "best-effort"
|
|
|
|
# --- I/O THROTTLES: bound every Longhorn write source that competes with etcd's WAL fsync ---
|
|
# WHY THIS BLOCK IS BIGGER THAN THE ONE LINE IT REPLACES. On 2026-09-07 k3s-prod-1 went
|
|
# NotReady ("Kubelet stopped posting node status") with the host up and SSH-able: its embedded
|
|
# etcd could not finish a WAL fdatasync (took=3.56s/6.71s/4.95s/3.40s against a 1 s
|
|
# expectation), "apply request took too long took=8.30s", then "etcdserver: too many
|
|
# requests". /proc/pressure/io FULL avg300 read 11.36 there against 0.75 (node 2, NVMe) and
|
|
# 1.29 (node 3). Cause: EVERY prod node puts /data/k3s (etcd WAL + containerd) and
|
|
# /data/longhorn (every replica) on ONE partition, and node 1's was a 2011 consumer SATA SSD
|
|
# with 61,440 reallocated sectors (infra/PROD-PROVISIONING.md §2 and §0c).
|
|
#
|
|
# THE HONEST HEADLINE: the rebuild path was ALREADY throttled and it is not what this block
|
|
# fixes. concurrentReplicaRebuildPerNodeLimit was 1 during that outage and it HELD — the 18
|
|
# volumes rebuilt one at a time — and the node stalled anyway. Longhorn offers no further
|
|
# rebuild lever on the v1 data engine: replicaRebuildingBandwidthLimit is V2/SPDK ONLY
|
|
# (verified live 2026-09-13, the Setting reads a v2-only map with no v1 key) and there is no
|
|
# rebuild-concurrent-sync-limit Setting on 1.12.1 at all (NotFound). What is bounded below is
|
|
# therefore every OTHER Longhorn write source, each of which measured 5 per node.
|
|
#
|
|
# WHAT THIS BLOCK CANNOT DO. Node 1's drive HAS since been replaced (§0c), and THAT is the
|
|
# real mitigation: under the same load class — an 18-replica rebuild with node 1 as the write
|
|
# target — IO PSI full fell 11.36 -> 3.79 and the node stayed Ready. These settings only bound
|
|
# how much competing I/O Longhorn is ALLOWED to generate. etcd and Longhorn still share one
|
|
# ext4 filesystem, and no scheduler or cgroup control arbitrates the jbd2 journal that
|
|
# fdatasync waits on. 3.79 against the 5% NodeIOPressureHigh threshold is about 1.2 points of
|
|
# headroom, which is exactly why a SECOND concurrent heavy writer is still worth refusing.
|
|
#
|
|
# EVERY KEY BELOW IS PROVEN TO LAND, not assumed. `helm template longhorn/longhorn --version
|
|
# 1.12.1` (the version this chart range resolves to, per the HelmRelease's own
|
|
# lastAttemptedRevision) emits each one into the rendered longhorn-default-setting ConfigMap
|
|
# in kebab-case, and a live Setting of that name exists for each. This matters because the
|
|
# chart DROPS unknown defaultSettings keys with no error — see the orphanAutoDeletion note
|
|
# further down for the setting that was inert here for two minor versions because nobody
|
|
# checked the render.
|
|
|
|
# Rebuild ONE replica per node at a time. Unchanged, and load-bearing: without it a node
|
|
# bounce replenishes every degraded volume at once. The freeze/thaw to "0"/"1" during an
|
|
# operator bounce (runbooks/longhorn-degraded.md, docs/plans/cluster-upgrade-plan.md) patches
|
|
# the Setting directly — deliberate and temporary; this line is the value it returns to.
|
|
concurrentReplicaRebuildPerNodeLimit: 1
|
|
|
|
# Seconds Longhorn waits before replenishing a replica on a node that went away. This is
|
|
# Longhorn's own default and the live Setting already reads 600 — PINNED, not changed, so no
|
|
# future chart or manager default can silently shorten the window in which a node can be
|
|
# recovered by hand without triggering a rebuild storm.
|
|
replicaReplenishmentWaitInterval: 600
|
|
|
|
# WAS 5, MEASURED LIVE ON PROD 2026-09-13. Restores read from the backup target and write
|
|
# whole volumes to the local disk — the heaviest sustained write Longhorn can produce. 5 means
|
|
# five parallel full-volume writes onto the partition etcd is fsyncing to.
|
|
# THE COST OF 1, STATED HONESTLY: today it is zero. Longhorn has NO backup target configured
|
|
# (the backup-target Setting does not even exist on the live cluster — it is still the
|
|
# WS-STORAGE Phase 4 TODO at the foot of this file), and velero's nightly run is NOT a
|
|
# Longhorn backup: its schedules set snapshotVolumes:false with defaultVolumesToFsBackup:true,
|
|
# i.e. kopia file-system backup through velero's own node-agent, which this Setting does not
|
|
# gate. CNPG base backups are barman/Postgres-level and are not gated either. The only
|
|
# consumer is an R-STATE-03 DR restore of ~24 volumes, where serialising is the correct trade
|
|
# against losing etcd mid-restore — and an operator who needs restore throughput can raise the
|
|
# Setting for the duration and watch NodeIOPressureHigh. REVISIT when a Longhorn backup target
|
|
# actually lands: from that day this knob starts costing real recovery time.
|
|
concurrentVolumeBackupRestorePerNodeLimit: 1
|
|
|
|
# WAS 5, MEASURED LIVE ON PROD 2026-09-13. Snapshot "heavy tasks" are purge and clone: full
|
|
# reads and rewrites of snapshot disk files.
|
|
# WHY 1 AND NOT 2 — this balance was judged, not copied. The work that generates purges here
|
|
# is rebuild cleanup: auto-cleanup-system-generated-snapshot is true, there are NO
|
|
# RecurringJobs on this cluster, and the 24 volumes hold 24 snapshots between them. Purge
|
|
# therefore arrives at the rate rebuilds retire snapshots, which is ALREADY serialised to 1
|
|
# per node — so a purge concurrency of 1 keeps pace with its own generator. 2 would only drain
|
|
# an existing backlog faster, at the price of a second heavy writer on the disk during
|
|
# precisely the rebuild that starved etcd once, with about 1.2 points of measured PSI headroom
|
|
# to spend.
|
|
# THE COST OF 1: space is reclaimed serially after a rebuild storm. That is a REAL cost here —
|
|
# a 37.7 GiB system snapshot from prod-1's rebuild once pinned a volume at 97.3% ALLOCATED —
|
|
# but it is the cost that is already instrumented (LonghornDiskAllocationHigh,
|
|
# LonghornDiskUnschedulable) and recoverable by hand, where a wedged etcd is neither.
|
|
snapshotHeavyTaskConcurrentLimit: 1
|
|
|
|
# Worker threads WITHIN one backup / one restore. Both sit at Longhorn's default of 2; PINNED
|
|
# rather than lowered — combined with the per-node limit of 1 above, one volume at a time with
|
|
# two threads is already conservative, and pinning stops an upstream bump from widening it.
|
|
backupConcurrentLimit: 2
|
|
restoreConcurrentLimit: 2
|
|
|
|
# Automatic engine upgrades roll every volume's engine after a Longhorn minor bump — a
|
|
# cluster-wide restart-and-resync storm on the same shared disk. 0 disables it, making engine
|
|
# upgrade a deliberate operator action, which matches this HelmRelease's own
|
|
# remediateLastFailure:false posture on minor upgrades. It is Longhorn's default; PINNED so
|
|
# both halves of that one decision live in one file.
|
|
concurrentAutomaticEngineUpgradePerNodeLimit: 0
|
|
|
|
# Never drain a node that holds the last healthy replica of a volume. This was ALREADY the
|
|
# live value on 2026-09-13 — but only because it is Longhorn's own default: it appeared in NO
|
|
# manifest in this repo and in NO rendered ConfigMap, so an upstream default change would have
|
|
# moved it silently. PINNED, value unchanged. runbooks/longhorn-degraded.md and
|
|
# docs/plans/cluster-upgrade-plan.md both reason about drain safety, so it has to be a
|
|
# declaration rather than a coincidence.
|
|
nodeDrainPolicy: "block-if-contains-last-replica"
|
|
|
|
# --- capacity guardrails (docs/04 "reserve disk headroom") ---
|
|
# Reserve 20% of the dedicated data disk so a node never fills to 100% (which wedges the
|
|
# kubelet and Longhorn alike). Dev disks are small, so headroom matters.
|
|
storageReservedPercentageForDefaultDisk: "20"
|
|
# Stop scheduling new replicas onto a disk once free space drops below 25%.
|
|
storageMinimalAvailablePercentage: "25"
|
|
# Dev: 200% thin over-provisioning — config-only volumes use little actual space, so the small
|
|
# 39Gi data disks host the full app catalog. Physical fill risk is low (watch monitoring PVCs).
|
|
# Prod-grade alternative: grow the Longhorn data disks and return to 1:1 (WS-STORAGE).
|
|
storageOverProvisioningPercentage: "200"
|
|
|
|
# --- misc dev-appropriate defaults ---
|
|
# Keep replicas spread for durability rather than co-locating with the pod.
|
|
defaultDataLocality: "disabled"
|
|
# `orphanAutoDeletion: true` USED TO BE HERE AND WAS INERT. Removed 2026-09-13, having been
|
|
# verified dead two independent ways against the DEPLOYED 1.12.1:
|
|
# 1. RENDER: `helm template longhorn/longhorn --version 1.12.1` with that key set emits NO
|
|
# orphan key whatsoever into longhorn-default-setting. Longhorn 1.9 replaced
|
|
# orphanAutoDeletion (a bool) with orphanResourceAutoDeletion (a semicolon-separated list
|
|
# such as "replica-data;instance"), and the chart template drops unknown defaultSettings
|
|
# keys with no error.
|
|
# 2. LIVE, from the other end: the rendered longhorn-default-setting ConfigMap in
|
|
# longhorn-system carries no orphan key at all, `get setting orphan-auto-deletion` is
|
|
# NotFound, and `get setting orphan-resource-auto-deletion` reads "" — Longhorn's default.
|
|
# DELIBERATELY NOT REPLACED with orphanResourceAutoDeletion: orphan deletion is extra write
|
|
# work on the one partition that is this cluster's constraint (see the I/O block above) and it
|
|
# buys only reclaimed space, of which prod has hundreds of GB free. Leaving it unset keeps
|
|
# Longhorn's default of not auto-deleting, which is the conservative side here — orphan
|
|
# cleanup stays manual, as runbooks/longhorn-capacity.md and runbooks/node-disk.md say.
|
|
# Left as a comment rather than deleted silently, because "config that looks applied and is
|
|
# not" is the failure this repo keeps re-learning, and two runbooks cite this key by name.
|
|
|
|
# --- SCRAPE THE STORAGE LAYER, or every Longhorn alert is INERT (added 2026-09-13) ----------
|
|
# platform/monitoring/alerts.yaml ships the platform-health.longhorn group (as of 2026-09-13:
|
|
# LonghornVolumeDegraded, LonghornVolumeFaulted, LonghornDiskAllocationHigh and
|
|
# LonghornDiskUnschedulable) — and NONE of them could ever fire, because the chart defaults
|
|
# `metrics.serviceMonitor.enabled: false` and no ServiceMonitor existed in ns longhorn-system.
|
|
# VERIFIED live on prod 2026-09-13: `count by (job) (up)` listed 19 jobs and none was Longhorn;
|
|
# `count(longhorn_volume_robustness)` and `count({__name__=~"longhorn.*"})` both returned NOTHING.
|
|
# So the entire longhorn_* family was absent from the TSDB and four alert rules were decoration.
|
|
#
|
|
# WHY THIS IS PART OF THE STORAGE PATH, NOT AN OBSERVABILITY NICE-TO-HAVE: this cluster's
|
|
# recurring failure mode is a disk. prod-1's failing SSD starved etcd's WAL fsync and wedged the
|
|
# node (DECISIONS-LOG; the prod-1 drive replacement), and a degraded/faulted replica is precisely
|
|
# the first symptom Longhorn publishes. R-STATE-02 promises 3-way replication survives a node
|
|
# loss; with no scrape, a volume can sit at 2 replicas — or faulted — indefinitely, unalerted.
|
|
# platform/monitoring/pvc-capacity-alerts.yaml's header already flagged this exact gap as "a
|
|
# separate defect (a missing Longhorn ServiceMonitor/metrics config)". This closes it.
|
|
#
|
|
# Chart mechanics, verified by rendering longhorn 1.12.1 (the version this range resolves to):
|
|
# * templates/servicemonitor.yaml is gated on `{{- if .Values.metrics.serviceMonitor.enabled }}`
|
|
# and NOTHING else — no `.Capabilities` probe. Measured: `helm template --set
|
|
# metrics.serviceMonitor.enabled=true` emits exactly 1 ServiceMonitor, and 0 without the flag.
|
|
# So this line is both necessary and sufficient; unlike velero's, it needs no API capability.
|
|
# * the rendered ServiceMonitor selects `app: longhorn-manager` in the release namespace on the
|
|
# port named `manager`. CONFIRMED against prod: Service/longhorn-backend carries
|
|
# `app: longhorn-manager` and a port `manager` (9500) — the selector resolves to real
|
|
# endpoints, it is not a dangling monitor (which is how the apps/ SLI monitors fail: they name
|
|
# a `metrics` port their Service does not have, so they discover zero targets and their
|
|
# `up{job=...}` alerts are inert in the very same way).
|
|
# * no `additionalLabels` needed: platform/monitoring sets serviceMonitorSelectorNilUsesHelmValues
|
|
# false, so Prometheus selects every ServiceMonitor regardless of labels.
|
|
# Cost: one scrape target per manager pod (3), the longhorn_* families — low hundreds of series
|
|
# on this cluster's volume count.
|
|
#
|
|
# PROVEN END TO END ON PROD, 2026-09-13 21:0xZ. The rendered ServiceMonitor was applied to prod
|
|
# by hand (then removed again, so the merge's Helm upgrade creates and OWNS it rather than
|
|
# colliding with an unowned object) with this file's networkpolicy-metrics.yaml already in place:
|
|
# count(up{job="longhorn-backend"}) => 3 (all three managers, up==1)
|
|
# count({__name__=~"longhorn.*"}) => 8717 (was EMPTY)
|
|
# count(longhorn_volume_robustness) => 96 (24 volumes x 4 `state` labels)
|
|
# count by (state) (longhorn_volume_robustness == 1) => {state="healthy"} 24
|
|
# CORRECTION, re-measured 2026-09-13 later the same day: the line originally recorded here read
|
|
# "44 series … 24 healthy(1), 72 unknown(0, detached)". That was a MISREADING of a one-hot
|
|
# metric. `longhorn_volume_robustness` is NOT a 0-3 enum on longhorn 1.12.1; it carries a
|
|
# `state` label (healthy/degraded/faulted/unknown) and its value is only ever 0 or 1. The 72
|
|
# zeros were the other three state labels of the SAME 24 healthy volumes, not 72 detached
|
|
# volumes. The consequence was that enabling this scrape did NOT un-inert
|
|
# LonghornVolumeDegraded/Faulted: their `== 2` / `== 3` expressions still matched no sample.
|
|
# Fixed in platform/monitoring/alerts.yaml and pinned by
|
|
# platform/monitoring/tests/longhorn.test.yaml.
|
|
#
|
|
# Of the rules that WERE evaluating for real, one immediately found a true positive and one
|
|
# permanently-firing defect in the same reading: LonghornVolumeCapacityHigh went pending on the
|
|
# Prometheus TSDB volume at 97.3% ALLOCATED while its filesystem was 20.8% used. Longhorn does
|
|
# not reclaim freed blocks without discard/TRIM, and a 37.7 GiB system snapshot from prod-1's
|
|
# replica rebuild was pinning most of it. That rule has since been DELETED (not re-thresholded):
|
|
# it measured allocation and called it capacity, the filesystem question is already answered by
|
|
# kubelet_volume_stats_*, and the allocation question now lives per disk as
|
|
# LonghornDiskAllocationHigh / LonghornDiskUnschedulable.
|
|
metrics:
|
|
serviceMonitor:
|
|
enabled: true
|
|
|
|
# Longhorn's own manager/UI stay in-cluster; UI exposure (if any) is gated behind
|
|
# oauth2-proxy forward-auth (ADR-0014) by WS-SEC — not exposed here.
|
|
|
|
# TODO(WS-STORAGE, Phase 4): off-cluster backup target (defaultSettings.backupTarget +
|
|
# backupTargetCredentialSecret) is configured alongside Velero (ADR-0003, R-STATE-03).
|
|
# Credentials referenced BY NAME only, SOPS-encrypted (WS-SEC) — never inline here.
|