---
# Longhorn — distributed replicated block storage across the 3 nodes (ADR-0003).
# Serves R-STATE-01 (state survives destroy/rebuild) and R-STATE-02 (3-way replication).
#
# Dev-appropriate values. Environment differences enter ONLY via $${...} substitution from
# cluster-vars (R-ENV-02); everything else is identical base config for dev and prod.
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: longhorn
  namespace: longhorn-system
spec:
  interval: 30m
  # Longhorn upgrades roll every workload + engine image on slow HDD-backed nodes and can exceed the
  # 5m Helm default; a short timeout caused a W5 mid-upgrade wedge (data migrated to the new minor but
  # workloads rolled back, and the new-version data refuses the old-version manager). Give it room.
  timeout: 15m0s
  chart:
    spec:
      chart: longhorn
      version: ">=1.12.0 <1.13.0"   # W5 upgrade 1.11->1.12 (resolves 1.12.1; supports k8s 1.36). Renovate maintains
      sourceRef:
        kind: HelmRepository
        name: longhorn
        namespace: longhorn-system
  install:
    createNamespace: false        # namespace.yaml owns it (GitOps-managed, labelled)
    remediation:
      retries: 3
  upgrade:
    remediation:
      retries: 3
      # NEVER auto-rollback a Longhorn minor upgrade: once the on-disk data migrates to the new minor,
      # the older manager fatally refuses it ("downgrading from vX to vY is not supported"). Rolling
      # back the workloads while data is already migrated wedges the cluster. Fail forward instead.
      remediateLastFailure: false
  values:
    # We own the StorageClasses (storageclasses.yaml) so the default 3-replica class and the bulk
    # tier are explicit and GitOps-managed. Disable the chart's built-in default SC so there is
    # exactly one default-class annotation in the cluster.
    persistence:
      defaultClass: false

    defaultSettings:
      # --- data path (R-STATE-02, R-ENV-02) ---
      # The on-node directory every replica is written into. A cluster-vars KNOB, not a literal,
      # because the two environments mount their data disk differently and a hardcoded path would
      # put one of them on the OS filesystem:
      #   dev  /var/lib/longhorn  — a dedicated ext4 disk (/dev/vdb) mounted there by
      #                             roles/node_baseline. The mount root IS the dataPath: one disk,
      #                             one tenant. 18 live volumes; this value must never change.
      #   prod /data/longhorn     — a SUBDIRECTORY of the claimed ~424 GB partition mounted at
      #                             /data, whose sibling /data/k3s holds k3s's entire data dir
      #                             (etcd + the containerd image store, moved off the 11.8 GiB
      #                             /var). Two tenants, so Longhorn gets a subdirectory rather
      #                             than the mount root. infra/PROD-PROVISIONING.md §2.
      # [parity] MUST equal roles/node_baseline's node_baseline_longhorn_data_path for the same
      # environment — Ansible creates and mounts the directory, Helm tells Longhorn to use it.
      # Set in BOTH overlays (clusters/dev, clusters/prod); base default is the dev value.
      defaultDataPath: "${LONGHORN_DATA_PATH}"

      # --- replication (R-STATE-02) ---
      # Global fallback replica count. The `longhorn` StorageClass sets numberOfReplicas explicitly
      # (authoritative); this keeps any volume created without an explicit count consistent.
      # Substituted from cluster-vars.longhorn_default_replica_count (=3 in dev).
      defaultReplicaCount: ${LONGHORN_DEFAULT_REPLICA_COUNT}

      # --- replica anti-affinity across nodes (R-STATE-02 durability guarantee) ---
      # HARD anti-affinity: never place two replicas of the same volume on the same node, so a
      # single node loss cannot take down more than one replica. On a 3-node / 3-replica cluster
      # this yields exactly one replica per node.
      replicaSoftAntiAffinity: false
      # Single zone (3 mini-PCs, no failure zones): allow replicas within the one zone.
      replicaZoneSoftAntiAffinity: true
      # When a downed node returns, rebalance replicas back toward one-per-node (best-effort so it
      # never blocks scheduling).
      replicaAutoBalance: "best-effort"

      # --- rebuild I/O throttle for slow disks (etcd WAL fsync protection) ---
      # The k3s VM disks sit on HDDs, so a full-replica rebuild's sequential I/O can saturate the
      # spindle and starve etcd's WAL fsync — which crash-loops etcd and flaps the kube-vip VIP.
      # Cap Longhorn to rebuilding ONE replica per node at a time so a node bounce / replica
      # replenishment can never storm the HDDs. Deliberately conservative for this slow-disk cluster
      # (Longhorn's chart default is higher). Pair with replica-replenishment-wait-interval=600
      # (Longhorn default) so brief node bounces don't trigger replenishment at all.
      concurrentReplicaRebuildPerNodeLimit: 1

      # --- capacity guardrails (docs/04 "reserve disk headroom") ---
      # Reserve 20% of the dedicated data disk so a node never fills to 100% (which wedges the
      # kubelet and Longhorn alike). Dev disks are small, so headroom matters.
      storageReservedPercentageForDefaultDisk: "20"
      # Stop scheduling new replicas onto a disk once free space drops below 25%.
      storageMinimalAvailablePercentage: "25"
      # Dev: 200% thin over-provisioning — config-only volumes use little actual space, so the small
      # 39Gi data disks host the full app catalog. Physical fill risk is low (watch monitoring PVCs).
      # Prod-grade alternative: grow the Longhorn data disks and return to 1:1 (WS-STORAGE).
      storageOverProvisioningPercentage: "200"

      # --- misc dev-appropriate defaults ---
      # Keep replicas spread for durability rather than co-locating with the pod.
      defaultDataLocality: "disabled"
      # Reclaim replica space of deleted/failed volumes promptly in dev.
      orphanAutoDeletion: true

    # --- SCRAPE THE STORAGE LAYER, or every Longhorn alert is INERT (added 2026-09-13) ----------
    # platform/monitoring/alerts.yaml ships LonghornVolumeDegraded (critical),
    # LonghornVolumeFaulted (critical), LonghornVolumeCapacityHigh and LonghornNodeStorageHigh —
    # and NONE of them could ever fire, because the chart defaults
    # `metrics.serviceMonitor.enabled: false` and no ServiceMonitor existed in ns longhorn-system.
    # VERIFIED live on prod 2026-09-13: `count by (job) (up)` listed 19 jobs and none was Longhorn;
    # `count(longhorn_volume_robustness)` and `count({__name__=~"longhorn.*"})` both returned NOTHING.
    # So the entire longhorn_* family was absent from the TSDB and four alert rules were decoration.
    #
    # WHY THIS IS PART OF THE STORAGE PATH, NOT AN OBSERVABILITY NICE-TO-HAVE: this cluster's
    # recurring failure mode is a disk. prod-1's failing SSD starved etcd's WAL fsync and wedged the
    # node (DECISIONS-LOG; the prod-1 drive replacement), and a degraded/faulted replica is precisely
    # the first symptom Longhorn publishes. R-STATE-02 promises 3-way replication survives a node
    # loss; with no scrape, a volume can sit at 2 replicas — or faulted — indefinitely, unalerted.
    # platform/monitoring/pvc-capacity-alerts.yaml's header already flagged this exact gap as "a
    # separate defect (a missing Longhorn ServiceMonitor/metrics config)". This closes it.
    #
    # Chart mechanics, verified by rendering longhorn 1.12.1 (the version this range resolves to):
    #   * templates/servicemonitor.yaml is gated on `{{- if .Values.metrics.serviceMonitor.enabled }}`
    #     and NOTHING else — no `.Capabilities` probe. Measured: `helm template --set
    #     metrics.serviceMonitor.enabled=true` emits exactly 1 ServiceMonitor, and 0 without the flag.
    #     So this line is both necessary and sufficient; unlike velero's, it needs no API capability.
    #   * the rendered ServiceMonitor selects `app: longhorn-manager` in the release namespace on the
    #     port named `manager`. CONFIRMED against prod: Service/longhorn-backend carries
    #     `app: longhorn-manager` and a port `manager` (9500) — the selector resolves to real
    #     endpoints, it is not a dangling monitor (which is how the apps/ SLI monitors fail: they name
    #     a `metrics` port their Service does not have, so they discover zero targets and their
    #     `up{job=...}` alerts are inert in the very same way).
    #   * no `additionalLabels` needed: platform/monitoring sets serviceMonitorSelectorNilUsesHelmValues
    #     false, so Prometheus selects every ServiceMonitor regardless of labels.
    # Cost: one scrape target per manager pod (3), the longhorn_* families — low hundreds of series
    # on this cluster's volume count.
    #
    # PROVEN END TO END ON PROD, 2026-09-13 21:0xZ. The rendered ServiceMonitor was applied to prod
    # by hand (then removed again, so the merge's Helm upgrade creates and OWNS it rather than
    # colliding with an unowned object) with this file's networkpolicy-metrics.yaml already in place:
    #   count(up{job="longhorn-backend"})              => 3      (all three managers, up==1)
    #   count({__name__=~"longhorn.*"})                => 8717   (was EMPTY)
    #   count(longhorn_volume_robustness)              => 44
    #   count_values("v", longhorn_volume_robustness)  => 24 healthy(1), 72 unknown(0, detached)
    # and the four alert rules went from "no series, cannot fire" to evaluating for real — one of
    # them immediately found a true positive (LonghornVolumeCapacityHigh on the Prometheus TSDB
    # volume at 97.1% ALLOCATED while its filesystem is only 20.4% used: Longhorn is not reclaiming
    # freed blocks. Reported, NOT silenced by moving the threshold).
    metrics:
      serviceMonitor:
        enabled: true

    # Longhorn's own manager/UI stay in-cluster; UI exposure (if any) is gated behind
    # oauth2-proxy forward-auth (ADR-0014) by WS-SEC — not exposed here.

    # TODO(WS-STORAGE, Phase 4): off-cluster backup target (defaultSettings.backupTarget +
    #   backupTargetCredentialSecret) is configured alongside Velero (ADR-0003, R-STATE-03).
    #   Credentials referenced BY NAME only, SOPS-encrypted (WS-SEC) — never inline here.