home-network-k3s-impl/infra/maintenance/quiesce.sh

#!/usr/bin/env bash
# =================================================================================================
# quiesce.sh — bring the prod cluster to a safe standstill before powering nodes off, and bring it
#              back afterwards. Companion to runbooks/cluster-maintenance-shutdown.md.
# =================================================================================================
# WHY THIS EXISTS. On 2026-10-17 all three nodes had to be powered off to move them onto a UPS, and
# there was NO procedure for it: `infra/cluster.sh stop` is `virsh`-based and only ever applied to
# the decommissioned dev cluster. The whole sequence was improvised live. This script is that
# sequence, so the next person does not improvise it.
#
# WHY IT IS NOT JUST `shutdown -h`. Graceful node shutdown was unconfigured on every prod node that
# day, so systemd stopped containerd and SIGKILLed every pod where it stood. roles/k3s_server now
# sets shutdown-grace-period (+ the logind InhibitDelayMaxSec cap that actually enforces it), which
# makes a bare reboot safe. This script is still the right tool for a PLANNED whole-cluster
# shutdown, for three reasons the kubelet's drain does not cover:
#   1. The UniFi VM needs an ACPI guest shutdown, not an eviction.
#   2. CloudNativePG wants hibernation, not pod deletion -- and hibernation is IMPERATIVE state
#      that Flux will not set or clear for you.
#   3. Draining three nodes one at a time makes pods RESCHEDULE onto the survivors, churning
#      Longhorn replicas for no reason when the whole cluster is going down anyway. Scaling in
#      place avoids all of it.
#
# SAFE TO RE-RUN. Every step is idempotent and `resume` is the exact inverse of `quiesce`.
#
# WHAT IT DELIBERATELY DOES NOT DO: power anything off. Node poweroff is an operator action
# (CLAUDE.md §5) and needs the become password; the runbook gives the ansible one-liner. It also
# does not touch TrueNAS (not in the inventory, SSH refused -- web UI only) or the router.
set -euo pipefail

CONTEXT="${CONTEXT:-k3s-prod}"
KUBECTL=(kubectl --context "$CONTEXT" --request-timeout=30s)
TIMEOUT="${TIMEOUT:-600}"

# The three CloudNativePG clusters, as namespace/name. Hibernation is per-Cluster.
CNPG=(keycloak/keycloak-db nextcloud-db/nextcloud-db semaphore-db/semaphore-db)

log()  { printf '\033[34m[quiesce]\033[0m %s\n' "$*"; }
ok()   { printf '\033[32m[ ok ]\033[0m %s\n' "$*"; }
warn() { printf '\033[33m[warn]\033[0m %s\n' "$*"; }
die()  { printf '\033[31m[fail]\033[0m %s\n' "$*" >&2; exit 1; }

need() { command -v "$1" >/dev/null || die "$1 not on PATH — run this inside ./infra/devcontainer/dev.sh"; }

# ---------------------------------------------------------------- discovery
# Every Deployment/StatefulSet whose pods mount a PVC — Longhorn OR the TrueNAS NFS bulk tier.
# Computed from the LIVE cluster rather than hardcoded, so a newly onboarded service is covered
# without anyone remembering to add it here. PROTECTED namespaces are excluded because scaling
# them down would remove the very machinery that performs the detach (longhorn-system), keep the
# API reachable (kube-system), or finish the hibernation and the VM shutdown (cnpg-system,
# kubevirt).
pvc_workloads() {
  "${KUBECTL[@]}" get pods -A -o json | python3 -c '
import json,sys
PROTECT={"longhorn-system","kube-system","cilium-secrets","cnpg-system","kubevirt","flux-system","cdi"}
out={}
for p in json.load(sys.stdin)["items"]:
    ns=p["metadata"]["namespace"]
    if ns in PROTECT: continue
    if not any(v.get("persistentVolumeClaim") for v in (p["spec"].get("volumes") or [])): continue
    ow=(p["metadata"].get("ownerReferences") or [])
    if not ow: continue
    k,n=ow[0]["kind"],ow[0]["name"]
    if k=="ReplicaSet": k,n="deployment",n.rsplit("-",1)[0]
    elif k=="StatefulSet": k="statefulset"
    else: continue          # Jobs, and CNPG Clusters, are handled separately
    out[(ns,k,n)]=1
for ns,k,n in sorted(out): print(ns,k,n)
'
}

# ---------------------------------------------------------------- the PVC-deletion guard
# *** DO NOT REMOVE THIS CHECK. IT EXISTS BECAUSE SCALING A StatefulSet DESTROYED DATA. ***
# A StatefulSet may carry `persistentVolumeClaimRetentionPolicy.whenScaled: Delete`, and then
# scaling it to 0 DELETES ITS PVCs -- and with them the Longhorn volumes. MEASURED 2026-10-17: the
# quiesce for the UPS move scaled `monitoring/loki` (whenScaled: Delete) to 0 and its
# `storage-loki-0` PVC and volume were deleted outright. Loki stores with `type: filesystem`, so
# that PVC was the only copy; 72h retention bounded the loss to a rolling window, but nothing about
# the mechanism is bounded -- the same action on a Delete-policy StatefulSet holding real state
# would be unrecoverable. The volume count going 25 -> 24 was the only visible symptom, and it was
# first misread as a miscount.
#
# So: such a StatefulSet is NEVER scaled. It is left RUNNING and its volume hard-detaches at
# poweroff, which costs a replica rebuild on restart (degraded -> healthy, automatic). A rebuild is
# an enormously better outcome than a deletion, and that is the whole trade being made here.
sts_deletes_pvc_on_scale() { # ns name -> 0 if scaling it would delete its PVCs
  local policy
  policy=$("${KUBECTL[@]}" -n "$1" get statefulset "$2" \
            -o jsonpath='{.spec.persistentVolumeClaimRetentionPolicy.whenScaled}' 2>/dev/null || true)
  [ "$policy" = "Delete" ]
}

wait_for() { # wait_for <description> <shell-condition>
  local desc="$1" cond="$2" waited=0
  log "waiting: $desc (up to ${TIMEOUT}s)"
  until eval "$cond"; do
    (( waited += 10 )); (( waited >= TIMEOUT )) && { warn "TIMED OUT waiting: $desc"; return 1; }
    sleep 10
  done
  ok "$desc"
}

# ---------------------------------------------------------------- quiesce
do_quiesce() {
  need kubectl; need python3

  # Refuse to start if Longhorn is already unhealthy. Powering off on top of a degraded volume is
  # how one replica becomes zero; better to stop here and let it rebuild first.
  local unhealthy
  unhealthy=$("${KUBECTL[@]}" -n longhorn-system get volumes.longhorn.io \
    -o jsonpath='{range .items[*]}{.status.robustness}{"\n"}{end}' | grep -cvE '^(healthy|unknown)$' || true)
  [ "$unhealthy" = "0" ] || die "$unhealthy Longhorn volume(s) are not healthy — fix or wait for rebuild BEFORE powering off"
  ok "Longhorn: all volumes healthy"

  # 1. Flux first, or it reverts every scale-down below within its reconcile interval.
  log "suspending Flux Kustomizations"
  local k
  for k in $("${KUBECTL[@]}" -n flux-system get kustomizations -o name); do
    "${KUBECTL[@]}" -n flux-system patch "${k}" --type=merge -p '{"spec":{"suspend":true}}' >/dev/null
  done
  ok "all Kustomizations suspended"

  # 2. The VM needs an ACPI shutdown of the GUEST. KubeVirt does that on running:false; a plain
  #    pod eviction would be a power cut to the VM.
  if "${KUBECTL[@]}" -n unifi get vm unifi >/dev/null 2>&1; then
    log "ACPI-stopping the UniFi VM"
    "${KUBECTL[@]}" -n unifi patch vm unifi --type=merge -p '{"spec":{"running":false}}' >/dev/null
  fi

  # 3. Postgres: hibernation is a clean smart shutdown that KEEPS the PVCs. CNPG does one instance
  #    at a time, primary last-ish, so this is not instant.
  log "hibernating CloudNativePG clusters"
  local c ns name
  for c in "${CNPG[@]}"; do
    ns="${c%%/*}"; name="${c##*/}"
    "${KUBECTL[@]}" -n "$ns" annotate cluster.postgresql.cnpg.io "$name" \
      cnpg.io/hibernation=on --overwrite >/dev/null 2>&1 || warn "could not hibernate $c"
  done

  # 4. Everything holding a PVC, scaled in place. NFS consumers first so nothing holds a hard
  #    TrueNAS mount; PiHole last because it is the LAN resolver.
  log "scaling down PVC-holding workloads"
  local line ns2 kind nm deferred=() skipped_delete_policy=()
  while read -r ns2 kind nm; do
    [ -z "${ns2:-}" ] && continue
    if [ "$kind" = "statefulset" ] && sts_deletes_pvc_on_scale "$ns2" "$nm"; then
      warn "SKIPPING $ns2/$nm — whenScaled: Delete would DELETE its PVC (see the guard above)."
      warn "  Left running; its volume hard-detaches at poweroff and rebuilds on restart."
      skipped_delete_policy+=("$ns2/$nm")
      continue
    fi
    if [ "$ns2" = "pihole" ]; then deferred+=("$ns2 $kind $nm"); continue; fi
    "${KUBECTL[@]}" -n "$ns2" scale "$kind" "$nm" --replicas=0 >/dev/null 2>&1 \
      && printf '    scaled 0  %s/%s %s\n' "$ns2" "$kind" "$nm" \
      || warn "could not scale $ns2/$kind/$nm"
  done < <(pvc_workloads)
  for line in "${deferred[@]:-}"; do
    [ -z "${line:-}" ] && continue
    # shellcheck disable=SC2086
    set -- $line
    "${KUBECTL[@]}" -n "$1" scale "$2" "$3" --replicas=0 >/dev/null 2>&1 \
      && printf '    scaled 0  %s/%s %s  (LAN DNS — expect name resolution to stop)\n' "$1" "$2" "$3"
  done

  # 5. The only assertion that matters. An attached volume at poweroff comes back degraded and
  #    rebuilds; this is what we are buying with all of the above.
  wait_for "UniFi VMI gone" \
    '[ "$('"${KUBECTL[*]}"' -n unifi get vmi --no-headers 2>/dev/null | wc -l)" = "0" ]' || true
  wait_for "all Postgres instances stopped" \
    '[ "$('"${KUBECTL[*]}"' get pods -A --no-headers 2>/dev/null | grep -cE "(keycloak|nextcloud|semaphore)-db-[0-9]")" = "0" ]' || true
  wait_for "all Longhorn volumes detached" \
    '[ "$('"${KUBECTL[*]}"' -n longhorn-system get volumes.longhorn.io -o jsonpath="{range .items[*]}{.status.state}{\"\n\"}{end}" 2>/dev/null | grep -c attached)" = "0" ]' || \
      warn "some volumes are STILL ATTACHED — they will come back degraded and rebuild; acceptable, but note which"

  if [ "${#skipped_delete_policy[@]}" -gt 0 ]; then
    echo
    warn "NOT scaled down (whenScaled: Delete — scaling them would delete their PVCs):"
    printf '      %s\n' "${skipped_delete_policy[@]}"
    warn "Expect these volumes to come back 'degraded' and rebuild. That is correct and automatic."
  fi
  echo
  ok "QUIESCED. Safe to power off."
  cat <<'NEXT'

  Next, from inside the devcontainer (it holds the become password):
    cd /work/infra/ansible && ansible -i inventory/hosts.ini k3s_prod -b \
      -m shell -a 'shutdown -h +1 "maintenance"'

  THEN, in this order and not before:
    1. TrueNAS (192.168.2.30) — web UI only; not in the ansible inventory and SSH is refused.
    2. Router / switch last.
NEXT
}

# ---------------------------------------------------------------- resume
do_resume() {
  need kubectl

  # THE STEP THAT IS ALWAYS FORGOTTEN. Hibernation is imperative state: Flux neither sets nor
  # clears it, so resuming Flux alone leaves all three databases down and every app that needs
  # one looking broken for reasons that point nowhere near Postgres.
  log "un-hibernating CloudNativePG clusters"
  local c ns name
  for c in "${CNPG[@]}"; do
    ns="${c%%/*}"; name="${c##*/}"
    "${KUBECTL[@]}" -n "$ns" annotate cluster.postgresql.cnpg.io "$name" \
      cnpg.io/hibernation=off --overwrite >/dev/null 2>&1 || warn "could not un-hibernate $c"
  done

  # Flux restores every replica count AND the VM's running:true from git, so nothing here needs to
  # undo the scale-downs by hand.
  log "resuming Flux Kustomizations"
  local k
  for k in $("${KUBECTL[@]}" -n flux-system get kustomizations -o name); do
    "${KUBECTL[@]}" -n flux-system patch "${k}" --type=merge -p '{"spec":{"suspend":false}}' >/dev/null
  done
  ok "all Kustomizations resumed"

  wait_for "Flux fully reconciled" \
    '[ "$('"${KUBECTL[*]}"' -n flux-system get kustomizations --no-headers 2>/dev/null | grep -vc True)" = "0" ]' || \
      warn "some Kustomizations are not Ready — the graph serialises behind platform-config, so give it a few minutes before worrying"

  echo
  ok "RESUMED."
  cat <<'NEXT'

  Known cold-start race, not a fault in this script: oauth2-proxy performs OIDC discovery at
  startup and exits 1 on the 503 it gets while Keycloak is still coming up, then sits in
  exponential CrashLoopBackOff. If a replica is stuck, delete the pod once:
    kubectl --context k3s-prod -n oauth2-proxy delete pod <name>
NEXT
}

do_status() {
  need kubectl
  printf 'nodes:      %s Ready\n' "$("${KUBECTL[@]}" get nodes --no-headers | grep -c ' Ready')"
  printf 'flux:       %s/%s Ready\n' \
    "$("${KUBECTL[@]}" -n flux-system get kustomizations --no-headers | grep -c True)" \
    "$("${KUBECTL[@]}" -n flux-system get kustomizations --no-headers | wc -l)"
  printf 'longhorn:   %s attached, %s detached, %s not healthy\n' \
    "$("${KUBECTL[@]}" -n longhorn-system get volumes.longhorn.io -o jsonpath='{range .items[*]}{.status.state}{"\n"}{end}' | grep -c attached)" \
    "$("${KUBECTL[@]}" -n longhorn-system get volumes.longhorn.io -o jsonpath='{range .items[*]}{.status.state}{"\n"}{end}' | grep -c detached)" \
    "$("${KUBECTL[@]}" -n longhorn-system get volumes.longhorn.io -o jsonpath='{range .items[*]}{.status.robustness}{"\n"}{end}' | grep -cvE '^(healthy|unknown)$' || true)"
  printf 'postgres:   %s instance pod(s)\n' \
    "$("${KUBECTL[@]}" get pods -A --no-headers | grep -cE '(keycloak|nextcloud|semaphore)-db-[0-9]' || true)"
  printf 'unifi VM:   %s VMI\n' "$("${KUBECTL[@]}" -n unifi get vmi --no-headers 2>/dev/null | wc -l)"
}

case "${1:-}" in
  quiesce) do_quiesce ;;
  resume)  do_resume ;;
  status)  do_status ;;
  *) cat >&2 <<USAGE
usage: $0 {quiesce|resume|status}

  quiesce  suspend Flux, ACPI-stop the UniFi VM, hibernate Postgres, scale down every
           PVC-holding workload, and wait for all Longhorn volumes to detach.
           Does NOT power anything off — see runbooks/cluster-maintenance-shutdown.md.
  resume   the exact inverse: un-hibernate Postgres, resume Flux, wait for reconcile.
  status   one-screen readout of nodes / Flux / Longhorn / Postgres / VM.

Run it inside the pinned toolbox, which supplies kubectl and the context:
  ./infra/devcontainer/dev.sh run bash -c './infra/maintenance/quiesce.sh status'
USAGE
     exit 2 ;;
esac