diff --git a/clusters/base/cluster-vars.yaml b/clusters/base/cluster-vars.yaml index b2f0bf9..5a1ba0f 100644 --- a/clusters/base/cluster-vars.yaml +++ b/clusters/base/cluster-vars.yaml @@ -35,6 +35,11 @@ data: # --- observability sizing (docs/10) --- MONITORING_PROMETHEUS_RETENTION: "7d" # dev short | prod long MONITORING_PROMETHEUS_PVC_SIZE: "10Gi" + # SIZE-based retention cap for Prometheus (Prometheus *iB units, NOT the K8s `Gi` of the PVC knob). + # MUST stay below MONITORING_PROMETHEUS_PVC_SIZE — leaves headroom for the WAL + head + compaction + # scratch (retentionSize counts only persisted blocks). Prevents a busy TSDB from filling /prometheus + # (a full disk wedges all WAL commits and makes every absent()/==0 alert fire falsely). Base 10Gi PVC. + MONITORING_PROMETHEUS_RETENTION_SIZE: "8GiB" MONITORING_LOKI_PVC_SIZE: "10Gi" # --- admission enforcement (Kyverno, ADR-0011). Base defaults are the STRICT/prod posture; diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml index 511c662..ec227fc 100644 --- a/clusters/dev/kustomization.yaml +++ b/clusters/dev/kustomization.yaml @@ -134,9 +134,17 @@ patches: - op: replace path: /data/MONITORING_PROMETHEUS_RETENTION value: "3d" + # 5Gi was too tight: 3d of this cluster's series volume (~1.2Gi/day of blocks) plus a WAL that + # ballooned past 1Gi overran the disk, wedging WAL commits and firing every absent()/==0 alert + # falsely. Bumped to 8Gi (still lighter than base 10Gi) so the size-cap below has real headroom. - op: replace path: /data/MONITORING_PROMETHEUS_PVC_SIZE - value: "5Gi" + value: "8Gi" + # Size-cap kept well below the 8Gi PVC (headroom for WAL + head + compaction scratch) so + # Prometheus prunes oldest blocks by size and never fills the disk on this young cluster. + - op: replace + path: /data/MONITORING_PROMETHEUS_RETENTION_SIZE + value: "5GiB" - op: replace path: /data/MONITORING_LOKI_PVC_SIZE value: "5Gi" diff --git a/clusters/prod/kustomization.yaml b/clusters/prod/kustomization.yaml index c0902bd..3eaec4f 100644 --- a/clusters/prod/kustomization.yaml +++ b/clusters/prod/kustomization.yaml @@ -31,6 +31,11 @@ patches: - op: replace path: /data/MONITORING_PROMETHEUS_PVC_SIZE value: "50Gi" + # Size-cap below the 50Gi PVC (headroom for WAL + head + compaction scratch): prod may hit the + # size cap before 30d if series volume is high, keeping the disk from filling either way. + - op: replace + path: /data/MONITORING_PROMETHEUS_RETENTION_SIZE + value: "45GiB" - op: replace path: /data/MONITORING_LOKI_PVC_SIZE value: "50Gi" diff --git a/platform/monitoring/README.md b/platform/monitoring/README.md index bb494d7..2e24805 100644 --- a/platform/monitoring/README.md +++ b/platform/monitoring/README.md @@ -90,9 +90,17 @@ Only three substitution vars touch this subtree; the dev overlay (`clusters/dev` | Var | base | dev | |---|---|---| | `${MONITORING_PROMETHEUS_RETENTION}` | `7d` | `3d` | -| `${MONITORING_PROMETHEUS_PVC_SIZE}` | `10Gi` | `5Gi` | +| `${MONITORING_PROMETHEUS_PVC_SIZE}` | `10Gi` | `8Gi` | +| `${MONITORING_PROMETHEUS_RETENTION_SIZE}` | `8GiB` | `5GiB` | | `${MONITORING_LOKI_PVC_SIZE}` | `10Gi` | `5Gi` | +`MONITORING_PROMETHEUS_RETENTION_SIZE` is a **size-based** retention cap (Prometheus `*iB` units, +distinct from the K8s `Gi` PVC knob) and must stay **below** `MONITORING_PROMETHEUS_PVC_SIZE`. Time-only +retention prunes nothing until the age window elapses, so a young/busy cluster fills `/prometheus` first; +a full disk wedges every WAL commit (scrapes still 200 but samples are dropped, the head freezes) and +makes every `absent()`/`==0` platform alert fire falsely. The size cap drops the oldest blocks before the +disk fills. (prod overlay: PVC `50Gi`, cap `45GiB`.) + PVCs use the replicated `longhorn` StorageClass (ADR-0003). ## Notes / known gaps for bring-up diff --git a/platform/monitoring/kube-prometheus-stack.yaml b/platform/monitoring/kube-prometheus-stack.yaml index ec660b5..835e528 100644 --- a/platform/monitoring/kube-prometheus-stack.yaml +++ b/platform/monitoring/kube-prometheus-stack.yaml @@ -45,6 +45,17 @@ spec: prometheusSpec: replicas: 1 retention: ${MONITORING_PROMETHEUS_RETENTION} # overlay knob (dev 3d / prod long) + # SIZE-based retention cap (docs/10, R-OBS). `retention` alone prunes only by AGE — on a young + # cluster (< retention window) NOTHING is pruned yet, so a busy TSDB fills the PVC before the + # time window elapses. A full /prometheus wedges ALL WAL commits ("no space left on device"): + # scrapes still 200 but samples are never stored, the head freezes, and every absent()/==0 + # platform alert (Keycloak/KubeAPI/Kubelet/VPN/KubeVirt/CDI…) fires falsely because instant + # queries at "now" return empty. retentionSize makes Prometheus drop the OLDEST blocks once the + # block set approaches this cap, so the disk never fills regardless of cluster age or churn. + # Kept BELOW the PVC size (leaves headroom for the WAL + head + compaction scratch, which are + # NOT counted by retentionSize). Prometheus size units are *iB (GiB), distinct from the K8s PVC + # `Gi`, so it is a separate overlay knob rather than a reuse of the PVC-size knob. + retentionSize: ${MONITORING_PROMETHEUS_RETENTION_SIZE} # overlay knob (dev 5GiB / base 8GiB / prod 45GiB) # Discover ServiceMonitors/PodMonitors/Rules from ALL namespaces (flux, cert-manager, # longhorn, velero, keycloak, cilium/hubble ship their own) — not just this release's. ruleSelectorNilUsesHelmValues: false