From 3e61f817b6dc28c4e003b0499b897380c39f13ce Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Sun, 30 Aug 2026 21:33:40 -0500 Subject: [PATCH] kps: raise Prometheus (2->3Gi) + Grafana (512->768Mi) memory limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were pinned at ~98% of their limits (Prometheus 2002/2048Mi, Grafana 499/512Mi) with no headroom for TSDB compaction / dashboard-render spikes — the classic OOM setups. Nodes are 24Gi at 38-44% used (~14Gi free each; the '~22 GiB ceiling' comment predates the 16->24Gi node bump), so the extra headroom is comfortable. Grafana request 256->384Mi, Prometheus request unchanged. Co-Authored-By: Claude Opus 4.8 --- platform/monitoring/kube-prometheus-stack.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/platform/monitoring/kube-prometheus-stack.yaml b/platform/monitoring/kube-prometheus-stack.yaml index 4aea9c0..2ed6968 100644 --- a/platform/monitoring/kube-prometheus-stack.yaml +++ b/platform/monitoring/kube-prometheus-stack.yaml @@ -78,8 +78,10 @@ spec: cpu: 100m memory: 512Mi limits: - memory: 2Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint - # spikes; 2Gi holds. Nodes have headroom (~2Gi free each). + memory: 3Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint; + # 2Gi ran pinned at ~98% (2002Mi) with no room for compaction/WAL-replay + # spikes (the classic Prometheus OOM). Nodes are 24Gi at 38-44% used + # (~14Gi free each), so 3Gi gives real headroom. storageSpec: volumeClaimTemplate: spec: @@ -172,11 +174,13 @@ spec: resources: requests: cpu: 50m - memory: 256Mi # Grafana 13.x (image 13.2.0, kps 88) baseline is well above Grafana 11.x (kps 65) + memory: 384Mi # Grafana 13.x (image 13.2.0, kps 88) baseline is well above Grafana 11.x (kps 65) limits: - memory: 512Mi # 256Mi OOMKilled (exit 137) Grafana 13.x during apiserver/unified-storage + memory: 768Mi # 256Mi OOMKilled (exit 137) Grafana 13.x during apiserver/unified-storage # startup — the silent SIGKILL that made the whole 65->88 Helm upgrade - # time out (the new Grafana pod never went Ready). 512Mi holds. + # time out (the new Grafana pod never went Ready). 512Mi then ran pinned at + # ~98% (499Mi) with no headroom — one render/GC spike from OOM. 768Mi holds + # with real headroom (nodes 24Gi, ~14Gi free each). # --- Keycloak native-OIDC SSO (ADR-0014, deliverable #4 SSO proof) --- # Grafana is a confidential OIDC client of the Keycloak `homelab` realm. The client SECRET comes # from the SOPS-encrypted grafana-oidc-credentials Secret via env (referenced by name only,