From 7b619f8bb0e08d81ee2e01ccbb37a03075d90b44 Mon Sep 17 00:00:00 2001 From: Keith Jackson Date: Mon, 24 Aug 2026 00:04:17 -0500 Subject: [PATCH] fix(monitoring): Prometheus memory limit 1Gi -> 2Gi (OOMKilled on compaction) prometheus-...-0 was OOMKilled (exit 137) every ~4min during TSDB compaction/checkpoint (261 restarts/37h). Raise limit to 2Gi + request to 512Mi; nodes have headroom. Co-Authored-By: Claude Opus 4.8 --- platform/monitoring/kube-prometheus-stack.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platform/monitoring/kube-prometheus-stack.yaml b/platform/monitoring/kube-prometheus-stack.yaml index 199dfc2..b42063f 100644 --- a/platform/monitoring/kube-prometheus-stack.yaml +++ b/platform/monitoring/kube-prometheus-stack.yaml @@ -56,9 +56,10 @@ spec: resources: requests: cpu: 100m - memory: 384Mi + memory: 512Mi limits: - memory: 1Gi + memory: 2Gi # 1Gi OOMKilled (exit 137) during TSDB compaction/checkpoint + # spikes; 2Gi holds. Nodes have headroom (~2Gi free each). storageSpec: volumeClaimTemplate: spec: