diff --git a/platform/monitoring/kube-prometheus-stack.yaml b/platform/monitoring/kube-prometheus-stack.yaml index da34233..53019cb 100644 --- a/platform/monitoring/kube-prometheus-stack.yaml +++ b/platform/monitoring/kube-prometheus-stack.yaml @@ -328,12 +328,25 @@ spec: labelsFromPath: cron: [spec, schedule] phase: [status, phase] + # MEMORY RAISED 2026-09-13 after 91 OOMKills. Measured, not guessed: working set 111 MiB + # with a 6h peak of 123.3 MiB against the old 128Mi limit — KSM was dying on every spike, + # roughly every 11 minutes (exit 137). That matters beyond the restarts: THIRTEEN alert + # rules read kube_* series, so each kill made them intermittently inert, and an alert that + # is only sometimes able to fire is the quiet version of the inert-alert class this repo + # keeps getting bitten by (KubePodNotReady and KubeDeploymentReplicasMismatch were both + # stuck pending on it). The trigger was enabling customResourceState below for the Velero + # Schedule metrics — it adds a CRD informer plus the mandatory CRD list/watch, and that + # marginal cost crossed a limit already sitting ~4 MiB above the working set. + # 384Mi is ~3x the observed peak and costs nothing here: the nodes report 18.7-22.3 GiB + # available. Re-measure with + # max_over_time(container_memory_working_set_bytes{container="kube-state-metrics"}[6h]) + # before trimming it back. resources: requests: cpu: 20m - memory: 64Mi + memory: 192Mi limits: - memory: 128Mi + memory: 384Mi prometheus-node-exporter: resources: requests: