From 28a34c4477f86c2f64d33e58ddac7def5414c5e7 Mon Sep 17 00:00:00 2001
From: Keith Jackson <keithallenjackson@gmail.com>
Date: Tue, 25 Aug 2026 16:43:43 -0500
Subject: [PATCH] fix(kyverno): raise admission-controller resources + dev
 replicas 2 to stop fail-closed crashloop/502s

Admission controller was crashlooping (9 restarts/4h, exit 255, NOT OOM:
live usage ~65Mi/384Mi limit, 7m cpu). Root cause: chart-default 100m CPU
request starved the process under node contention, so client-go calls hit
context-deadline-exceeded, leader-election lease renewal timed out, leadership
was lost and the webhook server closed. Raise the CPU request to 250m (no CPU
limit, to avoid CFS throttling) with modest memory headroom (256Mi/512Mi), and
raise dev admission replicas 1->2 so a single fail-closed pod restart no longer
502s cluster-wide admission-gated writes.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 clusters/dev/kustomization.yaml             |  5 ++++-
 platform/kyverno/admission/helmrelease.yaml | 13 +++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/clusters/dev/kustomization.yaml b/clusters/dev/kustomization.yaml
index 10c83cc..160835c 100644
--- a/clusters/dev/kustomization.yaml
+++ b/clusters/dev/kustomization.yaml
@@ -104,6 +104,9 @@ patches:
       - op: replace
         path: /data/KYVERNO_FORCE_FAILURE_POLICY_IGNORE
         value: "true"
+      # 2 (not base-default 1): the webhook is fail-closed, so a single-pod restart 502s every
+      # admission-gated write cluster-wide. Two replicas give a fail-closed webhook availability
+      # headroom during a restart/upgrade while staying dev-modest (prod runs 3).
       - op: replace
         path: /data/KYVERNO_ADMISSION_REPLICAS
-        value: "1"
+        value: "2"
diff --git a/platform/kyverno/admission/helmrelease.yaml b/platform/kyverno/admission/helmrelease.yaml
index e451e0f..7dce551 100644
--- a/platform/kyverno/admission/helmrelease.yaml
+++ b/platform/kyverno/admission/helmrelease.yaml
@@ -49,6 +49,19 @@ spec:
     # restart cannot open an enforcement gap; dev drops to 1 to save RAM during bootstrap.
     admissionController:
       replicas: ${KYVERNO_ADMISSION_REPLICAS}
+      # Resources — chart default (limits.memory 384Mi / requests.cpu 100m,memory 128Mi) starves this
+      # latency-sensitive, fail-closed webhook: under node CPU contention the 100m request gave too few
+      # CFS shares, so client-go calls hit "context deadline exceeded" → leader-election lease renewal
+      # timed out → "leadership lost, stopped leading" → server closed (exit 255, NOT OOM: live usage
+      # was ~65Mi/7m). Raise the CPU REQUEST for scheduling headroom and keep NO CPU limit (a limit would
+      # reintroduce throttling and make lease-renewal timeouts worse). Memory bumped for headroom only.
+      container:
+        resources:
+          requests:
+            cpu: 250m
+            memory: 256Mi
+          limits:
+            memory: 512Mi
     backgroundController:
       enabled: true               # needed for generate (default-deny baseline) + sync
     reportsController: