Merge pull request 'feat(gitea-actions): act_runner capacity 2, dind sized for two concurrent jobs' (#146) from feat/runner-capacity-2 into main

Reviewed-on: #146

apps/wave-0/jackett.yaml

@@ -85,7 +85,7 @@ spec:
         - name: config
           mountPath: /config
       - name: oauth2-proxy
-        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
         imagePullPolicy: IfNotPresent
         args:
         - --http-address=0.0.0.0:4180

apps/wave-0/pihole.yaml

@@ -90,7 +90,7 @@ spec:
               - -c
               - "# Assert the declared adlists in gravity.db. Idempotent: steady state writes nothing and\n# does NOT run a gravity rebuild.\nset -u\nGRAVITY=/etc/pihole/gravity.db\n\n# THE DECLARED SET \u2014 one URL per line, the source of truth for what PiHole blocks.\n# To add a list: add a line here, commit, let Flux reconcile. Nothing else to do.\nDECLARED='https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'\n\n# On a fresh PVC, FTL creates gravity.db a moment after the container starts. Wait, bounded,\n# and NEVER block or fail the start: a missing DB here must not crashloop DNS for the LAN.\ni=0\nwhile [ ! -s \"$GRAVITY\" ] && [ \"$i\" -lt 90 ]; do sleep 2; i=$(expr $i + 1); done\nif [ ! -s \"$GRAVITY\" ]; then\n  echo \"pihole-adlists: gravity.db absent after 180s - leaving adlists untouched\" >&2\n  exit 0\nfi\n\nsql() { pihole-FTL sqlite3 \"$GRAVITY\" \"$1\"; }\nCHANGED=0\n\n# 1) Every declared URL must be present AND enabled.\necho \"$DECLARED\" | while IFS= read -r url; do\n  [ -n \"$url\" ] || continue\n  printf '%s\\n' \"$url\"\ndone > /tmp/declared.txt\nwhile IFS= read -r url; do\n  [ -n \"$url\" ] || continue\n  # Single quotes inside a URL would break this SQL; adlist URLs do not contain them.\n  have=$(sql \"select count(*) from adlist where address='$url';\")\n  if [ \"$have\" = \"0\" ]; then\n    echo \"pihole-adlists: adding $url\"\n    sql \"insert into adlist (address,enabled,comment) values ('$url',1,'declared in catalog/pihole.yaml');\"\n    CHANGED=1\n  else\n    en=$(sql \"select enabled from adlist where address='$url';\")\n    if [ \"$en\" != \"1\" ]; then\n      echo \"pihole-adlists: re-enabling $url\"\n      sql \"update adlist set enabled=1 where address='$url';\"\n      CHANGED=1\n    fi\n  fi\ndone < /tmp/declared.txt\n\n# 2) Anything enabled in the DB but NOT declared gets disabled (never deleted - see above).\nsql \"select address from adlist where enabled=1;\" | while IFS= read -r addr; do\n  [ -n \"$addr\" ] || continue\n  if ! grep -qxF \"$addr\" /tmp/declared.txt; then\n    echo \"pihole-adlists: disabling undeclared $addr\"\n    sql \"update adlist set enabled=0 where address='$addr';\"\n    echo changed >> /tmp/adlist-changed\n  fi\ndone\n[ -s /tmp/adlist-changed ] && CHANGED=1\nrm -f /tmp/adlist-changed /tmp/declared.txt\n\n# 3) Rebuild gravity ONLY if the declared set actually moved.\nif [ \"$CHANGED\" = \"1\" ]; then\n  echo \"pihole-adlists: declared set changed - running gravity update\"\n  pihole -g || echo \"pihole-adlists: gravity update failed; existing gravity.db retained\" >&2\nelse\n  echo \"pihole-adlists: already matches the declared set - nothing to do\"\nfi\nexit 0\n"
       - name: oauth2-proxy
-        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
         imagePullPolicy: IfNotPresent
         args:
         - --http-address=0.0.0.0:4180

apps/wave-0/qbittorrent.yaml

@@ -156,7 +156,7 @@ spec:
                 , \"[Preferences]\"\n\ndef matches(raw):\n    \"\"\"True iff the stored digest is one OUR password produces (salt is read from it).\"\"\"\n    v = raw.strip().strip('\"')\n    if v.startswith(\"@ByteArray(\") and v.endswith(\")\"):\n        v = v[len(\"@ByteArray(\"):-1]\n    try:\n        salt_b64, digest_b64 = v.split(\":\", 1)\n        salt = base64.b64decode(salt_b64, validate=True)\n        digest = base64.b64decode(digest_b64, validate=True)\n    except Exception:\n        return False\n    if not salt or not digest:\n        return False\n    return hmac.compare_digest(\n        hashlib.pbkdf2_hmac(\"sha512\", pw.encode(), salt, ITERATIONS, len(digest)), digest)\n\ntry:\n    lines = open(conf, encoding=\"utf-8\", errors=\"replace\").read().splitlines()\nexcept FileNotFoundError:\n    lines = []\n\ndef value_of(key):\n    for i, line in enumerate(lines):\n        if line.startswith(key + \"=\"):\n            return i, line.split(\"=\", 1)[1]\n    return -1, None\n\nui, uval = value_of(UKEY)\npi, pval = value_of(PKEY)\nif uval == user and pval is not None and matches(pval):\n    raise SystemExit(0)          # converged - leave the file (and its salt) alone\n\nsalt = os.urandom(SALT_LEN)\ndigest = hashlib.pbkdf2_hmac(\"sha512\", pw.encode(), salt, ITERATIONS, DIGEST_LEN)\nstored = base64.b64encode(salt).decode() + \":\" + base64.b64encode(digest).decode()\ndesired = [UKEY + \"=\" + user, PKEY + '=\"@ByteArray(' + stored + ')\"']\n\nout = [line for i, line in enumerate(lines) if i not in (ui, pi)]\nif SECTION in out:\n    at = out.index(SECTION) + 1\nelse:\n    if out and out[-1].strip():\n        out.append(\"\")\n    out.append(SECTION)\n    at = len(out)\nout[at:at] = desired\n\ntry:\n    os.makedirs(os.path.dirname(conf) or \".\", exist_ok=True)\n    tmp = conf + \".webui-cred.new\"\n    with open(tmp, \"w\", encoding=\"utf-8\") as fh:\n        fh.write(\"\\n\".join(out) + \"\\n\")\n        fh.flush()\n        os.fsync(fh.fileno())\n    # This hook runs as ROOT; the service runs as PUID:PGID. A root-owned 0600 conf would\n    # cost the app the ability to save its own settings, so carry the ownership over.\n    try:\n        st = os.stat(conf)\n        os.chown(tmp, st.st_uid, st.st_gid)\n        os.chmod(tmp, stat.S_IMODE(st.st_mode))\n    except FileNotFoundError:\n        os.chown(tmp, int(os.environ.get(\"PUID\", \"1000\")),\n                 int(os.environ.get(\"PGID\", \"1000\")))\n        os.chmod(tmp, 0o664)\n    os.replace(tmp, conf)        # atomic: the app never reads a half-written conf\nexcept OSError as exc:\n    sys.stderr.write(\"[qbt-webui-cred] cannot write %s: %s\\n\" % (conf, exc))\n    raise SystemExit(1)\nraise SystemExit(10)\nPY\n  rc=$?\n\n  if [ \"$rc\" = 0 ]; then\n    log \"conf already holds the durable credential for '$QBT_WEBUI_USERNAME' (no write)\"\n  elif [ \"$rc\" = 10 ]; then\n    log \"asserted WebUI Username + Password_PBKDF2 into $CONF\"\n    # postStart runs CONCURRENTLY with the s6 init chain, so qbittorrent-nox may ALREADY\n    # have read the old, credential-less conf and be serving a random temporary password.\n    # Restart the SUPERVISED SERVICE (not the pod) so it re-reads what was just written.\n    # Nothing to do when the service has not started yet - it will read the new file.\n    # Its own counter, NOT the shared `waited` API budget: sharing one counter is how the\n    # 2026-09-13 failure spent the entire verification budget before the first login and\n    # then fired attempts 2 and 3 with no wait left at all.\n    sv=0\n    while [ \"$sv\" -lt 60 ] && [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" != \"true\" ]; do\n      sv=$((sv + 1)); sleep 1\n    done\n    if [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" = \"true\" ]; then\n      log \"qbittorrent-nox was already running - restarting it to load the credential\"\n      s6-svc -r \"$SVC\" || log \"s6-svc -r failed (continuing to verify anyway)\"\n      sleep 3\n    fi\n  else\n    log \"FAILED to write the credential into $CONF (helper exit $rc). The WebUI will keep\" >&2\n    log \"minting a random password\
                 \ every start and BOTH arrs will break. Fix and restart.\" >&2\n    exit 0\n  fi\n\n  # (3) GROUND TRUTH: the WebUI itself must accept it. Wait for the API to answer at ALL\n  #     (shared ~120s budget across attempts), then spend exactly ONE login per attempt -\n  #     at most 3 rejected logins, which stays under qBittorrent's default ban threshold.\n  #\n  # \"THE API NEVER ANSWERED\" IS A DIFFERENT FAULT FROM \"THE PASSWORD WAS REJECTED\", and\n  # conflating them is what made this hook lie once already (see the PORT comment). An\n  # unreachable WebUI is reported as unreachable, and the hook stops rather than spending\n  # its remaining login attempts to produce two more identical wrong diagnoses.\n  api_ok=0\n  while [ \"$waited\" -lt 120 ]; do\n    code=$(curl -s -o /dev/null -m 3 -w '%{http_code}' \"http://127.0.0.1:$PORT/\" 2>/dev/null)\n    if [ -n \"$code\" ] && [ \"$code\" != \"000\" ]; then api_ok=1; break; fi\n    waited=$((waited + 1)); sleep 1\n  done\n  if [ \"$api_ok\" != 1 ]; then\n    log \"the WebUI never answered on 127.0.0.1:$PORT within the wait budget, so the\" >&2\n    log \"credential in $CONF is UNVERIFIED. This is NOT a rejected password - it was\" >&2\n    log \"never submitted. Do not rotate the credential on the strength of this line.\" >&2\n    exit 0\n  fi\n  body=$(curl -sS -m 10 \\\n           --data-urlencode \"username=$QBT_WEBUI_USERNAME\" \\\n           --data-urlencode \"password=$QBT_WEBUI_PASSWORD\" \\\n           \"http://127.0.0.1:$PORT/api/v2/auth/login\" 2>/dev/null)\n  if [ \"$body\" = \"Ok.\" ]; then\n    log \"VERIFIED: the WebUI accepts the durable credential for '$QBT_WEBUI_USERNAME'\"\n    exit 0\n  fi\n  log \"WebUI rejected the durable credential (login said '$body')\" >&2\n  # A rejection when the conf was ALREADY converged (rc=0, nothing written) can only mean\n  # the running qbittorrent-nox is serving something it read from an EARLIER conf, so\n  # rewriting the same bytes on the next attempt would change nothing. The remedy is to\n  # make the process RE-READ the file, so restart the supervised service and let the next\n  # attempt re-verify. Without this the retry loop could never change its own outcome.\n  if [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" = \"true\" ]; then\n    log \"restarting qbittorrent-nox so it re-reads $CONF, then re-verifying\" >&2\n    s6-svc -r \"$SVC\" || log \"s6-svc -r failed (continuing to verify anyway)\" >&2\n    sleep 3\n  fi\ndone\n\nlog \"GAVE UP after $attempt attempts: the WebUI does not accept the credential from git.\" >&2\n# A postStart hook MUST exit 0 - a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), turning a credential fault into an outage.\n# Both failure paths above therefore log LOUDLY and exit 0; the `up` SLI still watches the\n# app, and the arrs' own download-client health check surfaces a broken credential.\nexit 0\n"
       - name: oauth2-proxy
-        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
         imagePullPolicy: IfNotPresent
         args:
         - --http-address=0.0.0.0:4180

apps/wave-1/gitea-actions.yaml

@@ -77,6 +77,42 @@ spec:
     requests:
       storage: 1Gi
 ---
+# act_runner config file — DELTAS ONLY. act_runner v0.6.1 LoadDefault()
+# (internal/pkg/config/config.go) unmarshals this file onto a ZERO Config and then fills every
+# unset field with the same default it uses when there is no file at all, so a file holding
+# only `runner.capacity` changes exactly that and nothing else. Do NOT paste in the output of
+# `act_runner generate-config`: that example is NOT the defaults. Its `runner.labels` would
+# OVERRIDE the registered labels in /data/.runner (daemon.go: config labels win when non-empty)
+# and repoint ubuntu-latest at a different job image; its `force_pull: true` differs from the
+# no-file default (false); and its sample `envs` would be injected into every job.
+#
+# Using a file also makes act_runner IGNORE the legacy GITEA_RUNNER_CAPACITY / _FILE /
+# _ENVIRON / _ENV_FILE / GITEA_DEBUG / GITEA_TRACE env vars (deprecated.go). None of them is
+# set on this Deployment; the GITEA_INSTANCE_URL / _REGISTRATION_TOKEN / _NAME / _LABELS env
+# below is consumed by the image's run.sh for registration, not by that legacy path.
+#
+# A ConfigMap edit alone does NOT restart the runner (it reads the file once at start). If a
+# future change touches ONLY this ConfigMap, also bump the Deployment's pod template (e.g. the
+# config-revision annotation below) or the new value stays inert until the next restart.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: act-runner-config
+  namespace: gitea-actions
+  labels:
+    app.kubernetes.io/name: act-runner
+data:
+  config.yaml: |
+    runner:
+      # Two concurrent jobs (default 1). Measured 2026-11-08: queue wait ~1s normally, but
+      # 29-759s whenever gate / images / renovate runs collide (run 472 waited 759s behind a
+      # renovate burst). Both jobs share the ONE dind daemon below; act_runner names each job's
+      # containers GITEA-ACTIONS-TASK-<task id> and creates a network per job, so they do not
+      # collide. Workflow `concurrency:` groups are enforced by Gitea server-side and are
+      # unaffected: gate.yml's gate-<ref> group still cancels a superseded run on the SAME ref,
+      # and renovate's group still serialises renovate. The dind resources are sized for TWO jobs.
+      capacity: 2
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -95,6 +131,10 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/name: act-runner
+      annotations:
+        # Bump when act-runner-config changes without any other pod-template change, so the
+        # Deployment rolls and act_runner re-reads its config (see the ConfigMap header).
+        homelab.keithjackson.dev/act-runner-config-revision: "1"   # capacity 2
     spec:
       # CI holds no cluster credentials — do not mount a SA token into the runner or
       # (via the shared netns) the job containers (R-SEC-02, docs/14).
@@ -146,12 +186,23 @@ spec:
               mountPath: /certs
             - name: dind-storage
               mountPath: /var/lib/docker
+          # Sized for runner.capacity: 2. Every job's containers run UNDER this daemon, so this
+          # container's cgroup carries the whole CI workload. Measured 2026-11-08 (Prometheus,
+          # capacity 1): peak CPU 0.69 core (1m rate, 30d), peak working set 447Mi (30d), 0 OOMs.
+          #   cpu request 2: two jobs at today's peak is ~1.4 core, and the gate is being
+          #     parallelised, so per-job demand rises. The request is what the CFS share is
+          #     weighted by under contention (100m was a 1/20 share); NO cpu limit, deliberately,
+          #     so an idle node's spare cores are still usable. k3s-prod-2 (8 CPU) had 2945m
+          #     requested; +1.9 core = ~4.85 (61%), and prod-1/prod-3 fit too if it reschedules.
+          #   memory request 1Gi / limit 4Gi: two jobs at today's peak is ~0.9Gi. An OOM kill here
+          #     takes down dockerd and with it BOTH running jobs, so the limit carries ~4x headroom
+          #     over that for the parallelised gate. Nodes have ~31Gi allocatable.
           resources:
             requests:
-              cpu: 100m
-              memory: 256Mi
+              cpu: "2"
+              memory: 1Gi
             limits:
-              memory: 2Gi
+              memory: 4Gi
           startupProbe:
             exec:
               command:
@@ -195,23 +246,36 @@ spec:
               value: /certs/client
             - name: DOCKER_TLS_VERIFY
               value: "1"
+            # The image's run.sh passes `--config <this>` to BOTH `act_runner register` and
+            # `act_runner daemon` when CONFIG_FILE is set (act_runner v0.6.1 scripts/run.sh).
+            - name: CONFIG_FILE
+              value: /etc/act_runner/config.yaml
           volumeMounts:
             - name: data
               mountPath: /data
             - name: certs
               mountPath: /certs
               readOnly: true
+            - name: config
+              mountPath: /etc/act_runner
+              readOnly: true
+          # Measured 2026-11-08 at capacity 1: peak CPU 0.24 core, peak working set 314Mi (30d)
+          # against the old 512Mi limit. It holds per-task state, log buffers and the embedded
+          # actions/cache server for every concurrent task, so the limit is doubled for capacity 2.
           resources:
             requests:
               cpu: 100m
-              memory: 128Mi
+              memory: 256Mi
             limits:
-              memory: 512Mi
+              memory: 1Gi
       volumes:
         - name: data
           persistentVolumeClaim:
             claimName: act-runner-data
         - name: certs
           emptyDir: {}
+        - name: config
+          configMap:
+            name: act-runner-config
         - name: dind-storage
           emptyDir: {}

apps/wave-1/radarr.yaml

@@ -138,7 +138,7 @@ spec:
         - name: media
           mountPath: /media
       - name: oauth2-proxy
-        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
         imagePullPolicy: IfNotPresent
         args:
         - --http-address=0.0.0.0:4180

apps/wave-1/sonarr.yaml

@@ -138,7 +138,7 @@ spec:
         - name: media
           mountPath: /media
       - name: oauth2-proxy
-        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
         imagePullPolicy: IfNotPresent
         args:
         - --http-address=0.0.0.0:4180

catalog/render.py

@@ -129,7 +129,7 @@ AUTH_MODES = ("oidc", "forward-auth", "native", "none")
 # the sidecar authenticates the browser against Keycloak (homelab realm, the app's auth.groups)
 # and only THEN proxies to the co-located app UI at http://127.0.0.1:<uiport>. The gate is IN
 # the data path (fail-closed: no valid session in an authorized group => no upstream; R-IDP-02).
-OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.8.1"
+OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.15.6"
 OAUTH2_PROXY_PORT = 4180
 KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab"
 

conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml

@@ -62,7 +62,7 @@ spec:
         ports:
         - containerPort: 8080
       - name: oauth2-proxy
-        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
         imagePullPolicy: IfNotPresent
         args:
         - --http-address=0.0.0.0:4180

conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml

@@ -85,7 +85,7 @@ spec:
         - name: config
           mountPath: /data/config
       - name: oauth2-proxy
-        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
         imagePullPolicy: IfNotPresent
         args:
         - --http-address=0.0.0.0:4180

conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml

@@ -85,7 +85,7 @@ spec:
         - name: config
           mountPath: /data/config
       - name: oauth2-proxy
-        image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
         imagePullPolicy: IfNotPresent
         args:
         - --http-address=0.0.0.0:4180

decisions/2026-11-08-keycloak-operator-crds.md

@@ -0,0 +1,74 @@
+---
+date: 2026-11-08
+branch: fix/keycloak-operator-crds
+title: "Keycloak 26.8.2 operator CrashLoop: install its two new CRDs, raise its memory"
+---
+
+## What changed
+
+**Incident.** PR #121 (Renovate) bumped `platform/keycloak/kustomization.yaml`'s three
+`keycloak-k8s-resources` URLs 26.0.6 -> 26.8.2. Upstream `kubernetes/` at 26.8.2 holds **four**
+CRDs, not two (GitHub contents API, `?ref=26.8.2`):
+
+| file | at 26.0.6 | at 26.8.2 |
+|---|---|---|
+| `keycloaks.k8s.keycloak.org-v1.yml` | yes | yes |
+| `keycloakrealmimports.k8s.keycloak.org-v1.yml` | yes | yes |
+| `keycloakoidcclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
+| `keycloaksamlclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
+| `kubernetes.yml` | yes | yes (now carries RBAC for both new kinds) |
+
+(`cluster-wide/` is a separate variant; upstream's own `kubernetes/kustomization.yml` lists exactly
+the five files above.) The 26.8.2 operator starts an informer for every controller it has and dies
+on the first missing kind: `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1
+... GET .../namespaces/keycloak/keycloaksamlclients ... Not Found`, exit 1, CrashLoopBackOff. Flux
+`platform` (`wait: true`, 5m) sat in health checks and every dependent Kustomization reported
+`dependency ... is not ready`. The old 26.0.6 ReplicaSet kept serving (RollingUpdate 25%/25%), so SSO
+stayed up.
+
+**Fix.**
+- `platform/keycloak/kustomization.yaml`: added the two CRD URLs, each with the same
+  `# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources` marker (the repo's
+  custom.regex manager matches all five — checked by running its regex over the file). Header
+  comment updated to v26.8.2, plus a warning that Renovate cannot add a CRD a release introduces.
+- Same file: a `patches:` JSON6902 patch on Deployment `keycloak-operator`: memory limit
+  450Mi -> **768Mi**, request 450Mi -> **512Mi**. 450Mi is upstream's default (unchanged at 26.8.2,
+  so not something the bump changed). Evidence it is too small here: the 26.0.6 pod had **154
+  restarts**, `lastState.terminated.reason: OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and
+  `kubectl top` showed **421Mi / 450Mi**. 26.8.2 runs four controllers instead of two on Quarkus
+  3.40.1.1 (was 3.15.1). No variable in the operator to disable watching unused kinds was used —
+  installing the CRDs is upstream's supported shape.
+- `platform/keycloak/README.md` "Upgrading Keycloak": the diff-the-CRD-list step; "three URLs" ->
+  "the release URLs". `renovate.json5`: same wording fix in a comment only.
+
+## What I verified
+
+- Upstream file lists at 26.0.6 and 26.8.2 (GitHub API) and `diff` of `kubernetes.yml` between the
+  two: the new kinds' ClusterRoles + RoleBindings ship in it, and the Deployment's resources are
+  still 450Mi/450Mi.
+- Live, read-only (`kubectl --context k3s-prod`, 2026-11-08 ~05:57Z): only `keycloaks` and
+  `keycloakrealmimports` CRDs installed; Deployment strategy RollingUpdate maxSurge/maxUnavailable
+  25%; RS `keycloak-operator-58dff6c4` (26.8.2) 0/1 CrashLoopBackOff, RS `78dbb75f64` (26.0.6) 1/1;
+  operator log is the informer error above; Flux `platform` `Healthy=Unknown` "Running health
+  checks for revision main@sha1:e8d2666"; no NetworkPolicy and no CiliumNetworkPolicy in ns
+  `keycloak`.
+- No sensitive path touched (`infra/gitea/approval-policy.json`), so this should merge on green.
+- Gate: see the PR body for the exact rc and stages of the run against the final commit.
+
+## What I did NOT verify
+
+- **That the 26.8.2 operator becomes Ready after this merges.** Only the two missing kinds were
+  observed failing; another start-up failure behind them is possible.
+- **That 768Mi is enough** — no 26.8.2 memory measurement exists yet; re-check `kubectl top` after
+  a week.
+- **The Keycloak server upgrade 26.0.6 -> 26.8.2 itself.** Once the operator runs it will roll
+  keycloak-0/1 and Keycloak migrates the DB schema. The CR's `spec.update.strategy` defaults to
+  `RecreateOnImageChange` — expect a **full SSO outage** while both pods recreate, not a rolling
+  one. Back up / confirm the latest CNPG backup first if that matters.
+- **The 26.8.2 Keycloak CRD's `spec.networkPolicy.enabled` defaults to `true`.** The operator will
+  create a NetworkPolicy selecting the Keycloak pods (first policy in that namespace). With no
+  `from` lists set it allows all sources on the HTTP/management ports, so Gateway, ServiceMonitor
+  and the reconciler Jobs should still reach it — but this was read from the CRD text, not observed.
+- **The passkey reconciler's 26.0.6 Admin API assumptions against 26.8.2** (README "Upgrading
+  Keycloak" list) — not re-checked here; PR #121 merged without it.
+- `catalog/keycloak.yaml`'s documentation-only image tag was not updated.

decisions/2026-11-08-keycloak-servicemonitor-owner.md

@@ -0,0 +1,60 @@
+---
+date: 2026-11-08
+branch: fix/keycloak-servicemonitor-owner
+title: One owner for the keycloak ServiceMonitor — disable the 26.8 operator's (409 loop stalled platform-config)
+---
+
+## What changed
+
+- `platform/keycloak/keycloak-instance.yaml`: added `spec.serviceMonitor.enabled: false`.
+  The 26.x Keycloak operator creates its own ServiceMonitor, named after the CR (`keycloak/keycloak`),
+  whenever `metrics-enabled=true` and the ServiceMonitor CRD exists. `serviceMonitor.enabled` defaults
+  to `true` in the 26.8.2 CRD, both in v2alpha1 and in v2beta1 (v2beta1 is now the storage version).
+  That is the same name and namespace as the repo's Flux-owned `platform/keycloak/servicemonitor.yaml`.
+  After the 26.0.6 -> 26.8.2 bump (#121/#144) the operator's PATCH against it returned
+  `409 Conflict` and it ran out of retries. The CR then sat at `Ready=False`/`HasErrors=True`, and
+  `platform-config` (`wait: true`, which health-checks every object it applies, this CR included)
+  blocked everything that `dependsOn` it: velero, apps-vms, apps-wave-0 (and so waves 1/2 and
+  pm-agent), homepage, oauth2-proxy, headlamp, conformance, vpn-ingress, crafty, runbooks.
+- `platform/keycloak/servicemonitor.yaml`: a comment saying it is the sole owner.
+- **Why ours is kept:** these are written against it: the L0 `servicemonitor-ports-resolve`
+  allow-list entry `platform/keycloak/servicemonitor.yaml:keycloak`, the L2 `keycloak-monitored`
+  chainsaw test, the README's ownership table, and the dashboard's `up{namespace="keycloak"}`.
+  The operator's version (upstream `KeycloakServiceMonitorDependentResource` @26.8.2) scrapes the same
+  target: port name `management` (9000 on `keycloak-service`), path `/metrics`, interval 30s,
+  selector = the operator's instance labels. It offers nothing ours lacks, so having two owners
+  would add a fight and gain nothing.
+
+## What I verified
+
+- Live, read-only, `k3s-prod` ~06:25Z: the CRD schema field `spec.serviceMonitor.enabled`
+  (`default: true`) exists in both served versions. The live `keycloak` ServiceMonitor has exactly
+  **one** managedFields manager, `kustomize-controller` (Apply, 2026-09-05), **no ownerReferences**,
+  and labels `kustomize.toolkit.fluxcd.io/name=platform-config`, `release=kube-prometheus-stack`. So
+  the operator's apply never landed, and nothing on the object ties it to the Keycloak CR.
+- The ServiceMonitor will not be deleted when the operator is switched off: the operator's dependent
+  informer is filtered on `Constants.DEFAULT_LABELS_AS_STRING`
+  (`app.kubernetes.io/managed-by=keycloak-operator`). Our object does not carry that label and has
+  no ownerReference, so the operator cannot see it as its secondary and cannot delete or GC it.
+  This conclusion comes from reading the code. It was not observed.
+- The other operator dependents in ns `keycloak` do not collide with anything: `keycloak-service`,
+  `keycloak-discovery`, the StatefulSet `keycloak`, `keycloak-initial-admin`, and
+  `keycloak-network-policy` (owner Keycloak, no Flux label). The repo declares none of these names.
+  It only *references* `keycloak-service` by name (HTTPRoute, reconcilers). There are no PDBs in the
+  namespace, and the Ingress is already `enabled: false`. The only NetworkPolicy in the namespace is
+  the operator's. The repo has no Keycloak NetworkPolicy, and no Kyverno-generated default-deny is
+  present there. NetworkPolicies add up, so the operator's allow-policy cannot fight another owner.
+  Left at its default.
+- Local gate: see the PR body. It records the rc and stages.
+
+## What I did NOT verify
+
+- That the CR reaches `Ready=True`/`HasErrors=False` after merge. The status also says
+  "Performing Keycloak upgrade, scaling down the deployment". I am inferring that this half clears
+  once the workflow stops erroring. It is not proven.
+- That the operator stops issuing the PATCH once `enabled: false` is applied. Watch the operator
+  logs for 409.
+- That `platform-config` and its downstream Kustomizations go Ready on the new revision, and that
+  Prometheus keeps exactly one `serviceMonitor/keycloak/keycloak/0` pool with targets up.
+- The operator's actual label set for `DEFAULT_LABELS_AS_STRING`. I took it from the observed labels
+  on operator-created objects, not from `Constants.java`.

decisions/2026-11-08-rework-136.md

@@ -0,0 +1,48 @@
+---
+date: 2026-11-08
+branch: rework/136
+title: "PR #136 rework: regenerate L1 render goldens for the oauth2-proxy v7.15.6 sidecar bump"
+---
+
+## What changed
+
+- Renovate PR #136 bumps `OAUTH2_PROXY_IMAGE` in `catalog/render.py` (and the generated
+  `apps/wave-*` / `platform/crafty` pins) from `v7.8.1` to `v7.15.6`. Its gate failed at L1
+  because three byte-for-byte render goldens still carried the old tag:
+  - `conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml`
+  - `conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml`
+  - `conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml`
+    (CI only reported the vpn one because `render-check.sh` stops at the first failing diff;
+    the internet golden had drifted too.)
+- Regenerated all three in the pinned toolbox with the exact invocation the check scripts use:
+  `python3 catalog/render.py <fixture> > <golden>`. The diff in each is **one line**:
+  `image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1` -> `...:v7.15.6`. Nothing else moved.
+- `platform/crafty/README.md`: the "Sidecar" row described the current pin; updated to v7.15.6.
+- Deliberately NOT changed (they record history, not the current pin):
+  - `catalog/render.py` comments / assertion messages saying a flag was "confirmed present in the
+    PINNED v7.8.1 binary" or that `--api-routes` "does not exist in v7.8.1". Rewriting them to
+    v7.15.6 would assert a `--help` verification nobody has done against v7.15.6.
+  - `conformance/tests/l0-static/oauth2-proxy-api-routes-per-app/check.sh`, the schema
+    description, `runbooks/oauth2-proxy-session.md`, `docs/plans/...` — same reasoning.
+  - `platform/oauth2-proxy/auth-visibility-alerts.promtool-test.yaml` — v7.8.1 appears in
+    synthetic kube-state-metrics series (test input data), not as a pin.
+  - `.gitea/workflows/renovate.yaml` comment — a dated measurement of the pre-Renovate state.
+- Branch was 55 commits behind `origin/main`, but main touches neither `catalog/render.py` nor
+  the L1 goldens, and the merge is conflict-free; no merge of main was needed for the gate.
+
+## What I verified
+
+- The golden diff is exactly the image tag in each of the three files (`git diff -U0`).
+- No remaining `oauth2-proxy:v7.8.1` in any rendered/generated/golden file (`git grep`).
+- Local toolbox gate result: see the PR #136 comment posted with this push (rc and per-stage
+  results are quoted there and in the worker report).
+
+## What I did NOT verify
+
+- That the v7.15.6 sidecar actually starts and authenticates in the cluster with the rendered
+  args. Earlier research said no flag used here was removed/renamed between 7.8.1 and 7.15.6;
+  I did not re-run `oauth2-proxy --help` against the v7.15.6 image.
+- L2/L3 conformance (does not run in CI: `CLUSTER_AVAILABLE` unset).
+- `approval-policy`: `conformance/**` is a sensitive path, so this PR is CI-red until the
+  operator posts `/approve-override <reason>` as a PR review. The local gate cannot evaluate it.
+- Whether Renovate keeps rebasing this branch now that it carries a non-Renovate commit.

decisions/2026-11-08-runner-capacity-2.md

@@ -0,0 +1,155 @@
+---
+date: 2026-11-08
+branch: feat/runner-capacity-2
+title: act_runner runs two jobs at once, and dind is sized for two
+---
+
+## What changed
+
+`apps/wave-1/gitea-actions.yaml` only. That file is a **hand-owned** non-catalog unit
+(`EXTRA_UNITS["gitea-actions"]` in `catalog/render.py`): the renderer only decides which wave
+it lands in, it does not generate its bytes, and `ci/render-golden.sh` compares only files the
+catalog renders. So it is edited in place, and no re-render was needed or run.
+
+1. **New ConfigMap `act-runner-config`** holding a config file with ONE key,
+   `runner.capacity: 2`, mounted read-only at `/etc/act_runner/` and selected with
+   `CONFIG_FILE=/etc/act_runner/config.yaml`. The image's `run.sh` (act_runner v0.6.1) turns
+   that into `--config` for both `register` and `daemon`.
+   - Deltas only, on evidence: v0.6.1 `LoadDefault()` unmarshals the file onto a zero Config and
+     then applies the same defaults as the no-file path, so the other settings are unchanged.
+   - The `generate-config` example was deliberately NOT used. Its `runner.labels` would override
+     the registered labels in `/data/.runner` and repoint `ubuntu-latest` at
+     `docker.gitea.com/runner-images`. Its `force_pull: true` is not the default. Its sample `envs`
+     would be injected into every job.
+   - With a file present, act_runner ignores the legacy `GITEA_RUNNER_CAPACITY` /
+     `_FILE` / `_ENVIRON` / `_ENV_FILE` / `GITEA_DEBUG` / `GITEA_TRACE` env vars. None of
+     them is set on this Deployment.
+2. **Pod-template annotation `homelab.keithjackson.dev/act-runner-config-revision: "1"`.**
+   act_runner reads its config once, at start, so a change to the ConfigMap alone would never
+   take effect. Bump this annotation whenever you change only the ConfigMap.
+3. **dind resources:** requests `cpu 100m -> 2` and `memory 256Mi -> 1Gi`; limit
+   `memory 2Gi -> 4Gi`. There is still no CPU limit, on purpose.
+4. **runner resources:** request `memory 128Mi -> 256Mi`, limit `memory 512Mi -> 1Gi`. CPU stays
+   at `100m`.
+
+Why these sizes. Measured 2026-11-08 through Prometheus, at capacity 1:
+
+| container | peak CPU (1m rate, 30d) | peak working set (30d) | OOM events (7d) |
+|---|---|---|---|
+| dind | 0.69 core | 447 MiB | 0 |
+| runner | 0.24 core | 314 MiB | 0 |
+
+At the moment of measurement the pod was idle (`top`: dind 1m/138Mi, runner 1m/83Mi), on
+k3s-prod-2.
+
+- **CPU request 2.** Two jobs at today's peak is about 1.4 cores. The gate is being parallelised
+  in other work, so each job will use more than that. The request sets dind's CFS weight under
+  contention; at 100m it got a 1/20 share. With no limit, dind can still use spare cores on an
+  idle node.
+- **Memory 1Gi / 4Gi.** Two jobs at today's peak is about 0.9 GiB. If dind is OOM-killed,
+  dockerd goes down and takes both running jobs with it, so the limit is about 4x that.
+- **Runner limit 512Mi -> 1Gi.** Its 30-day peak was already 61% of the old limit, and with
+  capacity 2 it holds state for two tasks.
+
+Scheduling headroom: CPU requests per node were measured with `describe node`. The pod's total
+CPU request goes from 200m to 2100m (+1.9 cores).
+
+| node | CPU requested now | after the change | allocatable |
+|---|---|---|---|
+| prod-2 (current) | 2945m | ~4845m (61%) | 8 |
+| prod-1 | 3175m | ~5075m | 8 |
+| prod-3 | 3840m | ~5740m (72%) | 8 |
+
+The pod fits on any of the three if the RWO PVC moves it.
+
+Concurrency semantics. Workflow `concurrency:` groups are enforced by Gitea on the server, not by
+the runner, so capacity does not change them:
+
+- `gate.yml` `gate-${{ github.ref }}` with cancel-in-progress still cancels a superseded run on
+  the same ref.
+- `renovate` (group `renovate`, no cancel) and `images-${{ github.ref }}` still serialise
+  themselves.
+
+What capacity 2 does change: runs on *different* refs (a PR gate next to a renovate or images
+run, or two PRs) now run side by side instead of queueing. The required check
+`gate / gate (pull_request)` is still one job per PR head, and gate.yml's other job
+(`conformance`) is `if: false`, so it never runs.
+
+Both jobs share one dockerd. act_runner gives each task its own container prefix
+(`GITEA-ACTIONS-TASK-<task id>`) and its own network, and no workflow uses fixed container names
+or host ports.
+
+## What I verified
+
+- **Source of truth.** `catalog/render.py` `EXTRA_UNITS` places `gitea-actions.yaml` by
+  anchor/offset only. `ci/render-golden.sh` iterates over rendered files only, so this file is
+  outside render==golden. No `--emit-waves` was run.
+- **act_runner v0.6.1 behaviour, read in its source** (tag v0.6.1):
+  - `config.LoadDefault()` treats a partial file as described above.
+  - `daemon.go`: config labels override `.runner` labels only when non-empty.
+  - `poller.go`: a semaphore of `cfg.Runner.Capacity` bounds concurrent tasks.
+  - `runner.go`: per-task container name prefix.
+  - The image's `run.sh`, read from `gitea/act_runner:0.6.1`, honours `CONFIG_FILE`.
+- **The live pod has no config file today.** `/data` holds only `.runner` and `lost+found`.
+  `act_runner --version` reports v0.6.1. No LimitRange or ResourceQuota exists in
+  `gitea-actions`.
+- **Resource numbers.** All figures above come from read-only `kubectl top`, `describe node` and
+  Prometheus queries against k3s-prod.
+- **dind disk.** The emptyDir sits on the node's 11.8G ephemeral filesystem: 4.0G used, 7.2G
+  free. Image cache is 2.18GB, which is the one `catthehacker/ubuntu:act-22.04` image. Two
+  concurrent jobs share that image.
+- **Embedded config.** It parses as YAML to `{'runner': {'capacity': 2}}`. The only `$` in the
+  manifest is the existing `${BASE_DOMAIN}`, so nothing new is exposed to Flux postBuild
+  envsubst.
+- **Approval policy.** `apps/wave-1/` matches no `sensitive_paths` glob in
+  `infra/gitea/approval-policy.json`, so this PR merges on a green gate alone.
+- **Local gate.** Pinned toolbox, run from the worktree on the committed tree. The rc and skipped
+  stages are given in the PR description.
+
+## What I did NOT verify
+
+- **UNVERIFIED: that the runner actually starts with the config and runs two jobs at once.**
+  Nothing is applied until Flux reconciles `main`.
+- **Capacity cannot be seen through Gitea's API.** act_runner v0.6.1 does not report capacity to
+  Gitea: `Declare` sends only version and labels. `GET /api/v1/admin/actions/runners` therefore
+  cannot show "capacity 2". Concurrency has to be observed instead.
+- **UNVERIFIED: whether the parallelised gate fits in these sizes.** That gate has not landed.
+  Re-measure after it does, with
+  `max_over_time(sum by (container)(rate(container_cpu_usage_seconds_total{namespace="gitea-actions"}[1m]))[7d:30s])`.
+- **Rollout cancels in-flight jobs.** The rollout restarts the runner pod (strategy Recreate,
+  `shutdown_timeout: 0s`), and any job running at that moment is cancelled, possibly including
+  the `push`-to-main gate run triggered by this very merge. Re-run it if that happens.
+- **dind disk is not bounded by a limit.** With two jobs it holds two jobs' workspaces and layers
+  on the same 11.8G node filesystem. I did not add an `ephemeral-storage` limit, because an
+  eviction would kill both jobs. Watch kubelet DiskPressure on the runner's node.
+- **Stale comments in sensitive paths.** `.gitea/workflows/gate.yml` (line ~52) and
+  `.gitea/workflows/README.md` (line ~86) still say "`act_runner` runs at capacity 1". I left
+  them alone because `.gitea/**` is a sensitive path and editing them would put this PR behind
+  human approval. Fix them in a separate PR.
+
+### Operator post-merge checklist
+
+1. Flux applied it:
+   `kubectl --context k3s-prod -n flux-system get kustomization apps-wave-1 -o jsonpath='{.status.lastAppliedRevision}'`
+   equals the merge commit.
+2. The pod restarted with the config:
+   - `kubectl --context k3s-prod -n gitea-actions get pod -l app.kubernetes.io/name=act-runner`
+     shows a new pod, 2/2 Running.
+   - `kubectl ... exec deploy/act-runner -c runner -- cat /etc/act_runner/config.yaml` shows
+     `capacity: 2`.
+   - `kubectl ... exec deploy/act-runner -c runner -- ps` shows `act_runner daemon --config /etc/act_runner/config.yaml`.
+   - The logs show "declare successfully" with the two expected labels (`ubuntu-latest`,
+     `homelab` -> `catthehacker/ubuntu:act-22.04`). That confirms the labels did not change.
+3. The runner is registered and online:
+   - `GET /api/v1/admin/actions/runners` lists `incluster-dind`, online, with the same ID as
+     before (no duplicate row).
+   - Capacity is NOT shown there; see above.
+4. Concurrency is real: trigger two runs on different refs (e.g. two PR gates, or a PR gate plus a
+   `workflow_dispatch` of renovate). Then either:
+   - in `GET /api/v1/repos/Home-Network/home-network-k3s-impl/actions/tasks`, confirm their
+     `run_started_at` .. `updated_at` windows overlap; or
+   - check
+     `docker --host=tcp://127.0.0.1:2376 --tlsverify ... ps` in the dind container lists two
+     `GITEA-ACTIONS-TASK-*` container sets at once.
+5. Resources: `kubectl --context k3s-prod -n gitea-actions describe pod` shows dind requests
+   cpu 2 / memory 1Gi, limit memory 4Gi, and runner limit memory 1Gi.

platform/crafty/README.md

@@ -16,7 +16,7 @@ Cilium L2 LoadBalancer (game protocols authenticate players themselves).
 |---|---|
 | Image | `registry.gitlab.com/crafty-controller/crafty-4:4.10.8` (current stable; bare-semver tags, no `v`) |
 | Web UI | HTTPS **self-signed** on `:8443` (reached only via the sidecar / same-pod) |
-| Sidecar | `quay.io/oauth2-proxy/oauth2-proxy:v7.8.1`, reverse-proxy on `:4180` |
+| Sidecar | `quay.io/oauth2-proxy/oauth2-proxy:v7.15.6`, reverse-proxy on `:4180` |
 | Data | ONE Longhorn RWO PVC `crafty-data` (20Gi), mounted 5× by subPath |
 | PVC subPaths | `servers`→`/crafty/servers`, `backups`→`/crafty/backups`, `logs`→`/crafty/logs`, `import`→`/crafty/import`, `config`→`/crafty/app/config` |
 | Identity | `runAsUser:1000 runAsGroup:0 fsGroup:0 runAsNonRoot:true` (see below) |

platform/crafty/deployment.yaml

@@ -130,7 +130,7 @@ spec:
         - name: oauth2-proxy
           # quay.io/oauth2-proxy/oauth2-proxy — same authenticator family as platform/oauth2-proxy, but
           # here in per-app REVERSE-PROXY mode (--upstream is the co-located Crafty UI, not static://202).
-          image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
           imagePullPolicy: IfNotPresent
           args:
             - --http-address=0.0.0.0:4180

platform/keycloak/README.md

@@ -367,9 +367,17 @@ is involved — so step 2 failing with no log entry is the expected signature of
 The `Keycloak` CR has no `spec.image`, so the server image is whatever the operator's
 `RELATED_IMAGE_KEYCLOAK` names — the same tag as the operator (prod, 2026-10-30: operator
 `26.0.6`, `keycloak-0/1` on `quay.io/keycloak/keycloak:26.0.6`). Operator, CRDs and server are
-released in lockstep from `keycloak/keycloak-k8s-resources`, and the three URLs in
+released in lockstep from `keycloak/keycloak-k8s-resources`, and the release URLs in
 `kustomization.yaml` are the one knob. Renovate raises them as ONE PR (`keycloak-operator` group,
 labels `needs-review` + `blast-radius`), **never auto-merged**; majors wait for a dashboard tick.
+
+**Renovate bumps the URLs that exist; it cannot add a CRD a release introduced.** Before merging a
+bump, diff `kubernetes/` (and its `kustomization.yml`) at the new tag against the URL list. 26.0.6
+shipped 2 CRDs, 26.8.2 ships 4 (`keycloakoidcclients`, `keycloaksamlclients` added); PR #121 bumped
+only the existing URLs and the 26.8.2 operator CrashLooped on the missing `keycloaksamlclients`
+informer, stalling Flux `platform` and every tier below it (2026-11-08,
+`decisions/2026-11-08-keycloak-operator-crds.md`). The operator's memory is also patched in
+`kustomization.yaml` (upstream's 450Mi OOMKilled 26.0.6 here); keep the patch across bumps.
 `catalog/keycloak.yaml`'s image tag is documentation only (its render deploys nothing) and
 Renovate is disabled for it — update it by hand to match after an operator bump merges.
 

platform/keycloak/keycloak-instance.yaml

@@ -38,6 +38,20 @@ spec:
   ingress:
     enabled: false                         # exposed via Gateway API HTTPRoute (httproute.yaml), not the
                                            # operator's built-in Ingress
+  # --- ONE OWNER for the `keycloak` ServiceMonitor: this repo (servicemonitor.yaml), not the operator.
+  #     Since 26.x the operator creates its OWN ServiceMonitor, named after this CR (`keycloak`, i.e.
+  #     the SAME name/namespace as ours), whenever metrics-enabled=true and the CRD exists —
+  #     spec.serviceMonitor.enabled defaults TRUE. On 2026-11-08, right after the 26.0.6 -> 26.8.2
+  #     bump, its server-side apply onto our Flux-owned object failed `409 Conflict`, the operator
+  #     exhausted retries, the CR sat Ready=False/HasErrors=True, and `platform-config` (wait: true,
+  #     so it health-checks this CR) stalled every Kustomization downstream of it. Ours is kept
+  #     because the L0 servicemonitor-ports-resolve and L2 keycloak-monitored conformance tests and
+  #     the dashboard's up{namespace="keycloak"} are written against it; the operator's would scrape
+  #     the same thing (management port 9000, /metrics, 30s). Do NOT flip this to true without
+  #     deleting servicemonitor.yaml in the same change — two owners of one object is a fight
+  #     (cf. the default-deny NetworkPolicy drift loop).
+  serviceMonitor:
+    enabled: false
   # --- Hardening + observability options (docs/15). Brute-force detection is a REALM setting
   #     (bruteForceProtected in the realm import); these enable health + metrics endpoints. ---
   additionalOptions:

platform/keycloak/kustomization.yaml

@@ -20,18 +20,32 @@ kind: Kustomization
 namespace: keycloak
 resources:
   - namespace.yaml
-  # --- Keycloak Operator v26.0.6 (pinned upstream release manifests) ---
+  # --- Keycloak Operator v26.8.2 (pinned upstream release manifests) ---
   # The operator, its CRDs and the SERVER IMAGE move in lockstep: the Keycloak CR has no
   # spec.image, so the operator's RELATED_IMAGE_KEYCLOAK (quay.io/keycloak/keycloak:<same tag>)
-  # is what runs. These three URLs are therefore THE Keycloak version. Each carries a Renovate
-  # marker (custom.regex manager, renovate.json5) under ONE depName, so all three bump in ONE
+  # is what runs. These URLs are therefore THE Keycloak version. Each carries a Renovate
+  # marker (custom.regex manager, renovate.json5) under ONE depName, so all of them bump in ONE
   # branch, never auto-merged (packageRule "keycloak-operator"). Before merging a bump, re-check
   # passkey-reconciler.yaml's 26.0.6 Admin API assumptions — see README.md "Upgrading Keycloak".
+  #
+  # THE URL LIST MUST MATCH UPSTREAM'S kubernetes/kustomization.yml AT THE NEW TAG — Renovate only
+  # rewrites the version in the URLs that are here; it cannot add a CRD the new release introduced.
+  # 26.0.6 shipped 2 CRDs; 26.8.2 ships 4 (+ keycloakoidcclients, keycloaksamlclients, both
+  # v2alpha1). PR #121 bumped only the existing three URLs, and the 26.8.2 operator — which starts an
+  # informer for EVERY kind it controls, used or not — CrashLooped on
+  # `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1 ... Not Found`,
+  # stalling Flux `platform` and everything downstream (2026-11-08). On every bump, diff
+  #   https://github.com/keycloak/keycloak-k8s-resources/tree/<tag>/kubernetes
+  # against this list before merging.
   # renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
   - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
   # renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
   - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
   # renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
+  - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakoidcclients.k8s.keycloak.org-v1.yml
+  # renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
+  - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaksamlclients.k8s.keycloak.org-v1.yml
+  # renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
   - https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/kubernetes.yml
   # --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
   - keycloak-admin-credentials.sops.yaml
@@ -51,3 +65,23 @@ resources:
   - keycloak-db-backup-credentials-prod.sops.yaml
   # --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
   - grafana-dashboard.yaml
+# --- Operator memory: upstream ships requests=limits=450Mi, which this cluster outgrew ON 26.0.6 ---
+# Measured on prod 2026-11-08: the 26.0.6 operator pod had 154 restarts in 64d, lastState
+# `OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and sat at 421Mi of its 450Mi (94%) in `kubectl top`.
+# 26.8.2 keeps the same 450Mi default but runs FOUR controllers/informers instead of two (adds
+# keycloakoidcclient + keycloaksamlclient) on a newer Quarkus (3.15.1 -> 3.40.1.1), so it would
+# plausibly start above a ceiling 26.0.6 already crossed. Limit 768Mi = ~1.8x the observed 421Mi
+# working set; request 512Mi = the observed working set plus headroom, so the scheduler reserves what
+# it actually uses. Revisit with `kubectl top` once 26.8.2 has run for a week. Patched here (not in
+# the remote kubernetes.yml) so a Renovate bump keeps it.
+patches:
+  - target:
+      kind: Deployment
+      name: keycloak-operator
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/resources/limits/memory
+        value: 768Mi
+      - op: replace
+        path: /spec/template/spec/containers/0/resources/requests/memory
+        value: 512Mi

platform/keycloak/servicemonitor.yaml

@@ -8,6 +8,9 @@
 # /metrics) once metrics-enabled/health-enabled are set (see keycloak-instance.yaml additionalOptions).
 # We select the operator-created keycloak-service (which selects the Keycloak pods) and scrape the pods'
 # management port by number via targetPort.
+# SOLE OWNER: the operator's own ServiceMonitor (same name `keycloak`) is switched OFF in
+#   keycloak-instance.yaml (spec.serviceMonitor.enabled: false) — 26.x defaults it ON and its apply
+#   409-looped against this object on 2026-11-08. Keep exactly one of the two.
 # TODO(WS-SEC, bring-up): confirm the management port number/name the operator exposes (9000 default)
 #   and adjust targetPort if the pinned operator changes it. The KeycloakDown / KeycloakDatabaseDown
 #   alerts (platform/monitoring/alerts.yaml) key on kube-state-metrics and fire regardless of this SM.

renovate.json5

@@ -287,7 +287,7 @@
     // and the server image are released in lockstep from keycloak/keycloak-k8s-resources, and the
     // Keycloak CR carries no spec.image, so the operator's RELATED_IMAGE_KEYCLOAK picks the server
     // (measured on prod 2026-10-30: operator 26.0.6 -> keycloak-0/1 run quay.io/keycloak/keycloak:
-    // 26.0.6). The three release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
+    // 26.0.6). The release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
     // markers under this ONE depName, so they move together in one branch (groupName makes that
     // explicit). NEVER AUTO-MERGED, not even a patch: Keycloak is the IdP that gates SSO for every
     // app AND kubectl OIDC, and platform/keycloak/passkey-reconciler.yaml encodes Keycloak 26.0.6