Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b548224db8 |
No files matched your search
@@ -77,6 +77,42 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
storage: 1Gi
|
storage: 1Gi
|
||||||
---
|
---
|
||||||
|
# act_runner config file — DELTAS ONLY. act_runner v0.6.1 LoadDefault()
|
||||||
|
# (internal/pkg/config/config.go) unmarshals this file onto a ZERO Config and then fills every
|
||||||
|
# unset field with the same default it uses when there is no file at all, so a file holding
|
||||||
|
# only `runner.capacity` changes exactly that and nothing else. Do NOT paste in the output of
|
||||||
|
# `act_runner generate-config`: that example is NOT the defaults. Its `runner.labels` would
|
||||||
|
# OVERRIDE the registered labels in /data/.runner (daemon.go: config labels win when non-empty)
|
||||||
|
# and repoint ubuntu-latest at a different job image; its `force_pull: true` differs from the
|
||||||
|
# no-file default (false); and its sample `envs` would be injected into every job.
|
||||||
|
#
|
||||||
|
# Using a file also makes act_runner IGNORE the legacy GITEA_RUNNER_CAPACITY / _FILE /
|
||||||
|
# _ENVIRON / _ENV_FILE / GITEA_DEBUG / GITEA_TRACE env vars (deprecated.go). None of them is
|
||||||
|
# set on this Deployment; the GITEA_INSTANCE_URL / _REGISTRATION_TOKEN / _NAME / _LABELS env
|
||||||
|
# below is consumed by the image's run.sh for registration, not by that legacy path.
|
||||||
|
#
|
||||||
|
# A ConfigMap edit alone does NOT restart the runner (it reads the file once at start). If a
|
||||||
|
# future change touches ONLY this ConfigMap, also bump the Deployment's pod template (e.g. the
|
||||||
|
# config-revision annotation below) or the new value stays inert until the next restart.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: act-runner-config
|
||||||
|
namespace: gitea-actions
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: act-runner
|
||||||
|
data:
|
||||||
|
config.yaml: |
|
||||||
|
runner:
|
||||||
|
# Two concurrent jobs (default 1). Measured 2026-11-08: queue wait ~1s normally, but
|
||||||
|
# 29-759s whenever gate / images / renovate runs collide (run 472 waited 759s behind a
|
||||||
|
# renovate burst). Both jobs share the ONE dind daemon below; act_runner names each job's
|
||||||
|
# containers GITEA-ACTIONS-TASK-<task id> and creates a network per job, so they do not
|
||||||
|
# collide. Workflow `concurrency:` groups are enforced by Gitea server-side and are
|
||||||
|
# unaffected: gate.yml's gate-<ref> group still cancels a superseded run on the SAME ref,
|
||||||
|
# and renovate's group still serialises renovate. The dind resources are sized for TWO jobs.
|
||||||
|
capacity: 2
|
||||||
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
@@ -95,6 +131,10 @@ spec:
|
|||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: act-runner
|
app.kubernetes.io/name: act-runner
|
||||||
|
annotations:
|
||||||
|
# Bump when act-runner-config changes without any other pod-template change, so the
|
||||||
|
# Deployment rolls and act_runner re-reads its config (see the ConfigMap header).
|
||||||
|
homelab.keithjackson.dev/act-runner-config-revision: "1" # capacity 2
|
||||||
spec:
|
spec:
|
||||||
# CI holds no cluster credentials — do not mount a SA token into the runner or
|
# CI holds no cluster credentials — do not mount a SA token into the runner or
|
||||||
# (via the shared netns) the job containers (R-SEC-02, docs/14).
|
# (via the shared netns) the job containers (R-SEC-02, docs/14).
|
||||||
@@ -146,12 +186,23 @@ spec:
|
|||||||
mountPath: /certs
|
mountPath: /certs
|
||||||
- name: dind-storage
|
- name: dind-storage
|
||||||
mountPath: /var/lib/docker
|
mountPath: /var/lib/docker
|
||||||
|
# Sized for runner.capacity: 2. Every job's containers run UNDER this daemon, so this
|
||||||
|
# container's cgroup carries the whole CI workload. Measured 2026-11-08 (Prometheus,
|
||||||
|
# capacity 1): peak CPU 0.69 core (1m rate, 30d), peak working set 447Mi (30d), 0 OOMs.
|
||||||
|
# cpu request 2: two jobs at today's peak is ~1.4 core, and the gate is being
|
||||||
|
# parallelised, so per-job demand rises. The request is what the CFS share is
|
||||||
|
# weighted by under contention (100m was a 1/20 share); NO cpu limit, deliberately,
|
||||||
|
# so an idle node's spare cores are still usable. k3s-prod-2 (8 CPU) had 2945m
|
||||||
|
# requested; +1.9 core = ~4.85 (61%), and prod-1/prod-3 fit too if it reschedules.
|
||||||
|
# memory request 1Gi / limit 4Gi: two jobs at today's peak is ~0.9Gi. An OOM kill here
|
||||||
|
# takes down dockerd and with it BOTH running jobs, so the limit carries ~4x headroom
|
||||||
|
# over that for the parallelised gate. Nodes have ~31Gi allocatable.
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 100m
|
cpu: "2"
|
||||||
memory: 256Mi
|
memory: 1Gi
|
||||||
limits:
|
limits:
|
||||||
memory: 2Gi
|
memory: 4Gi
|
||||||
startupProbe:
|
startupProbe:
|
||||||
exec:
|
exec:
|
||||||
command:
|
command:
|
||||||
@@ -195,23 +246,36 @@ spec:
|
|||||||
value: /certs/client
|
value: /certs/client
|
||||||
- name: DOCKER_TLS_VERIFY
|
- name: DOCKER_TLS_VERIFY
|
||||||
value: "1"
|
value: "1"
|
||||||
|
# The image's run.sh passes `--config <this>` to BOTH `act_runner register` and
|
||||||
|
# `act_runner daemon` when CONFIG_FILE is set (act_runner v0.6.1 scripts/run.sh).
|
||||||
|
- name: CONFIG_FILE
|
||||||
|
value: /etc/act_runner/config.yaml
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: data
|
- name: data
|
||||||
mountPath: /data
|
mountPath: /data
|
||||||
- name: certs
|
- name: certs
|
||||||
mountPath: /certs
|
mountPath: /certs
|
||||||
readOnly: true
|
readOnly: true
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/act_runner
|
||||||
|
readOnly: true
|
||||||
|
# Measured 2026-11-08 at capacity 1: peak CPU 0.24 core, peak working set 314Mi (30d)
|
||||||
|
# against the old 512Mi limit. It holds per-task state, log buffers and the embedded
|
||||||
|
# actions/cache server for every concurrent task, so the limit is doubled for capacity 2.
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 100m
|
cpu: 100m
|
||||||
memory: 128Mi
|
memory: 256Mi
|
||||||
limits:
|
limits:
|
||||||
memory: 512Mi
|
memory: 1Gi
|
||||||
volumes:
|
volumes:
|
||||||
- name: data
|
- name: data
|
||||||
persistentVolumeClaim:
|
persistentVolumeClaim:
|
||||||
claimName: act-runner-data
|
claimName: act-runner-data
|
||||||
- name: certs
|
- name: certs
|
||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: act-runner-config
|
||||||
- name: dind-storage
|
- name: dind-storage
|
||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
@@ -1,74 +0,0 @@
|
|||||||
---
|
|
||||||
date: 2026-11-08
|
|
||||||
branch: fix/keycloak-operator-crds
|
|
||||||
title: "Keycloak 26.8.2 operator CrashLoop: install its two new CRDs, raise its memory"
|
|
||||||
---
|
|
||||||
|
|
||||||
## What changed
|
|
||||||
|
|
||||||
**Incident.** PR #121 (Renovate) bumped `platform/keycloak/kustomization.yaml`'s three
|
|
||||||
`keycloak-k8s-resources` URLs 26.0.6 -> 26.8.2. Upstream `kubernetes/` at 26.8.2 holds **four**
|
|
||||||
CRDs, not two (GitHub contents API, `?ref=26.8.2`):
|
|
||||||
|
|
||||||
| file | at 26.0.6 | at 26.8.2 |
|
|
||||||
|---|---|---|
|
|
||||||
| `keycloaks.k8s.keycloak.org-v1.yml` | yes | yes |
|
|
||||||
| `keycloakrealmimports.k8s.keycloak.org-v1.yml` | yes | yes |
|
|
||||||
| `keycloakoidcclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
|
|
||||||
| `keycloaksamlclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
|
|
||||||
| `kubernetes.yml` | yes | yes (now carries RBAC for both new kinds) |
|
|
||||||
|
|
||||||
(`cluster-wide/` is a separate variant; upstream's own `kubernetes/kustomization.yml` lists exactly
|
|
||||||
the five files above.) The 26.8.2 operator starts an informer for every controller it has and dies
|
|
||||||
on the first missing kind: `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1
|
|
||||||
... GET .../namespaces/keycloak/keycloaksamlclients ... Not Found`, exit 1, CrashLoopBackOff. Flux
|
|
||||||
`platform` (`wait: true`, 5m) sat in health checks and every dependent Kustomization reported
|
|
||||||
`dependency ... is not ready`. The old 26.0.6 ReplicaSet kept serving (RollingUpdate 25%/25%), so SSO
|
|
||||||
stayed up.
|
|
||||||
|
|
||||||
**Fix.**
|
|
||||||
- `platform/keycloak/kustomization.yaml`: added the two CRD URLs, each with the same
|
|
||||||
`# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources` marker (the repo's
|
|
||||||
custom.regex manager matches all five — checked by running its regex over the file). Header
|
|
||||||
comment updated to v26.8.2, plus a warning that Renovate cannot add a CRD a release introduces.
|
|
||||||
- Same file: a `patches:` JSON6902 patch on Deployment `keycloak-operator`: memory limit
|
|
||||||
450Mi -> **768Mi**, request 450Mi -> **512Mi**. 450Mi is upstream's default (unchanged at 26.8.2,
|
|
||||||
so not something the bump changed). Evidence it is too small here: the 26.0.6 pod had **154
|
|
||||||
restarts**, `lastState.terminated.reason: OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and
|
|
||||||
`kubectl top` showed **421Mi / 450Mi**. 26.8.2 runs four controllers instead of two on Quarkus
|
|
||||||
3.40.1.1 (was 3.15.1). No variable in the operator to disable watching unused kinds was used —
|
|
||||||
installing the CRDs is upstream's supported shape.
|
|
||||||
- `platform/keycloak/README.md` "Upgrading Keycloak": the diff-the-CRD-list step; "three URLs" ->
|
|
||||||
"the release URLs". `renovate.json5`: same wording fix in a comment only.
|
|
||||||
|
|
||||||
## What I verified
|
|
||||||
|
|
||||||
- Upstream file lists at 26.0.6 and 26.8.2 (GitHub API) and `diff` of `kubernetes.yml` between the
|
|
||||||
two: the new kinds' ClusterRoles + RoleBindings ship in it, and the Deployment's resources are
|
|
||||||
still 450Mi/450Mi.
|
|
||||||
- Live, read-only (`kubectl --context k3s-prod`, 2026-11-08 ~05:57Z): only `keycloaks` and
|
|
||||||
`keycloakrealmimports` CRDs installed; Deployment strategy RollingUpdate maxSurge/maxUnavailable
|
|
||||||
25%; RS `keycloak-operator-58dff6c4` (26.8.2) 0/1 CrashLoopBackOff, RS `78dbb75f64` (26.0.6) 1/1;
|
|
||||||
operator log is the informer error above; Flux `platform` `Healthy=Unknown` "Running health
|
|
||||||
checks for revision main@sha1:e8d2666"; no NetworkPolicy and no CiliumNetworkPolicy in ns
|
|
||||||
`keycloak`.
|
|
||||||
- No sensitive path touched (`infra/gitea/approval-policy.json`), so this should merge on green.
|
|
||||||
- Gate: see the PR body for the exact rc and stages of the run against the final commit.
|
|
||||||
|
|
||||||
## What I did NOT verify
|
|
||||||
|
|
||||||
- **That the 26.8.2 operator becomes Ready after this merges.** Only the two missing kinds were
|
|
||||||
observed failing; another start-up failure behind them is possible.
|
|
||||||
- **That 768Mi is enough** — no 26.8.2 memory measurement exists yet; re-check `kubectl top` after
|
|
||||||
a week.
|
|
||||||
- **The Keycloak server upgrade 26.0.6 -> 26.8.2 itself.** Once the operator runs it will roll
|
|
||||||
keycloak-0/1 and Keycloak migrates the DB schema. The CR's `spec.update.strategy` defaults to
|
|
||||||
`RecreateOnImageChange` — expect a **full SSO outage** while both pods recreate, not a rolling
|
|
||||||
one. Back up / confirm the latest CNPG backup first if that matters.
|
|
||||||
- **The 26.8.2 Keycloak CRD's `spec.networkPolicy.enabled` defaults to `true`.** The operator will
|
|
||||||
create a NetworkPolicy selecting the Keycloak pods (first policy in that namespace). With no
|
|
||||||
`from` lists set it allows all sources on the HTTP/management ports, so Gateway, ServiceMonitor
|
|
||||||
and the reconciler Jobs should still reach it — but this was read from the CRD text, not observed.
|
|
||||||
- **The passkey reconciler's 26.0.6 Admin API assumptions against 26.8.2** (README "Upgrading
|
|
||||||
Keycloak" list) — not re-checked here; PR #121 merged without it.
|
|
||||||
- `catalog/keycloak.yaml`'s documentation-only image tag was not updated.
|
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
---
|
||||||
|
date: 2026-11-08
|
||||||
|
branch: feat/runner-capacity-2
|
||||||
|
title: act_runner runs two jobs at once, and dind is sized for two
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
`apps/wave-1/gitea-actions.yaml` only. That file is a **hand-owned** non-catalog unit
|
||||||
|
(`EXTRA_UNITS["gitea-actions"]` in `catalog/render.py`): the renderer only decides which wave
|
||||||
|
it lands in, it does not generate its bytes, and `ci/render-golden.sh` compares only files the
|
||||||
|
catalog renders. So it is edited in place, and no re-render was needed or run.
|
||||||
|
|
||||||
|
1. **New ConfigMap `act-runner-config`** holding a config file with ONE key,
|
||||||
|
`runner.capacity: 2`, mounted read-only at `/etc/act_runner/` and selected with
|
||||||
|
`CONFIG_FILE=/etc/act_runner/config.yaml`. The image's `run.sh` (act_runner v0.6.1) turns
|
||||||
|
that into `--config` for both `register` and `daemon`.
|
||||||
|
- Deltas only, on evidence: v0.6.1 `LoadDefault()` unmarshals the file onto a zero Config and
|
||||||
|
then applies the same defaults as the no-file path, so the other settings are unchanged.
|
||||||
|
- The `generate-config` example was deliberately NOT used. Its `runner.labels` would override
|
||||||
|
the registered labels in `/data/.runner` and repoint `ubuntu-latest` at
|
||||||
|
`docker.gitea.com/runner-images`. Its `force_pull: true` is not the default. Its sample `envs`
|
||||||
|
would be injected into every job.
|
||||||
|
- With a file present, act_runner ignores the legacy `GITEA_RUNNER_CAPACITY` /
|
||||||
|
`_FILE` / `_ENVIRON` / `_ENV_FILE` / `GITEA_DEBUG` / `GITEA_TRACE` env vars. None of
|
||||||
|
them is set on this Deployment.
|
||||||
|
2. **Pod-template annotation `homelab.keithjackson.dev/act-runner-config-revision: "1"`.**
|
||||||
|
act_runner reads its config once, at start, so a change to the ConfigMap alone would never
|
||||||
|
take effect. Bump this annotation whenever you change only the ConfigMap.
|
||||||
|
3. **dind resources:** requests `cpu 100m -> 2` and `memory 256Mi -> 1Gi`; limit
|
||||||
|
`memory 2Gi -> 4Gi`. There is still no CPU limit, on purpose.
|
||||||
|
4. **runner resources:** request `memory 128Mi -> 256Mi`, limit `memory 512Mi -> 1Gi`. CPU stays
|
||||||
|
at `100m`.
|
||||||
|
|
||||||
|
Why these sizes. Measured 2026-11-08 through Prometheus, at capacity 1:
|
||||||
|
|
||||||
|
| container | peak CPU (1m rate, 30d) | peak working set (30d) | OOM events (7d) |
|
||||||
|
|---|---|---|---|
|
||||||
|
| dind | 0.69 core | 447 MiB | 0 |
|
||||||
|
| runner | 0.24 core | 314 MiB | 0 |
|
||||||
|
|
||||||
|
At the moment of measurement the pod was idle (`top`: dind 1m/138Mi, runner 1m/83Mi), on
|
||||||
|
k3s-prod-2.
|
||||||
|
|
||||||
|
- **CPU request 2.** Two jobs at today's peak is about 1.4 cores. The gate is being parallelised
|
||||||
|
in other work, so each job will use more than that. The request sets dind's CFS weight under
|
||||||
|
contention; at 100m it got a 1/20 share. With no limit, dind can still use spare cores on an
|
||||||
|
idle node.
|
||||||
|
- **Memory 1Gi / 4Gi.** Two jobs at today's peak is about 0.9 GiB. If dind is OOM-killed,
|
||||||
|
dockerd goes down and takes both running jobs with it, so the limit is about 4x that.
|
||||||
|
- **Runner limit 512Mi -> 1Gi.** Its 30-day peak was already 61% of the old limit, and with
|
||||||
|
capacity 2 it holds state for two tasks.
|
||||||
|
|
||||||
|
Scheduling headroom: CPU requests per node were measured with `describe node`. The pod's total
|
||||||
|
CPU request goes from 200m to 2100m (+1.9 cores).
|
||||||
|
|
||||||
|
| node | CPU requested now | after the change | allocatable |
|
||||||
|
|---|---|---|---|
|
||||||
|
| prod-2 (current) | 2945m | ~4845m (61%) | 8 |
|
||||||
|
| prod-1 | 3175m | ~5075m | 8 |
|
||||||
|
| prod-3 | 3840m | ~5740m (72%) | 8 |
|
||||||
|
|
||||||
|
The pod fits on any of the three if the RWO PVC moves it.
|
||||||
|
|
||||||
|
Concurrency semantics. Workflow `concurrency:` groups are enforced by Gitea on the server, not by
|
||||||
|
the runner, so capacity does not change them:
|
||||||
|
|
||||||
|
- `gate.yml` `gate-${{ github.ref }}` with cancel-in-progress still cancels a superseded run on
|
||||||
|
the same ref.
|
||||||
|
- `renovate` (group `renovate`, no cancel) and `images-${{ github.ref }}` still serialise
|
||||||
|
themselves.
|
||||||
|
|
||||||
|
What capacity 2 does change: runs on *different* refs (a PR gate next to a renovate or images
|
||||||
|
run, or two PRs) now run side by side instead of queueing. The required check
|
||||||
|
`gate / gate (pull_request)` is still one job per PR head, and gate.yml's other job
|
||||||
|
(`conformance`) is `if: false`, so it never runs.
|
||||||
|
|
||||||
|
Both jobs share one dockerd. act_runner gives each task its own container prefix
|
||||||
|
(`GITEA-ACTIONS-TASK-<task id>`) and its own network, and no workflow uses fixed container names
|
||||||
|
or host ports.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **Source of truth.** `catalog/render.py` `EXTRA_UNITS` places `gitea-actions.yaml` by
|
||||||
|
anchor/offset only. `ci/render-golden.sh` iterates over rendered files only, so this file is
|
||||||
|
outside render==golden. No `--emit-waves` was run.
|
||||||
|
- **act_runner v0.6.1 behaviour, read in its source** (tag v0.6.1):
|
||||||
|
- `config.LoadDefault()` treats a partial file as described above.
|
||||||
|
- `daemon.go`: config labels override `.runner` labels only when non-empty.
|
||||||
|
- `poller.go`: a semaphore of `cfg.Runner.Capacity` bounds concurrent tasks.
|
||||||
|
- `runner.go`: per-task container name prefix.
|
||||||
|
- The image's `run.sh`, read from `gitea/act_runner:0.6.1`, honours `CONFIG_FILE`.
|
||||||
|
- **The live pod has no config file today.** `/data` holds only `.runner` and `lost+found`.
|
||||||
|
`act_runner --version` reports v0.6.1. No LimitRange or ResourceQuota exists in
|
||||||
|
`gitea-actions`.
|
||||||
|
- **Resource numbers.** All figures above come from read-only `kubectl top`, `describe node` and
|
||||||
|
Prometheus queries against k3s-prod.
|
||||||
|
- **dind disk.** The emptyDir sits on the node's 11.8G ephemeral filesystem: 4.0G used, 7.2G
|
||||||
|
free. Image cache is 2.18GB, which is the one `catthehacker/ubuntu:act-22.04` image. Two
|
||||||
|
concurrent jobs share that image.
|
||||||
|
- **Embedded config.** It parses as YAML to `{'runner': {'capacity': 2}}`. The only `$` in the
|
||||||
|
manifest is the existing `${BASE_DOMAIN}`, so nothing new is exposed to Flux postBuild
|
||||||
|
envsubst.
|
||||||
|
- **Approval policy.** `apps/wave-1/` matches no `sensitive_paths` glob in
|
||||||
|
`infra/gitea/approval-policy.json`, so this PR merges on a green gate alone.
|
||||||
|
- **Local gate.** Pinned toolbox, run from the worktree on the committed tree. The rc and skipped
|
||||||
|
stages are given in the PR description.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **UNVERIFIED: that the runner actually starts with the config and runs two jobs at once.**
|
||||||
|
Nothing is applied until Flux reconciles `main`.
|
||||||
|
- **Capacity cannot be seen through Gitea's API.** act_runner v0.6.1 does not report capacity to
|
||||||
|
Gitea: `Declare` sends only version and labels. `GET /api/v1/admin/actions/runners` therefore
|
||||||
|
cannot show "capacity 2". Concurrency has to be observed instead.
|
||||||
|
- **UNVERIFIED: whether the parallelised gate fits in these sizes.** That gate has not landed.
|
||||||
|
Re-measure after it does, with
|
||||||
|
`max_over_time(sum by (container)(rate(container_cpu_usage_seconds_total{namespace="gitea-actions"}[1m]))[7d:30s])`.
|
||||||
|
- **Rollout cancels in-flight jobs.** The rollout restarts the runner pod (strategy Recreate,
|
||||||
|
`shutdown_timeout: 0s`), and any job running at that moment is cancelled, possibly including
|
||||||
|
the `push`-to-main gate run triggered by this very merge. Re-run it if that happens.
|
||||||
|
- **dind disk is not bounded by a limit.** With two jobs it holds two jobs' workspaces and layers
|
||||||
|
on the same 11.8G node filesystem. I did not add an `ephemeral-storage` limit, because an
|
||||||
|
eviction would kill both jobs. Watch kubelet DiskPressure on the runner's node.
|
||||||
|
- **Stale comments in sensitive paths.** `.gitea/workflows/gate.yml` (line ~52) and
|
||||||
|
`.gitea/workflows/README.md` (line ~86) still say "`act_runner` runs at capacity 1". I left
|
||||||
|
them alone because `.gitea/**` is a sensitive path and editing them would put this PR behind
|
||||||
|
human approval. Fix them in a separate PR.
|
||||||
|
|
||||||
|
### Operator post-merge checklist
|
||||||
|
|
||||||
|
1. Flux applied it:
|
||||||
|
`kubectl --context k3s-prod -n flux-system get kustomization apps-wave-1 -o jsonpath='{.status.lastAppliedRevision}'`
|
||||||
|
equals the merge commit.
|
||||||
|
2. The pod restarted with the config:
|
||||||
|
- `kubectl --context k3s-prod -n gitea-actions get pod -l app.kubernetes.io/name=act-runner`
|
||||||
|
shows a new pod, 2/2 Running.
|
||||||
|
- `kubectl ... exec deploy/act-runner -c runner -- cat /etc/act_runner/config.yaml` shows
|
||||||
|
`capacity: 2`.
|
||||||
|
- `kubectl ... exec deploy/act-runner -c runner -- ps` shows `act_runner daemon --config /etc/act_runner/config.yaml`.
|
||||||
|
- The logs show "declare successfully" with the two expected labels (`ubuntu-latest`,
|
||||||
|
`homelab` -> `catthehacker/ubuntu:act-22.04`). That confirms the labels did not change.
|
||||||
|
3. The runner is registered and online:
|
||||||
|
- `GET /api/v1/admin/actions/runners` lists `incluster-dind`, online, with the same ID as
|
||||||
|
before (no duplicate row).
|
||||||
|
- Capacity is NOT shown there; see above.
|
||||||
|
4. Concurrency is real: trigger two runs on different refs (e.g. two PR gates, or a PR gate plus a
|
||||||
|
`workflow_dispatch` of renovate). Then either:
|
||||||
|
- in `GET /api/v1/repos/Home-Network/home-network-k3s-impl/actions/tasks`, confirm their
|
||||||
|
`run_started_at` .. `updated_at` windows overlap; or
|
||||||
|
- check
|
||||||
|
`docker --host=tcp://127.0.0.1:2376 --tlsverify ... ps` in the dind container lists two
|
||||||
|
`GITEA-ACTIONS-TASK-*` container sets at once.
|
||||||
|
5. Resources: `kubectl --context k3s-prod -n gitea-actions describe pod` shows dind requests
|
||||||
|
cpu 2 / memory 1Gi, limit memory 4Gi, and runner limit memory 1Gi.
|
||||||
@@ -367,17 +367,9 @@ is involved — so step 2 failing with no log entry is the expected signature of
|
|||||||
The `Keycloak` CR has no `spec.image`, so the server image is whatever the operator's
|
The `Keycloak` CR has no `spec.image`, so the server image is whatever the operator's
|
||||||
`RELATED_IMAGE_KEYCLOAK` names — the same tag as the operator (prod, 2026-10-30: operator
|
`RELATED_IMAGE_KEYCLOAK` names — the same tag as the operator (prod, 2026-10-30: operator
|
||||||
`26.0.6`, `keycloak-0/1` on `quay.io/keycloak/keycloak:26.0.6`). Operator, CRDs and server are
|
`26.0.6`, `keycloak-0/1` on `quay.io/keycloak/keycloak:26.0.6`). Operator, CRDs and server are
|
||||||
released in lockstep from `keycloak/keycloak-k8s-resources`, and the release URLs in
|
released in lockstep from `keycloak/keycloak-k8s-resources`, and the three URLs in
|
||||||
`kustomization.yaml` are the one knob. Renovate raises them as ONE PR (`keycloak-operator` group,
|
`kustomization.yaml` are the one knob. Renovate raises them as ONE PR (`keycloak-operator` group,
|
||||||
labels `needs-review` + `blast-radius`), **never auto-merged**; majors wait for a dashboard tick.
|
labels `needs-review` + `blast-radius`), **never auto-merged**; majors wait for a dashboard tick.
|
||||||
|
|
||||||
**Renovate bumps the URLs that exist; it cannot add a CRD a release introduced.** Before merging a
|
|
||||||
bump, diff `kubernetes/` (and its `kustomization.yml`) at the new tag against the URL list. 26.0.6
|
|
||||||
shipped 2 CRDs, 26.8.2 ships 4 (`keycloakoidcclients`, `keycloaksamlclients` added); PR #121 bumped
|
|
||||||
only the existing URLs and the 26.8.2 operator CrashLooped on the missing `keycloaksamlclients`
|
|
||||||
informer, stalling Flux `platform` and every tier below it (2026-11-08,
|
|
||||||
`decisions/2026-11-08-keycloak-operator-crds.md`). The operator's memory is also patched in
|
|
||||||
`kustomization.yaml` (upstream's 450Mi OOMKilled 26.0.6 here); keep the patch across bumps.
|
|
||||||
`catalog/keycloak.yaml`'s image tag is documentation only (its render deploys nothing) and
|
`catalog/keycloak.yaml`'s image tag is documentation only (its render deploys nothing) and
|
||||||
Renovate is disabled for it — update it by hand to match after an operator bump merges.
|
Renovate is disabled for it — update it by hand to match after an operator bump merges.
|
||||||
|
|
||||||
|
|||||||
@@ -20,32 +20,18 @@ kind: Kustomization
|
|||||||
namespace: keycloak
|
namespace: keycloak
|
||||||
resources:
|
resources:
|
||||||
- namespace.yaml
|
- namespace.yaml
|
||||||
# --- Keycloak Operator v26.8.2 (pinned upstream release manifests) ---
|
# --- Keycloak Operator v26.0.6 (pinned upstream release manifests) ---
|
||||||
# The operator, its CRDs and the SERVER IMAGE move in lockstep: the Keycloak CR has no
|
# The operator, its CRDs and the SERVER IMAGE move in lockstep: the Keycloak CR has no
|
||||||
# spec.image, so the operator's RELATED_IMAGE_KEYCLOAK (quay.io/keycloak/keycloak:<same tag>)
|
# spec.image, so the operator's RELATED_IMAGE_KEYCLOAK (quay.io/keycloak/keycloak:<same tag>)
|
||||||
# is what runs. These URLs are therefore THE Keycloak version. Each carries a Renovate
|
# is what runs. These three URLs are therefore THE Keycloak version. Each carries a Renovate
|
||||||
# marker (custom.regex manager, renovate.json5) under ONE depName, so all of them bump in ONE
|
# marker (custom.regex manager, renovate.json5) under ONE depName, so all three bump in ONE
|
||||||
# branch, never auto-merged (packageRule "keycloak-operator"). Before merging a bump, re-check
|
# branch, never auto-merged (packageRule "keycloak-operator"). Before merging a bump, re-check
|
||||||
# passkey-reconciler.yaml's 26.0.6 Admin API assumptions — see README.md "Upgrading Keycloak".
|
# passkey-reconciler.yaml's 26.0.6 Admin API assumptions — see README.md "Upgrading Keycloak".
|
||||||
#
|
|
||||||
# THE URL LIST MUST MATCH UPSTREAM'S kubernetes/kustomization.yml AT THE NEW TAG — Renovate only
|
|
||||||
# rewrites the version in the URLs that are here; it cannot add a CRD the new release introduced.
|
|
||||||
# 26.0.6 shipped 2 CRDs; 26.8.2 ships 4 (+ keycloakoidcclients, keycloaksamlclients, both
|
|
||||||
# v2alpha1). PR #121 bumped only the existing three URLs, and the 26.8.2 operator — which starts an
|
|
||||||
# informer for EVERY kind it controls, used or not — CrashLooped on
|
|
||||||
# `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1 ... Not Found`,
|
|
||||||
# stalling Flux `platform` and everything downstream (2026-11-08). On every bump, diff
|
|
||||||
# https://github.com/keycloak/keycloak-k8s-resources/tree/<tag>/kubernetes
|
|
||||||
# against this list before merging.
|
|
||||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
||||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
||||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakoidcclients.k8s.keycloak.org-v1.yml
|
|
||||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaksamlclients.k8s.keycloak.org-v1.yml
|
|
||||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/kubernetes.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/kubernetes.yml
|
||||||
# --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
|
# --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
|
||||||
- keycloak-admin-credentials.sops.yaml
|
- keycloak-admin-credentials.sops.yaml
|
||||||
@@ -65,23 +51,3 @@ resources:
|
|||||||
- keycloak-db-backup-credentials-prod.sops.yaml
|
- keycloak-db-backup-credentials-prod.sops.yaml
|
||||||
# --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
|
# --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
|
||||||
- grafana-dashboard.yaml
|
- grafana-dashboard.yaml
|
||||||
# --- Operator memory: upstream ships requests=limits=450Mi, which this cluster outgrew ON 26.0.6 ---
|
|
||||||
# Measured on prod 2026-11-08: the 26.0.6 operator pod had 154 restarts in 64d, lastState
|
|
||||||
# `OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and sat at 421Mi of its 450Mi (94%) in `kubectl top`.
|
|
||||||
# 26.8.2 keeps the same 450Mi default but runs FOUR controllers/informers instead of two (adds
|
|
||||||
# keycloakoidcclient + keycloaksamlclient) on a newer Quarkus (3.15.1 -> 3.40.1.1), so it would
|
|
||||||
# plausibly start above a ceiling 26.0.6 already crossed. Limit 768Mi = ~1.8x the observed 421Mi
|
|
||||||
# working set; request 512Mi = the observed working set plus headroom, so the scheduler reserves what
|
|
||||||
# it actually uses. Revisit with `kubectl top` once 26.8.2 has run for a week. Patched here (not in
|
|
||||||
# the remote kubernetes.yml) so a Renovate bump keeps it.
|
|
||||||
patches:
|
|
||||||
- target:
|
|
||||||
kind: Deployment
|
|
||||||
name: keycloak-operator
|
|
||||||
patch: |-
|
|
||||||
- op: replace
|
|
||||||
path: /spec/template/spec/containers/0/resources/limits/memory
|
|
||||||
value: 768Mi
|
|
||||||
- op: replace
|
|
||||||
path: /spec/template/spec/containers/0/resources/requests/memory
|
|
||||||
value: 512Mi
|
|
||||||
+1
-1
@@ -264,7 +264,7 @@
|
|||||||
// and the server image are released in lockstep from keycloak/keycloak-k8s-resources, and the
|
// and the server image are released in lockstep from keycloak/keycloak-k8s-resources, and the
|
||||||
// Keycloak CR carries no spec.image, so the operator's RELATED_IMAGE_KEYCLOAK picks the server
|
// Keycloak CR carries no spec.image, so the operator's RELATED_IMAGE_KEYCLOAK picks the server
|
||||||
// (measured on prod 2026-10-30: operator 26.0.6 -> keycloak-0/1 run quay.io/keycloak/keycloak:
|
// (measured on prod 2026-10-30: operator 26.0.6 -> keycloak-0/1 run quay.io/keycloak/keycloak:
|
||||||
// 26.0.6). The release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
// 26.0.6). The three release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
||||||
// markers under this ONE depName, so they move together in one branch (groupName makes that
|
// markers under this ONE depName, so they move together in one branch (groupName makes that
|
||||||
// explicit). NEVER AUTO-MERGED, not even a patch: Keycloak is the IdP that gates SSO for every
|
// explicit). NEVER AUTO-MERGED, not even a patch: Keycloak is the IdP that gates SSO for every
|
||||||
// app AND kubectl OIDC, and platform/keycloak/passkey-reconciler.yaml encodes Keycloak 26.0.6
|
// app AND kubectl OIDC, and platform/keycloak/passkey-reconciler.yaml encodes Keycloak 26.0.6
|
||||||
|
|||||||
Reference in new issue
Block a user