Compare commits
54
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea30ba876c | ||
|
|
e8d26669a4 | ||
|
|
048254bfaf | ||
|
|
dd94101647 | ||
|
|
60e3a24652 | ||
|
|
8a9baa92bc | ||
|
|
2d32b9bd7d | ||
|
|
9f82b2accf | ||
|
|
990b309cbb | ||
|
|
cedcb01975 | ||
|
|
2b279d4f21 | ||
|
|
12c28adf4d | ||
|
|
b0230ba736 | ||
|
|
9c51f60a35 | ||
|
|
267446c68f | ||
|
|
d1994b16e8 | ||
|
|
d7317606e7 | ||
|
|
970d65d193 | ||
|
|
6719082e5c | ||
|
|
428b77f08c | ||
|
|
2ee7d5036a | ||
|
|
5736f42e22 | ||
|
|
773254e3a1 | ||
|
|
264cd7a46d | ||
|
|
979cc4b21d | ||
|
|
be6742c22b | ||
|
|
c8b915d8f2 | ||
|
|
6cd3bd29a9 | ||
|
|
2040d35222 | ||
|
|
47fc27ff08 | ||
|
|
5e8b080298 | ||
|
|
24a0e1e4c5 | ||
|
|
3aec6886d8 | ||
|
|
1b2ec82a9d | ||
|
|
3724d3068b | ||
|
|
c6ad6b559c | ||
|
|
6efc49d415 | ||
|
|
a6dc2d1adb | ||
|
|
75a472cf67 | ||
|
|
24b02f37df | ||
|
|
be3ce4645b | ||
|
|
b7a77a7d1e | ||
|
|
1a94b0535c | ||
|
|
0e7d0204b3 | ||
|
|
b77572f843 | ||
|
|
76b5d038fb | ||
|
|
306f6f4fb0 | ||
|
|
3ceaad15e0 | ||
|
|
ae772e050d | ||
|
|
517dfc9030 | ||
|
|
0605ecf593 | ||
|
|
16de787557 | ||
|
|
b7cb70955c | ||
|
|
9bcea04b1b |
No files matched your search
@@ -80,7 +80,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0 # full history so ci/touched-requirements.sh can diff vs main
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
timeout-minutes: 75
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0 # the per-image change detection diffs against the base commit
|
||||
|
||||
|
||||
@@ -60,9 +60,9 @@ jobs:
|
||||
spec-immutability:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 # the implementation repo
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 # the implementation repo
|
||||
- name: Check out the read-only spec (sibling path)
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
repository: ${{ vars.SPEC_REPO_SLUG }} # e.g. Home-Network/home-network-k3s (set in Gitea)
|
||||
path: ../home-network-k3s
|
||||
@@ -74,7 +74,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [spec-immutability]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 } # full history so touched-requirement diff works
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh lint
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l0
|
||||
@@ -96,7 +96,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [l0]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [render-golden]
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh l1
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
run: make -C ci l1
|
||||
- name: Upload L1 artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: l1-junit-coverage
|
||||
path: |
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
needs: [l1]
|
||||
if: ${{ vars.CLUSTER_AVAILABLE == 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Install pinned tools
|
||||
run: ./.gitea/workflows/scripts/install-tools.sh conformance
|
||||
@@ -142,7 +142,7 @@ jobs:
|
||||
run: make -C ci conformance
|
||||
- name: Upload L2/L3 artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: conformance-junit-coverage
|
||||
path: |
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
needs: [lint, l0, render-golden, l1, conformance]
|
||||
if: always()
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- name: Decide
|
||||
env:
|
||||
R_LINT: ${{ needs.lint.result }}
|
||||
|
||||
@@ -78,7 +78,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: gitea
|
||||
image: gitea/gitea:1.24.3
|
||||
image: gitea/gitea:28.3.0
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
env:
|
||||
@@ -89,14 +89,18 @@ spec:
|
||||
secretKeyRef:
|
||||
name: gitea-bootstrap
|
||||
key: secret-key
|
||||
- name: GITEA__server__DOMAIN
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
- name: GITEA__server__ROOT_URL
|
||||
value: https://gitea.${BASE_DOMAIN}/
|
||||
- name: GITEA__actions__ENABLED
|
||||
value: 'true'
|
||||
- name: GITEA__actions__DEFAULT_ACTIONS_URL
|
||||
value: github
|
||||
- name: GITEA__actions__RUN_RETENTION_DAYS
|
||||
value: '400'
|
||||
- name: GITEA__security__EGRESS_MODE
|
||||
value: lax
|
||||
- name: GITEA__service__DISABLE_REGISTRATION
|
||||
value: 'true'
|
||||
- name: GITEA_ADMIN_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -78,7 +78,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: jackett
|
||||
image: lscr.io/linuxserver/jackett:v0.24.3009-ls26
|
||||
image: lscr.io/linuxserver/jackett:v0.24.3025-ls29
|
||||
ports:
|
||||
- containerPort: 9117
|
||||
volumeMounts:
|
||||
|
||||
@@ -48,7 +48,7 @@ spec:
|
||||
# Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
|
||||
# pg_basebackup from the running primary, which is neither restarted nor disconnected.
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.15
|
||||
# Stamp app.kubernetes.io/name=nextcloud-db onto every generated pod/service so the rendered
|
||||
# nextcloud egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
|
||||
inheritedMetadata:
|
||||
@@ -300,7 +300,7 @@ spec:
|
||||
# is already in it and it is already present on every node, so this
|
||||
# adds no new image and no new supply-chain surface. Keep the tag in
|
||||
# step with spec.imageName above when Renovate bumps it.
|
||||
image: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
image: ghcr.io/cloudnative-pg/postgresql:16.15
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- psql
|
||||
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: qbittorrent
|
||||
image: lscr.io/linuxserver/qbittorrent:5.1.2
|
||||
image: lscr.io/linuxserver/qbittorrent:5.3.0
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
env:
|
||||
|
||||
@@ -49,7 +49,7 @@ spec:
|
||||
# Scaling out is non-disruptive: CNPG creates -2 then -3 sequentially, each seeded by
|
||||
# pg_basebackup from the running primary, which is neither restarted nor disconnected.
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.15
|
||||
# Stamp app.kubernetes.io/name=semaphore-db onto every generated pod/service so the rendered
|
||||
# semaphore egress CiliumNetworkPolicy's toEndpoints selector matches the primary pod (R-NET-02).
|
||||
inheritedMetadata:
|
||||
|
||||
@@ -133,7 +133,7 @@ spec:
|
||||
initContainers:
|
||||
- name: dind
|
||||
restartPolicy: Always # <- this field is what makes it a SIDECAR
|
||||
image: docker:27-dind
|
||||
image: docker:29-dind
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
@@ -167,7 +167,7 @@ spec:
|
||||
containers:
|
||||
# ---- act_runner: registers with Gitea, dispatches jobs to the DinD daemon ----
|
||||
- name: runner
|
||||
image: gitea/act_runner:0.2.11
|
||||
image: gitea/act_runner:0.6.1
|
||||
# No `command:` override. The image's own entrypoint (/sbin/tini -- /opt/act/run.sh)
|
||||
# registers from env on first boot and runs `act_runner daemon`. The two wait-loops
|
||||
# that used to live here are gone: kubelet now holds this container until the dind
|
||||
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: radarr
|
||||
image: lscr.io/linuxserver/radarr:5.21.1
|
||||
image: lscr.io/linuxserver/radarr:5.28.0
|
||||
ports:
|
||||
- containerPort: 7878
|
||||
env:
|
||||
|
||||
@@ -78,7 +78,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: semaphore
|
||||
image: semaphoreui/semaphore:v2.19.10
|
||||
image: semaphoreui/semaphore:v2.19.17
|
||||
ports:
|
||||
- containerPort: 3000
|
||||
env:
|
||||
|
||||
@@ -75,7 +75,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: seerr
|
||||
image: ghcr.io/seerr-team/seerr:v3.4.1
|
||||
image: ghcr.io/seerr-team/seerr:v3.5.1
|
||||
ports:
|
||||
- containerPort: 5055
|
||||
volumeMounts:
|
||||
|
||||
+39
-5
@@ -10,7 +10,7 @@
|
||||
name: gitea
|
||||
kind: container
|
||||
source:
|
||||
image: gitea/gitea:1.24.3 # pinned recent stable tag (R-UPD-01); update.range governs advancement
|
||||
image: gitea/gitea:28.3.0 # pinned recent stable tag (R-UPD-01); update.range governs advancement
|
||||
# Headless-install bootstrap (formerly apps/gitea-bootstrap-patch.yaml; now first-class via the
|
||||
# ServiceDefinition env/envFrom/lifecycle fields, schema >=1.2.0). Gitea must come up INSTALLED
|
||||
# without the interactive web wizard and with a FIXED, persistent SECRET_KEY, so all of this is
|
||||
@@ -23,14 +23,43 @@ env:
|
||||
secretKeyRef:
|
||||
name: gitea-bootstrap
|
||||
key: secret-key
|
||||
- name: GITEA__server__DOMAIN # correct public clone/redirect URLs for the TLS-terminated host
|
||||
value: gitea.${BASE_DOMAIN}
|
||||
# ROOT_URL is the ONLY source of the instance domain since Gitea 28 (go-gitea/gitea#39400): [server]
|
||||
# DOMAIN is no longer read, and the SSH domain defaults from ROOT_URL too. GITEA__server__DOMAIN was
|
||||
# removed in the 28 upgrade; the stale DOMAIN line environment-to-ini already wrote into app.ini is
|
||||
# inert. Correct public clone/redirect URLs for the TLS-terminated host come from this one value.
|
||||
- name: GITEA__server__ROOT_URL
|
||||
value: https://gitea.${BASE_DOMAIN}/
|
||||
- name: GITEA__actions__ENABLED # Phase 6 CI/CD: declared (not implicit upstream default) so the
|
||||
value: "true" # in-cluster act_runner (ns gitea-actions) registers against it
|
||||
- name: GITEA__actions__DEFAULT_ACTIONS_URL # resolve `uses: <org>/<action>@ref` from github.com (cluster
|
||||
value: github # has internet egress; not air-gapped)
|
||||
# Gitea 28 added run retention (go-gitea/gitea#38855): completed runs, with their jobs, logs and
|
||||
# artifacts, are deleted by the cleanup_action_runs cron after RUN_RETENTION_DAYS. Upstream's default
|
||||
# (400) is kept but DECLARED, so the deletion horizon is visible here rather than implicit. The spec
|
||||
# repo states no CI-history retention requirement; 0 would mean keep forever.
|
||||
- name: GITEA__actions__RUN_RETENTION_DAYS
|
||||
value: "400"
|
||||
# Gitea 28 (go-gitea/gitea#39426) puts webhooks, the OAuth2 avatar fetch and OpenID 2.0 behind an
|
||||
# egress policy: in the default lax mode, PUBLIC targets are allowed and private/loopback/CGNAT
|
||||
# targets need an IP or CIDR entry in [security] ALLOWED_HOST_LIST (a hostname entry is NOT enough
|
||||
# for a private address). DECLARED lax with an EMPTY allow list, deliberately:
|
||||
# * the Keycloak OIDC login source is NOT gated by it. In 28.3.0 the openidConnect provider is
|
||||
# goth's, built with its default HTTP client (services/auth/source/oauth2/providers_openid.go);
|
||||
# egress.NewSecurityPolicy is used only by routers/web/auth/oauth.go (avatar fetch, and
|
||||
# [oauth2_client] UPDATE_AVATAR is off here) and modules/auth/openid (OpenID 2.0, unused).
|
||||
# So keycloak.<domain> resolving to the private Gateway address in-cluster does not matter.
|
||||
# * this instance has no webhooks and no mirrors (measured 2026-11-07), and 1.27 already refused
|
||||
# private webhook targets by default, so nothing that worked before is newly denied.
|
||||
# Adding an entry here would only WIDEN the webhook SSRF surface. Add the narrowest CIDR (e.g. the
|
||||
# Gateway address /32) only if a real private webhook target appears.
|
||||
- name: GITEA__security__EGRESS_MODE
|
||||
value: lax
|
||||
# Gitea 28 (go-gitea/gitea#39400) disables LOCAL self-registration unless DISABLE_REGISTRATION is
|
||||
# explicitly false; 1.27 defaulted it open. Declared true: accounts arrive via Keycloak only.
|
||||
# OAuth2 auto-registration below is unaffected — routers/web/auth/oauth.go gates it on
|
||||
# ALLOW_ONLY_INTERNAL_REGISTRATION and ENABLE_AUTO_REGISTRATION, not on DISABLE_REGISTRATION.
|
||||
- name: GITEA__service__DISABLE_REGISTRATION
|
||||
value: "true"
|
||||
- name: GITEA_ADMIN_USERNAME # break-glass local admin creds (consumed by the postStart hook)
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -233,9 +262,14 @@ auth:
|
||||
# recovery tools (docs/15 "Availability caveat"). No secret is embedded here.
|
||||
update:
|
||||
channel: stable # R-UPD-01
|
||||
range: ">=1.22.0 <2.0.0" # stay on the 1.x stable line
|
||||
range: ">=28.0.0 <29.0.0" # Gitea dropped the "1." prefix at 28.0.0 (successor of 1.27); stay on 28.x
|
||||
cadence: "0 4 * * *"
|
||||
autoRollbackOnFailedHealth: true # R-UPD-03
|
||||
# R-UPD-03. INERT FOR THIS SERVICE, and it must stay that way: this flag renders only an annotation
|
||||
# on a Flux ImagePolicy, and apps/ is rendered --no-image-automation (ci/render-golden.sh), so no
|
||||
# ImagePolicy exists and the Deployment (strategy Recreate) is never auto-reverted. That matters
|
||||
# because a Gitea upgrade (1.27 -> 28 included) MIGRATES THE SQLITE DB ONE-WAY: an image revert onto
|
||||
# a migrated DB is not a rollback. Roll Gitea back by RESTORING THE DATA PVC from its pre-upgrade backup.
|
||||
autoRollbackOnFailedHealth: true
|
||||
observability:
|
||||
metrics: true
|
||||
slis:
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
name: jackett
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/jackett:v0.24.3009-ls26
|
||||
image: lscr.io/linuxserver/jackett:v0.24.3025-ls29
|
||||
exposure: internal # LAN-only web UI at jackett.${BASE_DOMAIN} (docs/13 row 4; R-DNS/R-NET)
|
||||
egress: vpn # indexer scraping via node PIA tunnel + kill-switch (docs/03; ADR-0015; R-NET-05/06)
|
||||
storage:
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
name: qbittorrent
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/qbittorrent:5.1.2 # pinned stable tag (R-UPD-01); update.range governs advancement
|
||||
image: lscr.io/linuxserver/qbittorrent:5.3.0 # pinned stable tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING FOR THE BULK MOUNT, not cosmetic — exactly as on sonarr/radarr. Measured on prod
|
||||
# 2026-09-12 BEFORE this change: `id` inside the container reported the qbittorrent-nox process as
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
name: radarr
|
||||
kind: container
|
||||
source:
|
||||
image: lscr.io/linuxserver/radarr:5.21.1 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
|
||||
image: lscr.io/linuxserver/radarr:5.28.0 # pinned stable v5 tag (R-UPD-01); update.range governs advancement
|
||||
# LinuxServer images run their service as the user `abc`, whose uid/gid these two set. They are
|
||||
# LOAD-BEARING for the bulk mount, not cosmetic: see the storage note below. The default is 911,
|
||||
# which cannot write a 1000:3000-owned 0755 tree.
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@
|
||||
name: seerr
|
||||
kind: container
|
||||
source:
|
||||
image: ghcr.io/seerr-team/seerr:v3.4.1 # Seerr: maintained Overseerr/Jellyseerr successor, Plex-native request UI (docs/13 row 5); served on 5055
|
||||
image: ghcr.io/seerr-team/seerr:v3.5.1 # Seerr: maintained Overseerr/Jellyseerr successor, Plex-native request UI (docs/13 row 5); served on 5055
|
||||
# The ghcr.io/seerr-team/seerr image runs as the non-root user `node:node` (UID 1000 / GID 1000 —
|
||||
# the standard node base-image user; confirmed from the image config's User field). On boot the app
|
||||
# creates /app/config/logs and writes its settings/DB under the mounted /app/config PVC. A fresh
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
name: semaphore
|
||||
kind: container
|
||||
source:
|
||||
image: semaphoreui/semaphore:v2.19.10 # pinned recent stable tag (R-UPD-01); update.range governs advancement
|
||||
image: semaphoreui/semaphore:v2.19.17 # pinned recent stable tag (R-UPD-01); update.range governs advancement
|
||||
env:
|
||||
# --- PostgreSQL connection (the CNPG-DB pattern). DIALECT/HOST/PORT/DB/USER are literals; PASS is
|
||||
# sourced from the in-namespace SOPS secret (the CNPG -app secret lives in ns semaphore-db,
|
||||
|
||||
@@ -26,7 +26,10 @@
|
||||
# KNOWN_NON_CATALOG_PUBLISHERS — shrink-only, stale entries FAIL;
|
||||
# (7) the check is LOAD-BEARING: whole-repo scope declared in chainsaw-test.yaml, honoured by
|
||||
# ci/l0.sh's --include-scope union, and ci/gate.sh blocks on an l0 failure. NO hand-written
|
||||
# ci/l0.sh stage (whole-repo-invariants-wired fails the gate if one appears).
|
||||
# ci/l0.sh stage (whole-repo-invariants-wired fails the gate if one appears);
|
||||
# (8) the external-dns HelmRelease declares `annotationPrefix` explicitly and it matches the
|
||||
# prefix of EVERY external-dns annotation in the tree and in catalog/render.py — added after
|
||||
# the 2026-11-07 v0.23.0 default-prefix change silently voided (1)-(3) at runtime.
|
||||
#
|
||||
# WHAT THIS DOES *NOT* PROVE (honest gaps — it is STATIC, offline, cluster-free):
|
||||
# - That the record is gone from Cloudflare. Live read; belongs beside
|
||||
@@ -90,9 +93,18 @@ CHECKS = 0
|
||||
# ControllerValue = "dns-controller" -> the value that CLAIMS an object
|
||||
# source/annotations/processors.go IsControllerMismatch skips a source object when the key is
|
||||
# PRESENT and the value is NOT ControllerValue. So "disowned" == key present, value != claimed.
|
||||
CONTROLLER_KEY = "external-dns.alpha.kubernetes.io/controller"
|
||||
#
|
||||
# THE PREFIX IS NOT A CONSTANT OF NATURE. Every key external-dns reads is `<annotationPrefix><name>`,
|
||||
# and the prefix is a runtime flag whose DEFAULT moved in v0.22.0 (#6424: alpha -> GA, "with no
|
||||
# fallback"). On 2026-11-07 the chart bump to v0.23.0 made every key below invisible to the running
|
||||
# controller while this check — which only asserted the keys were PRESENT — stayed green. (8) now
|
||||
# ties ANNOTATION_PREFIX to the HelmRelease's explicit `annotationPrefix`.
|
||||
ANNOTATION_PREFIX = "external-dns.alpha.kubernetes.io/"
|
||||
CONTROLLER_KEY = ANNOTATION_PREFIX + "controller"
|
||||
CONTROLLER_CLAIMED = "dns-controller"
|
||||
HOSTNAME_KEY = "external-dns.alpha.kubernetes.io/hostname"
|
||||
HOSTNAME_KEY = ANNOTATION_PREFIX + "hostname"
|
||||
# Any key that looks like an external-dns annotation, under ANY prefix variant (alpha, GA, typos).
|
||||
EXTERNAL_DNS_KEY_RE = re.compile(r"^external-dns(\.[a-z0-9-]+)*\.kubernetes\.io/")
|
||||
|
||||
|
||||
def check(cond, msg):
|
||||
@@ -231,9 +243,7 @@ for sub in SCAN_DIRS:
|
||||
ann = md.get("annotations") or {}
|
||||
if kind == "HTTPRoute":
|
||||
routes.append((rel, d))
|
||||
elif kind == "Service" and any(
|
||||
a.startswith("external-dns.alpha.kubernetes.io/") for a in ann
|
||||
):
|
||||
elif kind == "Service" and any(EXTERNAL_DNS_KEY_RE.match(a) for a in ann):
|
||||
dns_services.append((rel, d))
|
||||
|
||||
check(len(routes) > 0, "inventory: no HTTPRoute found under apps/ platform/ runbooks/ — parse "
|
||||
@@ -444,6 +454,67 @@ check(
|
||||
"and the whole-repo-invariants-wired meta-test fails the gate on a hand-written stage.",
|
||||
)
|
||||
|
||||
# =============================================================================================
|
||||
# (8) THE RUNNING CONTROLLER READS THE PREFIX THIS TREE WRITES.
|
||||
# (a) the external-dns HelmRelease sets `values.annotationPrefix` EXPLICITLY (an upstream
|
||||
# default is exactly what moved under us) and it equals ANNOTATION_PREFIX;
|
||||
# (b) it does not ALSO enable `--enable-legacy-annotation-prefix` (dual-read hides a
|
||||
# half-migration; upstream says the flag will be removed);
|
||||
# (c) every external-dns-looking annotation key on any object under apps/ platform/ runbooks/
|
||||
# uses ANNOTATION_PREFIX — a key under another prefix is silently ignored at runtime;
|
||||
# (d) catalog/render.py emits no external-dns key under another prefix.
|
||||
# Migrating to the GA prefix = change ANNOTATION_PREFIX, the HelmRelease value, the renderer
|
||||
# and every manifest in ONE commit; this assertion fails on any partial state.
|
||||
# =============================================================================================
|
||||
HR_PATH = ROOT / "platform/externaldns/helmrelease.yaml"
|
||||
hr = next((d for d in docs_of(HR_PATH) if d.get("kind") == "HelmRelease"
|
||||
and (d.get("metadata") or {}).get("name") == "external-dns"), None)
|
||||
check(hr is not None, f"(8) {HR_PATH.relative_to(ROOT)} holds no HelmRelease `external-dns` — "
|
||||
f"cannot prove which annotation prefix the controller reads")
|
||||
values = ((hr or {}).get("spec") or {}).get("values") or {}
|
||||
hr_prefix = values.get("annotationPrefix")
|
||||
check(
|
||||
hr_prefix == ANNOTATION_PREFIX,
|
||||
f"(8a) external-dns HelmRelease `values.annotationPrefix` is {hr_prefix!r}, but every "
|
||||
f"external-dns annotation in this tree uses {ANNOTATION_PREFIX!r}. A mismatch (or relying on "
|
||||
f"the upstream default, which changed in v0.22.0) makes the controller IGNORE `controller: "
|
||||
f"none`, the Gateway `target` and Service `hostname` — internal names go public, public names "
|
||||
f"answer RFC1918 (incident 2026-11-07).",
|
||||
)
|
||||
extra = values.get("extraArgs") or []
|
||||
extra_s = " ".join(f"{k}={v}" for k, v in extra.items()) if isinstance(extra, dict) \
|
||||
else " ".join(map(str, extra))
|
||||
check("enable-legacy-annotation-prefix" not in extra_s and "annotation-prefix" not in extra_s,
|
||||
"(8b) external-dns extraArgs must not set --annotation-prefix or "
|
||||
"--enable-legacy-annotation-prefix; the prefix is declared ONLY via values.annotationPrefix "
|
||||
"so this check can see it")
|
||||
|
||||
foreign: list[str] = []
|
||||
seen_keys = 0
|
||||
for sub in SCAN_DIRS:
|
||||
base = ROOT / sub
|
||||
if not base.is_dir():
|
||||
continue
|
||||
for p in sorted(base.rglob("*.yaml")):
|
||||
rel = p.relative_to(ROOT).as_posix()
|
||||
for d in docs_of(p):
|
||||
for k in annotations_of(d):
|
||||
if isinstance(k, str) and EXTERNAL_DNS_KEY_RE.match(k):
|
||||
seen_keys += 1
|
||||
if not k.startswith(ANNOTATION_PREFIX):
|
||||
foreign.append(f"{rel}:{'/'.join(ident(d))} {k}")
|
||||
check(seen_keys > 0, "(8c) no external-dns annotation found anywhere — parse contract broken "
|
||||
"(the Gateway target alone guarantees at least one)")
|
||||
check(not foreign,
|
||||
f"(8c) external-dns annotations under a prefix the controller does NOT read "
|
||||
f"({ANNOTATION_PREFIX!r} is configured): {foreign}")
|
||||
render_src = read(CATALOG / "render.py")
|
||||
bad_render = sorted({m for m in re.findall(r"external-dns(?:\.[a-z0-9-]+)*\.kubernetes\.io/",
|
||||
render_src) if m != ANNOTATION_PREFIX})
|
||||
check(not bad_render,
|
||||
f"(8d) catalog/render.py emits external-dns keys under {bad_render}, but the controller "
|
||||
f"reads {ANNOTATION_PREFIX!r}")
|
||||
|
||||
# =============================================================================================
|
||||
if FAILURES:
|
||||
for f in FAILURES:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
date: 2026-11-07
|
||||
branch: feat/devcontainer-velero-cli
|
||||
title: Bake the velero CLI into the dev toolbox, pinned to the prod server version
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
- `infra/devcontainer/versions.env`: new `VELERO_VERSION=1.18.2`, with a comment explaining that
|
||||
the CLI and server are version-coupled, so this pin follows the server and is not bumped on
|
||||
its own. It has **no** `# renovate:` marker on purpose. When a `velero` chart bump in
|
||||
`platform/velero/helmrelease.yaml` changes the appVersion, bump this pin in the same PR.
|
||||
- `infra/devcontainer/Containerfile`: `ARG VELERO_VERSION`; the binary comes from the
|
||||
vmware-tanzu GitHub release tarball and is installed in the same RUN as flux/helm. There is
|
||||
no checksum check, which matches the other tools in that file (none of them verify one).
|
||||
`velero=` is recorded in `/etc/dev-toolchain-versions`. The verify stage now checks that
|
||||
`velero` is on PATH, and `check velero v${VELERO_VERSION}` asserts the output of
|
||||
`velero version --client-only`.
|
||||
- `infra/devcontainer/build-image.sh`: `VELERO_VERSION` added to `BUILD_ARGS`. CI reads that
|
||||
list generically through `--print-build-args` (`.gitea/workflows/scripts/images.sh`), so
|
||||
**no sensitive path (`.gitea/**`, `ci/**`) was touched**.
|
||||
- `infra/devcontainer/README.md`: added a Backups row to the tool table.
|
||||
- `runbooks/backup-failure.md`: under "First commands", a read-only
|
||||
`velero backup describe --details` / `backup logs` example run through dev.sh.
|
||||
|
||||
## What I verified
|
||||
|
||||
- Live server version, read-only: `kubectl --context k3s-prod -n velero get deploy velero`
|
||||
shows image `docker.io/velero/velero:v1.18.2`, and HelmRelease `velero` has
|
||||
lastAttemptedRevision `12.2.3`.
|
||||
- Local build: `IMAGE=localhost/home-k3s-dev:velero-wt ./build-image.sh` returned rc 0. The
|
||||
verify stage printed `ok velero` (on PATH), `ok velero v1.18.2` (pin check) and
|
||||
`TOOLBOX VERIFIED`. The image was built under a separate local tag and **not pushed**.
|
||||
- Smoke test with that image against prod (read-only): `velero version` reports Client v1.18.2
|
||||
and Server v1.18.2. `velero backup get` lists the daily backups, all `Completed` with 0 errors.
|
||||
- Recurring warning (reported only, not fixed): every daily backup since
|
||||
`velero-daily-20261031030050` has 1 warning. Backups up to 20261030 have 0. The warning text is:
|
||||
`gitea-actions: resource: /pods name: /act-runner-5944dfdfb8-69bvq message: /Skip pod volume
|
||||
data error: /pod is not in the expected status, ... phase=Failed: pod is not running`.
|
||||
It is the same pod in 20261031, 20261104 and 20261108. `gitea-actions` holds 11 `Failed`
|
||||
pods with reason `Evicted` from ReplicaSet `act-runner-5944dfdfb8`, ten of them started
|
||||
2026-10-30T08:26Z. The running runner is `act-runner-fc597f6b5-vsvkl`. So the warning comes from a
|
||||
leftover evicted (`Failed`) pod that Velero still tries to back up volumes for. Cleaning up the evicted pods is an
|
||||
operator action.
|
||||
- Gate: see the PR description for the toolbox `ci/gate.sh` rc and the skipped stages.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- The CI image build (`.gitea/workflows/images.yaml`) on this branch. It runs on push/PR and I
|
||||
have not seen its result while writing this.
|
||||
- An arm64 build. Only amd64 was built (`TARGETARCH` defaults to amd64).
|
||||
- Any mutating velero command (`backup create`, `restore`, `delete`). None were run.
|
||||
- Why only `-69bvq` of the 11 evicted act-runner pods shows up in the warning. A likely guess is
|
||||
that only that pod's volumes match the backup's pod-volume selection, but I did not check.
|
||||
@@ -0,0 +1,62 @@
|
||||
---
|
||||
date: 2026-11-07
|
||||
branch: feat/gitea-28
|
||||
title: "Gitea 1.27.3 -> 28.3.0 (upstream dropped the 1. prefix)"
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
- `catalog/gitea.yaml`: `source.image` `gitea/gitea:1.27.3` -> `gitea/gitea:28.3.0` (still the
|
||||
newest stable tag on Docker Hub and GitHub on 2026-11-07: 28.0.0, 28.1.0, 28.2.0, 28.3.0).
|
||||
`update.range` `">=1.22.0 <2.0.0"` -> `">=28.0.0 <29.0.0"`.
|
||||
- 28.x breaking changes handled in the env block:
|
||||
- `GITEA__server__DOMAIN` **removed**: 28 no longer reads `[server] DOMAIN` (go-gitea/gitea#39400);
|
||||
`ROOT_URL` was already set and is now the only source of the domain. Nothing else in the repo
|
||||
reads the variable (the bootstrap script sets ROOT_URL/SSH_PORT only).
|
||||
- `GITEA__actions__RUN_RETENTION_DAYS=400` **declared** (go-gitea/gitea#38855 added the 400-day
|
||||
default; the spec repo states no CI-history retention requirement).
|
||||
- `GITEA__security__EGRESS_MODE=lax` **declared, with no allow list.** Read from the 28.3.0
|
||||
source: the Keycloak OIDC login source does NOT go through the new egress policy (goth's
|
||||
openidConnect provider, default HTTP client). The policy covers webhooks, the OAuth2 avatar
|
||||
fetch (`UPDATE_AVATAR` is off) and OpenID 2.0. A hostname entry would not even help for a
|
||||
private address — `WithLocalNeedsIPAllow` requires an IP/CIDR entry. So the brief's
|
||||
"Keycloak SSO breaks" premise does not hold for 28.3.0, and an allow-list entry would only
|
||||
widen webhook SSRF reach.
|
||||
- `GITEA__service__DISABLE_REGISTRATION=true` **declared** (28's new default; 1.27 defaulted it
|
||||
open). OAuth2 auto-registration is gated separately and keeps working.
|
||||
- `update.autoRollbackOnFailedHealth`: kept `true`, commented as **inert**. It renders only an
|
||||
annotation on a Flux ImagePolicy, and `apps/` is rendered `--no-image-automation`, so no
|
||||
ImagePolicy exists. The Deployment is `Recreate` with no probes, so nothing reverts the image
|
||||
automatically. Rollback = restore the PVC, never an image revert onto the migrated DB.
|
||||
- `apps/wave-0/gitea.yaml` re-rendered with the canonical command
|
||||
(`render.py --all catalog -o apps --wave-layout --no-image-automation`, in the toolbox). Only
|
||||
gitea changed.
|
||||
- `renovate.json5`: gitea `allowedVersions` `<2.0.0` -> `<29.0.0`; the description no longer calls
|
||||
28.x a stale tag.
|
||||
|
||||
## What I verified
|
||||
|
||||
- Docker Hub tag list and GitHub tags: 28.3.0 is the newest 28.x stable tag.
|
||||
- Read the v28.3.0 source: `modules/egress/policy/policy.go`, `modules/egress/policies.go`, every
|
||||
`egress.New*` caller, `services/auth/source/oauth2/providers_openid.go`,
|
||||
`modules/setting/{security,actions,service}.go`, `routers/web/auth/{oauth,linkaccount}.go`,
|
||||
`cmd/admin_user_list.go`, `cmd/admin_auth*.go`, the Dockerfile, and `docker/root` (the s6 setup
|
||||
still runs `environment-to-ini`). The postStart hook's CLI is unchanged in 28.3.0: `admin user
|
||||
list --admin` still has Username in column 2, `admin auth list` still has ID/Name in columns 1/2,
|
||||
`add-oauth`/`update-oauth` keep the same flags, and curl and su-exec are still in the image.
|
||||
- Live Gitea (API, 2026-11-07): version 1.27.3. Zero repo, org and system webhooks, no mirrors.
|
||||
- `renovate-config-validator` at 44.167.0 (the tag renovate.yaml pins): "Config validated
|
||||
successfully".
|
||||
- Ran the toolbox gate on the committed tree (numbers are in the PR).
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **The upgrade itself.** Nothing booted 28.3.0 against a copy of the production DB. Migration
|
||||
time, migration success and the post-upgrade SSO login are all UNVERIFIED, and the PR's
|
||||
post-merge checklist covers them.
|
||||
- **The age of the latest gitea-data Longhorn/off-cluster backup.** The cluster read failed from
|
||||
the worktree (no k3s-prod context with the kubeconfig available), so the operator must confirm
|
||||
it before merging.
|
||||
- How the Envoy Gateway handles the new `/-/ws` WebSocket. If upgrades are not forwarded, live
|
||||
notifications fall back to polling (not breaking). UNVERIFIED.
|
||||
- Whether the L2/L3 conformance suite passes. It does not run in CI.
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
date: 2026-11-07
|
||||
branch: feat/l0-externaldns-annotation-prefix
|
||||
title: L0 guard — external-dns annotationPrefix must match every external-dns annotation in the tree
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
`conformance/tests/l0-static/internal-services-publish-no-public-dns/check.sh` gains assertion
|
||||
(8). The check is already whole-repo-scoped, so it runs on every gate.
|
||||
|
||||
- **(8a)** `platform/externaldns/helmrelease.yaml` must set `values.annotationPrefix` explicitly,
|
||||
and it must equal the check's `ANNOTATION_PREFIX` (`external-dns.alpha.kubernetes.io/`). The
|
||||
check does not trust the upstream default, because that default is what changed.
|
||||
- **(8b)** `extraArgs` must not carry `--annotation-prefix` or `--enable-legacy-annotation-prefix`,
|
||||
so the prefix has one declared place.
|
||||
- **(8c)** Every external-dns-looking annotation key under apps/, platform/ and runbooks/ must use
|
||||
that prefix. The match is any `external-dns(.x)*.kubernetes.io/` key.
|
||||
- **(8d)** catalog/render.py must emit no external-dns key under any other prefix.
|
||||
|
||||
`CONTROLLER_KEY` and `HOSTNAME_KEY` are now derived from `ANNOTATION_PREFIX`. The Service
|
||||
inventory matches any external-dns prefix variant, not only the alpha one.
|
||||
|
||||
Why: on 2026-11-07, external-dns v0.23.0 (whose default prefix is GA) ignored every alpha
|
||||
annotation. Assertions (1) to (3) stayed green, because they prove the annotations are present,
|
||||
not that the controller reads them. See `decisions/2026-11-07-public-dns-wan-a-record.md`.
|
||||
|
||||
This branch is stacked on `fix/public-dns-wan-a-record` (PR #140). Without that fix, (8a) fails,
|
||||
which is correct. Merge #140 first.
|
||||
|
||||
## What I verified
|
||||
|
||||
Run on the host (PyYAML):
|
||||
|
||||
- the check passes on this tree: `check OK: 62 assertions passed`.
|
||||
- it **fails (8a)** with origin/main's HelmRelease, which has no `annotationPrefix`.
|
||||
- it **fails (8c)** when `platform/gateway/gateway.yaml`'s `target` key is changed to the GA
|
||||
prefix.
|
||||
|
||||
Toolbox gate: see the PR description.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- Whether a future chart could read the prefix from somewhere other than `values.annotationPrefix`,
|
||||
for example a renamed chart value. (8a) would then pass while the flag silently vanished from the
|
||||
rendered args. Rendering the chart in the gate would close that gap. Not done: it needs network
|
||||
access or a vendored chart in L0.
|
||||
- `conformance/**` is a **sensitive path**. This PR needs the operator's approving review or
|
||||
`/approve-override`. The agent did not post one.
|
||||
- The header of the sibling chainsaw-test.yaml still says "external-dns v0.21.0". That is cosmetic
|
||||
and I left it alone.
|
||||
@@ -0,0 +1,99 @@
|
||||
---
|
||||
date: 2026-11-07
|
||||
branch: fix/public-dns-wan-a-record
|
||||
title: external-dns v0.23 ignored every alpha-prefixed annotation — pin annotationPrefix back
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
`platform/externaldns/helmrelease.yaml`:
|
||||
|
||||
- adds `values.annotationPrefix: "external-dns.alpha.kubernetes.io/"`. The chart renders it as
|
||||
`--annotation-prefix=`.
|
||||
- tightens the chart range from `<1.24.0` to `>=1.23.0 <1.24.0`, because that flag needs app
|
||||
>= 0.22.
|
||||
- adds a comment block explaining why.
|
||||
|
||||
**Root cause.** Renovate PR #129 (merged 12:12:00Z) moved the chart from `>=1.21.0 <1.22.0` to
|
||||
`<1.24.0`. Helm deployed chart 1.23.0 at 12:13:08Z, which moved the app from v0.21.0 to v0.23.0.
|
||||
The upstream v0.22.0 release notes (#6424) say: "The default annotation prefix is now
|
||||
`external-dns.kubernetes.io/` with no fallback". Every external-dns annotation in this repo uses
|
||||
`external-dns.alpha.kubernetes.io/`. At 12:13:12Z v0.23.0 therefore:
|
||||
|
||||
- **ignored the Gateway's `target`.** Every public name flipped from `CNAME
|
||||
wan.home.keithjackson.dev` to `A 192.168.2.20`, an RFC1918 address served in public DNS.
|
||||
- **ignored `controller: none`.** The internal names jackett, pihole and semaphore were published.
|
||||
That breaks R-DNS-03.
|
||||
- **ignored the L4 Services' `hostname`.** The unifi and crafty-mc A records were deleted.
|
||||
|
||||
cloudflare-ddns (#123, v1.18.0) is not involved. It logs only `IPv4-enabled domains:
|
||||
wan.home.keithjackson.dev` and has made no changes.
|
||||
|
||||
**Why pin the prefix instead of migrating the annotations.** The pin is one value in one file, and
|
||||
that file is not on the sensitive-path list. It restores the exact v0.21 reading of `target`,
|
||||
`controller` and `hostname` together. A migration would touch `catalog/render.py`, every
|
||||
`apps/wave-*` file, the hand-written platform manifests, and the conformance goldens and checks.
|
||||
`conformance/**` needs an operator override. And a migration half-applied in either direction
|
||||
reproduces this incident.
|
||||
|
||||
I rejected `--enable-legacy-annotation-prefix`. Upstream calls it a migration aid that will be
|
||||
removed.
|
||||
|
||||
**Follow-up.** The L0 guard that the tree's annotation prefix matches this value is a separate PR
|
||||
(`feat/l0-externaldns-annotation-prefix`), because it lives in `conformance/**`. A full migration
|
||||
to the GA prefix remains open. It must change the annotations and this value in one commit.
|
||||
|
||||
## What I verified
|
||||
|
||||
- **Live HelmRelease history.** `1.23.0 0.23.0 lastDeployed 2026-11-07T12:13:08Z deployed`, and
|
||||
before that `1.21.1 0.21.0 ... superseded`. The Deployment image is `external-dns:v0.23.0`. The
|
||||
startup config logs `AnnotationPrefix:external-dns.kubernetes.io/
|
||||
EnableLegacyAnnotationPrefix:false`.
|
||||
- **external-dns log, 12:13:12Z:**
|
||||
- `action=CREATE ... type=A` and `a-` TXT for gitea, crafty, the apex, headlamp, auth, jackett,
|
||||
nextcloud, keycloak, grafana, seerr, plex, pihole and semaphore.
|
||||
- `action=DELETE ... type=CNAME` and `cname-` TXT for all of those except jackett, pihole and
|
||||
semaphore.
|
||||
- `action=DELETE ... type=A` for unifi and crafty-mc.
|
||||
- **Before the upgrade.** Loki covers about 11-05 13:00Z to 12:13Z. The old pod logged no `Changing
|
||||
record` line in that window. Its only error was one Cloudflare 502 on 11-05.
|
||||
- **Gateway.** `gateway/shared` still carries `external-dns.alpha.kubernetes.io/target:
|
||||
wan.home.keithjackson.dev`.
|
||||
- **Public DNS.** At the authority (betty.ns.cloudflare.com), 1.1.1.1 and 8.8.8.8:
|
||||
- plex, gitea, auth, the apex, jackett, pihole and semaphore answer A `192.168.2.20`.
|
||||
- unifi and crafty-mc have no record.
|
||||
- `wan.` answers A `136.51.115.252`.
|
||||
- There is no AAAA for any of these names.
|
||||
- The `a-plex` and `a-jackett` TXT records read `external-dns/owner=prod`.
|
||||
- **Upstream v0.23.0 source.**
|
||||
- `controller/execute.go` calls `annotations.SetAnnotationPrefix(cfg.AnnotationPrefix)`.
|
||||
- `SetAnnotationPrefix` rebuilds `TargetKey`, `ControllerKey` and `HostnameKey`.
|
||||
- `source/gateway.go` still reads `TargetsFromTargetAnnotation(gw.Annotations)`.
|
||||
- Chart 1.23.0 `templates/deployment.yaml` renders `--annotation-prefix={{
|
||||
.Values.annotationPrefix }}`.
|
||||
- **Phone.** Pi-hole 192.168.2.23 answers correctly: A `192.168.2.20` with TTL 60, and AAAA
|
||||
NOERROR with no data. The router 192.168.2.1 answers `gitea.` with NOERROR and 0 answers, because
|
||||
it strips RFC1918 results. The workstation uses only `nameserver 192.168.2.23` and has
|
||||
`disable_ipv6 = 1`.
|
||||
- **Toolbox gate.** See the PR description.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **The fix live.** This depends on Flux applying it. Expected on the first reconcile, with
|
||||
`policy: sync` and owner `prod`, all of these records being prod-owned:
|
||||
- DELETE the A record and create CNAME `wan.` for the ten public names.
|
||||
- DELETE the A and TXT records for jackett, pihole and semaphore.
|
||||
- CREATE A for unifi (.21) and crafty-mc (.22).
|
||||
This is UNVERIFIED until someone reads the external-dns log and checks with `dig
|
||||
@betty.ns.cloudflare.com`. The A-to-CNAME type swap went through cleanly on 2026-10-03, and the
|
||||
reverse went through today, so I expect it to converge. If it does not, see the manual list in
|
||||
the PR.
|
||||
- **That the chart's schema accepts `annotationPrefix` under Flux.** I did not render the
|
||||
HelmRelease with helm here. The gate checks YAML and policy, not the chart render.
|
||||
- **The phone.** Not investigated, by decision. Likely cause: the phone bypasses Pi-hole on the
|
||||
client side. Candidates are IPv6 RA/RDNSS DNS from the router, Android Private DNS and iCloud
|
||||
Private Relay. Answers it gets that way are rebind-stripped, or point to the WAN IP, which needs
|
||||
hairpin NAT. Hairpin looked absent on 2026-10-03 (a self-signed cert on WAN:443). This is
|
||||
router- or client-side, not in-repo.
|
||||
- The AAAA records the operator saw on 8.8.8.8. I did not reproduce them.
|
||||
- The cilium (#104) and pihole (#109) bumps were not examined beyond Pi-hole answering correctly.
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
date: 2026-11-08
|
||||
branch: fix/keycloak-operator-crds
|
||||
title: "Keycloak 26.8.2 operator CrashLoop: install its two new CRDs, raise its memory"
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
**Incident.** PR #121 (Renovate) bumped `platform/keycloak/kustomization.yaml`'s three
|
||||
`keycloak-k8s-resources` URLs 26.0.6 -> 26.8.2. Upstream `kubernetes/` at 26.8.2 holds **four**
|
||||
CRDs, not two (GitHub contents API, `?ref=26.8.2`):
|
||||
|
||||
| file | at 26.0.6 | at 26.8.2 |
|
||||
|---|---|---|
|
||||
| `keycloaks.k8s.keycloak.org-v1.yml` | yes | yes |
|
||||
| `keycloakrealmimports.k8s.keycloak.org-v1.yml` | yes | yes |
|
||||
| `keycloakoidcclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
|
||||
| `keycloaksamlclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
|
||||
| `kubernetes.yml` | yes | yes (now carries RBAC for both new kinds) |
|
||||
|
||||
(`cluster-wide/` is a separate variant; upstream's own `kubernetes/kustomization.yml` lists exactly
|
||||
the five files above.) The 26.8.2 operator starts an informer for every controller it has and dies
|
||||
on the first missing kind: `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1
|
||||
... GET .../namespaces/keycloak/keycloaksamlclients ... Not Found`, exit 1, CrashLoopBackOff. Flux
|
||||
`platform` (`wait: true`, 5m) sat in health checks and every dependent Kustomization reported
|
||||
`dependency ... is not ready`. The old 26.0.6 ReplicaSet kept serving (RollingUpdate 25%/25%), so SSO
|
||||
stayed up.
|
||||
|
||||
**Fix.**
|
||||
- `platform/keycloak/kustomization.yaml`: added the two CRD URLs, each with the same
|
||||
`# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources` marker (the repo's
|
||||
custom.regex manager matches all five — checked by running its regex over the file). Header
|
||||
comment updated to v26.8.2, plus a warning that Renovate cannot add a CRD a release introduces.
|
||||
- Same file: a `patches:` JSON6902 patch on Deployment `keycloak-operator`: memory limit
|
||||
450Mi -> **768Mi**, request 450Mi -> **512Mi**. 450Mi is upstream's default (unchanged at 26.8.2,
|
||||
so not something the bump changed). Evidence it is too small here: the 26.0.6 pod had **154
|
||||
restarts**, `lastState.terminated.reason: OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and
|
||||
`kubectl top` showed **421Mi / 450Mi**. 26.8.2 runs four controllers instead of two on Quarkus
|
||||
3.40.1.1 (was 3.15.1). No variable in the operator to disable watching unused kinds was used —
|
||||
installing the CRDs is upstream's supported shape.
|
||||
- `platform/keycloak/README.md` "Upgrading Keycloak": the diff-the-CRD-list step; "three URLs" ->
|
||||
"the release URLs". `renovate.json5`: same wording fix in a comment only.
|
||||
|
||||
## What I verified
|
||||
|
||||
- Upstream file lists at 26.0.6 and 26.8.2 (GitHub API) and `diff` of `kubernetes.yml` between the
|
||||
two: the new kinds' ClusterRoles + RoleBindings ship in it, and the Deployment's resources are
|
||||
still 450Mi/450Mi.
|
||||
- Live, read-only (`kubectl --context k3s-prod`, 2026-11-08 ~05:57Z): only `keycloaks` and
|
||||
`keycloakrealmimports` CRDs installed; Deployment strategy RollingUpdate maxSurge/maxUnavailable
|
||||
25%; RS `keycloak-operator-58dff6c4` (26.8.2) 0/1 CrashLoopBackOff, RS `78dbb75f64` (26.0.6) 1/1;
|
||||
operator log is the informer error above; Flux `platform` `Healthy=Unknown` "Running health
|
||||
checks for revision main@sha1:e8d2666"; no NetworkPolicy and no CiliumNetworkPolicy in ns
|
||||
`keycloak`.
|
||||
- No sensitive path touched (`infra/gitea/approval-policy.json`), so this should merge on green.
|
||||
- Gate: see the PR body for the exact rc and stages of the run against the final commit.
|
||||
|
||||
## What I did NOT verify
|
||||
|
||||
- **That the 26.8.2 operator becomes Ready after this merges.** Only the two missing kinds were
|
||||
observed failing; another start-up failure behind them is possible.
|
||||
- **That 768Mi is enough** — no 26.8.2 memory measurement exists yet; re-check `kubectl top` after
|
||||
a week.
|
||||
- **The Keycloak server upgrade 26.0.6 -> 26.8.2 itself.** Once the operator runs it will roll
|
||||
keycloak-0/1 and Keycloak migrates the DB schema. The CR's `spec.update.strategy` defaults to
|
||||
`RecreateOnImageChange` — expect a **full SSO outage** while both pods recreate, not a rolling
|
||||
one. Back up / confirm the latest CNPG backup first if that matters.
|
||||
- **The 26.8.2 Keycloak CRD's `spec.networkPolicy.enabled` defaults to `true`.** The operator will
|
||||
create a NetworkPolicy selecting the Keycloak pods (first policy in that namespace). With no
|
||||
`from` lists set it allows all sources on the HTTP/management ports, so Gateway, ServiceMonitor
|
||||
and the reconciler Jobs should still reach it — but this was read from the CRD text, not observed.
|
||||
- **The passkey reconciler's 26.0.6 Admin API assumptions against 26.8.2** (README "Upgrading
|
||||
Keycloak" list) — not re-checked here; PR #121 merged without it.
|
||||
- `catalog/keycloak.yaml`'s documentation-only image tag was not updated.
|
||||
@@ -30,6 +30,9 @@ ARG CHECK_JSONSCHEMA_VERSION
|
||||
# Pinned to the Prometheus version prod actually runs — see versions.env for why that lockstep is
|
||||
# the point, and for the live-cluster command that establishes it.
|
||||
ARG PROMTOOL_VERSION
|
||||
# velero: operator CLI for the backup server. Pinned to the server version prod runs — the CLI
|
||||
# and server are version-coupled; see versions.env.
|
||||
ARG VELERO_VERSION
|
||||
ARG TARGETARCH=amd64
|
||||
|
||||
# Distro packages: libvirt client libs (harmless disk-image utilities), ansible, ssh,
|
||||
@@ -66,6 +69,10 @@ RUN set -eux; \
|
||||
curl -fsSL "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl; chmod +x /usr/local/bin/kubectl; \
|
||||
curl -fsSL "https://get.helm.sh/helm-v${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" | tar -xz -C /tmp; \
|
||||
install -m0755 "/tmp/linux-${TARGETARCH}/helm" /usr/local/bin/helm; \
|
||||
VELERO_TARBALL="velero-v${VELERO_VERSION}-linux-${TARGETARCH}"; \
|
||||
curl -fsSL "https://github.com/vmware-tanzu/velero/releases/download/v${VELERO_VERSION}/${VELERO_TARBALL}.tar.gz" \
|
||||
| tar -xz -C /usr/local/bin --strip-components=1 "${VELERO_TARBALL}/velero"; \
|
||||
chmod +x /usr/local/bin/velero; \
|
||||
rm -rf /tmp/*
|
||||
|
||||
# Pinned conformance test toolchain (L0/L1 static gate + the L2/L3 Chainsaw runner).
|
||||
@@ -93,7 +100,7 @@ RUN { echo "opentofu=${OPENTOFU_VERSION}"; echo "age=${AGE_VERSION}"; echo "sops
|
||||
echo "kyverno=${KYVERNO_VERSION}"; echo "kyverno-json=${KYVERNO_JSON_VERSION}"; \
|
||||
echo "chainsaw=${CHAINSAW_VERSION}"; echo "yamllint=${YAMLLINT_VERSION}"; \
|
||||
echo "yq=${YQ_VERSION}"; echo "check-jsonschema=${CHECK_JSONSCHEMA_VERSION}"; \
|
||||
echo "promtool=${PROMTOOL_VERSION}"; \
|
||||
echo "promtool=${PROMTOOL_VERSION}"; echo "velero=${VELERO_VERSION}"; \
|
||||
} > /etc/dev-toolchain-versions
|
||||
|
||||
WORKDIR /work
|
||||
@@ -112,6 +119,7 @@ ARG KUBECTL_VERSION
|
||||
ARG CHAINSAW_VERSION
|
||||
ARG KYVERNO_VERSION
|
||||
ARG SOPS_VERSION
|
||||
ARG VELERO_VERSION
|
||||
RUN --network=none /bin/bash -euo pipefail <<'VERIFY'
|
||||
echo "--- no network (the build-time stand-in for run --network none) ---"
|
||||
if getent hosts registry.fedoraproject.org >/dev/null 2>&1 || curl -sS --max-time 3 -o /dev/null https://1.1.1.1 2>/dev/null; then
|
||||
@@ -122,13 +130,13 @@ echo "--- baked versions ---"; cat /etc/dev-toolchain-versions
|
||||
echo "--- every tool is on PATH ---"
|
||||
for t in tofu age age-keygen sops flux kubectl helm kyverno kyverno-json chainsaw yq \
|
||||
yamllint check-jsonschema promtool ansible-playbook git make jq diff python3 \
|
||||
wg qrencode; do
|
||||
wg qrencode velero; do
|
||||
if command -v "$t" >/dev/null 2>&1; then printf " ok %s\n" "$t";
|
||||
else printf " MISSING %s\n" "$t"; missing=1; fi
|
||||
done
|
||||
[ -z "${missing:-}" ] || { echo "toolbox is missing tools"; exit 1; }
|
||||
echo "--- version-sensitive tools report their pins ---"
|
||||
: "${PROMTOOL_VERSION:?}" "${KUBECTL_VERSION:?}" "${CHAINSAW_VERSION:?}" "${KYVERNO_VERSION:?}" "${SOPS_VERSION:?}"
|
||||
: "${PROMTOOL_VERSION:?}" "${KUBECTL_VERSION:?}" "${CHAINSAW_VERSION:?}" "${KYVERNO_VERSION:?}" "${SOPS_VERSION:?}" "${VELERO_VERSION:?}"
|
||||
check() { # name pin actual-output
|
||||
if printf "%s" "$3" | grep -qF "$2"; then printf " ok %-10s %s\n" "$1" "$2";
|
||||
else printf " DRIFT %-10s want %s, got: %s\n" "$1" "$2" "$(printf "%s" "$3" | head -1)"; drift=1; fi
|
||||
@@ -138,6 +146,7 @@ check kubectl "${KUBECTL_VERSION}" "$(kubectl version --client 2>&1)"
|
||||
check chainsaw "${CHAINSAW_VERSION}" "$(chainsaw version 2>&1)"
|
||||
check kyverno "${KYVERNO_VERSION}" "$(kyverno version 2>&1)"
|
||||
check sops "${SOPS_VERSION}" "$(sops --version --disable-version-check 2>&1 || sops --version 2>&1)"
|
||||
check velero "v${VELERO_VERSION}" "$(velero version --client-only 2>&1)"
|
||||
[ -z "${drift:-}" ] || { echo "a tool does not report its pinned version"; exit 1; }
|
||||
echo "--- python can import what the renderer and the gate need ---"
|
||||
python3 -c "import yaml; print(' PyYAML', yaml.__version__)"
|
||||
|
||||
@@ -14,6 +14,7 @@ baseline, and the *same* binaries CI uses.
|
||||
| Provisioning | OpenTofu (`bpg/proxmox` provider), `ansible-core`, `openssh-clients` |
|
||||
| GitOps / cluster | `flux`, `kubectl`, `helm` |
|
||||
| Secrets | `sops`, `age` + `age-keygen` |
|
||||
| Backups (`platform/velero`) | `velero` CLI — pinned to the **server** version prod runs (version-coupled; see `versions.env`) |
|
||||
| VPN (`platform/vpn-ingress`) | `wg` (wireguard-tools) — mint the server key, derive client public keys; `qrencode` — render a client profile as a phone-scannable QR |
|
||||
| **CI gate** | `yamllint`, `check-jsonschema`, `kyverno` CLI, `kyverno-json`, `chainsaw`, `yq`, `diffutils`, `make` |
|
||||
| Misc | `qemu-img`, `cloud-utils`/`genisoimage`, `libvirt-client` (disk-image utilities only — leftovers from the libvirt era, harmless) |
|
||||
|
||||
@@ -54,6 +54,7 @@ BUILD_ARGS=(
|
||||
"KYVERNO_JSON_VERSION=$KYVERNO_JSON_VERSION" "CHAINSAW_VERSION=$CHAINSAW_VERSION"
|
||||
"YAMLLINT_VERSION=$YAMLLINT_VERSION" "YQ_VERSION=$YQ_VERSION"
|
||||
"CHECK_JSONSCHEMA_VERSION=$CHECK_JSONSCHEMA_VERSION" "PROMTOOL_VERSION=$PROMTOOL_VERSION"
|
||||
"VELERO_VERSION=$VELERO_VERSION"
|
||||
)
|
||||
if [ "${PRINT_ARGS}" -eq 1 ]; then printf '%s\n' "${BUILD_ARGS[@]}"; exit 0; fi
|
||||
|
||||
|
||||
@@ -44,4 +44,19 @@ CHECK_JSONSCHEMA_VERSION=0.29.4
|
||||
# promtool ships INSIDE the prometheus release tarball (prometheus-<v>.linux-<arch>.tar.gz), so
|
||||
# the version string here is a Prometheus release, not a separate promtool release.
|
||||
PROMTOOL_VERSION=3.14.0
|
||||
# velero — the operator CLI for the Velero backup server (platform/velero): `velero backup
|
||||
# describe <name> --details`, `velero backup logs`, `velero backup create --from-schedule`.
|
||||
#
|
||||
# THIS PIN IS NOT FREE-FLOATING: the CLI and server are VERSION-COUPLED (the CLI talks to the
|
||||
# server's CRDs and its download-request protocol; upstream supports only a matching minor), so
|
||||
# it MUST equal the Velero server version actually running on prod.
|
||||
# Verified against the live cluster on 2026-11-07:
|
||||
# kubectl --context k3s-prod -n velero get deploy velero \
|
||||
# -o jsonpath='{.spec.template.spec.containers[0].image}'
|
||||
# -> docker.io/velero/velero:v1.18.2 (HelmRelease velero, chart velero-12.2.3)
|
||||
# Deliberately carries NO `# renovate:` marker: it must move WITH the server, i.e. when a
|
||||
# vmware-tanzu `velero` chart bump (platform/velero/helmrelease.yaml, range >=12 <13) changes the
|
||||
# chart's appVersion, bump this in the same PR. Never bump it on its own. The verify stage
|
||||
# asserts `velero version --client-only` reports exactly this version.
|
||||
VELERO_VERSION=1.18.2
|
||||
# Proxmox provider (bpg/proxmox) is pinned in the tofu module (infra/tofu/versions.tf), not here.
|
||||
@@ -61,7 +61,7 @@ spec:
|
||||
- name: crafty
|
||||
# registry.gitlab.com/crafty-controller/crafty-4 — official image. Pinned to the current
|
||||
# stable release (4.10.8, 2026-07-29; tags are bare semver, NO `v` prefix). Renovate maintains.
|
||||
image: registry.gitlab.com/crafty-controller/crafty-4:4.10.8
|
||||
image: registry.gitlab.com/crafty-controller/crafty-4:4.11.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: TZ
|
||||
|
||||
@@ -51,4 +51,4 @@ images:
|
||||
# Controller image == CRD version. A version bump = re-vendor crds.yaml + snapshot-controller.yaml
|
||||
# at the new tag (see header) AND bump this newTag.
|
||||
- name: registry.k8s.io/sig-storage/snapshot-controller
|
||||
newTag: v8.2.0
|
||||
newTag: v8.7.0
|
||||
@@ -194,7 +194,7 @@ spec:
|
||||
- args:
|
||||
- --v=5
|
||||
- --leader-election=true
|
||||
image: registry.k8s.io/sig-storage/snapshot-controller:v8.0.1
|
||||
image: registry.k8s.io/sig-storage/snapshot-controller:v8.7.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: snapshot-controller
|
||||
serviceAccountName: snapshot-controller
|
||||
@@ -51,7 +51,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: cloudflare-ddns
|
||||
image: favonia/cloudflare-ddns:1.17.1
|
||||
image: favonia/cloudflare-ddns:1.18.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: CLOUDFLARE_API_TOKEN
|
||||
|
||||
@@ -11,7 +11,10 @@ spec:
|
||||
chart:
|
||||
spec:
|
||||
chart: external-dns
|
||||
version: ">=1.21.0 <1.22.0" # k8s-1.36 catch-up (app v0.21; policy already set). Renovate maintains
|
||||
# chart 1.23.x = app v0.23.0. Floor >=1.23.0 because `annotationPrefix` below needs app >=0.22.
|
||||
# Renovate maintains — but READ THE APP RELEASE NOTES on every 0.x minor: 0.22 changed the
|
||||
# annotation prefix with no fallback and silently re-published every internal name (2026-11-07).
|
||||
version: ">=1.23.0 <1.24.0"
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: external-dns
|
||||
@@ -25,6 +28,29 @@ spec:
|
||||
values:
|
||||
provider:
|
||||
name: cloudflare
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# ANNOTATION PREFIX — PINNED TO THE LEGACY `alpha` PREFIX (2026-11-07). Do not remove.
|
||||
#
|
||||
# external-dns v0.22.0 (#6424) changed the default annotation prefix from
|
||||
# `external-dns.alpha.kubernetes.io/` to `external-dns.kubernetes.io/` "with no fallback".
|
||||
# EVERY external-dns annotation in this repo uses the alpha prefix: the Gateway's `target`
|
||||
# (platform/gateway/gateway.yaml), the renderer's `controller: none` on `internal` routes
|
||||
# (catalog/render.py -> apps/wave-*), and the L4 Services' `hostname` (crafty-mc, unifi).
|
||||
# Renovate PR #129 moved the chart range to `<1.24.0` (app v0.21.0 -> v0.23.0) and on the first
|
||||
# reconcile (2026-11-07T12:13Z) v0.23.0, reading only the GA prefix, saw none of them:
|
||||
# * `target` ignored -> every public name flipped CNAME wan.<base> -> A 192.168.2.20 (RFC1918)
|
||||
# * `controller: none` ignored -> internal jackett, pihole, semaphore PUBLISHED (R-DNS-03 broken)
|
||||
# * `hostname` ignored -> unifi and crafty-mc A records DELETED
|
||||
# Logged as `action=CREATE ... type=A` / `action=DELETE ... type=CNAME` for every name at once.
|
||||
#
|
||||
# The chart's `annotationPrefix` renders `--annotation-prefix=`, which v0.23.0 applies to every
|
||||
# key it reads (source/annotations/annotations.go SetAnnotationPrefix: target, controller,
|
||||
# hostname, ttl, ...), restoring exactly the v0.21 behaviour. Chosen over
|
||||
# `--enable-legacy-annotation-prefix` (a dual-read migration aid upstream says will be removed)
|
||||
# and over migrating the annotations (renderer + goldens + conformance = sensitive paths, and a
|
||||
# flag-day across ~20 objects). Migrating to the GA prefix is the follow-up: change every
|
||||
# annotation AND this value in ONE commit, never one without the other.
|
||||
annotationPrefix: "external-dns.alpha.kubernetes.io/"
|
||||
# Watch Gateway API routes (Cilium Gateway) + Services. docs/09: records rendered from intent.
|
||||
#
|
||||
# BOTH SOURCES ARE UNFILTERED HERE, AND THAT IS DELIBERATE — see the WHICH RESOURCES block
|
||||
|
||||
@@ -40,7 +40,7 @@ spec:
|
||||
# values (-session-ttl + the -oidc-* args from the externalSecret; useCookie stays default-false
|
||||
# so `-oidc-use-cookie` is NOT emitted) is supported by the v0.41.0 backend. Renovate maintains
|
||||
# within 0.41.x; the appVersion continues to pin the Headlamp image.
|
||||
version: ">=0.41.0 <0.42.0"
|
||||
version: "<0.47.0"
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: headlamp
|
||||
|
||||
@@ -38,7 +38,7 @@ spec:
|
||||
containers:
|
||||
- name: homepage
|
||||
# Renovate maintains this pin (ghcr.io/gethomepage/homepage). Config lives in /app/config.
|
||||
image: ghcr.io/gethomepage/homepage:v2.1.2
|
||||
image: ghcr.io/gethomepage/homepage:v2.7.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
|
||||
@@ -367,9 +367,17 @@ is involved — so step 2 failing with no log entry is the expected signature of
|
||||
The `Keycloak` CR has no `spec.image`, so the server image is whatever the operator's
|
||||
`RELATED_IMAGE_KEYCLOAK` names — the same tag as the operator (prod, 2026-10-30: operator
|
||||
`26.0.6`, `keycloak-0/1` on `quay.io/keycloak/keycloak:26.0.6`). Operator, CRDs and server are
|
||||
released in lockstep from `keycloak/keycloak-k8s-resources`, and the three URLs in
|
||||
released in lockstep from `keycloak/keycloak-k8s-resources`, and the release URLs in
|
||||
`kustomization.yaml` are the one knob. Renovate raises them as ONE PR (`keycloak-operator` group,
|
||||
labels `needs-review` + `blast-radius`), **never auto-merged**; majors wait for a dashboard tick.
|
||||
|
||||
**Renovate bumps the URLs that exist; it cannot add a CRD a release introduced.** Before merging a
|
||||
bump, diff `kubernetes/` (and its `kustomization.yml`) at the new tag against the URL list. 26.0.6
|
||||
shipped 2 CRDs, 26.8.2 ships 4 (`keycloakoidcclients`, `keycloaksamlclients` added); PR #121 bumped
|
||||
only the existing URLs and the 26.8.2 operator CrashLooped on the missing `keycloaksamlclients`
|
||||
informer, stalling Flux `platform` and every tier below it (2026-11-08,
|
||||
`decisions/2026-11-08-keycloak-operator-crds.md`). The operator's memory is also patched in
|
||||
`kustomization.yaml` (upstream's 450Mi OOMKilled 26.0.6 here); keep the patch across bumps.
|
||||
`catalog/keycloak.yaml`'s image tag is documentation only (its render deploys nothing) and
|
||||
Renovate is disabled for it — update it by hand to match after an operator bump merges.
|
||||
|
||||
|
||||
@@ -1224,7 +1224,7 @@ spec:
|
||||
# curlimages elsewhere in the repo. Renovate does not manage plain
|
||||
# `image:` tags (renovate.json disables image-tag updates so it never
|
||||
# fights Flux image-automation), so this is bumped by hand.
|
||||
image: python:3.13.15-alpine3.23
|
||||
image: python:3.15.2-alpine3.23
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- python3
|
||||
|
||||
@@ -20,19 +20,33 @@ kind: Kustomization
|
||||
namespace: keycloak
|
||||
resources:
|
||||
- namespace.yaml
|
||||
# --- Keycloak Operator v26.0.6 (pinned upstream release manifests) ---
|
||||
# --- Keycloak Operator v26.8.2 (pinned upstream release manifests) ---
|
||||
# The operator, its CRDs and the SERVER IMAGE move in lockstep: the Keycloak CR has no
|
||||
# spec.image, so the operator's RELATED_IMAGE_KEYCLOAK (quay.io/keycloak/keycloak:<same tag>)
|
||||
# is what runs. These three URLs are therefore THE Keycloak version. Each carries a Renovate
|
||||
# marker (custom.regex manager, renovate.json5) under ONE depName, so all three bump in ONE
|
||||
# is what runs. These URLs are therefore THE Keycloak version. Each carries a Renovate
|
||||
# marker (custom.regex manager, renovate.json5) under ONE depName, so all of them bump in ONE
|
||||
# branch, never auto-merged (packageRule "keycloak-operator"). Before merging a bump, re-check
|
||||
# passkey-reconciler.yaml's 26.0.6 Admin API assumptions — see README.md "Upgrading Keycloak".
|
||||
#
|
||||
# THE URL LIST MUST MATCH UPSTREAM'S kubernetes/kustomization.yml AT THE NEW TAG — Renovate only
|
||||
# rewrites the version in the URLs that are here; it cannot add a CRD the new release introduced.
|
||||
# 26.0.6 shipped 2 CRDs; 26.8.2 ships 4 (+ keycloakoidcclients, keycloaksamlclients, both
|
||||
# v2alpha1). PR #121 bumped only the existing three URLs, and the 26.8.2 operator — which starts an
|
||||
# informer for EVERY kind it controls, used or not — CrashLooped on
|
||||
# `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1 ... Not Found`,
|
||||
# stalling Flux `platform` and everything downstream (2026-11-08). On every bump, diff
|
||||
# https://github.com/keycloak/keycloak-k8s-resources/tree/<tag>/kubernetes
|
||||
# against this list before merging.
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.0.6/kubernetes/kubernetes.yml
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakoidcclients.k8s.keycloak.org-v1.yml
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaksamlclients.k8s.keycloak.org-v1.yml
|
||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/kubernetes.yml
|
||||
# --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
|
||||
- keycloak-admin-credentials.sops.yaml
|
||||
# --- OIDC client secrets (grafana + oauth2-proxy) the realm import injects via placeholders ---
|
||||
@@ -51,3 +65,23 @@ resources:
|
||||
- keycloak-db-backup-credentials-prod.sops.yaml
|
||||
# --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
|
||||
- grafana-dashboard.yaml
|
||||
# --- Operator memory: upstream ships requests=limits=450Mi, which this cluster outgrew ON 26.0.6 ---
|
||||
# Measured on prod 2026-11-08: the 26.0.6 operator pod had 154 restarts in 64d, lastState
|
||||
# `OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and sat at 421Mi of its 450Mi (94%) in `kubectl top`.
|
||||
# 26.8.2 keeps the same 450Mi default but runs FOUR controllers/informers instead of two (adds
|
||||
# keycloakoidcclient + keycloaksamlclient) on a newer Quarkus (3.15.1 -> 3.40.1.1), so it would
|
||||
# plausibly start above a ceiling 26.0.6 already crossed. Limit 768Mi = ~1.8x the observed 421Mi
|
||||
# working set; request 512Mi = the observed working set plus headroom, so the scheduler reserves what
|
||||
# it actually uses. Revisit with `kubectl top` once 26.8.2 has run for a week. Patched here (not in
|
||||
# the remote kubernetes.yml) so a Renovate bump keeps it.
|
||||
patches:
|
||||
- target:
|
||||
kind: Deployment
|
||||
name: keycloak-operator
|
||||
patch: |-
|
||||
- op: replace
|
||||
path: /spec/template/spec/containers/0/resources/limits/memory
|
||||
value: 768Mi
|
||||
- op: replace
|
||||
path: /spec/template/spec/containers/0/resources/requests/memory
|
||||
value: 512Mi
|
||||
@@ -3205,7 +3205,7 @@ spec:
|
||||
# sibling's pod on prod runs a different digest under this same tag.
|
||||
# Renovate's `kubernetes` manager covers platform/** (renovate.json5), so
|
||||
# it proposes tag and digest bumps for this line like any other image.
|
||||
image: python:3.13.15-alpine3.23@sha256:6438599575cca0d1df94aeee0d2ae088d4d8846eab554b2ee7784a3a6df0d516
|
||||
image: python:3.15.2-alpine3.23@sha256:8d77d813a53e534dec9e1e5182cbd5b651b70d63b27e4ea7c1f616b37cf9aad2
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- python3
|
||||
|
||||
@@ -41,7 +41,7 @@ spec:
|
||||
# not disconnected. (The anti-affinity change is a separate commit precisely because that one DOES
|
||||
# cause a switchover — see it for the blast radius.)
|
||||
instances: 3
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16.15
|
||||
# HARD anti-affinity — the three instances MUST land on three DIFFERENT nodes (2026-09-07).
|
||||
#
|
||||
# CNPG's default is `preferred`, which is only a scheduler HINT: the scheduler may put two
|
||||
|
||||
@@ -95,7 +95,7 @@ spec:
|
||||
# OIDC bootstrap hooks. (ci: flux-postbuild-substitution enforces it.)
|
||||
extraInitContainers:
|
||||
- name: wait-for-idp
|
||||
image: docker.io/library/alpine:3.22
|
||||
image: docker.io/library/alpine:3.24
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
|
||||
@@ -88,7 +88,7 @@ spec:
|
||||
# rather than becoming a fifth per-environment knob. ---
|
||||
initContainers:
|
||||
- name: velero-plugin-for-aws
|
||||
image: velero/velero-plugin-for-aws:v1.14.3-rc.2
|
||||
image: velero/velero-plugin-for-aws:v1.14.4
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- mountPath: /target
|
||||
|
||||
+3
-3
@@ -264,7 +264,7 @@
|
||||
// and the server image are released in lockstep from keycloak/keycloak-k8s-resources, and the
|
||||
// Keycloak CR carries no spec.image, so the operator's RELATED_IMAGE_KEYCLOAK picks the server
|
||||
// (measured on prod 2026-10-30: operator 26.0.6 -> keycloak-0/1 run quay.io/keycloak/keycloak:
|
||||
// 26.0.6). The three release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
||||
// 26.0.6). The release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
||||
// markers under this ONE depName, so they move together in one branch (groupName makes that
|
||||
// explicit). NEVER AUTO-MERGED, not even a patch: Keycloak is the IdP that gates SSO for every
|
||||
// app AND kubectl OIDC, and platform/keycloak/passkey-reconciler.yaml encodes Keycloak 26.0.6
|
||||
@@ -316,9 +316,9 @@
|
||||
"enabled": false
|
||||
},
|
||||
{
|
||||
"description": "ADDED 2026-10-30. MIRRORS catalog/<svc>.yaml `update.range` — the operator's declared per-service policy (schemas/servicedefinition.schema.json), which Renovate does not read. Without it the dry run proposed gitea 28.2.0, qbittorrent 20.04.1 (both stale/odd tags, not real successors), radarr 6.x and nextcloud 35 (four majors past 31; Nextcloud cannot skip majors). KEEP IN STEP with the catalog ranges.",
|
||||
"description": "ADDED 2026-10-30. MIRRORS catalog/<svc>.yaml `update.range` — the operator's declared per-service policy (schemas/servicedefinition.schema.json), which Renovate does not read. Without it the dry run proposed gitea 28.2.0, qbittorrent 20.04.1 (the qbittorrent tag is a stale/odd one, not a real successor; gitea 28.x IS the real successor of 1.27 — upstream dropped the \"1.\" prefix at 28.0.0 — and the catalog moved to it on 2026-11-07, so the gitea ceiling is now <29.0.0: 28.x patches/minors are proposed, never auto-merged per the blast-radius rule, and 29 waits for a human), radarr 6.x and nextcloud 35 (four majors past 31; Nextcloud cannot skip majors). KEEP IN STEP with the catalog ranges.",
|
||||
"matchPackageNames": ["gitea/gitea"],
|
||||
"allowedVersions": "<2.0.0"
|
||||
"allowedVersions": "<29.0.0"
|
||||
},
|
||||
{ "matchPackageNames": ["nextcloud"], "allowedVersions": "<32.0.0" },
|
||||
{ "matchPackageNames": ["lscr.io/linuxserver/qbittorrent"], "allowedVersions": "<6.0.0" },
|
||||
|
||||
@@ -99,6 +99,16 @@ infra/devcontainer/dev.sh run bash -c \
|
||||
-o custom-columns=NAME:.metadata.name,BACKUP:.metadata.labels.velero\\.io/backup-name,PHASE:.status.phase,BYTES:.status.progress.bytesDone'
|
||||
```
|
||||
|
||||
The `velero` CLI is in the toolbox (pinned to the server version) and gives the per-item errors and
|
||||
warnings the CR only counts. Read-only; `velero backup create/delete` and `velero restore` are
|
||||
operator actions:
|
||||
|
||||
```
|
||||
infra/devcontainer/dev.sh run bash -c \
|
||||
'velero --kubecontext k3s-prod -n velero backup describe <name> --details; \
|
||||
velero --kubecontext k3s-prod -n velero backup logs <name> | grep -iE "level=(error|warn)" | tail -25'
|
||||
```
|
||||
|
||||
The plugin pin (cause 1 depends entirely on this):
|
||||
|
||||
```
|
||||
|
||||
@@ -87,7 +87,7 @@ spec:
|
||||
# Renovate maintains this pin. nginx-unprivileged (not nginx) so the container can honour
|
||||
# runAsNonRoot: the stock nginx image's entrypoint needs root to drop privileges and to
|
||||
# bind :80. `stable-alpine` branch, pinned exactly.
|
||||
image: nginxinc/nginx-unprivileged:1.30.4-alpine
|
||||
image: nginxinc/nginx-unprivileged:1.31.7-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
|
||||
Reference in new issue
Block a user