Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0b0b5abc8e | ||
|
|
6e2e353bcc | ||
|
|
045d81c7a2 | ||
|
|
7be693bf37 | ||
|
|
9e9a3ea8cc | ||
|
|
89f48e4a5f | ||
|
|
35ba875380 | ||
|
|
a775082ae1 | ||
|
|
1a33814522 | ||
|
|
4a3f6dee59 | ||
|
|
b548224db8 | ||
|
|
759a0e51c8 | ||
|
|
ea30ba876c | ||
|
|
3d7ecbcfe9 | ||
|
|
893c9dee4f |
No files matched your search
@@ -85,7 +85,7 @@ spec:
|
|||||||
- name: config
|
- name: config
|
||||||
mountPath: /config
|
mountPath: /config
|
||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ spec:
|
|||||||
- -c
|
- -c
|
||||||
- "# Assert the declared adlists in gravity.db. Idempotent: steady state writes nothing and\n# does NOT run a gravity rebuild.\nset -u\nGRAVITY=/etc/pihole/gravity.db\n\n# THE DECLARED SET \u2014 one URL per line, the source of truth for what PiHole blocks.\n# To add a list: add a line here, commit, let Flux reconcile. Nothing else to do.\nDECLARED='https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'\n\n# On a fresh PVC, FTL creates gravity.db a moment after the container starts. Wait, bounded,\n# and NEVER block or fail the start: a missing DB here must not crashloop DNS for the LAN.\ni=0\nwhile [ ! -s \"$GRAVITY\" ] && [ \"$i\" -lt 90 ]; do sleep 2; i=$(expr $i + 1); done\nif [ ! -s \"$GRAVITY\" ]; then\n echo \"pihole-adlists: gravity.db absent after 180s - leaving adlists untouched\" >&2\n exit 0\nfi\n\nsql() { pihole-FTL sqlite3 \"$GRAVITY\" \"$1\"; }\nCHANGED=0\n\n# 1) Every declared URL must be present AND enabled.\necho \"$DECLARED\" | while IFS= read -r url; do\n [ -n \"$url\" ] || continue\n printf '%s\\n' \"$url\"\ndone > /tmp/declared.txt\nwhile IFS= read -r url; do\n [ -n \"$url\" ] || continue\n # Single quotes inside a URL would break this SQL; adlist URLs do not contain them.\n have=$(sql \"select count(*) from adlist where address='$url';\")\n if [ \"$have\" = \"0\" ]; then\n echo \"pihole-adlists: adding $url\"\n sql \"insert into adlist (address,enabled,comment) values ('$url',1,'declared in catalog/pihole.yaml');\"\n CHANGED=1\n else\n en=$(sql \"select enabled from adlist where address='$url';\")\n if [ \"$en\" != \"1\" ]; then\n echo \"pihole-adlists: re-enabling $url\"\n sql \"update adlist set enabled=1 where address='$url';\"\n CHANGED=1\n fi\n fi\ndone < /tmp/declared.txt\n\n# 2) Anything enabled in the DB but NOT declared gets disabled (never deleted - see above).\nsql \"select address from adlist where enabled=1;\" | while IFS= read -r addr; do\n [ -n \"$addr\" ] || continue\n if ! grep -qxF \"$addr\" /tmp/declared.txt; then\n echo \"pihole-adlists: disabling undeclared $addr\"\n sql \"update adlist set enabled=0 where address='$addr';\"\n echo changed >> /tmp/adlist-changed\n fi\ndone\n[ -s /tmp/adlist-changed ] && CHANGED=1\nrm -f /tmp/adlist-changed /tmp/declared.txt\n\n# 3) Rebuild gravity ONLY if the declared set actually moved.\nif [ \"$CHANGED\" = \"1\" ]; then\n echo \"pihole-adlists: declared set changed - running gravity update\"\n pihole -g || echo \"pihole-adlists: gravity update failed; existing gravity.db retained\" >&2\nelse\n echo \"pihole-adlists: already matches the declared set - nothing to do\"\nfi\nexit 0\n"
|
- "# Assert the declared adlists in gravity.db. Idempotent: steady state writes nothing and\n# does NOT run a gravity rebuild.\nset -u\nGRAVITY=/etc/pihole/gravity.db\n\n# THE DECLARED SET \u2014 one URL per line, the source of truth for what PiHole blocks.\n# To add a list: add a line here, commit, let Flux reconcile. Nothing else to do.\nDECLARED='https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'\n\n# On a fresh PVC, FTL creates gravity.db a moment after the container starts. Wait, bounded,\n# and NEVER block or fail the start: a missing DB here must not crashloop DNS for the LAN.\ni=0\nwhile [ ! -s \"$GRAVITY\" ] && [ \"$i\" -lt 90 ]; do sleep 2; i=$(expr $i + 1); done\nif [ ! -s \"$GRAVITY\" ]; then\n echo \"pihole-adlists: gravity.db absent after 180s - leaving adlists untouched\" >&2\n exit 0\nfi\n\nsql() { pihole-FTL sqlite3 \"$GRAVITY\" \"$1\"; }\nCHANGED=0\n\n# 1) Every declared URL must be present AND enabled.\necho \"$DECLARED\" | while IFS= read -r url; do\n [ -n \"$url\" ] || continue\n printf '%s\\n' \"$url\"\ndone > /tmp/declared.txt\nwhile IFS= read -r url; do\n [ -n \"$url\" ] || continue\n # Single quotes inside a URL would break this SQL; adlist URLs do not contain them.\n have=$(sql \"select count(*) from adlist where address='$url';\")\n if [ \"$have\" = \"0\" ]; then\n echo \"pihole-adlists: adding $url\"\n sql \"insert into adlist (address,enabled,comment) values ('$url',1,'declared in catalog/pihole.yaml');\"\n CHANGED=1\n else\n en=$(sql \"select enabled from adlist where address='$url';\")\n if [ \"$en\" != \"1\" ]; then\n echo \"pihole-adlists: re-enabling $url\"\n sql \"update adlist set enabled=1 where address='$url';\"\n CHANGED=1\n fi\n fi\ndone < /tmp/declared.txt\n\n# 2) Anything enabled in the DB but NOT declared gets disabled (never deleted - see above).\nsql \"select address from adlist where enabled=1;\" | while IFS= read -r addr; do\n [ -n \"$addr\" ] || continue\n if ! grep -qxF \"$addr\" /tmp/declared.txt; then\n echo \"pihole-adlists: disabling undeclared $addr\"\n sql \"update adlist set enabled=0 where address='$addr';\"\n echo changed >> /tmp/adlist-changed\n fi\ndone\n[ -s /tmp/adlist-changed ] && CHANGED=1\nrm -f /tmp/adlist-changed /tmp/declared.txt\n\n# 3) Rebuild gravity ONLY if the declared set actually moved.\nif [ \"$CHANGED\" = \"1\" ]; then\n echo \"pihole-adlists: declared set changed - running gravity update\"\n pihole -g || echo \"pihole-adlists: gravity update failed; existing gravity.db retained\" >&2\nelse\n echo \"pihole-adlists: already matches the declared set - nothing to do\"\nfi\nexit 0\n"
|
||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
@@ -156,7 +156,7 @@ spec:
|
|||||||
, \"[Preferences]\"\n\ndef matches(raw):\n \"\"\"True iff the stored digest is one OUR password produces (salt is read from it).\"\"\"\n v = raw.strip().strip('\"')\n if v.startswith(\"@ByteArray(\") and v.endswith(\")\"):\n v = v[len(\"@ByteArray(\"):-1]\n try:\n salt_b64, digest_b64 = v.split(\":\", 1)\n salt = base64.b64decode(salt_b64, validate=True)\n digest = base64.b64decode(digest_b64, validate=True)\n except Exception:\n return False\n if not salt or not digest:\n return False\n return hmac.compare_digest(\n hashlib.pbkdf2_hmac(\"sha512\", pw.encode(), salt, ITERATIONS, len(digest)), digest)\n\ntry:\n lines = open(conf, encoding=\"utf-8\", errors=\"replace\").read().splitlines()\nexcept FileNotFoundError:\n lines = []\n\ndef value_of(key):\n for i, line in enumerate(lines):\n if line.startswith(key + \"=\"):\n return i, line.split(\"=\", 1)[1]\n return -1, None\n\nui, uval = value_of(UKEY)\npi, pval = value_of(PKEY)\nif uval == user and pval is not None and matches(pval):\n raise SystemExit(0) # converged - leave the file (and its salt) alone\n\nsalt = os.urandom(SALT_LEN)\ndigest = hashlib.pbkdf2_hmac(\"sha512\", pw.encode(), salt, ITERATIONS, DIGEST_LEN)\nstored = base64.b64encode(salt).decode() + \":\" + base64.b64encode(digest).decode()\ndesired = [UKEY + \"=\" + user, PKEY + '=\"@ByteArray(' + stored + ')\"']\n\nout = [line for i, line in enumerate(lines) if i not in (ui, pi)]\nif SECTION in out:\n at = out.index(SECTION) + 1\nelse:\n if out and out[-1].strip():\n out.append(\"\")\n out.append(SECTION)\n at = len(out)\nout[at:at] = desired\n\ntry:\n os.makedirs(os.path.dirname(conf) or \".\", exist_ok=True)\n tmp = conf + \".webui-cred.new\"\n with open(tmp, \"w\", encoding=\"utf-8\") as fh:\n fh.write(\"\\n\".join(out) + \"\\n\")\n fh.flush()\n os.fsync(fh.fileno())\n # This hook runs as ROOT; the service runs as PUID:PGID. A root-owned 0600 conf would\n # cost the app the ability to save its own settings, so carry the ownership over.\n try:\n st = os.stat(conf)\n os.chown(tmp, st.st_uid, st.st_gid)\n os.chmod(tmp, stat.S_IMODE(st.st_mode))\n except FileNotFoundError:\n os.chown(tmp, int(os.environ.get(\"PUID\", \"1000\")),\n int(os.environ.get(\"PGID\", \"1000\")))\n os.chmod(tmp, 0o664)\n os.replace(tmp, conf) # atomic: the app never reads a half-written conf\nexcept OSError as exc:\n sys.stderr.write(\"[qbt-webui-cred] cannot write %s: %s\\n\" % (conf, exc))\n raise SystemExit(1)\nraise SystemExit(10)\nPY\n rc=$?\n\n if [ \"$rc\" = 0 ]; then\n log \"conf already holds the durable credential for '$QBT_WEBUI_USERNAME' (no write)\"\n elif [ \"$rc\" = 10 ]; then\n log \"asserted WebUI Username + Password_PBKDF2 into $CONF\"\n # postStart runs CONCURRENTLY with the s6 init chain, so qbittorrent-nox may ALREADY\n # have read the old, credential-less conf and be serving a random temporary password.\n # Restart the SUPERVISED SERVICE (not the pod) so it re-reads what was just written.\n # Nothing to do when the service has not started yet - it will read the new file.\n # Its own counter, NOT the shared `waited` API budget: sharing one counter is how the\n # 2026-09-13 failure spent the entire verification budget before the first login and\n # then fired attempts 2 and 3 with no wait left at all.\n sv=0\n while [ \"$sv\" -lt 60 ] && [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" != \"true\" ]; do\n sv=$((sv + 1)); sleep 1\n done\n if [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" = \"true\" ]; then\n log \"qbittorrent-nox was already running - restarting it to load the credential\"\n s6-svc -r \"$SVC\" || log \"s6-svc -r failed (continuing to verify anyway)\"\n sleep 3\n fi\n else\n log \"FAILED to write the credential into $CONF (helper exit $rc). The WebUI will keep\" >&2\n log \"minting a random password\
|
, \"[Preferences]\"\n\ndef matches(raw):\n \"\"\"True iff the stored digest is one OUR password produces (salt is read from it).\"\"\"\n v = raw.strip().strip('\"')\n if v.startswith(\"@ByteArray(\") and v.endswith(\")\"):\n v = v[len(\"@ByteArray(\"):-1]\n try:\n salt_b64, digest_b64 = v.split(\":\", 1)\n salt = base64.b64decode(salt_b64, validate=True)\n digest = base64.b64decode(digest_b64, validate=True)\n except Exception:\n return False\n if not salt or not digest:\n return False\n return hmac.compare_digest(\n hashlib.pbkdf2_hmac(\"sha512\", pw.encode(), salt, ITERATIONS, len(digest)), digest)\n\ntry:\n lines = open(conf, encoding=\"utf-8\", errors=\"replace\").read().splitlines()\nexcept FileNotFoundError:\n lines = []\n\ndef value_of(key):\n for i, line in enumerate(lines):\n if line.startswith(key + \"=\"):\n return i, line.split(\"=\", 1)[1]\n return -1, None\n\nui, uval = value_of(UKEY)\npi, pval = value_of(PKEY)\nif uval == user and pval is not None and matches(pval):\n raise SystemExit(0) # converged - leave the file (and its salt) alone\n\nsalt = os.urandom(SALT_LEN)\ndigest = hashlib.pbkdf2_hmac(\"sha512\", pw.encode(), salt, ITERATIONS, DIGEST_LEN)\nstored = base64.b64encode(salt).decode() + \":\" + base64.b64encode(digest).decode()\ndesired = [UKEY + \"=\" + user, PKEY + '=\"@ByteArray(' + stored + ')\"']\n\nout = [line for i, line in enumerate(lines) if i not in (ui, pi)]\nif SECTION in out:\n at = out.index(SECTION) + 1\nelse:\n if out and out[-1].strip():\n out.append(\"\")\n out.append(SECTION)\n at = len(out)\nout[at:at] = desired\n\ntry:\n os.makedirs(os.path.dirname(conf) or \".\", exist_ok=True)\n tmp = conf + \".webui-cred.new\"\n with open(tmp, \"w\", encoding=\"utf-8\") as fh:\n fh.write(\"\\n\".join(out) + \"\\n\")\n fh.flush()\n os.fsync(fh.fileno())\n # This hook runs as ROOT; the service runs as PUID:PGID. A root-owned 0600 conf would\n # cost the app the ability to save its own settings, so carry the ownership over.\n try:\n st = os.stat(conf)\n os.chown(tmp, st.st_uid, st.st_gid)\n os.chmod(tmp, stat.S_IMODE(st.st_mode))\n except FileNotFoundError:\n os.chown(tmp, int(os.environ.get(\"PUID\", \"1000\")),\n int(os.environ.get(\"PGID\", \"1000\")))\n os.chmod(tmp, 0o664)\n os.replace(tmp, conf) # atomic: the app never reads a half-written conf\nexcept OSError as exc:\n sys.stderr.write(\"[qbt-webui-cred] cannot write %s: %s\\n\" % (conf, exc))\n raise SystemExit(1)\nraise SystemExit(10)\nPY\n rc=$?\n\n if [ \"$rc\" = 0 ]; then\n log \"conf already holds the durable credential for '$QBT_WEBUI_USERNAME' (no write)\"\n elif [ \"$rc\" = 10 ]; then\n log \"asserted WebUI Username + Password_PBKDF2 into $CONF\"\n # postStart runs CONCURRENTLY with the s6 init chain, so qbittorrent-nox may ALREADY\n # have read the old, credential-less conf and be serving a random temporary password.\n # Restart the SUPERVISED SERVICE (not the pod) so it re-reads what was just written.\n # Nothing to do when the service has not started yet - it will read the new file.\n # Its own counter, NOT the shared `waited` API budget: sharing one counter is how the\n # 2026-09-13 failure spent the entire verification budget before the first login and\n # then fired attempts 2 and 3 with no wait left at all.\n sv=0\n while [ \"$sv\" -lt 60 ] && [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" != \"true\" ]; do\n sv=$((sv + 1)); sleep 1\n done\n if [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" = \"true\" ]; then\n log \"qbittorrent-nox was already running - restarting it to load the credential\"\n s6-svc -r \"$SVC\" || log \"s6-svc -r failed (continuing to verify anyway)\"\n sleep 3\n fi\n else\n log \"FAILED to write the credential into $CONF (helper exit $rc). The WebUI will keep\" >&2\n log \"minting a random password\
|
||||||
\ every start and BOTH arrs will break. Fix and restart.\" >&2\n exit 0\n fi\n\n # (3) GROUND TRUTH: the WebUI itself must accept it. Wait for the API to answer at ALL\n # (shared ~120s budget across attempts), then spend exactly ONE login per attempt -\n # at most 3 rejected logins, which stays under qBittorrent's default ban threshold.\n #\n # \"THE API NEVER ANSWERED\" IS A DIFFERENT FAULT FROM \"THE PASSWORD WAS REJECTED\", and\n # conflating them is what made this hook lie once already (see the PORT comment). An\n # unreachable WebUI is reported as unreachable, and the hook stops rather than spending\n # its remaining login attempts to produce two more identical wrong diagnoses.\n api_ok=0\n while [ \"$waited\" -lt 120 ]; do\n code=$(curl -s -o /dev/null -m 3 -w '%{http_code}' \"http://127.0.0.1:$PORT/\" 2>/dev/null)\n if [ -n \"$code\" ] && [ \"$code\" != \"000\" ]; then api_ok=1; break; fi\n waited=$((waited + 1)); sleep 1\n done\n if [ \"$api_ok\" != 1 ]; then\n log \"the WebUI never answered on 127.0.0.1:$PORT within the wait budget, so the\" >&2\n log \"credential in $CONF is UNVERIFIED. This is NOT a rejected password - it was\" >&2\n log \"never submitted. Do not rotate the credential on the strength of this line.\" >&2\n exit 0\n fi\n body=$(curl -sS -m 10 \\\n --data-urlencode \"username=$QBT_WEBUI_USERNAME\" \\\n --data-urlencode \"password=$QBT_WEBUI_PASSWORD\" \\\n \"http://127.0.0.1:$PORT/api/v2/auth/login\" 2>/dev/null)\n if [ \"$body\" = \"Ok.\" ]; then\n log \"VERIFIED: the WebUI accepts the durable credential for '$QBT_WEBUI_USERNAME'\"\n exit 0\n fi\n log \"WebUI rejected the durable credential (login said '$body')\" >&2\n # A rejection when the conf was ALREADY converged (rc=0, nothing written) can only mean\n # the running qbittorrent-nox is serving something it read from an EARLIER conf, so\n # rewriting the same bytes on the next attempt would change nothing. The remedy is to\n # make the process RE-READ the file, so restart the supervised service and let the next\n # attempt re-verify. Without this the retry loop could never change its own outcome.\n if [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" = \"true\" ]; then\n log \"restarting qbittorrent-nox so it re-reads $CONF, then re-verifying\" >&2\n s6-svc -r \"$SVC\" || log \"s6-svc -r failed (continuing to verify anyway)\" >&2\n sleep 3\n fi\ndone\n\nlog \"GAVE UP after $attempt attempts: the WebUI does not accept the credential from git.\" >&2\n# A postStart hook MUST exit 0 - a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), turning a credential fault into an outage.\n# Both failure paths above therefore log LOUDLY and exit 0; the `up` SLI still watches the\n# app, and the arrs' own download-client health check surfaces a broken credential.\nexit 0\n"
|
\ every start and BOTH arrs will break. Fix and restart.\" >&2\n exit 0\n fi\n\n # (3) GROUND TRUTH: the WebUI itself must accept it. Wait for the API to answer at ALL\n # (shared ~120s budget across attempts), then spend exactly ONE login per attempt -\n # at most 3 rejected logins, which stays under qBittorrent's default ban threshold.\n #\n # \"THE API NEVER ANSWERED\" IS A DIFFERENT FAULT FROM \"THE PASSWORD WAS REJECTED\", and\n # conflating them is what made this hook lie once already (see the PORT comment). An\n # unreachable WebUI is reported as unreachable, and the hook stops rather than spending\n # its remaining login attempts to produce two more identical wrong diagnoses.\n api_ok=0\n while [ \"$waited\" -lt 120 ]; do\n code=$(curl -s -o /dev/null -m 3 -w '%{http_code}' \"http://127.0.0.1:$PORT/\" 2>/dev/null)\n if [ -n \"$code\" ] && [ \"$code\" != \"000\" ]; then api_ok=1; break; fi\n waited=$((waited + 1)); sleep 1\n done\n if [ \"$api_ok\" != 1 ]; then\n log \"the WebUI never answered on 127.0.0.1:$PORT within the wait budget, so the\" >&2\n log \"credential in $CONF is UNVERIFIED. This is NOT a rejected password - it was\" >&2\n log \"never submitted. Do not rotate the credential on the strength of this line.\" >&2\n exit 0\n fi\n body=$(curl -sS -m 10 \\\n --data-urlencode \"username=$QBT_WEBUI_USERNAME\" \\\n --data-urlencode \"password=$QBT_WEBUI_PASSWORD\" \\\n \"http://127.0.0.1:$PORT/api/v2/auth/login\" 2>/dev/null)\n if [ \"$body\" = \"Ok.\" ]; then\n log \"VERIFIED: the WebUI accepts the durable credential for '$QBT_WEBUI_USERNAME'\"\n exit 0\n fi\n log \"WebUI rejected the durable credential (login said '$body')\" >&2\n # A rejection when the conf was ALREADY converged (rc=0, nothing written) can only mean\n # the running qbittorrent-nox is serving something it read from an EARLIER conf, so\n # rewriting the same bytes on the next attempt would change nothing. The remedy is to\n # make the process RE-READ the file, so restart the supervised service and let the next\n # attempt re-verify. Without this the retry loop could never change its own outcome.\n if [ \"$(s6-svstat -u \"$SVC\" 2>/dev/null)\" = \"true\" ]; then\n log \"restarting qbittorrent-nox so it re-reads $CONF, then re-verifying\" >&2\n s6-svc -r \"$SVC\" || log \"s6-svc -r failed (continuing to verify anyway)\" >&2\n sleep 3\n fi\ndone\n\nlog \"GAVE UP after $attempt attempts: the WebUI does not accept the credential from git.\" >&2\n# A postStart hook MUST exit 0 - a non-zero exit makes the kubelet KILL the container\n# (FailedPostStartHook -> CrashLoopBackOff), turning a credential fault into an outage.\n# Both failure paths above therefore log LOUDLY and exit 0; the `up` SLI still watches the\n# app, and the arrs' own download-client health check surfaces a broken credential.\nexit 0\n"
|
||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
@@ -77,6 +77,42 @@ spec:
|
|||||||
requests:
|
requests:
|
||||||
storage: 1Gi
|
storage: 1Gi
|
||||||
---
|
---
|
||||||
|
# act_runner config file — DELTAS ONLY. act_runner v0.6.1 LoadDefault()
|
||||||
|
# (internal/pkg/config/config.go) unmarshals this file onto a ZERO Config and then fills every
|
||||||
|
# unset field with the same default it uses when there is no file at all, so a file holding
|
||||||
|
# only `runner.capacity` changes exactly that and nothing else. Do NOT paste in the output of
|
||||||
|
# `act_runner generate-config`: that example is NOT the defaults. Its `runner.labels` would
|
||||||
|
# OVERRIDE the registered labels in /data/.runner (daemon.go: config labels win when non-empty)
|
||||||
|
# and repoint ubuntu-latest at a different job image; its `force_pull: true` differs from the
|
||||||
|
# no-file default (false); and its sample `envs` would be injected into every job.
|
||||||
|
#
|
||||||
|
# Using a file also makes act_runner IGNORE the legacy GITEA_RUNNER_CAPACITY / _FILE /
|
||||||
|
# _ENVIRON / _ENV_FILE / GITEA_DEBUG / GITEA_TRACE env vars (deprecated.go). None of them is
|
||||||
|
# set on this Deployment; the GITEA_INSTANCE_URL / _REGISTRATION_TOKEN / _NAME / _LABELS env
|
||||||
|
# below is consumed by the image's run.sh for registration, not by that legacy path.
|
||||||
|
#
|
||||||
|
# A ConfigMap edit alone does NOT restart the runner (it reads the file once at start). If a
|
||||||
|
# future change touches ONLY this ConfigMap, also bump the Deployment's pod template (e.g. the
|
||||||
|
# config-revision annotation below) or the new value stays inert until the next restart.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: act-runner-config
|
||||||
|
namespace: gitea-actions
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: act-runner
|
||||||
|
data:
|
||||||
|
config.yaml: |
|
||||||
|
runner:
|
||||||
|
# Two concurrent jobs (default 1). Measured 2026-11-08: queue wait ~1s normally, but
|
||||||
|
# 29-759s whenever gate / images / renovate runs collide (run 472 waited 759s behind a
|
||||||
|
# renovate burst). Both jobs share the ONE dind daemon below; act_runner names each job's
|
||||||
|
# containers GITEA-ACTIONS-TASK-<task id> and creates a network per job, so they do not
|
||||||
|
# collide. Workflow `concurrency:` groups are enforced by Gitea server-side and are
|
||||||
|
# unaffected: gate.yml's gate-<ref> group still cancels a superseded run on the SAME ref,
|
||||||
|
# and renovate's group still serialises renovate. The dind resources are sized for TWO jobs.
|
||||||
|
capacity: 2
|
||||||
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
@@ -95,6 +131,10 @@ spec:
|
|||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/name: act-runner
|
app.kubernetes.io/name: act-runner
|
||||||
|
annotations:
|
||||||
|
# Bump when act-runner-config changes without any other pod-template change, so the
|
||||||
|
# Deployment rolls and act_runner re-reads its config (see the ConfigMap header).
|
||||||
|
homelab.keithjackson.dev/act-runner-config-revision: "1" # capacity 2
|
||||||
spec:
|
spec:
|
||||||
# CI holds no cluster credentials — do not mount a SA token into the runner or
|
# CI holds no cluster credentials — do not mount a SA token into the runner or
|
||||||
# (via the shared netns) the job containers (R-SEC-02, docs/14).
|
# (via the shared netns) the job containers (R-SEC-02, docs/14).
|
||||||
@@ -146,12 +186,23 @@ spec:
|
|||||||
mountPath: /certs
|
mountPath: /certs
|
||||||
- name: dind-storage
|
- name: dind-storage
|
||||||
mountPath: /var/lib/docker
|
mountPath: /var/lib/docker
|
||||||
|
# Sized for runner.capacity: 2. Every job's containers run UNDER this daemon, so this
|
||||||
|
# container's cgroup carries the whole CI workload. Measured 2026-11-08 (Prometheus,
|
||||||
|
# capacity 1): peak CPU 0.69 core (1m rate, 30d), peak working set 447Mi (30d), 0 OOMs.
|
||||||
|
# cpu request 2: two jobs at today's peak is ~1.4 core, and the gate is being
|
||||||
|
# parallelised, so per-job demand rises. The request is what the CFS share is
|
||||||
|
# weighted by under contention (100m was a 1/20 share); NO cpu limit, deliberately,
|
||||||
|
# so an idle node's spare cores are still usable. k3s-prod-2 (8 CPU) had 2945m
|
||||||
|
# requested; +1.9 core = ~4.85 (61%), and prod-1/prod-3 fit too if it reschedules.
|
||||||
|
# memory request 1Gi / limit 4Gi: two jobs at today's peak is ~0.9Gi. An OOM kill here
|
||||||
|
# takes down dockerd and with it BOTH running jobs, so the limit carries ~4x headroom
|
||||||
|
# over that for the parallelised gate. Nodes have ~31Gi allocatable.
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 100m
|
cpu: "2"
|
||||||
memory: 256Mi
|
memory: 1Gi
|
||||||
limits:
|
limits:
|
||||||
memory: 2Gi
|
memory: 4Gi
|
||||||
startupProbe:
|
startupProbe:
|
||||||
exec:
|
exec:
|
||||||
command:
|
command:
|
||||||
@@ -195,23 +246,36 @@ spec:
|
|||||||
value: /certs/client
|
value: /certs/client
|
||||||
- name: DOCKER_TLS_VERIFY
|
- name: DOCKER_TLS_VERIFY
|
||||||
value: "1"
|
value: "1"
|
||||||
|
# The image's run.sh passes `--config <this>` to BOTH `act_runner register` and
|
||||||
|
# `act_runner daemon` when CONFIG_FILE is set (act_runner v0.6.1 scripts/run.sh).
|
||||||
|
- name: CONFIG_FILE
|
||||||
|
value: /etc/act_runner/config.yaml
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: data
|
- name: data
|
||||||
mountPath: /data
|
mountPath: /data
|
||||||
- name: certs
|
- name: certs
|
||||||
mountPath: /certs
|
mountPath: /certs
|
||||||
readOnly: true
|
readOnly: true
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/act_runner
|
||||||
|
readOnly: true
|
||||||
|
# Measured 2026-11-08 at capacity 1: peak CPU 0.24 core, peak working set 314Mi (30d)
|
||||||
|
# against the old 512Mi limit. It holds per-task state, log buffers and the embedded
|
||||||
|
# actions/cache server for every concurrent task, so the limit is doubled for capacity 2.
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 100m
|
cpu: 100m
|
||||||
memory: 128Mi
|
memory: 256Mi
|
||||||
limits:
|
limits:
|
||||||
memory: 512Mi
|
memory: 1Gi
|
||||||
volumes:
|
volumes:
|
||||||
- name: data
|
- name: data
|
||||||
persistentVolumeClaim:
|
persistentVolumeClaim:
|
||||||
claimName: act-runner-data
|
claimName: act-runner-data
|
||||||
- name: certs
|
- name: certs
|
||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: act-runner-config
|
||||||
- name: dind-storage
|
- name: dind-storage
|
||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
@@ -138,7 +138,7 @@ spec:
|
|||||||
- name: media
|
- name: media
|
||||||
mountPath: /media
|
mountPath: /media
|
||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
@@ -138,7 +138,7 @@ spec:
|
|||||||
- name: media
|
- name: media
|
||||||
mountPath: /media
|
mountPath: /media
|
||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
+1
-1
@@ -129,7 +129,7 @@ AUTH_MODES = ("oidc", "forward-auth", "native", "none")
|
|||||||
# the sidecar authenticates the browser against Keycloak (homelab realm, the app's auth.groups)
|
# the sidecar authenticates the browser against Keycloak (homelab realm, the app's auth.groups)
|
||||||
# and only THEN proxies to the co-located app UI at http://127.0.0.1:<uiport>. The gate is IN
|
# and only THEN proxies to the co-located app UI at http://127.0.0.1:<uiport>. The gate is IN
|
||||||
# the data path (fail-closed: no valid session in an authorized group => no upstream; R-IDP-02).
|
# the data path (fail-closed: no valid session in an authorized group => no upstream; R-IDP-02).
|
||||||
OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.8.1"
|
OAUTH2_PROXY_IMAGE = "quay.io/oauth2-proxy/oauth2-proxy:v7.15.6"
|
||||||
OAUTH2_PROXY_PORT = 4180
|
OAUTH2_PROXY_PORT = 4180
|
||||||
KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab"
|
KEYCLOAK_ISSUER = f"https://keycloak.{ZONE}/realms/homelab"
|
||||||
|
|
||||||
|
|||||||
+38
-2
@@ -25,8 +25,44 @@ if [ "${#files[@]}" -eq 0 ]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
yamllint -c "${CI_DIR}/yamllint.yaml" -f "${YAMLLINT_FORMAT:-standard}" "${files[@]}"
|
# yamllint is single-threaded, so the file list is split into LINT_JOBS CONTIGUOUS slices of the
|
||||||
ok "yaml/lint passed (${#files[@]} files)"
|
# sorted list, one yamllint per slice, run concurrently. Each slice's output goes to its own file
|
||||||
|
# and the files are printed in slice order afterwards: the log is the same text, in the same
|
||||||
|
# file order, as one serial `yamllint "${files[@]}"`, and a finding is never interleaved with
|
||||||
|
# another process's output (yamllint's standard format puts the FILE NAME on its own line above
|
||||||
|
# that file's `line:col` findings, so interleaving would misattribute them). The stage's exit
|
||||||
|
# code is the highest any slice returned (yamllint: 1 = error, 2 = warning under strict), so a
|
||||||
|
# single failing slice fails the stage exactly as the serial call did. LINT_JOBS=1 is serial.
|
||||||
|
lint_jobs="${LINT_JOBS:-$(n="$(nproc 2>/dev/null || echo 1)"; [ "$n" -gt 8 ] && n=8; echo "$n")}"
|
||||||
|
case "$lint_jobs" in ''|*[!0-9]*|0) err "LINT_JOBS must be a positive integer, got '${lint_jobs}'"; exit 2 ;; esac
|
||||||
|
[ "$lint_jobs" -gt "${#files[@]}" ] && lint_jobs="${#files[@]}"
|
||||||
|
lint_tmp="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$lint_tmp"' EXIT
|
||||||
|
per=$(( (${#files[@]} + lint_jobs - 1) / lint_jobs ))
|
||||||
|
pids=()
|
||||||
|
for (( i = 0; i < lint_jobs; i++ )); do
|
||||||
|
slice=("${files[@]:$(( i * per )):${per}}")
|
||||||
|
[ "${#slice[@]}" -gt 0 ] || break
|
||||||
|
( rc=0
|
||||||
|
yamllint -c "${CI_DIR}/yamllint.yaml" -f "${YAMLLINT_FORMAT:-standard}" "${slice[@]}" \
|
||||||
|
>"${lint_tmp}/${i}.out" 2>&1 || rc=$?
|
||||||
|
echo "$rc" >"${lint_tmp}/${i}.rc" ) &
|
||||||
|
pids+=("$!")
|
||||||
|
done
|
||||||
|
wait "${pids[@]}" || true
|
||||||
|
lint_rc=0
|
||||||
|
for (( i = 0; i < ${#pids[@]}; i++ )); do
|
||||||
|
cat "${lint_tmp}/${i}.out"
|
||||||
|
# A slice that left no rc file died before yamllint returned: that is a failure, not a pass.
|
||||||
|
slice_rc="$(cat "${lint_tmp}/${i}.rc" 2>/dev/null || true)"
|
||||||
|
case "$slice_rc" in ''|*[!0-9]*) slice_rc=1 ;; esac
|
||||||
|
[ "$slice_rc" -gt "$lint_rc" ] && lint_rc="$slice_rc"
|
||||||
|
done
|
||||||
|
if [ "$lint_rc" -ne 0 ]; then
|
||||||
|
err "yaml/lint failed (yamllint exit ${lint_rc}; ${#files[@]} files in ${#pids[@]} parallel slice(s))"
|
||||||
|
exit "$lint_rc"
|
||||||
|
fi
|
||||||
|
ok "yaml/lint passed (${#files[@]} files, ${#pids[@]} parallel slice(s))"
|
||||||
|
|
||||||
# A workflow's `configurationFile:` must name a file that exists. Renovate's did not after #79
|
# A workflow's `configurationFile:` must name a file that exists. Renovate's did not after #79
|
||||||
# renamed renovate.json -> renovate.json5, and every scheduled run failed with ENOENT for 11 days
|
# renamed renovate.json -> renovate.json5, and every scheduled run failed with ENOENT for 11 days
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ Pure standard library (matches the harness's no-pip-dependency posture); the che
|
|||||||
themselves shell out (render checks to catalog/render.py, whose PyYAML dependency lives
|
themselves shell out (render checks to catalog/render.py, whose PyYAML dependency lives
|
||||||
there; L0 checks to git + PyYAML).
|
there; L0 checks to git + PyYAML).
|
||||||
|
|
||||||
Usage: run_l1_offline.py --junit <out.xml> [--label-prefix P] <test-dir> [<test-dir> ...]
|
Usage: run_l1_offline.py --junit <out.xml> [--label-prefix P] [--jobs N] <test-dir> [<test-dir> ...]
|
||||||
Exit 0 iff every selected offline check passes.
|
Exit 0 iff every selected offline check passes.
|
||||||
|
|
||||||
The per-dir check script is whichever of `render-check.sh` / `check.sh` the directory
|
The per-dir check script is whichever of `render-check.sh` / `check.sh` the directory
|
||||||
@@ -43,13 +43,35 @@ Exit codes: 0 all selected checks passed / 1 at least one FAILED / 3 nothing fai
|
|||||||
scope-declared check was UNWIRED. conformance/run.sh propagates 3 verbatim; ci/l0.sh turns it
|
scope-declared check was UNWIRED. conformance/run.sh propagates 3 verbatim; ci/l0.sh turns it
|
||||||
into the gate's own rc 3. This is the ONE place the "unwired, not green" rule lives now,
|
into the gate's own rc 3. This is the ONE place the "unwired, not green" rule lives now,
|
||||||
instead of once per hand-written stage.
|
instead of once per hand-written stage.
|
||||||
|
|
||||||
|
PARALLELISM (--jobs, env CONF_OFFLINE_JOBS)
|
||||||
|
-------------------------------------------
|
||||||
|
The checks are independent processes, so they run in a bounded thread pool (one `bash` child
|
||||||
|
per worker). Default: min(8, usable CPUs); `--jobs 1` is the old serial runner exactly. Each
|
||||||
|
check's stdout+stderr is CAPTURED and only reported after every check has finished, in the
|
||||||
|
ORDER THE DIRS WERE GIVEN, so the log, the JUnit and the step summary are byte-for-byte
|
||||||
|
independent of which check finished first (only the trailing "[N.Ns]" wall-time on each
|
||||||
|
status line varies). The 0/1/3 aggregation runs over the same ordered case list as before.
|
||||||
|
|
||||||
|
Parallel-safety is a property of the CHECKS, not of this runner: every check must keep its
|
||||||
|
scratch state in a private mktemp/mkdtemp dir and must not write into the tree or a fixed
|
||||||
|
path. The audit that established that for the current set is recorded in
|
||||||
|
decisions/2026-11-08-parallel-l0-checks.md; a new check that needs shared state must say so.
|
||||||
|
|
||||||
|
Nested pools: the keycloak reconciler checks run their own mutation suites in a pool
|
||||||
|
(mutations.py, env MUTATION_JOBS). When this runner is itself parallel it exports a SMALLER
|
||||||
|
MUTATION_JOBS (unless the caller already set one), so the two suites share the CPUs the
|
||||||
|
other jobs-2 checks are not using instead of each taking min(8, CPUs). See inner_jobs().
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from xml.dom import minidom
|
from xml.dom import minidom
|
||||||
@@ -97,6 +119,46 @@ def collect_meta(test_yaml: Path, prefix: str):
|
|||||||
return name, reqs
|
return name, reqs
|
||||||
|
|
||||||
|
|
||||||
|
def usable_cpus() -> int:
|
||||||
|
"""CPUs this process may run on (the cgroup/affinity view, not the host's count)."""
|
||||||
|
try:
|
||||||
|
return max(1, len(os.sched_getaffinity(0)))
|
||||||
|
except (AttributeError, OSError):
|
||||||
|
return max(1, os.cpu_count() or 1)
|
||||||
|
|
||||||
|
|
||||||
|
def positive_int(text: str) -> int:
|
||||||
|
n = int(text)
|
||||||
|
if n < 1:
|
||||||
|
raise argparse.ArgumentTypeError(f"must be >= 1, got {n}")
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def inner_jobs(cpus: int, outer: int) -> int:
|
||||||
|
"""MUTATION_JOBS to hand the checks when this runner runs `outer` of them at once.
|
||||||
|
|
||||||
|
Two checks (the keycloak client-secret and passkey reconcilers) fan out internally; the
|
||||||
|
other outer-2 workers are single processes. Give the two suites an equal share of the CPUs
|
||||||
|
those others leave free, never fewer than 2 each (a 1-wide pool would make the 40-mutation
|
||||||
|
passkey suite the critical path of the whole stage), never more than 8. Peak process count
|
||||||
|
is then about (outer-2) + 2*inner, i.e. <= cpus on any machine with cpus >= outer + 2, and
|
||||||
|
at worst ~1.25-1.5x cpus on a small runner — CPU-bound oversubscription of that size costs
|
||||||
|
latency per process, not throughput.
|
||||||
|
"""
|
||||||
|
if outer <= 1:
|
||||||
|
return min(8, cpus)
|
||||||
|
return max(2, min(8, (cpus - outer + 2) // 2))
|
||||||
|
|
||||||
|
|
||||||
|
def run_check(check: Path, env: dict) -> tuple[int, str, float]:
|
||||||
|
t0 = time.monotonic()
|
||||||
|
proc = subprocess.run(
|
||||||
|
["bash", str(check)], capture_output=True, text=True,
|
||||||
|
stdin=subprocess.DEVNULL, env=env,
|
||||||
|
)
|
||||||
|
return proc.returncode, (proc.stdout + proc.stderr).strip(), time.monotonic() - t0
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
ap = argparse.ArgumentParser(description=__doc__)
|
ap = argparse.ArgumentParser(description=__doc__)
|
||||||
ap.add_argument("--junit", type=Path, required=True)
|
ap.add_argument("--junit", type=Path, required=True)
|
||||||
@@ -108,26 +170,54 @@ def main() -> int:
|
|||||||
"(makes this runner exit 3); repeatable")
|
"(makes this runner exit 3); repeatable")
|
||||||
ap.add_argument("--summary-out", type=Path, default=None,
|
ap.add_argument("--summary-out", type=Path, default=None,
|
||||||
help="write one Markdown line per case here, for the CI step summary")
|
help="write one Markdown line per case here, for the CI step summary")
|
||||||
|
ap.add_argument("--jobs", type=positive_int, default=None,
|
||||||
|
help="checks to run at once (default: env CONF_OFFLINE_JOBS, else "
|
||||||
|
"min(8, usable CPUs)); 1 = serial")
|
||||||
ap.add_argument("dirs", nargs="+", type=Path)
|
ap.add_argument("dirs", nargs="+", type=Path)
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
strict_scopes = set(args.unwired_scope)
|
strict_scopes = set(args.unwired_scope)
|
||||||
|
|
||||||
cases = [] # (name, reqs, dir, rc, output, scope)
|
cpus = usable_cpus()
|
||||||
|
if args.jobs is not None:
|
||||||
|
jobs = args.jobs
|
||||||
|
elif os.environ.get("CONF_OFFLINE_JOBS", "").strip():
|
||||||
|
try:
|
||||||
|
jobs = positive_int(os.environ["CONF_OFFLINE_JOBS"].strip())
|
||||||
|
except (ValueError, argparse.ArgumentTypeError) as e:
|
||||||
|
ap.error(f"CONF_OFFLINE_JOBS={os.environ['CONF_OFFLINE_JOBS']!r}: {e}")
|
||||||
|
else:
|
||||||
|
jobs = min(8, cpus)
|
||||||
|
child_env = dict(os.environ)
|
||||||
|
if not child_env.get("MUTATION_JOBS", "").strip():
|
||||||
|
child_env["MUTATION_JOBS"] = str(inner_jobs(cpus, jobs))
|
||||||
|
print(f"[offline] running {len(args.dirs)} check(s), jobs={jobs} "
|
||||||
|
f"(cpus={cpus}, MUTATION_JOBS={child_env['MUTATION_JOBS']})", file=sys.stderr)
|
||||||
|
|
||||||
|
# Metadata first, in the given order; then the checks themselves through the pool. The
|
||||||
|
# results are consumed in SUBMISSION order, so `cases` is identical to the serial runner's.
|
||||||
|
planned = [] # (name, reqs, dir, scope, check-or-None)
|
||||||
for d in args.dirs:
|
for d in args.dirs:
|
||||||
check = next((d / s for s in CHECK_SCRIPTS if (d / s).exists()), None)
|
check = next((d / s for s in CHECK_SCRIPTS if (d / s).exists()), None)
|
||||||
name, reqs = collect_meta(d / "chainsaw-test.yaml", args.label_prefix)
|
name, reqs = collect_meta(d / "chainsaw-test.yaml", args.label_prefix)
|
||||||
scope = scope_of(d / "chainsaw-test.yaml", args.scope_annotation)
|
scope = scope_of(d / "chainsaw-test.yaml", args.scope_annotation)
|
||||||
if check is None:
|
planned.append((name, reqs, d, scope, check))
|
||||||
cases.append((name, reqs, d, 3,
|
|
||||||
f"no offline check script ({' or '.join(CHECK_SCRIPTS)}) in {d}",
|
cases = [] # (name, reqs, dir, rc, output, scope)
|
||||||
scope))
|
elapsed = [] # wall seconds per case, same order (None: nothing ran)
|
||||||
continue
|
with ThreadPoolExecutor(max_workers=jobs) as pool:
|
||||||
proc = subprocess.run(
|
futures = [pool.submit(run_check, check, child_env) if check is not None else None
|
||||||
["bash", str(check)], capture_output=True, text=True
|
for (_n, _r, _d, _s, check) in planned]
|
||||||
)
|
for (name, reqs, d, scope, check), fut in zip(planned, futures):
|
||||||
out = (proc.stdout + proc.stderr).strip()
|
if fut is None:
|
||||||
cases.append((name, reqs, d, proc.returncode, out, scope))
|
cases.append((name, reqs, d, 3,
|
||||||
|
f"no offline check script ({' or '.join(CHECK_SCRIPTS)}) in {d}",
|
||||||
|
scope))
|
||||||
|
elapsed.append(None)
|
||||||
|
continue
|
||||||
|
rc, out, secs = fut.result()
|
||||||
|
cases.append((name, reqs, d, rc, out, scope))
|
||||||
|
elapsed.append(secs)
|
||||||
|
|
||||||
failures = sum(1 for c in cases if c[3] not in (0, 3))
|
failures = sum(1 for c in cases if c[3] not in (0, 3))
|
||||||
skipped = sum(1 for c in cases if c[3] == 3)
|
skipped = sum(1 for c in cases if c[3] == 3)
|
||||||
@@ -145,7 +235,7 @@ def main() -> int:
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
summary_lines = []
|
summary_lines = []
|
||||||
for name, reqs, d, rc, out, scope in cases:
|
for (name, reqs, d, rc, out, scope), secs in zip(cases, elapsed):
|
||||||
tc = ET.SubElement(
|
tc = ET.SubElement(
|
||||||
suite,
|
suite,
|
||||||
"testcase",
|
"testcase",
|
||||||
@@ -164,8 +254,9 @@ def main() -> int:
|
|||||||
fail.text = out
|
fail.text = out
|
||||||
status = "PASS" if rc == 0 else ("SKIP" if rc == 3 else "FAIL")
|
status = "PASS" if rc == 0 else ("SKIP" if rc == 3 else "FAIL")
|
||||||
mark = f" scope={scope}" if scope else ""
|
mark = f" scope={scope}" if scope else ""
|
||||||
|
took = f" [{secs:.1f}s]" if secs is not None else ""
|
||||||
print(f"[offline] {layer_of(d):3s} {status:4s} {name} "
|
print(f"[offline] {layer_of(d):3s} {status:4s} {name} "
|
||||||
f"({','.join(reqs) or 'untagged'}){mark}",
|
f"({','.join(reqs) or 'untagged'}){mark}{took}",
|
||||||
file=sys.stderr)
|
file=sys.stderr)
|
||||||
if rc not in (0, 3):
|
if rc not in (0, 3):
|
||||||
print(out, file=sys.stderr)
|
print(out, file=sys.stderr)
|
||||||
|
|||||||
@@ -25,15 +25,19 @@
|
|||||||
#
|
#
|
||||||
# python3 mutations.py <path-to-extracted-reconciler.py>
|
# python3 mutations.py <path-to-extracted-reconciler.py>
|
||||||
#
|
#
|
||||||
# Runtime is ~20s (one subprocess per mutation, each running the full selftest
|
# Runtime is ~11s serial (one subprocess per mutation, each running the full selftest
|
||||||
# against an in-process stub Keycloak; no network, no cluster).
|
# against an in-process stub Keycloak; no network, no cluster). The selftests run in a
|
||||||
|
# bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and
|
||||||
|
# verdicts are independent of the pool size.
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
@@ -136,43 +140,99 @@ MUTATIONS: list[tuple[str, str, str, str]] = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def mutation_jobs() -> int:
|
||||||
|
"""Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs).
|
||||||
|
|
||||||
|
conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself
|
||||||
|
running several checks in parallel, so the nested pools do not oversubscribe the host.
|
||||||
|
MUTATION_JOBS=1 reproduces the old serial loop exactly.
|
||||||
|
"""
|
||||||
|
raw = os.environ.get("MUTATION_JOBS", "").strip()
|
||||||
|
if raw:
|
||||||
|
try:
|
||||||
|
n = int(raw)
|
||||||
|
except ValueError:
|
||||||
|
n = 0
|
||||||
|
if n < 1:
|
||||||
|
raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw)
|
||||||
|
return n
|
||||||
|
try:
|
||||||
|
cpus = len(os.sched_getaffinity(0))
|
||||||
|
except (AttributeError, OSError):
|
||||||
|
cpus = os.cpu_count() or 1
|
||||||
|
return max(1, min(8, cpus))
|
||||||
|
|
||||||
|
|
||||||
|
def run_selftest(mutated_src: str) -> subprocess.CompletedProcess:
|
||||||
|
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||||
|
fh.write(mutated_src)
|
||||||
|
path = fh.name
|
||||||
|
try:
|
||||||
|
return subprocess.run([sys.executable, path, "--selftest"],
|
||||||
|
capture_output=True, text=True, timeout=240,
|
||||||
|
stdin=subprocess.DEVNULL)
|
||||||
|
finally:
|
||||||
|
Path(path).unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
if len(sys.argv) != 2:
|
if len(sys.argv) != 2:
|
||||||
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
src = Path(sys.argv[1]).read_text()
|
src = Path(sys.argv[1]).read_text()
|
||||||
|
|
||||||
problems: list[str] = []
|
try:
|
||||||
killed = 0
|
jobs = mutation_jobs()
|
||||||
|
except ValueError as e:
|
||||||
|
print("mutations.py: %s" % e, file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
# Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the
|
||||||
|
# serial loop did, and each mutation's source is built here. None = anchor failure.
|
||||||
|
planned: list[tuple[str, str, str | None, str | None]] = []
|
||||||
for name, why, old, new in MUTATIONS:
|
for name, why, old, new in MUTATIONS:
|
||||||
old, new = dedent4(old), dedent4(new)
|
old, new = dedent4(old), dedent4(new)
|
||||||
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
||||||
# moved and this mutation silently stopped proving anything.
|
# moved and this mutation silently stopped proving anything.
|
||||||
n = src.count(old)
|
n = src.count(old)
|
||||||
if n != 1:
|
if n != 1:
|
||||||
problems.append("%s: anchor matches %d times (expected exactly 1) -- the "
|
planned.append((name, why,
|
||||||
|
"%s: anchor matches %d times (expected exactly 1) -- the "
|
||||||
"code moved and this mutation no longer proves anything. "
|
"code moved and this mutation no longer proves anything. "
|
||||||
"Re-anchor it against the current source." % (name, n))
|
"Re-anchor it against the current source." % (name, n), None))
|
||||||
continue
|
continue
|
||||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
planned.append((name, why, None, src.replace(old, new, 1)))
|
||||||
fh.write(src.replace(old, new, 1))
|
|
||||||
path = fh.name
|
# Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own
|
||||||
try:
|
# mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are
|
||||||
proc = subprocess.run([sys.executable, path, "--selftest"],
|
# consumed in MUTATIONS order, so every line below is printed in the same order, with the
|
||||||
capture_output=True, text=True, timeout=240)
|
# same verdict, as the serial loop produced -- whichever child finishes first.
|
||||||
finally:
|
problems: list[str] = []
|
||||||
Path(path).unlink(missing_ok=True)
|
killed = 0
|
||||||
if proc.returncode == 0:
|
pool = ThreadPoolExecutor(max_workers=jobs)
|
||||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
try:
|
||||||
"applied, so nothing in the suite actually checks it. The "
|
futures = [pool.submit(run_selftest, mutated) if mutated is not None else None
|
||||||
"mistake it models: %s" % (name, why))
|
for (_n, _w, _a, mutated) in planned]
|
||||||
continue
|
for (name, why, anchor_problem, _m), fut in zip(planned, futures):
|
||||||
killed += 1
|
if anchor_problem is not None:
|
||||||
out = proc.stdout + proc.stderr
|
problems.append(anchor_problem)
|
||||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
continue
|
||||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
proc = fut.result()
|
||||||
else "the selftest aborted (caught, less cleanly)")
|
if proc.returncode == 0:
|
||||||
print(" killed %-42s %s" % (name, how))
|
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||||
|
"applied, so nothing in the suite actually checks it. The "
|
||||||
|
"mistake it models: %s" % (name, why))
|
||||||
|
continue
|
||||||
|
killed += 1
|
||||||
|
out = proc.stdout + proc.stderr
|
||||||
|
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||||
|
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||||
|
else "the selftest aborted (caught, less cleanly)")
|
||||||
|
print(" killed %-42s %s" % (name, how))
|
||||||
|
finally:
|
||||||
|
# A timeout (or any other exception) still propagates and fails the check, as it did
|
||||||
|
# serially; queued mutations are cancelled rather than run for nothing.
|
||||||
|
pool.shutdown(wait=True, cancel_futures=True)
|
||||||
|
|
||||||
print()
|
print()
|
||||||
if problems:
|
if problems:
|
||||||
|
|||||||
@@ -40,15 +40,19 @@
|
|||||||
#
|
#
|
||||||
# python3 mutations.py <path-to-extracted-reconciler.py>
|
# python3 mutations.py <path-to-extracted-reconciler.py>
|
||||||
#
|
#
|
||||||
# Runtime is ~60s (one subprocess per mutation, each running the full selftest
|
# Runtime is ~85s serial (one subprocess per mutation, each running the full selftest
|
||||||
# against an in-process stub Keycloak; no network, no cluster).
|
# against an in-process stub Keycloak; no network, no cluster). The selftests run in a
|
||||||
|
# bounded pool (env MUTATION_JOBS, default min(8, CPUs); 1 = serial); output order and
|
||||||
|
# verdicts are independent of the pool size.
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# (name, the mistake being modelled, anchor, replacement)
|
# (name, the mistake being modelled, anchor, replacement)
|
||||||
@@ -403,42 +407,98 @@ MUTATIONS: list[tuple[str, str, str, str]] = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def mutation_jobs() -> int:
|
||||||
|
"""Selftests to run at once: env MUTATION_JOBS, else min(8, usable CPUs).
|
||||||
|
|
||||||
|
conformance/lib/run_l1_offline.py exports a smaller MUTATION_JOBS when it is itself
|
||||||
|
running several checks in parallel, so the nested pools do not oversubscribe the host.
|
||||||
|
MUTATION_JOBS=1 reproduces the old serial loop exactly.
|
||||||
|
"""
|
||||||
|
raw = os.environ.get("MUTATION_JOBS", "").strip()
|
||||||
|
if raw:
|
||||||
|
try:
|
||||||
|
n = int(raw)
|
||||||
|
except ValueError:
|
||||||
|
n = 0
|
||||||
|
if n < 1:
|
||||||
|
raise ValueError("MUTATION_JOBS must be a positive integer, got %r" % raw)
|
||||||
|
return n
|
||||||
|
try:
|
||||||
|
cpus = len(os.sched_getaffinity(0))
|
||||||
|
except (AttributeError, OSError):
|
||||||
|
cpus = os.cpu_count() or 1
|
||||||
|
return max(1, min(8, cpus))
|
||||||
|
|
||||||
|
|
||||||
|
def run_selftest(mutated_src: str) -> subprocess.CompletedProcess:
|
||||||
|
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
||||||
|
fh.write(mutated_src)
|
||||||
|
path = fh.name
|
||||||
|
try:
|
||||||
|
return subprocess.run([sys.executable, path, "--selftest"],
|
||||||
|
capture_output=True, text=True, timeout=300,
|
||||||
|
stdin=subprocess.DEVNULL)
|
||||||
|
finally:
|
||||||
|
Path(path).unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
if len(sys.argv) != 2:
|
if len(sys.argv) != 2:
|
||||||
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
print("usage: mutations.py <extracted-reconciler.py>", file=sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
src = Path(sys.argv[1]).read_text()
|
src = Path(sys.argv[1]).read_text()
|
||||||
|
|
||||||
problems: list[str] = []
|
try:
|
||||||
killed = 0
|
jobs = mutation_jobs()
|
||||||
|
except ValueError as e:
|
||||||
|
print("mutations.py: %s" % e, file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
# Pass 1, serial and cheap: every anchor is checked BEFORE anything runs, exactly as the
|
||||||
|
# serial loop did, and each mutation's source is built here. None = anchor failure.
|
||||||
|
planned: list[tuple[str, str, str | None, str | None]] = []
|
||||||
for name, why, old, new in MUTATIONS:
|
for name, why, old, new in MUTATIONS:
|
||||||
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
# An anchor that no longer matches is a FAILURE, not a skip: it means the code
|
||||||
# moved and this mutation silently stopped proving anything.
|
# moved and this mutation silently stopped proving anything.
|
||||||
n = src.count(old)
|
n = src.count(old)
|
||||||
if n != 1:
|
if n != 1:
|
||||||
problems.append("%s: anchor matches %d times (expected exactly 1) -- the "
|
planned.append((name, why,
|
||||||
|
"%s: anchor matches %d times (expected exactly 1) -- the "
|
||||||
"code moved and this mutation no longer proves anything. "
|
"code moved and this mutation no longer proves anything. "
|
||||||
"Re-anchor it against the current source." % (name, n))
|
"Re-anchor it against the current source." % (name, n), None))
|
||||||
continue
|
continue
|
||||||
with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as fh:
|
planned.append((name, why, None, src.replace(old, new, 1)))
|
||||||
fh.write(src.replace(old, new, 1))
|
|
||||||
path = fh.name
|
# Pass 2: the selftests, `jobs` at a time. Each child gets its own temp file, its own
|
||||||
try:
|
# mkdtemp dirs and its own 127.0.0.1:0 stub port, so they share nothing. Results are
|
||||||
proc = subprocess.run([sys.executable, path, "--selftest"],
|
# consumed in MUTATIONS order, so every line below is printed in the same order, with the
|
||||||
capture_output=True, text=True, timeout=300)
|
# same verdict, as the serial loop produced -- whichever child finishes first.
|
||||||
finally:
|
problems: list[str] = []
|
||||||
Path(path).unlink(missing_ok=True)
|
killed = 0
|
||||||
if proc.returncode == 0:
|
pool = ThreadPoolExecutor(max_workers=jobs)
|
||||||
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
try:
|
||||||
"applied, so nothing in the suite actually checks it. The "
|
futures = [pool.submit(run_selftest, mutated) if mutated is not None else None
|
||||||
"mistake it models: %s" % (name, why))
|
for (_n, _w, _a, mutated) in planned]
|
||||||
continue
|
for (name, why, anchor_problem, _m), fut in zip(planned, futures):
|
||||||
killed += 1
|
if anchor_problem is not None:
|
||||||
out = proc.stdout + proc.stderr
|
problems.append(anchor_problem)
|
||||||
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
continue
|
||||||
how = ("%s assertion(s) caught it" % m.group(1) if m
|
proc = fut.result()
|
||||||
else "the selftest aborted (caught, less cleanly)")
|
if proc.returncode == 0:
|
||||||
print(" killed %-54s %s" % (name, how))
|
problems.append("%s SURVIVED -- --selftest still passed with this regression "
|
||||||
|
"applied, so nothing in the suite actually checks it. The "
|
||||||
|
"mistake it models: %s" % (name, why))
|
||||||
|
continue
|
||||||
|
killed += 1
|
||||||
|
out = proc.stdout + proc.stderr
|
||||||
|
m = re.search(r"SELFTEST FAIL: (\d+) of \d+", out)
|
||||||
|
how = ("%s assertion(s) caught it" % m.group(1) if m
|
||||||
|
else "the selftest aborted (caught, less cleanly)")
|
||||||
|
print(" killed %-54s %s" % (name, how))
|
||||||
|
finally:
|
||||||
|
# A timeout (or any other exception) still propagates and fails the check, as it did
|
||||||
|
# serially; queued mutations are cancelled rather than run for nothing.
|
||||||
|
pool.shutdown(wait=True, cancel_futures=True)
|
||||||
|
|
||||||
print()
|
print()
|
||||||
if problems:
|
if problems:
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ spec:
|
|||||||
ports:
|
ports:
|
||||||
- containerPort: 8080
|
- containerPort: 8080
|
||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
+1
-1
@@ -85,7 +85,7 @@ spec:
|
|||||||
- name: config
|
- name: config
|
||||||
mountPath: /data/config
|
mountPath: /data/config
|
||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ spec:
|
|||||||
- name: config
|
- name: config
|
||||||
mountPath: /data/config
|
mountPath: /data/config
|
||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
---
|
||||||
|
date: 2026-11-08
|
||||||
|
branch: fix/keycloak-operator-crds
|
||||||
|
title: "Keycloak 26.8.2 operator CrashLoop: install its two new CRDs, raise its memory"
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
**Incident.** PR #121 (Renovate) bumped `platform/keycloak/kustomization.yaml`'s three
|
||||||
|
`keycloak-k8s-resources` URLs 26.0.6 -> 26.8.2. Upstream `kubernetes/` at 26.8.2 holds **four**
|
||||||
|
CRDs, not two (GitHub contents API, `?ref=26.8.2`):
|
||||||
|
|
||||||
|
| file | at 26.0.6 | at 26.8.2 |
|
||||||
|
|---|---|---|
|
||||||
|
| `keycloaks.k8s.keycloak.org-v1.yml` | yes | yes |
|
||||||
|
| `keycloakrealmimports.k8s.keycloak.org-v1.yml` | yes | yes |
|
||||||
|
| `keycloakoidcclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
|
||||||
|
| `keycloaksamlclients.k8s.keycloak.org-v1.yml` | — | **new** (v2alpha1) |
|
||||||
|
| `kubernetes.yml` | yes | yes (now carries RBAC for both new kinds) |
|
||||||
|
|
||||||
|
(`cluster-wide/` is a separate variant; upstream's own `kubernetes/kustomization.yml` lists exactly
|
||||||
|
the five files above.) The 26.8.2 operator starts an informer for every controller it has and dies
|
||||||
|
on the first missing kind: `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1
|
||||||
|
... GET .../namespaces/keycloak/keycloaksamlclients ... Not Found`, exit 1, CrashLoopBackOff. Flux
|
||||||
|
`platform` (`wait: true`, 5m) sat in health checks and every dependent Kustomization reported
|
||||||
|
`dependency ... is not ready`. The old 26.0.6 ReplicaSet kept serving (RollingUpdate 25%/25%), so SSO
|
||||||
|
stayed up.
|
||||||
|
|
||||||
|
**Fix.**
|
||||||
|
- `platform/keycloak/kustomization.yaml`: added the two CRD URLs, each with the same
|
||||||
|
`# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources` marker (the repo's
|
||||||
|
custom.regex manager matches all five — checked by running its regex over the file). Header
|
||||||
|
comment updated to v26.8.2, plus a warning that Renovate cannot add a CRD a release introduces.
|
||||||
|
- Same file: a `patches:` JSON6902 patch on Deployment `keycloak-operator`: memory limit
|
||||||
|
450Mi -> **768Mi**, request 450Mi -> **512Mi**. 450Mi is upstream's default (unchanged at 26.8.2,
|
||||||
|
so not something the bump changed). Evidence it is too small here: the 26.0.6 pod had **154
|
||||||
|
restarts**, `lastState.terminated.reason: OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and
|
||||||
|
`kubectl top` showed **421Mi / 450Mi**. 26.8.2 runs four controllers instead of two on Quarkus
|
||||||
|
3.40.1.1 (was 3.15.1). No variable in the operator to disable watching unused kinds was used —
|
||||||
|
installing the CRDs is upstream's supported shape.
|
||||||
|
- `platform/keycloak/README.md` "Upgrading Keycloak": the diff-the-CRD-list step; "three URLs" ->
|
||||||
|
"the release URLs". `renovate.json5`: same wording fix in a comment only.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- Upstream file lists at 26.0.6 and 26.8.2 (GitHub API) and `diff` of `kubernetes.yml` between the
|
||||||
|
two: the new kinds' ClusterRoles + RoleBindings ship in it, and the Deployment's resources are
|
||||||
|
still 450Mi/450Mi.
|
||||||
|
- Live, read-only (`kubectl --context k3s-prod`, 2026-11-08 ~05:57Z): only `keycloaks` and
|
||||||
|
`keycloakrealmimports` CRDs installed; Deployment strategy RollingUpdate maxSurge/maxUnavailable
|
||||||
|
25%; RS `keycloak-operator-58dff6c4` (26.8.2) 0/1 CrashLoopBackOff, RS `78dbb75f64` (26.0.6) 1/1;
|
||||||
|
operator log is the informer error above; Flux `platform` `Healthy=Unknown` "Running health
|
||||||
|
checks for revision main@sha1:e8d2666"; no NetworkPolicy and no CiliumNetworkPolicy in ns
|
||||||
|
`keycloak`.
|
||||||
|
- No sensitive path touched (`infra/gitea/approval-policy.json`), so this should merge on green.
|
||||||
|
- Gate: see the PR body for the exact rc and stages of the run against the final commit.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **That the 26.8.2 operator becomes Ready after this merges.** Only the two missing kinds were
|
||||||
|
observed failing; another start-up failure behind them is possible.
|
||||||
|
- **That 768Mi is enough** — no 26.8.2 memory measurement exists yet; re-check `kubectl top` after
|
||||||
|
a week.
|
||||||
|
- **The Keycloak server upgrade 26.0.6 -> 26.8.2 itself.** Once the operator runs it will roll
|
||||||
|
keycloak-0/1 and Keycloak migrates the DB schema. The CR's `spec.update.strategy` defaults to
|
||||||
|
`RecreateOnImageChange` — expect a **full SSO outage** while both pods recreate, not a rolling
|
||||||
|
one. Back up / confirm the latest CNPG backup first if that matters.
|
||||||
|
- **The 26.8.2 Keycloak CRD's `spec.networkPolicy.enabled` defaults to `true`.** The operator will
|
||||||
|
create a NetworkPolicy selecting the Keycloak pods (first policy in that namespace). With no
|
||||||
|
`from` lists set it allows all sources on the HTTP/management ports, so Gateway, ServiceMonitor
|
||||||
|
and the reconciler Jobs should still reach it — but this was read from the CRD text, not observed.
|
||||||
|
- **The passkey reconciler's 26.0.6 Admin API assumptions against 26.8.2** (README "Upgrading
|
||||||
|
Keycloak" list) — not re-checked here; PR #121 merged without it.
|
||||||
|
- `catalog/keycloak.yaml`'s documentation-only image tag was not updated.
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
---
|
||||||
|
date: 2026-11-08
|
||||||
|
branch: fix/keycloak-servicemonitor-owner
|
||||||
|
title: One owner for the keycloak ServiceMonitor — disable the 26.8 operator's (409 loop stalled platform-config)
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
- `platform/keycloak/keycloak-instance.yaml`: added `spec.serviceMonitor.enabled: false`.
|
||||||
|
The 26.x Keycloak operator creates its own ServiceMonitor, named after the CR (`keycloak/keycloak`),
|
||||||
|
whenever `metrics-enabled=true` and the ServiceMonitor CRD exists. `serviceMonitor.enabled` defaults
|
||||||
|
to `true` in the 26.8.2 CRD, both in v2alpha1 and in v2beta1 (v2beta1 is now the storage version).
|
||||||
|
That is the same name and namespace as the repo's Flux-owned `platform/keycloak/servicemonitor.yaml`.
|
||||||
|
After the 26.0.6 -> 26.8.2 bump (#121/#144) the operator's PATCH against it returned
|
||||||
|
`409 Conflict` and it ran out of retries. The CR then sat at `Ready=False`/`HasErrors=True`, and
|
||||||
|
`platform-config` (`wait: true`, which health-checks every object it applies, this CR included)
|
||||||
|
blocked everything that `dependsOn` it: velero, apps-vms, apps-wave-0 (and so waves 1/2 and
|
||||||
|
pm-agent), homepage, oauth2-proxy, headlamp, conformance, vpn-ingress, crafty, runbooks.
|
||||||
|
- `platform/keycloak/servicemonitor.yaml`: a comment saying it is the sole owner.
|
||||||
|
- **Why ours is kept:** these are written against it: the L0 `servicemonitor-ports-resolve`
|
||||||
|
allow-list entry `platform/keycloak/servicemonitor.yaml:keycloak`, the L2 `keycloak-monitored`
|
||||||
|
chainsaw test, the README's ownership table, and the dashboard's `up{namespace="keycloak"}`.
|
||||||
|
The operator's version (upstream `KeycloakServiceMonitorDependentResource` @26.8.2) scrapes the same
|
||||||
|
target: port name `management` (9000 on `keycloak-service`), path `/metrics`, interval 30s,
|
||||||
|
selector = the operator's instance labels. It offers nothing ours lacks, so having two owners
|
||||||
|
would add a fight and gain nothing.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- Live, read-only, `k3s-prod` ~06:25Z: the CRD schema field `spec.serviceMonitor.enabled`
|
||||||
|
(`default: true`) exists in both served versions. The live `keycloak` ServiceMonitor has exactly
|
||||||
|
**one** managedFields manager, `kustomize-controller` (Apply, 2026-09-05), **no ownerReferences**,
|
||||||
|
and labels `kustomize.toolkit.fluxcd.io/name=platform-config`, `release=kube-prometheus-stack`. So
|
||||||
|
the operator's apply never landed, and nothing on the object ties it to the Keycloak CR.
|
||||||
|
- The ServiceMonitor will not be deleted when the operator is switched off: the operator's dependent
|
||||||
|
informer is filtered on `Constants.DEFAULT_LABELS_AS_STRING`
|
||||||
|
(`app.kubernetes.io/managed-by=keycloak-operator`). Our object does not carry that label and has
|
||||||
|
no ownerReference, so the operator cannot see it as its secondary and cannot delete or GC it.
|
||||||
|
This conclusion comes from reading the code. It was not observed.
|
||||||
|
- The other operator dependents in ns `keycloak` do not collide with anything: `keycloak-service`,
|
||||||
|
`keycloak-discovery`, the StatefulSet `keycloak`, `keycloak-initial-admin`, and
|
||||||
|
`keycloak-network-policy` (owner Keycloak, no Flux label). The repo declares none of these names.
|
||||||
|
It only *references* `keycloak-service` by name (HTTPRoute, reconcilers). There are no PDBs in the
|
||||||
|
namespace, and the Ingress is already `enabled: false`. The only NetworkPolicy in the namespace is
|
||||||
|
the operator's. The repo has no Keycloak NetworkPolicy, and no Kyverno-generated default-deny is
|
||||||
|
present there. NetworkPolicies add up, so the operator's allow-policy cannot fight another owner.
|
||||||
|
Left at its default.
|
||||||
|
- Local gate: see the PR body. It records the rc and stages.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- That the CR reaches `Ready=True`/`HasErrors=False` after merge. The status also says
|
||||||
|
"Performing Keycloak upgrade, scaling down the deployment". I am inferring that this half clears
|
||||||
|
once the workflow stops erroring. It is not proven.
|
||||||
|
- That the operator stops issuing the PATCH once `enabled: false` is applied. Watch the operator
|
||||||
|
logs for 409.
|
||||||
|
- That `platform-config` and its downstream Kustomizations go Ready on the new revision, and that
|
||||||
|
Prometheus keeps exactly one `serviceMonitor/keycloak/keycloak/0` pool with targets up.
|
||||||
|
- The operator's actual label set for `DEFAULT_LABELS_AS_STRING`. I took it from the observed labels
|
||||||
|
on operator-created objects, not from `Constants.java`.
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
---
|
||||||
|
date: 2026-11-08
|
||||||
|
branch: feat/parallel-l0-checks
|
||||||
|
title: L0 offline checks, mutation suites and yamllint run in bounded parallel pools (gate 184s -> 32s locally)
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
The gate's wall time was dominated by serial loops over independent subprocesses. Three of
|
||||||
|
them now run in bounded pools; none of them changes a verdict, an exit code, or the order of
|
||||||
|
anything printed.
|
||||||
|
|
||||||
|
- `conformance/lib/run_l1_offline.py` — the per-check loop (L0 and L1 offline checks) runs in
|
||||||
|
a `ThreadPoolExecutor` of `bash <check>` children. `--jobs N` / env `CONF_OFFLINE_JOBS`,
|
||||||
|
default `min(8, usable CPUs)` (`sched_getaffinity`, so a cgroup cpuset is respected);
|
||||||
|
`--jobs 1` is the old serial runner. Results are consumed in SUBMISSION order, so `cases`,
|
||||||
|
the JUnit, the step summary and the 0/1/3 aggregation are built from exactly the same
|
||||||
|
ordered list as before. Additive output only: one `[offline] running N check(s), jobs=…`
|
||||||
|
line, and a trailing `[N.Ns]` wall time on each per-check status line. Children get
|
||||||
|
`stdin=/dev/null` (parallel children must not share the runner's stdin).
|
||||||
|
- `conformance/tests/l0-static/keycloak-{passkey,client-secret}-reconciler/mutations.py` —
|
||||||
|
pass 1 (serial) checks every anchor and builds each mutated source, exactly as before;
|
||||||
|
pass 2 runs the `--selftest` children in a pool (env `MUTATION_JOBS`, default
|
||||||
|
`min(8, CPUs)`, `1` = serial). Verdicts are consumed in `MUTATIONS` order. A timeout
|
||||||
|
(`TimeoutExpired`) still propagates and fails the check; queued mutations are cancelled.
|
||||||
|
An invalid `MUTATION_JOBS` exits 2 (fails the check, never skips it).
|
||||||
|
- **Nested-pool budget.** When the outer runner is parallel it exports
|
||||||
|
`MUTATION_JOBS = max(2, min(8, (cpus - jobs + 2) // 2))` unless the caller already set
|
||||||
|
one. Reason: only those two checks fan out; the other `jobs-2` workers are single
|
||||||
|
processes, so the two suites split the CPUs those leave free. Peak processes ≈
|
||||||
|
`(jobs-2) + 2*inner` ≤ cpus whenever cpus ≥ jobs+2 (24 CPUs: 6 + 16 = 22), and ~1.25–1.5×
|
||||||
|
cpus on a small runner (4 CPUs: 2 + 4 = 6). The floor of 2 is deliberate: a 1-wide inner
|
||||||
|
pool would make the 40-mutation passkey suite (~85s serial) the critical path of the stage.
|
||||||
|
- `ci/lint.sh` — yamllint runs over `LINT_JOBS` (default `min(8, nproc)`) CONTIGUOUS slices
|
||||||
|
of the sorted file list concurrently; each slice's output is buffered to its own file and
|
||||||
|
printed in slice order, so the text is the same as one serial call and a `file` header is
|
||||||
|
never separated from its `line:col` findings. Stage rc = the highest slice rc; a slice that
|
||||||
|
leaves no rc file counts as 1. Invalid `LINT_JOBS` exits 2.
|
||||||
|
|
||||||
|
### Parallel-safety audit (all 25 `l0-static/*/check.sh`, 3 `l1-rendering/*/render-check.sh`)
|
||||||
|
|
||||||
|
Grepped every check for fixed `/tmp` paths, `_artifacts`/`ARTIFACTS`/`COVERAGE_OUT`,
|
||||||
|
`write_text`/`open(…,"w")`, `mktemp`/`mkdtemp`/`TemporaryDirectory`, sockets/ports, `git`
|
||||||
|
writes and `run.sh` invocations, and read what each writer writes to:
|
||||||
|
|
||||||
|
- **Scratch state: all private.** Every writer uses its own `mktemp -d` / `mkdtemp` /
|
||||||
|
`TemporaryDirectory` (placeholder-guard-coupling, spec-immutability-enforced,
|
||||||
|
whole-repo-invariants-wired, auditability-proofs-producible, alert-rules-unit-tested and its
|
||||||
|
`platform/monitoring/tests/run.sh`, internal-services-publish-no-public-dns,
|
||||||
|
oauth2-proxy-{api-routes-per-app,cookie-names-distinct}, both keycloak reconcilers, the three
|
||||||
|
L1 render checks). No fixed `/tmp` path anywhere.
|
||||||
|
- **Git writes:** only spec-immutability-enforced, and only `git init/add/commit` inside its
|
||||||
|
own `mktemp` repo. Everything else is `git ls-files`/`ls-tree`/`log`/`rev-parse` (read-only).
|
||||||
|
- **Ports:** the reconciler selftests bind `127.0.0.1:0` (kernel-assigned); the passkey
|
||||||
|
selftest also dials `127.0.0.1:1` as a deliberate refused connection. No fixed listener.
|
||||||
|
- **Ansible** (dev-nodes-provisioned-as-code): `--syntax-check`/`--list-hosts` only,
|
||||||
|
`retry_files_enabled = False`; its local tmp is per-process.
|
||||||
|
- **The one shared writer:** auditability-proofs-producible runs
|
||||||
|
`./run.sh --coverage-only --requirements …`, which REWRITES
|
||||||
|
`conformance/_artifacts/coverage/coverage.{json,md}` (it inherits `CONF_ARTIFACTS`). That was
|
||||||
|
already true serially. No other offline check reads or writes that directory (the only
|
||||||
|
reader is `tests/meta/harness-selfcheck`, not an offline check), and the outer runner
|
||||||
|
writes coverage BEFORE and the JUnit AFTER the pool, so the final state is the same as
|
||||||
|
serial. Not serialised; left as is and recorded here. `lib/status.py` (run by that check)
|
||||||
|
reads `_artifacts/report.junit.xml` as an advisory — written only after the pool, unchanged.
|
||||||
|
- whole-repo-invariants-wired calls `run.sh --print-selection` (writes nothing) and the
|
||||||
|
runner itself against synthetic dirs in its own `mktemp`.
|
||||||
|
|
||||||
|
Result: no check needed fixing or serialising. A new check that needs shared state must say
|
||||||
|
so; the runner's docstring now states that contract.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
All in the pinned toolbox (`dev.sh run`, `DEV_KUBECONFIG=/dev/null`, ENGINE shim adding a ro
|
||||||
|
mount of the main checkout's `.git`; spec repo resolved as a sibling, spec-immutability
|
||||||
|
`pass`), host 24 CPUs, every `dev.sh` call under `flock`.
|
||||||
|
|
||||||
|
- **Gate, before** (origin/main `1a33814`, detached scratch worktree + spec mount):
|
||||||
|
`GATE_RC=3`, in-container wall **183.9s**; `l0` stage **168.3s**, `lint` **7.8s**.
|
||||||
|
"Stages that did not run: gitea-settings" (plus the expected on-base
|
||||||
|
touched-set-not-determinable note, since main has no branch commits).
|
||||||
|
- **Gate, after** (this branch, commit `a775082`): `GATE_RC=3`, in-container wall **32.1s**;
|
||||||
|
`l0` **21.1s**, `lint` **2.4s**, `l1` 2.3s. "Stages that did not run: gitea-settings" — the
|
||||||
|
only not-run stage. **rc 3, not green.** Caveat: the branch run selected 23 L0 checks (the
|
||||||
|
touched-requirement scoping dropped dev-nodes-provisioned-as-code and
|
||||||
|
placeholder-guard-coupling), the main run 25.
|
||||||
|
- **Same-tree L0 comparison** (all 25 l0 dirs, same tree, same container):
|
||||||
|
`CONF_OFFLINE_JOBS=1 MUTATION_JOBS=1` **164.0s / 165.3s** vs default (jobs=8,
|
||||||
|
MUTATION_JOBS=8) **19.2s / 18.7s**. JUnit `cmp`-identical, step summary `cmp`-identical,
|
||||||
|
stderr identical except the timing suffixes and the JUnit path in the last line.
|
||||||
|
- **Mutation suites, host:** passkey 83.2s → 10.5s (`MUTATION_JOBS=8`), client-secret
|
||||||
|
11.2s → 2.0s; output of `MUTATION_JOBS=1` and `=8` byte-identical to the pre-change script's,
|
||||||
|
rc 0 all round.
|
||||||
|
- **Falsification (not committed):** in a scratch worktree at `a775082`, made the passkey
|
||||||
|
mutation "the protected-subtree screen is never called" a no-op (replacement == anchor).
|
||||||
|
Host: `MUTATION_JOBS=1` and `=8` both rc 1, output byte-identical (with a second, broken
|
||||||
|
anchor also injected: both reported). Toolbox: runner rc 1 both serial and parallel,
|
||||||
|
`L0 FAIL keycloak-passkey-reconciler … 1 of 40 mutation(s) not proven … SURVIVED`; full
|
||||||
|
gate **rc 1**, `stage l0: FAIL`, `MERGE GATE: BLOCKED`. Reverted; scratch worktree removed.
|
||||||
|
- **lint.sh, host scratch repo** (31 YAML files, two broken): `LINT_JOBS=1/4/8` all rc 1,
|
||||||
|
stdout identical, findings named `y/f07.yaml` / `2:1 … key-duplicates` etc.; clean tree rc 0;
|
||||||
|
`LINT_JOBS=x` rc 2.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **CI timings at the time of writing.** The PR's CI run is measured after push and reported
|
||||||
|
on the PR / handoff message, not here. The CI runner's CPU count is UNVERIFIED — the new
|
||||||
|
`[offline] running … jobs=… cpus=…` line in the CI log is what will show it. On a small
|
||||||
|
runner the speedup will be smaller than locally.
|
||||||
|
- **approval-policy** will be red on CI: `ci/**` and `conformance/**` are sensitive paths and
|
||||||
|
need the operator's review. I did not and must not override it.
|
||||||
|
- Flakiness under load: the selftests have no wall-clock assertions I found, and 4 toolbox
|
||||||
|
runs + several host runs were all stable, but that is a handful of runs, not a soak.
|
||||||
|
- The coverage files under `conformance/_artifacts/coverage/` end in the same state as before
|
||||||
|
only because a single check rewrites them; I did not change that pre-existing quirk.
|
||||||
|
- L2/L3 conformance: not run (no cluster in CI, as before).
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
---
|
||||||
|
date: 2026-11-08
|
||||||
|
branch: chore/renovate-config-cleanup
|
||||||
|
title: Renovate config — hand-migrate the docker key, track gitea/runner, keep HelmRelease range lower bounds
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
- **`renovate.json5`: the deprecated top-level `"docker": {"enabled": true}` is hand-migrated** to
|
||||||
|
`packageRules[0] = {"matchCategories": ["docker"], "enabled": true}`, the exact shape Renovate's
|
||||||
|
own migration produced on `origin/renovate/migrate-config` (PR #139). Its comment moved with it.
|
||||||
|
This replaces #139, which was closed because its rewrite deleted every comment in the file. No
|
||||||
|
other comment was removed or reflowed (diff is additions plus the moved block).
|
||||||
|
- **`gitea/runner` added to the blast-radius rule** beside `gitea/act_runner`, with a comment:
|
||||||
|
act_runner was renamed upstream to gitea/runner after 0.6.1; Renovate does not follow renames,
|
||||||
|
so the actual image migration is a separate manual PR.
|
||||||
|
- **Two new packageRules for flux/helm `>=A <B` ranges:** `rangeStrategy: "bump"`, plus
|
||||||
|
`matchUpdateTypes: ["bump"] -> enabled: false`, so in-range releases (which Flux adopts on its
|
||||||
|
own) do not become PRs. Cause of the bug, from the 44.167.0 source: rangeStrategy is unset
|
||||||
|
(`auto`); the flux manager has no `getRangeStrategy`/`updateLockedDependency`, so
|
||||||
|
`lib/modules/manager/index.ts getRangeStrategy` resolves `auto` to `replace`; helm versioning is
|
||||||
|
the npm api verbatim, and npm `getNewValue` under `replace` turns `>=1.12.0 <1.13.0` into
|
||||||
|
`<1.14.0` (what PRs #131, #130 and #115 did).
|
||||||
|
- **`platform/headlamp/helmrelease.yaml`: `version: "<0.47.0"` -> `">=0.46.0 <0.47.0"`**, comment
|
||||||
|
refreshed. The live chart is 0.46.0, so the set of installable charts is unchanged.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **Semantic diff of #139:** I parsed `renovate.json5` on `origin/renovate/migrate-config` and on its
|
||||||
|
parent `5b484f6` with json5 (the copy inside the renovate image). With `docker` removed from the
|
||||||
|
parent and `packageRules[0]` removed from the migrated file, the two are identical. So the only
|
||||||
|
change was `{"matchCategories":["docker"],"enabled":true}`, inserted at packageRules[0].
|
||||||
|
- **Validator, `ghcr.io/renovatebot/renovate:44.167.0`** (the tag that `.gitea/workflows/renovate.yaml`
|
||||||
|
pins as `container.image`): `renovate-config-validator` gives `Config validated successfully`
|
||||||
|
(rc 0), and so does `--strict` (rc 0), both as a file path and in repo mode. **`--strict` on
|
||||||
|
origin/main's file gives rc 1** and prints the `docker` migration diff. Nothing is left to migrate.
|
||||||
|
- **`getNewValue` called in the 44.167.0 image** (helm versioning), currentValue
|
||||||
|
`>=1.12.0 <1.13.0`, newVersion 1.13.0: replace `<1.14.0`, widen `>=1.12.0 <1.14.0`,
|
||||||
|
bump `>=1.13.0 <1.14.0`, pin/update-lockfile `<1.14.0`. On `<0.47.0`, bump returns the value
|
||||||
|
unchanged, so it would never propose anything for that range. That is why the rule is scoped to
|
||||||
|
the `>=A <B` shape.
|
||||||
|
- **Local dry run** (`--platform=local`, `RENOVATE_DRY_RUN=lookup`, 44.167.0, live chart repos)
|
||||||
|
against a fixture repo of four HelmReleases under `platform/`:
|
||||||
|
- With this branch's config: `No config migration necessary`. longhorn `>=1.9.0 <1.10.0` ->
|
||||||
|
`>=1.13.0 <1.14.0` (minor). kube-prometheus-stack `>=88.0.0 <89.0.0` -> `>=93.1.0 <94.0.0`
|
||||||
|
(major). The in-range bump `>=88.6.5 <89.0.0` was dropped (`Filtered out 1 disabled update(s)`).
|
||||||
|
headlamp `>=0.46.0 <0.47.0` and `<0.47.0`: no update.
|
||||||
|
- Control run with origin/main's config: `Config migration necessary`, longhorn -> `<1.14.0`,
|
||||||
|
kube-prometheus-stack -> `<94.0.0`. This reproduces the reported bug.
|
||||||
|
- Live headlamp HelmRelease (read-only kubectl, k3s-prod): spec `<0.47.0`, history chartVersion
|
||||||
|
`0.46.0`, Ready `True`.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **I have not seen a real scheduled Renovate run against this repo with this config.** The claim that
|
||||||
|
Renovate stops reopening the migrate-config PR rests on the validator and on `No config migration
|
||||||
|
necessary` in the local dry run, not on the Gitea bot. After merge, watch the next run, and if
|
||||||
|
`renovate/migrate-config` is still on the remote, delete it.
|
||||||
|
- **Under bump, the new lower bound is the release Renovate proposes**, e.g. `>=1.13.2 <1.14.0`, not
|
||||||
|
`>=1.13.0`. Flux still installs the newest release in the window. Ranges with other shapes keep the
|
||||||
|
default `replace`: `3.9.x`, `~`, ceiling-only, and exact pins.
|
||||||
|
- **The open PRs #131, #115 and any others still carry unbounded `<X` values.** They were generated
|
||||||
|
under the old strategy, and Renovate should rebase or regenerate them under bump on its next run.
|
||||||
|
I have not checked that it does.
|
||||||
|
- The gitea/runner rename itself (a new image reference and any config changes) is not done here.
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
---
|
||||||
|
date: 2026-11-08
|
||||||
|
branch: rework/136
|
||||||
|
title: "PR #136 rework: regenerate L1 render goldens for the oauth2-proxy v7.15.6 sidecar bump"
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
- Renovate PR #136 bumps `OAUTH2_PROXY_IMAGE` in `catalog/render.py` (and the generated
|
||||||
|
`apps/wave-*` / `platform/crafty` pins) from `v7.8.1` to `v7.15.6`. Its gate failed at L1
|
||||||
|
because three byte-for-byte render goldens still carried the old tag:
|
||||||
|
- `conformance/tests/l1-rendering/container-workload-render/golden/expected.yaml`
|
||||||
|
- `conformance/tests/l1-rendering/egress-class-render/golden/egress-vpn.expected.yaml`
|
||||||
|
- `conformance/tests/l1-rendering/egress-class-render/golden/egress-internet.expected.yaml`
|
||||||
|
(CI only reported the vpn one because `render-check.sh` stops at the first failing diff;
|
||||||
|
the internet golden had drifted too.)
|
||||||
|
- Regenerated all three in the pinned toolbox with the exact invocation the check scripts use:
|
||||||
|
`python3 catalog/render.py <fixture> > <golden>`. The diff in each is **one line**:
|
||||||
|
`image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1` -> `...:v7.15.6`. Nothing else moved.
|
||||||
|
- `platform/crafty/README.md`: the "Sidecar" row described the current pin; updated to v7.15.6.
|
||||||
|
- Deliberately NOT changed (they record history, not the current pin):
|
||||||
|
- `catalog/render.py` comments / assertion messages saying a flag was "confirmed present in the
|
||||||
|
PINNED v7.8.1 binary" or that `--api-routes` "does not exist in v7.8.1". Rewriting them to
|
||||||
|
v7.15.6 would assert a `--help` verification nobody has done against v7.15.6.
|
||||||
|
- `conformance/tests/l0-static/oauth2-proxy-api-routes-per-app/check.sh`, the schema
|
||||||
|
description, `runbooks/oauth2-proxy-session.md`, `docs/plans/...` — same reasoning.
|
||||||
|
- `platform/oauth2-proxy/auth-visibility-alerts.promtool-test.yaml` — v7.8.1 appears in
|
||||||
|
synthetic kube-state-metrics series (test input data), not as a pin.
|
||||||
|
- `.gitea/workflows/renovate.yaml` comment — a dated measurement of the pre-Renovate state.
|
||||||
|
- Branch was 55 commits behind `origin/main`, but main touches neither `catalog/render.py` nor
|
||||||
|
the L1 goldens, and the merge is conflict-free; no merge of main was needed for the gate.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- The golden diff is exactly the image tag in each of the three files (`git diff -U0`).
|
||||||
|
- No remaining `oauth2-proxy:v7.8.1` in any rendered/generated/golden file (`git grep`).
|
||||||
|
- Local toolbox gate result: see the PR #136 comment posted with this push (rc and per-stage
|
||||||
|
results are quoted there and in the worker report).
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- That the v7.15.6 sidecar actually starts and authenticates in the cluster with the rendered
|
||||||
|
args. Earlier research said no flag used here was removed/renamed between 7.8.1 and 7.15.6;
|
||||||
|
I did not re-run `oauth2-proxy --help` against the v7.15.6 image.
|
||||||
|
- L2/L3 conformance (does not run in CI: `CLUSTER_AVAILABLE` unset).
|
||||||
|
- `approval-policy`: `conformance/**` is a sensitive path, so this PR is CI-red until the
|
||||||
|
operator posts `/approve-override <reason>` as a PR review. The local gate cannot evaluate it.
|
||||||
|
- Whether Renovate keeps rebasing this branch now that it carries a non-Renovate commit.
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
---
|
||||||
|
date: 2026-11-08
|
||||||
|
branch: feat/runner-capacity-2
|
||||||
|
title: act_runner runs two jobs at once, and dind is sized for two
|
||||||
|
---
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
`apps/wave-1/gitea-actions.yaml` only. That file is a **hand-owned** non-catalog unit
|
||||||
|
(`EXTRA_UNITS["gitea-actions"]` in `catalog/render.py`): the renderer only decides which wave
|
||||||
|
it lands in, it does not generate its bytes, and `ci/render-golden.sh` compares only files the
|
||||||
|
catalog renders. So it is edited in place, and no re-render was needed or run.
|
||||||
|
|
||||||
|
1. **New ConfigMap `act-runner-config`** holding a config file with ONE key,
|
||||||
|
`runner.capacity: 2`, mounted read-only at `/etc/act_runner/` and selected with
|
||||||
|
`CONFIG_FILE=/etc/act_runner/config.yaml`. The image's `run.sh` (act_runner v0.6.1) turns
|
||||||
|
that into `--config` for both `register` and `daemon`.
|
||||||
|
- Deltas only, on evidence: v0.6.1 `LoadDefault()` unmarshals the file onto a zero Config and
|
||||||
|
then applies the same defaults as the no-file path, so the other settings are unchanged.
|
||||||
|
- The `generate-config` example was deliberately NOT used. Its `runner.labels` would override
|
||||||
|
the registered labels in `/data/.runner` and repoint `ubuntu-latest` at
|
||||||
|
`docker.gitea.com/runner-images`. Its `force_pull: true` is not the default. Its sample `envs`
|
||||||
|
would be injected into every job.
|
||||||
|
- With a file present, act_runner ignores the legacy `GITEA_RUNNER_CAPACITY` /
|
||||||
|
`_FILE` / `_ENVIRON` / `_ENV_FILE` / `GITEA_DEBUG` / `GITEA_TRACE` env vars. None of
|
||||||
|
them is set on this Deployment.
|
||||||
|
2. **Pod-template annotation `homelab.keithjackson.dev/act-runner-config-revision: "1"`.**
|
||||||
|
act_runner reads its config once, at start, so a change to the ConfigMap alone would never
|
||||||
|
take effect. Bump this annotation whenever you change only the ConfigMap.
|
||||||
|
3. **dind resources:** requests `cpu 100m -> 2` and `memory 256Mi -> 1Gi`; limit
|
||||||
|
`memory 2Gi -> 4Gi`. There is still no CPU limit, on purpose.
|
||||||
|
4. **runner resources:** request `memory 128Mi -> 256Mi`, limit `memory 512Mi -> 1Gi`. CPU stays
|
||||||
|
at `100m`.
|
||||||
|
|
||||||
|
Why these sizes. Measured 2026-11-08 through Prometheus, at capacity 1:
|
||||||
|
|
||||||
|
| container | peak CPU (1m rate, 30d) | peak working set (30d) | OOM events (7d) |
|
||||||
|
|---|---|---|---|
|
||||||
|
| dind | 0.69 core | 447 MiB | 0 |
|
||||||
|
| runner | 0.24 core | 314 MiB | 0 |
|
||||||
|
|
||||||
|
At the moment of measurement the pod was idle (`top`: dind 1m/138Mi, runner 1m/83Mi), on
|
||||||
|
k3s-prod-2.
|
||||||
|
|
||||||
|
- **CPU request 2.** Two jobs at today's peak is about 1.4 cores. The gate is being parallelised
|
||||||
|
in other work, so each job will use more than that. The request sets dind's CFS weight under
|
||||||
|
contention; at 100m it got a 1/20 share. With no limit, dind can still use spare cores on an
|
||||||
|
idle node.
|
||||||
|
- **Memory 1Gi / 4Gi.** Two jobs at today's peak is about 0.9 GiB. If dind is OOM-killed,
|
||||||
|
dockerd goes down and takes both running jobs with it, so the limit is about 4x that.
|
||||||
|
- **Runner limit 512Mi -> 1Gi.** Its 30-day peak was already 61% of the old limit, and with
|
||||||
|
capacity 2 it holds state for two tasks.
|
||||||
|
|
||||||
|
Scheduling headroom: CPU requests per node were measured with `describe node`. The pod's total
|
||||||
|
CPU request goes from 200m to 2100m (+1.9 cores).
|
||||||
|
|
||||||
|
| node | CPU requested now | after the change | allocatable |
|
||||||
|
|---|---|---|---|
|
||||||
|
| prod-2 (current) | 2945m | ~4845m (61%) | 8 |
|
||||||
|
| prod-1 | 3175m | ~5075m | 8 |
|
||||||
|
| prod-3 | 3840m | ~5740m (72%) | 8 |
|
||||||
|
|
||||||
|
The pod fits on any of the three if the RWO PVC moves it.
|
||||||
|
|
||||||
|
Concurrency semantics. Workflow `concurrency:` groups are enforced by Gitea on the server, not by
|
||||||
|
the runner, so capacity does not change them:
|
||||||
|
|
||||||
|
- `gate.yml` `gate-${{ github.ref }}` with cancel-in-progress still cancels a superseded run on
|
||||||
|
the same ref.
|
||||||
|
- `renovate` (group `renovate`, no cancel) and `images-${{ github.ref }}` still serialise
|
||||||
|
themselves.
|
||||||
|
|
||||||
|
What capacity 2 does change: runs on *different* refs (a PR gate next to a renovate or images
|
||||||
|
run, or two PRs) now run side by side instead of queueing. The required check
|
||||||
|
`gate / gate (pull_request)` is still one job per PR head, and gate.yml's other job
|
||||||
|
(`conformance`) is `if: false`, so it never runs.
|
||||||
|
|
||||||
|
Both jobs share one dockerd. act_runner gives each task its own container prefix
|
||||||
|
(`GITEA-ACTIONS-TASK-<task id>`) and its own network, and no workflow uses fixed container names
|
||||||
|
or host ports.
|
||||||
|
|
||||||
|
## What I verified
|
||||||
|
|
||||||
|
- **Source of truth.** `catalog/render.py` `EXTRA_UNITS` places `gitea-actions.yaml` by
|
||||||
|
anchor/offset only. `ci/render-golden.sh` iterates over rendered files only, so this file is
|
||||||
|
outside render==golden. No `--emit-waves` was run.
|
||||||
|
- **act_runner v0.6.1 behaviour, read in its source** (tag v0.6.1):
|
||||||
|
- `config.LoadDefault()` treats a partial file as described above.
|
||||||
|
- `daemon.go`: config labels override `.runner` labels only when non-empty.
|
||||||
|
- `poller.go`: a semaphore of `cfg.Runner.Capacity` bounds concurrent tasks.
|
||||||
|
- `runner.go`: per-task container name prefix.
|
||||||
|
- The image's `run.sh`, read from `gitea/act_runner:0.6.1`, honours `CONFIG_FILE`.
|
||||||
|
- **The live pod has no config file today.** `/data` holds only `.runner` and `lost+found`.
|
||||||
|
`act_runner --version` reports v0.6.1. No LimitRange or ResourceQuota exists in
|
||||||
|
`gitea-actions`.
|
||||||
|
- **Resource numbers.** All figures above come from read-only `kubectl top`, `describe node` and
|
||||||
|
Prometheus queries against k3s-prod.
|
||||||
|
- **dind disk.** The emptyDir sits on the node's 11.8G ephemeral filesystem: 4.0G used, 7.2G
|
||||||
|
free. Image cache is 2.18GB, which is the one `catthehacker/ubuntu:act-22.04` image. Two
|
||||||
|
concurrent jobs share that image.
|
||||||
|
- **Embedded config.** It parses as YAML to `{'runner': {'capacity': 2}}`. The only `$` in the
|
||||||
|
manifest is the existing `${BASE_DOMAIN}`, so nothing new is exposed to Flux postBuild
|
||||||
|
envsubst.
|
||||||
|
- **Approval policy.** `apps/wave-1/` matches no `sensitive_paths` glob in
|
||||||
|
`infra/gitea/approval-policy.json`, so this PR merges on a green gate alone.
|
||||||
|
- **Local gate.** Pinned toolbox, run from the worktree on the committed tree. The rc and skipped
|
||||||
|
stages are given in the PR description.
|
||||||
|
|
||||||
|
## What I did NOT verify
|
||||||
|
|
||||||
|
- **UNVERIFIED: that the runner actually starts with the config and runs two jobs at once.**
|
||||||
|
Nothing is applied until Flux reconciles `main`.
|
||||||
|
- **Capacity cannot be seen through Gitea's API.** act_runner v0.6.1 does not report capacity to
|
||||||
|
Gitea: `Declare` sends only version and labels. `GET /api/v1/admin/actions/runners` therefore
|
||||||
|
cannot show "capacity 2". Concurrency has to be observed instead.
|
||||||
|
- **UNVERIFIED: whether the parallelised gate fits in these sizes.** That gate has not landed.
|
||||||
|
Re-measure after it does, with
|
||||||
|
`max_over_time(sum by (container)(rate(container_cpu_usage_seconds_total{namespace="gitea-actions"}[1m]))[7d:30s])`.
|
||||||
|
- **Rollout cancels in-flight jobs.** The rollout restarts the runner pod (strategy Recreate,
|
||||||
|
`shutdown_timeout: 0s`), and any job running at that moment is cancelled, possibly including
|
||||||
|
the `push`-to-main gate run triggered by this very merge. Re-run it if that happens.
|
||||||
|
- **dind disk is not bounded by a limit.** With two jobs it holds two jobs' workspaces and layers
|
||||||
|
on the same 11.8G node filesystem. I did not add an `ephemeral-storage` limit, because an
|
||||||
|
eviction would kill both jobs. Watch kubelet DiskPressure on the runner's node.
|
||||||
|
- **Stale comments in sensitive paths.** `.gitea/workflows/gate.yml` (line ~52) and
|
||||||
|
`.gitea/workflows/README.md` (line ~86) still say "`act_runner` runs at capacity 1". I left
|
||||||
|
them alone because `.gitea/**` is a sensitive path and editing them would put this PR behind
|
||||||
|
human approval. Fix them in a separate PR.
|
||||||
|
|
||||||
|
### Operator post-merge checklist
|
||||||
|
|
||||||
|
1. Flux applied it:
|
||||||
|
`kubectl --context k3s-prod -n flux-system get kustomization apps-wave-1 -o jsonpath='{.status.lastAppliedRevision}'`
|
||||||
|
equals the merge commit.
|
||||||
|
2. The pod restarted with the config:
|
||||||
|
- `kubectl --context k3s-prod -n gitea-actions get pod -l app.kubernetes.io/name=act-runner`
|
||||||
|
shows a new pod, 2/2 Running.
|
||||||
|
- `kubectl ... exec deploy/act-runner -c runner -- cat /etc/act_runner/config.yaml` shows
|
||||||
|
`capacity: 2`.
|
||||||
|
- `kubectl ... exec deploy/act-runner -c runner -- ps` shows `act_runner daemon --config /etc/act_runner/config.yaml`.
|
||||||
|
- The logs show "declare successfully" with the two expected labels (`ubuntu-latest`,
|
||||||
|
`homelab` -> `catthehacker/ubuntu:act-22.04`). That confirms the labels did not change.
|
||||||
|
3. The runner is registered and online:
|
||||||
|
- `GET /api/v1/admin/actions/runners` lists `incluster-dind`, online, with the same ID as
|
||||||
|
before (no duplicate row).
|
||||||
|
- Capacity is NOT shown there; see above.
|
||||||
|
4. Concurrency is real: trigger two runs on different refs (e.g. two PR gates, or a PR gate plus a
|
||||||
|
`workflow_dispatch` of renovate). Then either:
|
||||||
|
- in `GET /api/v1/repos/Home-Network/home-network-k3s-impl/actions/tasks`, confirm their
|
||||||
|
`run_started_at` .. `updated_at` windows overlap; or
|
||||||
|
- check
|
||||||
|
`docker --host=tcp://127.0.0.1:2376 --tlsverify ... ps` in the dind container lists two
|
||||||
|
`GITEA-ACTIONS-TASK-*` container sets at once.
|
||||||
|
5. Resources: `kubectl --context k3s-prod -n gitea-actions describe pod` shows dind requests
|
||||||
|
cpu 2 / memory 1Gi, limit memory 4Gi, and runner limit memory 1Gi.
|
||||||
@@ -16,7 +16,7 @@ Cilium L2 LoadBalancer (game protocols authenticate players themselves).
|
|||||||
|---|---|
|
|---|---|
|
||||||
| Image | `registry.gitlab.com/crafty-controller/crafty-4:4.10.8` (current stable; bare-semver tags, no `v`) |
|
| Image | `registry.gitlab.com/crafty-controller/crafty-4:4.10.8` (current stable; bare-semver tags, no `v`) |
|
||||||
| Web UI | HTTPS **self-signed** on `:8443` (reached only via the sidecar / same-pod) |
|
| Web UI | HTTPS **self-signed** on `:8443` (reached only via the sidecar / same-pod) |
|
||||||
| Sidecar | `quay.io/oauth2-proxy/oauth2-proxy:v7.8.1`, reverse-proxy on `:4180` |
|
| Sidecar | `quay.io/oauth2-proxy/oauth2-proxy:v7.15.6`, reverse-proxy on `:4180` |
|
||||||
| Data | ONE Longhorn RWO PVC `crafty-data` (20Gi), mounted 5× by subPath |
|
| Data | ONE Longhorn RWO PVC `crafty-data` (20Gi), mounted 5× by subPath |
|
||||||
| PVC subPaths | `servers`→`/crafty/servers`, `backups`→`/crafty/backups`, `logs`→`/crafty/logs`, `import`→`/crafty/import`, `config`→`/crafty/app/config` |
|
| PVC subPaths | `servers`→`/crafty/servers`, `backups`→`/crafty/backups`, `logs`→`/crafty/logs`, `import`→`/crafty/import`, `config`→`/crafty/app/config` |
|
||||||
| Identity | `runAsUser:1000 runAsGroup:0 fsGroup:0 runAsNonRoot:true` (see below) |
|
| Identity | `runAsUser:1000 runAsGroup:0 fsGroup:0 runAsNonRoot:true` (see below) |
|
||||||
|
|||||||
@@ -130,7 +130,7 @@ spec:
|
|||||||
- name: oauth2-proxy
|
- name: oauth2-proxy
|
||||||
# quay.io/oauth2-proxy/oauth2-proxy — same authenticator family as platform/oauth2-proxy, but
|
# quay.io/oauth2-proxy/oauth2-proxy — same authenticator family as platform/oauth2-proxy, but
|
||||||
# here in per-app REVERSE-PROXY mode (--upstream is the co-located Crafty UI, not static://202).
|
# here in per-app REVERSE-PROXY mode (--upstream is the co-located Crafty UI, not static://202).
|
||||||
image: quay.io/oauth2-proxy/oauth2-proxy:v7.8.1
|
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.6
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
args:
|
args:
|
||||||
- --http-address=0.0.0.0:4180
|
- --http-address=0.0.0.0:4180
|
||||||
|
|||||||
@@ -40,7 +40,11 @@ spec:
|
|||||||
# values (-session-ttl + the -oidc-* args from the externalSecret; useCookie stays default-false
|
# values (-session-ttl + the -oidc-* args from the externalSecret; useCookie stays default-false
|
||||||
# so `-oidc-use-cookie` is NOT emitted) is supported by the v0.41.0 backend. Renovate maintains
|
# so `-oidc-use-cookie` is NOT emitted) is supported by the v0.41.0 backend. Renovate maintains
|
||||||
# within 0.41.x; the appVersion continues to pin the Headlamp image.
|
# within 0.41.x; the appVersion continues to pin the Headlamp image.
|
||||||
version: "<0.47.0"
|
# 2026-11-08: NOW 0.46.x. PR #130 (Renovate, default rangeStrategy `replace`) rewrote
|
||||||
|
# `>=0.41.0 <0.42.0` to the unbounded-below `<0.47.0`; the lower bound is restored here at the
|
||||||
|
# chart that was live when it merged (0.46.0). renovate.json5 now bumps `>=A <B` ranges as a window,
|
||||||
|
# so the next widen PR proposes `>=0.47.x <0.48.0` rather than `<0.48.0`.
|
||||||
|
version: ">=0.46.0 <0.47.0"
|
||||||
sourceRef:
|
sourceRef:
|
||||||
kind: HelmRepository
|
kind: HelmRepository
|
||||||
name: headlamp
|
name: headlamp
|
||||||
|
|||||||
@@ -367,9 +367,17 @@ is involved — so step 2 failing with no log entry is the expected signature of
|
|||||||
The `Keycloak` CR has no `spec.image`, so the server image is whatever the operator's
|
The `Keycloak` CR has no `spec.image`, so the server image is whatever the operator's
|
||||||
`RELATED_IMAGE_KEYCLOAK` names — the same tag as the operator (prod, 2026-10-30: operator
|
`RELATED_IMAGE_KEYCLOAK` names — the same tag as the operator (prod, 2026-10-30: operator
|
||||||
`26.0.6`, `keycloak-0/1` on `quay.io/keycloak/keycloak:26.0.6`). Operator, CRDs and server are
|
`26.0.6`, `keycloak-0/1` on `quay.io/keycloak/keycloak:26.0.6`). Operator, CRDs and server are
|
||||||
released in lockstep from `keycloak/keycloak-k8s-resources`, and the three URLs in
|
released in lockstep from `keycloak/keycloak-k8s-resources`, and the release URLs in
|
||||||
`kustomization.yaml` are the one knob. Renovate raises them as ONE PR (`keycloak-operator` group,
|
`kustomization.yaml` are the one knob. Renovate raises them as ONE PR (`keycloak-operator` group,
|
||||||
labels `needs-review` + `blast-radius`), **never auto-merged**; majors wait for a dashboard tick.
|
labels `needs-review` + `blast-radius`), **never auto-merged**; majors wait for a dashboard tick.
|
||||||
|
|
||||||
|
**Renovate bumps the URLs that exist; it cannot add a CRD a release introduced.** Before merging a
|
||||||
|
bump, diff `kubernetes/` (and its `kustomization.yml`) at the new tag against the URL list. 26.0.6
|
||||||
|
shipped 2 CRDs, 26.8.2 ships 4 (`keycloakoidcclients`, `keycloaksamlclients` added); PR #121 bumped
|
||||||
|
only the existing URLs and the 26.8.2 operator CrashLooped on the missing `keycloaksamlclients`
|
||||||
|
informer, stalling Flux `platform` and every tier below it (2026-11-08,
|
||||||
|
`decisions/2026-11-08-keycloak-operator-crds.md`). The operator's memory is also patched in
|
||||||
|
`kustomization.yaml` (upstream's 450Mi OOMKilled 26.0.6 here); keep the patch across bumps.
|
||||||
`catalog/keycloak.yaml`'s image tag is documentation only (its render deploys nothing) and
|
`catalog/keycloak.yaml`'s image tag is documentation only (its render deploys nothing) and
|
||||||
Renovate is disabled for it — update it by hand to match after an operator bump merges.
|
Renovate is disabled for it — update it by hand to match after an operator bump merges.
|
||||||
|
|
||||||
|
|||||||
@@ -38,6 +38,20 @@ spec:
|
|||||||
ingress:
|
ingress:
|
||||||
enabled: false # exposed via Gateway API HTTPRoute (httproute.yaml), not the
|
enabled: false # exposed via Gateway API HTTPRoute (httproute.yaml), not the
|
||||||
# operator's built-in Ingress
|
# operator's built-in Ingress
|
||||||
|
# --- ONE OWNER for the `keycloak` ServiceMonitor: this repo (servicemonitor.yaml), not the operator.
|
||||||
|
# Since 26.x the operator creates its OWN ServiceMonitor, named after this CR (`keycloak`, i.e.
|
||||||
|
# the SAME name/namespace as ours), whenever metrics-enabled=true and the CRD exists —
|
||||||
|
# spec.serviceMonitor.enabled defaults TRUE. On 2026-11-08, right after the 26.0.6 -> 26.8.2
|
||||||
|
# bump, its server-side apply onto our Flux-owned object failed `409 Conflict`, the operator
|
||||||
|
# exhausted retries, the CR sat Ready=False/HasErrors=True, and `platform-config` (wait: true,
|
||||||
|
# so it health-checks this CR) stalled every Kustomization downstream of it. Ours is kept
|
||||||
|
# because the L0 servicemonitor-ports-resolve and L2 keycloak-monitored conformance tests and
|
||||||
|
# the dashboard's up{namespace="keycloak"} are written against it; the operator's would scrape
|
||||||
|
# the same thing (management port 9000, /metrics, 30s). Do NOT flip this to true without
|
||||||
|
# deleting servicemonitor.yaml in the same change — two owners of one object is a fight
|
||||||
|
# (cf. the default-deny NetworkPolicy drift loop).
|
||||||
|
serviceMonitor:
|
||||||
|
enabled: false
|
||||||
# --- Hardening + observability options (docs/15). Brute-force detection is a REALM setting
|
# --- Hardening + observability options (docs/15). Brute-force detection is a REALM setting
|
||||||
# (bruteForceProtected in the realm import); these enable health + metrics endpoints. ---
|
# (bruteForceProtected in the realm import); these enable health + metrics endpoints. ---
|
||||||
additionalOptions:
|
additionalOptions:
|
||||||
|
|||||||
@@ -20,18 +20,32 @@ kind: Kustomization
|
|||||||
namespace: keycloak
|
namespace: keycloak
|
||||||
resources:
|
resources:
|
||||||
- namespace.yaml
|
- namespace.yaml
|
||||||
# --- Keycloak Operator v26.0.6 (pinned upstream release manifests) ---
|
# --- Keycloak Operator v26.8.2 (pinned upstream release manifests) ---
|
||||||
# The operator, its CRDs and the SERVER IMAGE move in lockstep: the Keycloak CR has no
|
# The operator, its CRDs and the SERVER IMAGE move in lockstep: the Keycloak CR has no
|
||||||
# spec.image, so the operator's RELATED_IMAGE_KEYCLOAK (quay.io/keycloak/keycloak:<same tag>)
|
# spec.image, so the operator's RELATED_IMAGE_KEYCLOAK (quay.io/keycloak/keycloak:<same tag>)
|
||||||
# is what runs. These three URLs are therefore THE Keycloak version. Each carries a Renovate
|
# is what runs. These URLs are therefore THE Keycloak version. Each carries a Renovate
|
||||||
# marker (custom.regex manager, renovate.json5) under ONE depName, so all three bump in ONE
|
# marker (custom.regex manager, renovate.json5) under ONE depName, so all of them bump in ONE
|
||||||
# branch, never auto-merged (packageRule "keycloak-operator"). Before merging a bump, re-check
|
# branch, never auto-merged (packageRule "keycloak-operator"). Before merging a bump, re-check
|
||||||
# passkey-reconciler.yaml's 26.0.6 Admin API assumptions — see README.md "Upgrading Keycloak".
|
# passkey-reconciler.yaml's 26.0.6 Admin API assumptions — see README.md "Upgrading Keycloak".
|
||||||
|
#
|
||||||
|
# THE URL LIST MUST MATCH UPSTREAM'S kubernetes/kustomization.yml AT THE NEW TAG — Renovate only
|
||||||
|
# rewrites the version in the URLs that are here; it cannot add a CRD the new release introduced.
|
||||||
|
# 26.0.6 shipped 2 CRDs; 26.8.2 ships 4 (+ keycloakoidcclients, keycloaksamlclients, both
|
||||||
|
# v2alpha1). PR #121 bumped only the existing three URLs, and the 26.8.2 operator — which starts an
|
||||||
|
# informer for EVERY kind it controls, used or not — CrashLooped on
|
||||||
|
# `Couldn't start informer for keycloaksamlclients.k8s.keycloak.org/v2alpha1 ... Not Found`,
|
||||||
|
# stalling Flux `platform` and everything downstream (2026-11-08). On every bump, diff
|
||||||
|
# https://github.com/keycloak/keycloak-k8s-resources/tree/<tag>/kubernetes
|
||||||
|
# against this list before merging.
|
||||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
|
||||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml
|
||||||
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloakoidcclients.k8s.keycloak.org-v1.yml
|
||||||
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/keycloaksamlclients.k8s.keycloak.org-v1.yml
|
||||||
|
# renovate: datasource=github-tags depName=keycloak/keycloak-k8s-resources
|
||||||
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/kubernetes.yml
|
- https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/26.8.2/kubernetes/kubernetes.yml
|
||||||
# --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
|
# --- Admin (break-glass) credential — SOPS-encrypted, referenced by name (ADR-0007, R-IDP-04) ---
|
||||||
- keycloak-admin-credentials.sops.yaml
|
- keycloak-admin-credentials.sops.yaml
|
||||||
@@ -51,3 +65,23 @@ resources:
|
|||||||
- keycloak-db-backup-credentials-prod.sops.yaml
|
- keycloak-db-backup-credentials-prod.sops.yaml
|
||||||
# --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
|
# --- Grafana dashboard for Keycloak health (plain ConfigMap; R-OBS-03) ---
|
||||||
- grafana-dashboard.yaml
|
- grafana-dashboard.yaml
|
||||||
|
# --- Operator memory: upstream ships requests=limits=450Mi, which this cluster outgrew ON 26.0.6 ---
|
||||||
|
# Measured on prod 2026-11-08: the 26.0.6 operator pod had 154 restarts in 64d, lastState
|
||||||
|
# `OOMKilled` (exit 137, 2026-11-07T23:09:54Z), and sat at 421Mi of its 450Mi (94%) in `kubectl top`.
|
||||||
|
# 26.8.2 keeps the same 450Mi default but runs FOUR controllers/informers instead of two (adds
|
||||||
|
# keycloakoidcclient + keycloaksamlclient) on a newer Quarkus (3.15.1 -> 3.40.1.1), so it would
|
||||||
|
# plausibly start above a ceiling 26.0.6 already crossed. Limit 768Mi = ~1.8x the observed 421Mi
|
||||||
|
# working set; request 512Mi = the observed working set plus headroom, so the scheduler reserves what
|
||||||
|
# it actually uses. Revisit with `kubectl top` once 26.8.2 has run for a week. Patched here (not in
|
||||||
|
# the remote kubernetes.yml) so a Renovate bump keeps it.
|
||||||
|
patches:
|
||||||
|
- target:
|
||||||
|
kind: Deployment
|
||||||
|
name: keycloak-operator
|
||||||
|
patch: |-
|
||||||
|
- op: replace
|
||||||
|
path: /spec/template/spec/containers/0/resources/limits/memory
|
||||||
|
value: 768Mi
|
||||||
|
- op: replace
|
||||||
|
path: /spec/template/spec/containers/0/resources/requests/memory
|
||||||
|
value: 512Mi
|
||||||
@@ -8,6 +8,9 @@
|
|||||||
# /metrics) once metrics-enabled/health-enabled are set (see keycloak-instance.yaml additionalOptions).
|
# /metrics) once metrics-enabled/health-enabled are set (see keycloak-instance.yaml additionalOptions).
|
||||||
# We select the operator-created keycloak-service (which selects the Keycloak pods) and scrape the pods'
|
# We select the operator-created keycloak-service (which selects the Keycloak pods) and scrape the pods'
|
||||||
# management port by number via targetPort.
|
# management port by number via targetPort.
|
||||||
|
# SOLE OWNER: the operator's own ServiceMonitor (same name `keycloak`) is switched OFF in
|
||||||
|
# keycloak-instance.yaml (spec.serviceMonitor.enabled: false) — 26.x defaults it ON and its apply
|
||||||
|
# 409-looped against this object on 2026-11-08. Keep exactly one of the two.
|
||||||
# TODO(WS-SEC, bring-up): confirm the management port number/name the operator exposes (9000 default)
|
# TODO(WS-SEC, bring-up): confirm the management port number/name the operator exposes (9000 default)
|
||||||
# and adjust targetPort if the pinned operator changes it. The KeycloakDown / KeycloakDatabaseDown
|
# and adjust targetPort if the pinned operator changes it. The KeycloakDown / KeycloakDatabaseDown
|
||||||
# alerts (platform/monitoring/alerts.yaml) key on kube-state-metrics and fire regardless of this SM.
|
# alerts (platform/monitoring/alerts.yaml) key on kube-state-metrics and fire regardless of this SM.
|
||||||
|
|||||||
+64
-11
@@ -37,6 +37,8 @@
|
|||||||
// renovate-config-validator /tmp/renovate.json5
|
// renovate-config-validator /tmp/renovate.json5
|
||||||
// (Not `--strict`: that also fails on a pending migration of the pre-existing top-level
|
// (Not `--strict`: that also fails on a pending migration of the pre-existing top-level
|
||||||
// `"docker": {"enabled": true}` block at the bottom, which is a warning, not an error.)
|
// `"docker": {"enabled": true}` block at the bottom, which is a warning, not an error.)
|
||||||
|
// RESOLVED 2026-11-08: that block is now hand-migrated to the first packageRule (`matchCategories:
|
||||||
|
// ["docker"]`), the exact shape Renovate's own migration produced, so no migration is pending.
|
||||||
//
|
//
|
||||||
// `regex` vs `custom.regex` in enabledManagers, RESOLVED FROM SOURCE (Renovate 39.264.0 and
|
// `regex` vs `custom.regex` in enabledManagers, RESOLVED FROM SOURCE (Renovate 39.264.0 and
|
||||||
// later): both work. lib/config/migrations/custom/enabled-managers-migration.ts rewrites
|
// later): both work. lib/config/migrations/custom/enabled-managers-migration.ts rewrites
|
||||||
@@ -198,6 +200,22 @@
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
"packageRules": [
|
"packageRules": [
|
||||||
|
// The `docker` manager (Dockerfiles, docker-compose) and the `kubernetes` manager above were
|
||||||
|
// BOTH previously `enabled: false`, solely so Renovate would not fight Flux image-automation
|
||||||
|
// over a running tag. That controller was never enabled and never will be here (see _comment),
|
||||||
|
// so the only thing those two flags were achieving was leaving every plain `image:` tag in the
|
||||||
|
// repo unmanaged. `docker` is left out of enabledManagers because this repo contains no
|
||||||
|
// Dockerfile that ships to the cluster; `kubernetes` is now ON and is the one that matters.
|
||||||
|
// MOVED HERE 2026-11-08 from a top-level `"docker": {"enabled": true}` key, which Renovate
|
||||||
|
// deprecated (the top-level language objects became `matchCategories` rules). Hand-migrated to
|
||||||
|
// the exact output of Renovate's own config migration (branch renovate/migrate-config, PR #139:
|
||||||
|
// a json5 semantic diff of that file against its parent shows THIS rule, inserted at
|
||||||
|
// packageRules[0], as the ONLY change). #139 itself was closed unmerged because its rewrite
|
||||||
|
// dropped every comment in this file. With nothing left to migrate, Renovate stops reopening it.
|
||||||
|
{
|
||||||
|
"matchCategories": ["docker"],
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"description": "Auto-merge the allowed low-risk classes on green CI \u2014 no human authoring (R-UPD-02). The Gitea merge gate (make -C ci gate) is the required check.",
|
"description": "Auto-merge the allowed low-risk classes on green CI \u2014 no human authoring (R-UPD-02). The Gitea merge gate (make -C ci gate) is the required check.",
|
||||||
"matchUpdateTypes": [
|
"matchUpdateTypes": [
|
||||||
@@ -254,6 +272,11 @@
|
|||||||
// Keycloak's (the IdP) included, now that imageName is tracked.
|
// Keycloak's (the IdP) included, now that imageName is tracked.
|
||||||
"gitea/gitea",
|
"gitea/gitea",
|
||||||
"gitea/act_runner",
|
"gitea/act_runner",
|
||||||
|
// ADDED 2026-11-08. act_runner was RENAMED upstream to gitea/runner after 0.6.1
|
||||||
|
// (https://gitea.com/gitea/runner/releases). Renovate does not follow a rename — it will
|
||||||
|
// never propose gitea/act_runner -> gitea/runner — so that migration is a manual PR. Listed
|
||||||
|
// now so the new name is blast-radius from its first reference, not after an auto-merge.
|
||||||
|
"gitea/runner",
|
||||||
"ghcr.io/cloudnative-pg/postgresql"
|
"ghcr.io/cloudnative-pg/postgresql"
|
||||||
],
|
],
|
||||||
"automerge": false,
|
"automerge": false,
|
||||||
@@ -266,7 +289,7 @@
|
|||||||
// and the server image are released in lockstep from keycloak/keycloak-k8s-resources, and the
|
// and the server image are released in lockstep from keycloak/keycloak-k8s-resources, and the
|
||||||
// Keycloak CR carries no spec.image, so the operator's RELATED_IMAGE_KEYCLOAK picks the server
|
// Keycloak CR carries no spec.image, so the operator's RELATED_IMAGE_KEYCLOAK picks the server
|
||||||
// (measured on prod 2026-10-30: operator 26.0.6 -> keycloak-0/1 run quay.io/keycloak/keycloak:
|
// (measured on prod 2026-10-30: operator 26.0.6 -> keycloak-0/1 run quay.io/keycloak/keycloak:
|
||||||
// 26.0.6). The three release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
// 26.0.6). The release URLs in platform/keycloak/kustomization.yaml carry `# renovate:`
|
||||||
// markers under this ONE depName, so they move together in one branch (groupName makes that
|
// markers under this ONE depName, so they move together in one branch (groupName makes that
|
||||||
// explicit). NEVER AUTO-MERGED, not even a patch: Keycloak is the IdP that gates SSO for every
|
// explicit). NEVER AUTO-MERGED, not even a patch: Keycloak is the IdP that gates SSO for every
|
||||||
// app AND kubectl OIDC, and platform/keycloak/passkey-reconciler.yaml encodes Keycloak 26.0.6
|
// app AND kubectl OIDC, and platform/keycloak/passkey-reconciler.yaml encodes Keycloak 26.0.6
|
||||||
@@ -370,6 +393,45 @@
|
|||||||
"automerge": false,
|
"automerge": false,
|
||||||
"addLabels": ["needs-review", "range-widen"]
|
"addLabels": ["needs-review", "range-widen"]
|
||||||
},
|
},
|
||||||
|
// ADDED 2026-11-08 — KEEP THE LOWER BOUND WHEN A MINOR-LOCKED RANGE MOVES. MEASURED PROBLEM:
|
||||||
|
// PRs #131 (longhorn), #130 (headlamp) and #115 (kube-prometheus-stack) turned
|
||||||
|
// `">=1.12.0 <1.13.0"` into `"<1.14.0"` — an UNBOUNDED-below range that also admits every
|
||||||
|
// older chart (and #130 merged: platform/headlamp/helmrelease.yaml carried `<0.47.0`). WHY,
|
||||||
|
// from the 44.167.0 source this file is validated against: no rangeStrategy is set, so it is
|
||||||
|
// `auto`; the flux manager has no getRangeStrategy and no updateLockedDependency, so
|
||||||
|
// lib/modules/manager/index.ts getRangeStrategy resolves `auto` -> `replace`. The helm
|
||||||
|
// datasource's defaultVersioning is `helm`, which is the npm versioning api verbatim
|
||||||
|
// (lib/modules/versioning/helm/index.ts: `api = { ...npm }`), and npm getNewValue under
|
||||||
|
// `replace` rewrites `>=A <B` to just `<C`. Calling that getNewValue in the 44.167.0 image for
|
||||||
|
// currentValue `>=1.12.0 <1.13.0`, newVersion 1.13.0:
|
||||||
|
// replace -> `<1.14.0` widen -> `>=1.12.0 <1.14.0` bump -> `>=1.13.0 <1.14.0`
|
||||||
|
// Only `bump` keeps a one-minor window. (widen keeps the bound but grows the window by a minor
|
||||||
|
// on every PR — the opposite of minor-locking.) Under bump the new lower bound is the release
|
||||||
|
// proposed (e.g. 1.13.2 -> `>=1.13.2 <1.14.0`); Flux still picks the newest that satisfies it.
|
||||||
|
// Scoped to the bounded `>=A <B` shape only: on a ceiling-only range like `<0.47.0` bump
|
||||||
|
// returns the value unchanged, so Renovate would silently propose NOTHING for it, ever.
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-11-08. HelmRelease `version: \">=A <B\"` ranges move as a whole window (rangeStrategy bump), so a range-widen PR yields `>=1.13.0 <1.14.0`, not `<1.14.0`. See the comment above for the measured cause.",
|
||||||
|
"matchManagers": ["flux"],
|
||||||
|
"matchDatasources": ["helm"],
|
||||||
|
"matchCurrentValue": "/^>=\\S+ <\\S+$/",
|
||||||
|
"rangeStrategy": "bump"
|
||||||
|
},
|
||||||
|
// ...AND DO NOT LET bump ADD NOISE. Under bump Renovate also raises the lower bound for a
|
||||||
|
// release ALREADY INSIDE the range (lib/workers/repository/process/lookup/generate.ts sets
|
||||||
|
// isBump; e.g. 1.12.5 against `>=1.12.0 <1.13.0` -> `>=1.12.5 <1.13.0`), which `replace` never
|
||||||
|
// proposed. Flux already installs that release with no PR, and the range-widen rule above
|
||||||
|
// would hold every one for review. util/package-rules/update-types.ts matches isBump as
|
||||||
|
// updateType `bump`, so exactly those updates are switched off here; out-of-range moves are
|
||||||
|
// ordinary minor/major updates and are unaffected.
|
||||||
|
{
|
||||||
|
"description": "ADDED 2026-11-08. No PR for a chart release the HelmRelease range already admits — Flux adopts it on its own.",
|
||||||
|
"matchManagers": ["flux"],
|
||||||
|
"matchDatasources": ["helm"],
|
||||||
|
"matchCurrentValue": "/^>=\\S+ <\\S+$/",
|
||||||
|
"matchUpdateTypes": ["bump"],
|
||||||
|
"enabled": false
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"description": "ADDED 2026-10-30. velero-plugin-for-aws is pinned to a RELEASE CANDIDATE (v1.14.3-rc.2) and Renovate proposed NOTHING for it, although v1.14.3 and v1.14.4 are stable on Docker Hub. Docker versioning treats everything after the first `-` as a compatibility SUFFIX, so only other `*-rc.2` tags qualify and the RC is a fossil forever. Semver versioning reads it as a prerelease, and a prerelease upgrades to the stable release.",
|
"description": "ADDED 2026-10-30. velero-plugin-for-aws is pinned to a RELEASE CANDIDATE (v1.14.3-rc.2) and Renovate proposed NOTHING for it, although v1.14.3 and v1.14.4 are stable on Docker Hub. Docker versioning treats everything after the first `-` as a compatibility SUFFIX, so only other `*-rc.2` tags qualify and the RC is a fossil forever. Semver versioning reads it as a prerelease, and a prerelease upgrades to the stable release.",
|
||||||
"matchPackageNames": ["velero/velero-plugin-for-aws"],
|
"matchPackageNames": ["velero/velero-plugin-for-aws"],
|
||||||
@@ -441,14 +503,5 @@
|
|||||||
"conformance/tests/**",
|
"conformance/tests/**",
|
||||||
"platform/image-automation/per-service/**",
|
"platform/image-automation/per-service/**",
|
||||||
"platform/kyverno/tests/**"
|
"platform/kyverno/tests/**"
|
||||||
],
|
]
|
||||||
// The `docker` manager (Dockerfiles, docker-compose) and the `kubernetes` manager above were
|
|
||||||
// BOTH previously `enabled: false`, solely so Renovate would not fight Flux image-automation
|
|
||||||
// over a running tag. That controller was never enabled and never will be here (see _comment),
|
|
||||||
// so the only thing those two flags were achieving was leaving every plain `image:` tag in the
|
|
||||||
// repo unmanaged. `docker` is left out of enabledManagers because this repo contains no
|
|
||||||
// Dockerfile that ships to the cluster; `kubernetes` is now ON and is the one that matters.
|
|
||||||
"docker": {
|
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user