diff --git a/apps/wave-0/gitea.yaml b/apps/wave-0/gitea.yaml index 067949e..359b9df 100644 --- a/apps/wave-0/gitea.yaml +++ b/apps/wave-0/gitea.yaml @@ -127,8 +127,8 @@ spec: command: - /bin/sh - -c - - "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"$GITEA_ADMIN_USERNAME@home.local\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\nif [ -z \"${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\n done\n [ \"$disc_ok\" = 1 ] || gitea_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.\n r=0; reg_ok=0\n while [ \"$r\" -lt 10 ]; do\n oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{print $1; exit}')\n if [ -n \"$oidc_id\" ]; then\n su-exec git gitea admin auth update-oauth --id \"$oidc_id\" \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n else\n su-exec git gitea admin auth add-oauth \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name\ - \ groups --admin-group platform-admins >/dev/null 2>&1 || true\n fi\n # VERIFY: the \"keycloak\" source must actually be listed now (guards the exit-0-but-nothing-persisted race).\n if su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{f=1} END{exit !f}'; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\n done\n if [ \"$reg_ok\" = 1 ]; then\n echo \"[gitea-oidc] login source 'keycloak' registered and verified\"\n else\n gitea_oidc_fail \"add/update-oauth ran but the 'keycloak' source did not persist after retries\"\n fi\nfi\n" + - "# Idempotent break-glass admin bootstrap. Runs alongside the gitea entrypoint\n# (which, with INSTALL_LOCK=true, headless-installs + migrates the DB on first\n# boot). Waits for the DB, then creates the admin only if it does not exist.\nset -u\nexport HOME=/data/git\ni=0\nuntil su-exec git gitea admin user list >/dev/null 2>&1; do\n i=$((i + 1))\n [ \"$i\" -ge 150 ] && { echo \"[gitea-bootstrap] DB not ready after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\nif su-exec git gitea admin user list --admin 2>/dev/null | awk 'NR>1{print $2}' | grep -qx \"$GITEA_ADMIN_USERNAME\"; then\n echo \"[gitea-bootstrap] admin '$GITEA_ADMIN_USERNAME' already present\"\nelse\n n=0\n until su-exec git gitea admin user create \\\n --username \"$GITEA_ADMIN_USERNAME\" \\\n --password \"$GITEA_ADMIN_PASSWORD\" \\\n --email \"$GITEA_ADMIN_USERNAME@home.local\" \\\n --admin --must-change-password=false; do\n n=$((n + 1))\n [ \"$n\" -ge 10 ] && { echo \"[gitea-bootstrap] admin create failed after retries\" >&2; break; }\n sleep 3\n done\n echo \"[gitea-bootstrap] created break-glass admin '$GITEA_ADMIN_USERNAME'\"\nfi\n# --- Keycloak OIDC login source (idempotent add-or-update; R-IDP-02/06) ---\n# Registers the confidential `gitea` client as the OAuth2 login source \"keycloak\" against the\n# homelab realm's discovery doc. The `groups` claim maps platform-admins -> Gitea admin\n# (--admin-group). Missing secret => skip (break-glass admin still works), so an unfilled\n# secret never crashes the pod.\n#\n# HARDENED (was a silent first-boot RACE): this step used to fire before the app/network was\n# ready and swallow its failure (|| echo ... >&2, exit 0), so SSO silently never registered.\n# Now it (1) waits \u2014 bounded, ~90s \u2014 for the Keycloak discovery doc to serve HTTP 200,\n# (2) registers (update-in-place if present, else create), (3) VERIFIES the source actually\n# persisted and retries the whole cycle, and (4) on give-up logs LOUDLY to stderr AND a marker\n# file on the persistent /data volume WITHOUT crashlooping the pod (break-glass admin stays\n# usable). The verify-then-retry loop is the real guarantee against \"exit 0 but nothing stuck\".\nGITEA_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\ngitea_oidc_fail() {\n echo \"[gitea-oidc] FAILED to register 'keycloak' login source: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1\"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true\n}\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-0 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not\n# a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS TRUE\n# \u2014 making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on\n# every boot since 8738871 (2026-08-27). SSO kept working only because the login source was\n# already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all.\n# Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator forms by design.\n# `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell.\n# Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why the\n# `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${GITEA_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)\"\nelse\n # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200).\n d=0; disc_ok=0\n while [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$GITEA_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1;\ + \ break\n fi\n d=$((d + 1)); sleep 2\n done\n [ \"$disc_ok\" = 1 ] || gitea_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n # (2)+(3) Register (update-in-place or create) then VERIFY it persisted; retry the whole cycle.\n r=0; reg_ok=0\n while [ \"$r\" -lt 10 ]; do\n oidc_id=$(su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{print $1; exit}')\n if [ -n \"$oidc_id\" ]; then\n su-exec git gitea admin auth update-oauth --id \"$oidc_id\" \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n else\n su-exec git gitea admin auth add-oauth \\\n --name keycloak --provider openidConnect --key gitea \\\n --secret \"$GITEA_OIDC_CLIENT_SECRET\" \\\n --auto-discover-url \"$GITEA_DISCOVERY_URL\" \\\n --scopes \"openid email profile\" --group-claim-name groups --admin-group platform-admins >/dev/null 2>&1 || true\n fi\n # VERIFY: the \"keycloak\" source must actually be listed now (guards the exit-0-but-nothing-persisted race).\n if su-exec git gitea admin auth list 2>/dev/null | awk '$2==\"keycloak\"{f=1} END{exit !f}'; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\n done\n if [ \"$reg_ok\" = 1 ]; then\n echo \"[gitea-oidc] login source 'keycloak' registered and verified\"\n else\n gitea_oidc_fail \"add/update-oauth ran but the 'keycloak' source did not persist after retries\"\n fi\nfi\n" volumes: - name: data persistentVolumeClaim: diff --git a/apps/wave-1/nextcloud.yaml b/apps/wave-1/nextcloud.yaml index 6f68a40..b1a0f3d 100644 --- a/apps/wave-1/nextcloud.yaml +++ b/apps/wave-1/nextcloud.yaml @@ -125,8 +125,9 @@ spec: command: - /bin/sh - -c - - "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n# maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n# stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n i=$((i + 1))\n [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\nif [ -z \"${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"[nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)\"\n exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n# HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n# single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n# form fed the value back through the wrapper's $*, which SPLIT it into extra\ - \ positionals ->\n# occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n# (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n --clientid=nextcloud \\\n --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n --scope='openid email profile' \\\n --unique-uid=0 \\\n --mapping-uid=preferred_username \\\n --mapping-email=email \\\n --mapping-display-name=name \\\n --group-provisioning=1 \\\n --mapping-groups=groups\" >/dev/null 2>&1 || true\n if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n# NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n# login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n# built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n# start: occ group:adduser is a no-op for an existing member. A user is promotable only\n# AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n# rights on their next login + the following pod start (or run `occ group:adduser admin\n# ` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n# passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n current_admins=$(parse_group admin)\n missing=0\n for uid in $(parse_group \"$ADMIN_SRC_GROUP\"); do\n if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n continue\n fi\n missing=1\n if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n fi\n done\n [ \"$missing\" = 0 ] && break\n p=$((p + 1)); sleep 2\ndone\n[ \"$p\" -ge 15 ] && nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\n" + - "# Idempotent Keycloak OIDC wiring (R-IDP-02/06). Runs alongside the nextcloud entrypoint\n# (which installs/upgrades on first boot). Installs+enables the user_oidc app and creates OR\n# updates the \"keycloak\" login provider (occ user_oidc:provider is create-or-update by\n# identifier). Auto-discovery pulls the realm's endpoints; the `groups` claim drives Nextcloud\n# group provisioning. occ runs as www-data (owns config.php); the client secret is passed via\n# --clientsecret-env so it never lands on a command line. Safe to re-run every start; a missing\n# secret or not-yet-installed instance is skipped so the pod never crashloops (the local\n# break-glass admin still works).\nset -u\n# OCC runs occ as www-data (owns config.php). NOTE: this wrapper word-splits its args via $*,\n# so it is used ONLY for single-token subcommands (status/app:list/app:install/...). The\n# provider call below carries a MULTI-WORD value (--scope 'openid email profile') and is run as\n# ONE su -c with the scope single-quoted, so the inner shell keeps it a single argument.\nOCC() { su -s /bin/sh www-data -c \"php /var/www/html/occ $*\"; }\nNEXTCLOUD_DISCOVERY_URL=https://keycloak.home.keithjackson.dev/realms/homelab/.well-known/openid-configuration\nnextcloud_oidc_fail() {\n echo \"[nextcloud-oidc] FAILED to configure 'keycloak' provider: $1\" >&2\n { echo \"$(date -u +%Y-%m-%dT%H:%M:%SZ) [nextcloud-oidc] FAILED: $1\"; } >> /var/www/html/nextcloud-oidc-bootstrap.log 2>/dev/null || true\n}\n# (a) Wait for the instance to be installed AND out of maintenance mode. The entrypoint holds\n# maintenance mode during first-boot install/upgrade; occ writes made in that window do not\n# stick \u2014 a second race on top of the quoting bug fixed at (c).\ni=0\nuntil OCC status 2>/dev/null | grep -q \"installed: true\" && ! OCC status 2>/dev/null | grep -q \"maintenance: true\"; do\n i=$((i + 1))\n [ \"$i\" -ge 300 ] && { echo \"[nextcloud-oidc] occ not ready (installed/maintenance) after wait; skipping\" >&2; exit 0; }\n sleep 2\ndone\n# Let Nextcloud's OWN server-side HTTP client reach the private-network IdP. The Keycloak\n# FQDN resolves (split-horizon) to the PRIVATE Cilium gateway VIP (RFC1918), and Nextcloud's\n# SSRF guard otherwise REFUSES that request \u2014 OCP\\Http\\Client\\LocalServerException \"violates\n# local access rules\" \u2014 during user_oidc's discovery/JWKS/token fetch AT LOGIN, surfacing to\n# the user as \"Could not reach the provider\". (The (b) curl check below uses system libcurl,\n# NOT this guard, so it passes while the browser login still failed.) Idempotent; re-applied\n# every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27.\nOCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \\\n || echo \"[nextcloud-oidc] WARN: failed to set allow_local_remote_servers\" >&2\n# NOTE the `$$` escape below \u2014 it is load-bearing. apps-wave-1 declares postBuild.\n# substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE\n# time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is\n# not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z \"\" ]` \u2014 ALWAYS\n# TRUE \u2014 so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never\n# installing/enabling user_oidc nor registering the provider. SSO kept working only because\n# the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with\n# NO OIDC at all. Flux STRICT substitution cannot catch this \u2014 it EXEMPTS default-operator\n# forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion\n# for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping \u2014 which is why\n# `--clientsecret-env` and the bare uses below always worked and only the guard broke.\n# Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py.\nif [ -z \"$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}\" ]; then\n echo \"\ + [nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)\"\n exit 0\nfi\nif ! OCC app:list 2>/dev/null | grep -q \"user_oidc\"; then\n OCC app:install user_oidc || echo \"[nextcloud-oidc] app:install user_oidc failed\" >&2\nfi\nOCC app:enable user_oidc >/dev/null 2>&1 || true\n# (b) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200) before registering.\nd=0; disc_ok=0\nwhile [ \"$d\" -lt 45 ]; do\n if [ \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \"$NEXTCLOUD_DISCOVERY_URL\" 2>/dev/null)\" = \"200\" ]; then\n disc_ok=1; break\n fi\n d=$((d + 1)); sleep 2\ndone\n[ \"$disc_ok\" = 1 ] || nextcloud_oidc_fail \"Keycloak discovery endpoint not reachable (HTTP 200) within timeout\"\n# (c) Configure (create-or-update by identifier) then VERIFY it persisted; retry the whole cycle.\n# HARDENED: run as ONE su -c so www-data's shell parses --scope='openid email profile' as a\n# single argument. The previous `OCC user_oidc:provider ... --scope=\"openid email profile\"`\n# form fed the value back through the wrapper's $*, which SPLIT it into extra positionals ->\n# occ \"Too many arguments\" -> silent failure on EVERY boot. Secret via --clientsecret-env\n# (never on argv). The verify-then-retry loop guarantees the provider actually persisted.\nr=0; reg_ok=0\nwhile [ \"$r\" -lt 10 ]; do\n su -s /bin/sh www-data -c \"php /var/www/html/occ user_oidc:provider keycloak \\\n --clientid=nextcloud \\\n --clientsecret-env=NEXTCLOUD_OIDC_CLIENT_SECRET \\\n --discoveryuri=$NEXTCLOUD_DISCOVERY_URL \\\n --scope='openid email profile' \\\n --unique-uid=0 \\\n --mapping-uid=preferred_username \\\n --mapping-email=email \\\n --mapping-display-name=name \\\n --group-provisioning=1 \\\n --mapping-groups=groups\" >/dev/null 2>&1 || true\n if OCC user_oidc:provider 2>/dev/null | grep -q \"keycloak\"; then\n reg_ok=1; break\n fi\n r=$((r + 1)); sleep 3\ndone\nif [ \"$reg_ok\" = 1 ]; then\n echo \"[nextcloud-oidc] provider 'keycloak' configured and verified\"\nelse\n nextcloud_oidc_fail \"user_oidc:provider ran but the provider did not persist after retries\"\nfi\n# (d) Promote Keycloak 'platform-admins' members to Nextcloud admins (#5). user_oidc 8.11 has\n# NO native group->admin mapping (occ user_oidc:provider offers group PROVISIONING and a\n# login WHITELIST only), so mirror the provisioned 'platform-admins' membership into the\n# built-in 'admin' group (the sole source of admin rights in Nextcloud). Idempotent every\n# start: occ group:adduser is a no-op for an existing member. A user is promotable only\n# AFTER their first OIDC login provisions them into the group, so a brand-new admin gains\n# rights on their next login + the following pod start (or run `occ group:adduser admin\n# ` once). The LOCAL break-glass admin is unaffected (docs/15). The group name is\n# passed to php as argv (never interpolated) so group:list JSON parses robustly.\nADMIN_SRC_GROUP=platform-admins\n# HARDENED (verify-and-retry, like step (c)): the previous single pass ran `occ group:adduser`\n# exactly once with its output swallowed (>/dev/null 2>&1) and NO retry, so an early-boot\n# transient failure (DB / maintenance race in the first seconds after container start) silently\n# dropped the promotion until the NEXT pod restart. This loop re-reads group:list and re-adds\n# any platform-admins member still missing from 'admin' until all are verified (bounded ~30s);\n# it is a no-op for members already in 'admin'. Group names are passed to php as argv (never\n# interpolated) so the JSON parses robustly.\nparse_group() { su -s /bin/sh www-data -c \"php /var/www/html/occ group:list --output=json\" 2>/dev/null | php -r '$d=json_decode(stream_get_contents(STDIN),true);$g=$argv[1];if(is_array($d)&&!empty($d[$g])&&is_array($d[$g])){foreach($d[$g] as $u){echo $u,\"\\n\";}}' -- \"$1\" 2>/dev/null; }\np=0\nwhile [ \"$p\" -lt 15 ]; do\n current_admins=$(parse_group admin)\n missing=0\n for uid in $(parse_group \"\ + $ADMIN_SRC_GROUP\"); do\n if printf '%s\\n' \"$current_admins\" | grep -qxF -- \"$uid\"; then\n continue\n fi\n missing=1\n if OCC group:adduser admin \"$uid\" >/dev/null 2>&1; then\n echo \"[nextcloud-oidc] ensured '$uid' is a Nextcloud admin (member of $ADMIN_SRC_GROUP)\"\n fi\n done\n [ \"$missing\" = 0 ] && break\n p=$((p + 1)); sleep 2\ndone\n[ \"$p\" -ge 15 ] && nextcloud_oidc_fail \"could not promote all $ADMIN_SRC_GROUP members into 'admin' after retries\"\n" volumes: - name: data persistentVolumeClaim: diff --git a/catalog/README.md b/catalog/README.md index 4b5912a..271de76 100644 --- a/catalog/README.md +++ b/catalog/README.md @@ -28,6 +28,16 @@ inherit every guarantee automatically. `auth.groups`; `native`/`none` are documented exceptions only. - Irreplaceable state (`class: longhorn` configs/DBs) must set `backup.policy` to an off-cluster schedule — never `reacquirable` (R-STATE-03). Only bulk media may be `reacquirable`. +- **Shell braces must be escaped `$${...}`.** Anything rendered verbatim onto the workload — a + `lifecycle` postStart script, an `env` value — lands under `apps/wave-/`, and every + `apps-wave-` Flux Kustomization declares `postBuild.substituteFrom: cluster-vars`. Flux runs + envsubst over the WHOLE manifest at reconcile time, so an unescaped `${VAR:-}` meant for the + *shell* is replaced with the empty string before the container ever starts (this silently killed + gitea's and nextcloud's OIDC registration for days). Bare `$VAR` and `$(cmd)` are left alone and + need no escaping; a genuine environment knob is declared in `clusters/base/cluster-vars.yaml` and + written bare `${KNOB}`. Full contract: [`../clusters/base/apps-vms.yaml`](../clusters/base/apps-vms.yaml). + `render.py` REFUSES to emit a manifest that breaks this rule (Flux's *strict* substitution mode + cannot: it deliberately exempts the `${V:-}` / `${V:=x}` / `${V:?}` default-operator forms). ## Validate before commit — single command, no cluster (R-TEST-02) ```bash diff --git a/catalog/gitea.yaml b/catalog/gitea.yaml index ab07fba..f7a2885 100644 --- a/catalog/gitea.yaml +++ b/catalog/gitea.yaml @@ -109,7 +109,19 @@ lifecycle: echo "[gitea-oidc] FAILED to register 'keycloak' login source: $1" >&2 { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [gitea-oidc] FAILED: $1"; } >> /data/gitea-oidc-bootstrap.log 2>/dev/null || true } - if [ -z "${GITEA_OIDC_CLIENT_SECRET:-}" ]; then + # NOTE the `$$` escape below — it is load-bearing. apps-wave-0 declares postBuild. + # substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE + # time, before any container runs. Written unescaped, `$${GITEA_OIDC_CLIENT_SECRET:-}` is not + # a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z "" ]` — ALWAYS TRUE + # — making every line below (discovery wait, add/update-oauth, verify-then-retry) DEAD CODE on + # every boot since 8738871 (2026-08-27). SSO kept working only because the login source was + # already in the DB; a PVC rebuild or fresh cluster would have come up with NO OIDC at all. + # Flux STRICT substitution cannot catch this — it EXEMPTS default-operator forms by design. + # `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion for the shell. + # Bare `$VAR` and `$(cmd)` are left alone and need NO escaping — which is why the + # `$GITEA_OIDC_CLIENT_SECRET` uses below always worked and only the guards broke. + # Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py. + if [ -z "$${GITEA_OIDC_CLIENT_SECRET:-}" ]; then echo "[gitea-oidc] GITEA_OIDC_CLIENT_SECRET unset; skipping OIDC source (break-glass admin only)" else # (1) Wait up to ~90s for the Keycloak discovery doc to be reachable (HTTP 200). diff --git a/catalog/nextcloud.yaml b/catalog/nextcloud.yaml index f28aba1..3723d89 100644 --- a/catalog/nextcloud.yaml +++ b/catalog/nextcloud.yaml @@ -111,7 +111,19 @@ lifecycle: # every start so it survives pod rolls / PVC rebuilds. Confirmed root cause 2026-08-27. OCC config:system:set allow_local_remote_servers --value=true --type=boolean >/dev/null 2>&1 \ || echo "[nextcloud-oidc] WARN: failed to set allow_local_remote_servers" >&2 - if [ -z "${NEXTCLOUD_OIDC_CLIENT_SECRET:-}" ]; then + # NOTE the `$$` escape below — it is load-bearing. apps-wave-1 declares postBuild. + # substituteFrom cluster-vars, so Flux envsubst rewrites this whole manifest at RECONCILE + # time, before any container runs. Written unescaped, `$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}` is + # not a cluster-var, so envsubst blanked it and the guard reconciled as `[ -z "" ]` — ALWAYS + # TRUE — so this hook `exit 0`d here on EVERY boot since 8738871 (2026-08-27), never + # installing/enabling user_oidc nor registering the provider. SSO kept working only because + # the provider was already in the DB; a PVC rebuild or fresh cluster would have come up with + # NO OIDC at all. Flux STRICT substitution cannot catch this — it EXEMPTS default-operator + # forms by design. `$${VAR}` is the escape: envsubst emits it back as a plain brace expansion + # for the shell. Bare `$VAR` and `$(cmd)` are left alone and need NO escaping — which is why + # `--clientsecret-env` and the bare uses below always worked and only the guard broke. + # Contract: clusters/base/apps-vms.yaml. Enforced at render time by catalog/render.py. + if [ -z "$${NEXTCLOUD_OIDC_CLIENT_SECRET:-}" ]; then echo "[nextcloud-oidc] client secret unset; skipping OIDC (break-glass admin only)" exit 0 fi diff --git a/catalog/render.py b/catalog/render.py index 1873554..992f531 100644 --- a/catalog/render.py +++ b/catalog/render.py @@ -29,6 +29,7 @@ conformance harness stays stdlib-only and only shells out to this script + diffs from __future__ import annotations import argparse +import re import sys from pathlib import Path @@ -936,6 +937,124 @@ def r_virtualmachine(s: dict) -> dict: } +# ============================================================================= +# Substitution safety (R-ENV-02 / R-GITOPS-02) — refuse to emit a manifest that +# Flux's envsubst would silently eat before the workload ever runs. +# ============================================================================= +# Every manifest this renderer writes lands under apps/wave-/, and EVERY apps-wave- +# Flux Kustomization declares `postBuild.substituteFrom: cluster-vars` (see +# flux_wave_kustomization_text below). Flux therefore runs envsubst over the WHOLE rendered +# document — including any shell script carried verbatim in a `lifecycle` hook or an env +# value — at reconcile time, long before a container sees it. The escape contract is stated +# in clusters/base/apps-vms.yaml and proven with `flux envsubst`: +# +# `${VAR}` -> SUBSTITUTED (empty when unset) => a shell brace must be written `$${VAR}`, +# which envsubst emits back as `${VAR}`. +# `$VAR` -> left alone => bare shell vars need NO escaping. +# `$(cmd)` -> left alone => command substitution needs NO escaping. +# `${A[0]}` -> HARD ERROR ("missing closing brace"), failing the whole Kustomization. +# +# This bit us for real (fixed 2026-09-01): gitea/nextcloud guarded their whole Keycloak OIDC +# registration with `if [ -z "${..._OIDC_CLIENT_SECRET:-}" ]`. Neither name is a cluster-var, +# so envsubst rendered the guard as `if [ -z "" ]` — ALWAYS TRUE — and every boot skipped the +# registration entirely. Flux's STRICT substitution mode cannot catch this class: it +# deliberately EXEMPTS the default-operator forms (`${V:-}`, `${V:=x}`, `${V:?}`), which is +# exactly the form a defensive shell script uses. So the renderer checks it itself. +CLUSTER_VARS_PATH = ( + Path(__file__).resolve().parent.parent / "clusters" / "base" / "cluster-vars.yaml" +) + +# A run of `$` followed by `{...}`. envsubst consumes `$$` as a literal `$`, so a run of N +# dollars leaves the brace SUBSTITUTED when N is odd (`${X}`, `$$${X}`) and LITERAL when N is +# even (`$${X}` — the escape). Deliberately greedy about the body so malformed/indexed forms +# (`${A[0]}`) are caught too rather than skipped. +_BRACED_EXPANSION_RE = re.compile(r"(\$+)\{([^{}]*)\}") +_PLAIN_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") + +_CLUSTER_VARS_CACHE: dict = {} + + +def cluster_vars_keys(path: Path | None = None) -> set: + """The declared cluster-vars keys — the ONLY names envsubst may legitimately expand. + + Read from clusters/base/cluster-vars.yaml (the single knob surface, R-ENV-02) so the + renderer never carries a stale hardcoded copy. Unreadable/absent => empty set, i.e. + FAIL-CLOSED: with no known knobs, every braced expansion must be escaped for the shell. + """ + key = str(path or CLUSTER_VARS_PATH) + if key not in _CLUSTER_VARS_CACHE: + try: + doc = yaml.safe_load(Path(key).read_text()) + data = doc.get("data") if isinstance(doc, dict) else None + _CLUSTER_VARS_CACHE[key] = set(data) if isinstance(data, dict) else set() + except Exception: # pragma: no cover - absent/malformed => fail closed + _CLUSTER_VARS_CACHE[key] = set() + return _CLUSTER_VARS_CACHE[key] + + +def substitution_violations(text: str, known: set | None = None) -> list: + """Every unescaped `${...}` in `text` that is NOT a bare, declared cluster-vars key. + + Returns `(line_number, token, reason)` triples in line order. An expansion is ALLOWED only + when it is exactly `${KNOWN_KEY}` — a deliberate R-ENV-02 knob. Anything else is a bug: + a shell expansion that envsubst will eat (`${V:-}`), a typo'd/undeclared knob that + envsubst will blank, or an indexed form that hard-fails the whole Kustomization. + """ + keys = cluster_vars_keys() if known is None else known + out = [] + for lineno, line in enumerate(text.splitlines(), start=1): + for m in _BRACED_EXPANSION_RE.finditer(line): + if len(m.group(1)) % 2 == 0: + continue # even run of `$` => escaped; envsubst emits it back literally + expr = m.group(2) + if _PLAIN_IDENT_RE.match(expr): + if expr in keys: + continue # a declared cluster-vars knob, intentionally substituted + reason = ( + f"'{expr}' is not a declared cluster-vars key — envsubst will replace " + "it with the EMPTY STRING" + ) + elif "[" in expr: + reason = ( + "indexed expansion — envsubst hard-fails with 'missing closing brace' " + "and the whole Kustomization stops reconciling" + ) + else: + reason = ( + "shell expansion (default-operator/parameter form) — envsubst eats it " + "at reconcile time and STRICT mode exempts this form, so nothing else " + "will catch it" + ) + out.append((lineno, m.group(0), reason)) + return out + + +def assert_substitution_safe(text: str, origin: str = "") -> None: + """Refuse to emit a manifest carrying an unescaped braced expansion (see above). + + Raises ValueError listing every offending token. The fix is almost always to escape the + shell brace as `$${...}` (envsubst emits a literal `${...}` for the shell), or — if the + value really is an environment knob — to declare it in clusters/base/cluster-vars.yaml + and write it as a bare `${KNOB}`. + """ + bad = substitution_violations(text) + if not bad: + return + lines = [ + f"{origin}: refusing to emit — unescaped `${{...}}` in a manifest that Flux's " + "postBuild.substituteFrom (cluster-vars) will substitute at reconcile time:" + ] + for lineno, token, reason in bad: + lines.append(f" line {lineno}: {token} -> {reason}") + lines.append( + " FIX: a SHELL expansion must be escaped `$${...}` (envsubst emits a literal " + "`${...}`); bare `$VAR` / `$(cmd)` need no escaping. A genuine environment knob " + "must be declared in clusters/base/cluster-vars.yaml and written bare `${KNOB}`. " + "Contract: clusters/base/apps-vms.yaml." + ) + raise ValueError("\n".join(lines)) + + # ============================================================================= # Top-level render + emit # ============================================================================= @@ -1026,8 +1145,14 @@ def render(sd: dict, image_automation: bool = True, inbound: list | None = None) return docs -def emit(docs: list) -> str: - """Deterministic multi-document YAML text (golden-diffable).""" +def emit(docs: list, origin: str = "") -> str: + """Deterministic multi-document YAML text (golden-diffable). + + Every emitted document is checked against the substitution contract before it is handed + back (assert_substitution_safe): the rendered manifest lands on a Flux path that runs + envsubst over it, so an unescaped `${...}` meant for a SHELL is silently blanked. Raises + ValueError rather than emitting such a manifest. + """ parts = [] for comment, resource in docs: body = yaml.dump( @@ -1037,7 +1162,9 @@ def emit(docs: list) -> str: width=4096, ) parts.append(f"---\n# {comment}\n{body}".rstrip() + "\n") - return "".join(parts) + text = "".join(parts) + assert_substitution_safe(text, origin) + return text def load_sd(path: Path) -> dict: @@ -1076,7 +1203,9 @@ def render_file(path: Path, image_automation: bool = True, catalog_dir: Path | N src_dir = catalog_dir if catalog_dir is not None else path.parent rev = build_reverse_map(_load_catalog(src_dir)) inbound = rev.get(sd["name"], []) - return emit(render(sd, image_automation=image_automation, inbound=inbound)) + return emit( + render(sd, image_automation=image_automation, inbound=inbound), origin=str(path) + ) # ============================================================================= @@ -1729,6 +1858,90 @@ def _selftest() -> int: check(kt["kind"] == "Kustomization" and kt["resources"] == ["svc.yaml", "sonarr.yaml"], "wave kustomization must list exactly the wave's resources") + # 8) substitution safety (R-ENV-02): the renderer must REFUSE to emit a manifest whose + # shell script carries an unescaped `${...}`. Every rendered manifest lands under a + # Flux path with postBuild.substituteFrom cluster-vars, so envsubst rewrites it before + # the container runs — this is the gitea/nextcloud OIDC-guard bug (`[ -z "${X:-}" ]` + # -> `[ -z "" ]`, always true, whole registration block skipped on every boot). + known = {"BASE_DOMAIN", "GATEWAY_VIP"} + # allowed: a bare, DECLARED cluster-vars knob is a deliberate R-ENV-02 substitution. + check( + substitution_violations("host: app.${BASE_DOMAIN}", known) == [], + "a bare declared cluster-vars key must be allowed (that is what substitution is FOR)", + ) + # allowed: escaped shell braces, and the forms envsubst leaves alone. + check( + substitution_violations('if [ -z "$${X:-}" ]; then', known) == [], + "an ESCAPED `$${...}` must be allowed (envsubst emits it back as a literal `${...}`)", + ) + check( + substitution_violations('t="$(mktemp)"; echo "$HOME $1"', known) == [], + "bare `$VAR` / `$(cmd)` are left alone by envsubst and must NOT be flagged", + ) + # `$$` is a literal `$` to envsubst, so an ODD run of dollars still leaves the brace live. + check( + len(substitution_violations("$$${X:-}", known)) == 1, + "an ODD run of `$` still substitutes ($$ -> literal $, then `${X:-}` is eaten)", + ) + # flagged: the exact regression — a default-operator form STRICT mode deliberately exempts. + guard_bug = substitution_violations('if [ -z "${GITEA_OIDC_CLIENT_SECRET:-}" ]; then', known) + check( + len(guard_bug) == 1 and guard_bug[0][1] == "${GITEA_OIDC_CLIENT_SECRET:-}", + "an unescaped `${VAR:-}` shell guard must be flagged (strict mode exempts this form)", + ) + # flagged: an undeclared bare knob (envsubst blanks it), and an indexed form (hard error). + check( + len(substitution_violations("value: ${NOT_A_KNOB}", known)) == 1, + "an undeclared bare `${VAR}` must be flagged (envsubst blanks it silently)", + ) + idx = substitution_violations("echo ${PIPESTATUS[0]}", known) + check( + len(idx) == 1 and "missing closing brace" in idx[0][2], + "an indexed `${A[0]}` must be flagged (envsubst hard-fails the whole Kustomization)", + ) + # the checker must be wired into emit(): rendering an SD whose lifecycle hook carries the + # bug RAISES rather than writing a manifest whose guard is dead on arrival. + bad_sd = { + "name": "subbad", + "source": {"image": "a:1"}, + "lifecycle": { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + 'if [ -z "${SUBBAD_SECRET:-}" ]; then exit 0; fi', + ] + } + } + }, + } + try: + emit(render(bad_sd), origin="") + check(False, "emit must REFUSE a manifest with an unescaped `${...}` shell expansion") + except ValueError as exc: + check( + "SUBBAD_SECRET" in str(exc) and "$${" in str(exc), + "the refusal must name the offending token AND state the `$${...}` fix", + ) + # the ESCAPED form of the same hook renders normally (the fix must be emittable). + good_sd = dict(bad_sd, name="subgood") + good_sd["lifecycle"] = { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + 'if [ -z "$${SUBBAD_SECRET:-}" ]; then exit 0; fi', + ] + } + } + } + check( + "$${SUBBAD_SECRET:-}" in emit(render(good_sd), origin=""), + "the escaped `$${...}` form must render (and reach the shell as `${...}`)", + ) + if failures: for f in failures: sys.stderr.write(f"SELFTEST FAIL: {f}\n")